diff --git "a/multimodal/L2048/CuaS1Decoder_part3.mlmodelc/model.mil" "b/multimodal/L2048/CuaS1Decoder_part3.mlmodelc/model.mil"
new file mode 100644--- /dev/null
+++ "b/multimodal/L2048/CuaS1Decoder_part3.mlmodelc/model.mil"
@@ -0,0 +1,11996 @@
+program(1.3)
+[buildInfo = dict<string, string>({{"coremlc-component-MIL", "3600.16.1"}, {"coremlc-version", "3600.25.2"}, {"coremltools-component-torch", "2.7.0"}, {"coremltools-source-dialect", "TorchScript"}, {"coremltools-version", "9.0"}})]
+{
+    func main<ios18>(tensor<fp16, [2048, 64]> cos, tensor<fp16, [1, 2048, 2560]> hidden, tensor<fp16, [1, 2048]> last_onehot, tensor<fp16, [2048, 64]> sin) {
+            int32 var_37 = const()[name = string("op_37"), val = int32(-2)];
+            int32 var_66 = const()[name = string("op_66"), val = int32(-1)];
+            fp16 var_65_promoted_to_fp16 = const()[name = string("op_65_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_75_cast_fp16 = pow(x = hidden, y = var_65_promoted_to_fp16)[name = string("op_75_cast_fp16")];
+            tensor<int32, [1]> var_77_axes_0 = const()[name = string("op_77_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_77_keep_dims_0 = const()[name = string("op_77_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_77_cast_fp16 = reduce_mean(axes = var_77_axes_0, keep_dims = var_77_keep_dims_0, x = var_75_cast_fp16)[name = string("op_77_cast_fp16")];
+            fp16 var_78_to_fp16 = const()[name = string("op_78_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_79_cast_fp16 = add(x = var_77_cast_fp16, y = var_78_to_fp16)[name = string("op_79_cast_fp16")];
+            fp32 var_80_epsilon_0 = const()[name = string("op_80_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_80_cast_fp16 = rsqrt(epsilon = var_80_epsilon_0, x = var_79_cast_fp16)[name = string("op_80_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_1_cast_fp16 = mul(x = hidden, y = var_80_cast_fp16)[name = string("x_1_cast_fp16")];
+            tensor<fp16, [2560]> var_83_to_fp16 = const()[name = string("op_83_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64)))];
+            tensor<fp16, [1, 2048, 2560]> x_3_cast_fp16 = mul(x = x_1_cast_fp16, y = var_83_to_fp16)[name = string("x_3_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_0_linear_attn_in_proj_qkv_weight_to_fp16 = const()[name = string("layers_0_linear_attn_in_proj_qkv_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(5248)))];
+            tensor<fp16, [8192]> linear_0_bias_0_to_fp16 = const()[name = string("linear_0_bias_0_to_fp16"), val = tensor<fp16, [8192]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(41948352)))];
+            tensor<fp16, [1, 2048, 8192]> linear_0_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_qkv_weight_to_fp16, x = x_3_cast_fp16)[name = string("linear_0_cast_fp16")];
+            tensor<int32, [3]> input_1_perm_0 = const()[name = string("input_1_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_3_pad_0 = const()[name = string("input_3_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_3_mode_0 = const()[name = string("input_3_mode_0"), val = string("constant")];
+            fp16 const_1_to_fp16 = const()[name = string("const_1_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 2048]> input_1_cast_fp16 = transpose(perm = input_1_perm_0, x = linear_0_cast_fp16)[name = string("transpose_55")];
+            tensor<fp16, [1, 8192, 2051]> input_3_cast_fp16 = pad(constant_val = const_1_to_fp16, mode = input_3_mode_0, pad = input_3_pad_0, x = input_1_cast_fp16)[name = string("input_3_cast_fp16")];
+            string input_5_pad_type_0 = const()[name = string("input_5_pad_type_0"), val = string("valid")];
+            int32 input_5_groups_0 = const()[name = string("input_5_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_5_strides_0 = const()[name = string("input_5_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_5_pad_0 = const()[name = string("input_5_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_5_dilations_0 = const()[name = string("input_5_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_0_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_0_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(41964800)))];
+            tensor<fp16, [1, 8192, 2048]> input_5_cast_fp16 = conv(dilations = input_5_dilations_0, groups = input_5_groups_0, pad = input_5_pad_0, pad_type = input_5_pad_type_0, strides = input_5_strides_0, weight = layers_0_linear_attn_conv1d_weight_to_fp16, x = input_3_cast_fp16)[name = string("input_5_cast_fp16")];
+            tensor<fp16, [1, 8192, 2048]> var_118_cast_fp16 = silu(x = input_5_cast_fp16)[name = string("op_118_cast_fp16")];
+            tensor<int32, [3]> qkv_1_perm_0 = const()[name = string("qkv_1_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_120_begin_0 = const()[name = string("op_120_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_120_end_0 = const()[name = string("op_120_end_0"), val = tensor<int32, [3]>([1, 2048, 2048])];
+            tensor<bool, [3]> var_120_end_mask_0 = const()[name = string("op_120_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 8192]> qkv_1_cast_fp16 = transpose(perm = qkv_1_perm_0, x = var_118_cast_fp16)[name = string("transpose_54")];
+            tensor<fp16, [1, 2048, 2048]> var_120_cast_fp16 = slice_by_index(begin = var_120_begin_0, end = var_120_end_0, end_mask = var_120_end_mask_0, x = qkv_1_cast_fp16)[name = string("op_120_cast_fp16")];
+            tensor<int32, [4]> var_121 = const()[name = string("op_121"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> q_1_cast_fp16 = reshape(shape = var_121, x = var_120_cast_fp16)[name = string("q_1_cast_fp16")];
+            tensor<int32, [3]> var_123_begin_0 = const()[name = string("op_123_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_123_end_0 = const()[name = string("op_123_end_0"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<bool, [3]> var_123_end_mask_0 = const()[name = string("op_123_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 2048]> var_123_cast_fp16 = slice_by_index(begin = var_123_begin_0, end = var_123_end_0, end_mask = var_123_end_mask_0, x = qkv_1_cast_fp16)[name = string("op_123_cast_fp16")];
+            tensor<int32, [4]> var_124 = const()[name = string("op_124"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> k_1_cast_fp16 = reshape(shape = var_124, x = var_123_cast_fp16)[name = string("k_1_cast_fp16")];
+            tensor<int32, [3]> var_126_begin_0 = const()[name = string("op_126_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_126_end_0 = const()[name = string("op_126_end_0"), val = tensor<int32, [3]>([1, 2048, 8192])];
+            tensor<bool, [3]> var_126_end_mask_0 = const()[name = string("op_126_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 2048, 4096]> var_126_cast_fp16 = slice_by_index(begin = var_126_begin_0, end = var_126_end_0, end_mask = var_126_end_mask_0, x = qkv_1_cast_fp16)[name = string("op_126_cast_fp16")];
+            tensor<int32, [4]> var_127 = const()[name = string("op_127"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> v_1_cast_fp16 = reshape(shape = var_127, x = var_126_cast_fp16)[name = string("v_1_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_0_linear_attn_in_proj_z_weight_to_fp16 = const()[name = string("layers_0_linear_attn_in_proj_z_weight_to_fp16"), val = tensor<fp16, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(42030400)))];
+            tensor<fp16, [4096]> linear_1_bias_0_to_fp16 = const()[name = string("linear_1_bias_0_to_fp16"), val = tensor<fp16, [4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63001984)))];
+            tensor<fp16, [1, 2048, 4096]> linear_1_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_z_weight_to_fp16, x = x_3_cast_fp16)[name = string("linear_1_cast_fp16")];
+            tensor<int32, [4]> var_131 = const()[name = string("op_131"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> z_1_cast_fp16 = reshape(shape = var_131, x = linear_1_cast_fp16)[name = string("z_1_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_0_linear_attn_in_proj_b_weight_to_fp16 = const()[name = string("layers_0_linear_attn_in_proj_b_weight_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63010240)))];
+            tensor<fp16, [32]> linear_2_bias_0_to_fp16 = const()[name = string("linear_2_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63174144)))];
+            tensor<fp16, [1, 2048, 32]> linear_2_cast_fp16 = linear(bias = linear_2_bias_0_to_fp16, weight = layers_0_linear_attn_in_proj_b_weight_to_fp16, x = x_3_cast_fp16)[name = string("linear_2_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> beta_1_cast_fp16 = sigmoid(x = linear_2_cast_fp16)[name = string("beta_1_cast_fp16")];
+            tensor<fp16, [32, 2560]> var_140_weight_0_to_fp16 = const()[name = string("op_140_weight_0_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63174272)))];
+            tensor<fp16, [32]> var_140_bias_0_to_fp16 = const()[name = string("op_140_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63338176)))];
+            tensor<fp16, [1, 2048, 32]> var_140_cast_fp16 = linear(bias = var_140_bias_0_to_fp16, weight = var_140_weight_0_to_fp16, x = x_3_cast_fp16)[name = string("op_140_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> var_141_cast_fp16 = softplus(x = var_140_cast_fp16)[name = string("op_141_cast_fp16")];
+            tensor<fp16, [32]> var_137_to_fp16 = const()[name = string("op_137_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63338304)))];
+            tensor<fp16, [1, 2048, 32]> g_1_cast_fp16 = mul(x = var_137_to_fp16, y = var_141_cast_fp16)[name = string("g_1_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_143_cast_fp16 = mul(x = q_1_cast_fp16, y = q_1_cast_fp16)[name = string("op_143_cast_fp16")];
+            tensor<int32, [1]> var_145_axes_0 = const()[name = string("op_145_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_145_keep_dims_0 = const()[name = string("op_145_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_145_cast_fp16 = reduce_sum(axes = var_145_axes_0, keep_dims = var_145_keep_dims_0, x = var_143_cast_fp16)[name = string("op_145_cast_fp16")];
+            fp16 var_146_to_fp16 = const()[name = string("op_146_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_147_cast_fp16 = add(x = var_145_cast_fp16, y = var_146_to_fp16)[name = string("op_147_cast_fp16")];
+            fp32 var_148_epsilon_0 = const()[name = string("op_148_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_148_cast_fp16 = rsqrt(epsilon = var_148_epsilon_0, x = var_147_cast_fp16)[name = string("op_148_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> q_3_cast_fp16 = mul(x = q_1_cast_fp16, y = var_148_cast_fp16)[name = string("q_3_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_150_cast_fp16 = mul(x = k_1_cast_fp16, y = k_1_cast_fp16)[name = string("op_150_cast_fp16")];
+            tensor<int32, [1]> var_152_axes_0 = const()[name = string("op_152_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_152_keep_dims_0 = const()[name = string("op_152_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_152_cast_fp16 = reduce_sum(axes = var_152_axes_0, keep_dims = var_152_keep_dims_0, x = var_150_cast_fp16)[name = string("op_152_cast_fp16")];
+            fp16 var_153_to_fp16 = const()[name = string("op_153_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_154_cast_fp16 = add(x = var_152_cast_fp16, y = var_153_to_fp16)[name = string("op_154_cast_fp16")];
+            fp32 var_155_epsilon_0 = const()[name = string("op_155_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_155_cast_fp16 = rsqrt(epsilon = var_155_epsilon_0, x = var_154_cast_fp16)[name = string("op_155_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> k_3_cast_fp16 = mul(x = k_1_cast_fp16, y = var_155_cast_fp16)[name = string("k_3_cast_fp16")];
+            tensor<int32, [1]> var_160_axes_0 = const()[name = string("op_160_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_160_cast_fp16 = expand_dims(axes = var_160_axes_0, x = q_3_cast_fp16)[name = string("op_160_cast_fp16")];
+            tensor<int32, [5]> var_162_reps_0 = const()[name = string("op_162_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_162_cast_fp16 = tile(reps = var_162_reps_0, x = var_160_cast_fp16)[name = string("op_162_cast_fp16")];
+            tensor<int32, [4]> var_163 = const()[name = string("op_163"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> q_5_cast_fp16 = reshape(shape = var_163, x = var_162_cast_fp16)[name = string("q_5_cast_fp16")];
+            tensor<int32, [1]> var_168_axes_0 = const()[name = string("op_168_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_168_cast_fp16 = expand_dims(axes = var_168_axes_0, x = k_3_cast_fp16)[name = string("op_168_cast_fp16")];
+            tensor<int32, [5]> var_170_reps_0 = const()[name = string("op_170_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_170_cast_fp16 = tile(reps = var_170_reps_0, x = var_168_cast_fp16)[name = string("op_170_cast_fp16")];
+            tensor<int32, [4]> var_171 = const()[name = string("op_171"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> k_5_cast_fp16 = reshape(shape = var_171, x = var_170_cast_fp16)[name = string("k_5_cast_fp16")];
+            fp16 var_173_to_fp16 = const()[name = string("op_173_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 2048, 32, 128]> q_7_cast_fp16 = mul(x = q_5_cast_fp16, y = var_173_to_fp16)[name = string("q_7_cast_fp16")];
+            tensor<int32, [4]> var_175_begin_0 = const()[name = string("op_175_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_175_end_0 = const()[name = string("op_175_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_175_end_mask_0 = const()[name = string("op_175_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_175_squeeze_mask_0 = const()[name = string("op_175_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_175_cast_fp16 = slice_by_index(begin = var_175_begin_0, end = var_175_end_0, end_mask = var_175_end_mask_0, squeeze_mask = var_175_squeeze_mask_0, x = q_7_cast_fp16)[name = string("op_175_cast_fp16")];
+            tensor<int32, [3]> var_176 = const()[name = string("op_176"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_178 = const()[name = string("op_178"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_177_cast_fp16 = transpose(perm = var_176, x = var_175_cast_fp16)[name = string("transpose_53")];
+            tensor<fp16, [32, 32, 64, 128]> q_9_cast_fp16 = reshape(shape = var_178, x = var_177_cast_fp16)[name = string("q_9_cast_fp16")];
+            tensor<int32, [4]> var_180_begin_0 = const()[name = string("op_180_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_180_end_0 = const()[name = string("op_180_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_180_end_mask_0 = const()[name = string("op_180_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_180_squeeze_mask_0 = const()[name = string("op_180_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_180_cast_fp16 = slice_by_index(begin = var_180_begin_0, end = var_180_end_0, end_mask = var_180_end_mask_0, squeeze_mask = var_180_squeeze_mask_0, x = k_5_cast_fp16)[name = string("op_180_cast_fp16")];
+            tensor<int32, [3]> var_181 = const()[name = string("op_181"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_183 = const()[name = string("op_183"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_182_cast_fp16 = transpose(perm = var_181, x = var_180_cast_fp16)[name = string("transpose_52")];
+            tensor<fp16, [32, 32, 64, 128]> k_7_cast_fp16 = reshape(shape = var_183, x = var_182_cast_fp16)[name = string("k_7_cast_fp16")];
+            tensor<int32, [4]> var_185_begin_0 = const()[name = string("op_185_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_185_end_0 = const()[name = string("op_185_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_185_end_mask_0 = const()[name = string("op_185_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_185_squeeze_mask_0 = const()[name = string("op_185_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_185_cast_fp16 = slice_by_index(begin = var_185_begin_0, end = var_185_end_0, end_mask = var_185_end_mask_0, squeeze_mask = var_185_squeeze_mask_0, x = v_1_cast_fp16)[name = string("op_185_cast_fp16")];
+            tensor<int32, [3]> var_186 = const()[name = string("op_186"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_188 = const()[name = string("op_188"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_187_cast_fp16 = transpose(perm = var_186, x = var_185_cast_fp16)[name = string("transpose_51")];
+            tensor<fp16, [32, 32, 64, 128]> v_3_cast_fp16 = reshape(shape = var_188, x = var_187_cast_fp16)[name = string("v_3_cast_fp16")];
+            tensor<int32, [3]> var_190_begin_0 = const()[name = string("op_190_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_190_end_0 = const()[name = string("op_190_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_190_end_mask_0 = const()[name = string("op_190_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_190_squeeze_mask_0 = const()[name = string("op_190_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_190_cast_fp16 = slice_by_index(begin = var_190_begin_0, end = var_190_end_0, end_mask = var_190_end_mask_0, squeeze_mask = var_190_squeeze_mask_0, x = beta_1_cast_fp16)[name = string("op_190_cast_fp16")];
+            tensor<int32, [2]> var_191_perm_0 = const()[name = string("op_191_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_192 = const()[name = string("op_192"), val = tensor<int32, [4]>([32, 32, 64, 1])];
+            tensor<fp16, [32, 2048]> var_191_cast_fp16 = transpose(perm = var_191_perm_0, x = var_190_cast_fp16)[name = string("transpose_50")];
+            tensor<fp16, [32, 32, 64, 1]> beta_3_cast_fp16 = reshape(shape = var_192, x = var_191_cast_fp16)[name = string("beta_3_cast_fp16")];
+            tensor<int32, [3]> var_194_begin_0 = const()[name = string("op_194_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_194_end_0 = const()[name = string("op_194_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_194_end_mask_0 = const()[name = string("op_194_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_194_squeeze_mask_0 = const()[name = string("op_194_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_194_cast_fp16 = slice_by_index(begin = var_194_begin_0, end = var_194_end_0, end_mask = var_194_end_mask_0, squeeze_mask = var_194_squeeze_mask_0, x = g_1_cast_fp16)[name = string("op_194_cast_fp16")];
+            tensor<int32, [2]> var_195_perm_0 = const()[name = string("op_195_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_196 = const()[name = string("op_196"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<fp16, [32, 2048]> var_195_cast_fp16 = transpose(perm = var_195_perm_0, x = var_194_cast_fp16)[name = string("transpose_49")];
+            tensor<fp16, [32, 32, 64]> g_3_cast_fp16 = reshape(shape = var_196, x = var_195_cast_fp16)[name = string("g_3_cast_fp16")];
+            tensor<int32, [1]> g_col_1_axes_0 = const()[name = string("g_col_1_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64, 1]> g_col_1_cast_fp16 = expand_dims(axes = g_col_1_axes_0, x = g_3_cast_fp16)[name = string("g_col_1_cast_fp16")];
+            bool var_199_transpose_x_0 = const()[name = string("op_199_transpose_x_0"), val = bool(false)];
+            bool var_199_transpose_y_0 = const()[name = string("op_199_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_lower_incl_to_fp16 = const()[name = string("layers_0_linear_attn_lower_incl_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63338432)))];
+            tensor<fp16, [32, 32, 64, 1]> var_199_cast_fp16 = matmul(transpose_x = var_199_transpose_x_0, transpose_y = var_199_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_1_cast_fp16)[name = string("op_199_cast_fp16")];
+            tensor<int32, [1]> cum_1_axes_0 = const()[name = string("cum_1_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> cum_1_cast_fp16 = squeeze(axes = cum_1_axes_0, x = var_199_cast_fp16)[name = string("cum_1_cast_fp16")];
+            bool var_201_transpose_x_0 = const()[name = string("op_201_transpose_x_0"), val = bool(false)];
+            bool var_201_transpose_y_0 = const()[name = string("op_201_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_suffix_sum_t_to_fp16 = const()[name = string("layers_0_linear_attn_suffix_sum_t_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63346688)))];
+            tensor<fp16, [32, 32, 64, 1]> var_201_cast_fp16 = matmul(transpose_x = var_201_transpose_x_0, transpose_y = var_201_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_1_cast_fp16)[name = string("op_201_cast_fp16")];
+            tensor<int32, [1]> to_end_1_axes_0 = const()[name = string("to_end_1_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> to_end_1_cast_fp16 = squeeze(axes = to_end_1_axes_0, x = var_201_cast_fp16)[name = string("to_end_1_cast_fp16")];
+            bool var_203_transpose_x_0 = const()[name = string("op_203_transpose_x_0"), val = bool(false)];
+            bool var_203_transpose_y_0 = const()[name = string("op_203_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [4096, 64]> layers_0_linear_attn_pair_sum_t_to_fp16 = const()[name = string("layers_0_linear_attn_pair_sum_t_to_fp16"), val = tensor<fp16, [4096, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63354944)))];
+            tensor<fp16, [32, 32, 4096, 1]> var_203_cast_fp16 = matmul(transpose_x = var_203_transpose_x_0, transpose_y = var_203_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_1_cast_fp16)[name = string("op_203_cast_fp16")];
+            tensor<int32, [4]> var_204 = const()[name = string("op_204"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> pair_1_cast_fp16 = reshape(shape = var_204, x = var_203_cast_fp16)[name = string("pair_1_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_206_cast_fp16 = exp(x = pair_1_cast_fp16)[name = string("op_206_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> pair_decay_1_cast_fp16 = mul(x = var_206_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_1_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_beta_1_cast_fp16 = mul(x = k_7_cast_fp16, y = beta_3_cast_fp16)[name = string("k_beta_1_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> v_beta_1_cast_fp16 = mul(x = v_3_cast_fp16, y = beta_3_cast_fp16)[name = string("v_beta_1_cast_fp16")];
+            bool var_211_transpose_x_1 = const()[name = string("op_211_transpose_x_1"), val = bool(false)];
+            bool var_211_transpose_y_1 = const()[name = string("op_211_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_211_cast_fp16 = matmul(transpose_x = var_211_transpose_x_1, transpose_y = var_211_transpose_y_1, x = k_beta_1_cast_fp16, y = k_7_cast_fp16)[name = string("op_211_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> kkt_1_cast_fp16 = mul(x = var_211_cast_fp16, y = pair_decay_1_cast_fp16)[name = string("kkt_1_cast_fp16")];
+            bool var_214_transpose_x_1 = const()[name = string("op_214_transpose_x_1"), val = bool(false)];
+            bool var_214_transpose_y_1 = const()[name = string("op_214_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_214_cast_fp16 = matmul(transpose_x = var_214_transpose_x_1, transpose_y = var_214_transpose_y_1, x = q_9_cast_fp16, y = k_7_cast_fp16)[name = string("op_214_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> attn_intra_1_cast_fp16 = mul(x = var_214_cast_fp16, y = pair_decay_1_cast_fp16)[name = string("attn_intra_1_cast_fp16")];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_strict_lower_to_fp16 = const()[name = string("layers_0_linear_attn_strict_lower_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63879296)))];
+            tensor<fp16, [32, 32, 64, 64]> var_216_cast_fp16 = mul(x = kkt_1_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_216_cast_fp16")];
+            tensor<fp16, [64, 64]> layers_0_linear_attn_eye_to_fp16 = const()[name = string("layers_0_linear_attn_eye_to_fp16"), val = tensor<fp16, [64, 64]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63887552)))];
+            tensor<fp16, [32, 32, 64, 64]> t_1_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_216_cast_fp16)[name = string("t_1_cast_fp16")];
+            tensor<int32, [3]> var_220 = const()[name = string("op_220"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [1024, 64, 64]> t_3_cast_fp16 = reshape(shape = var_220, x = t_1_cast_fp16)[name = string("t_3_cast_fp16")];
+            tensor<int32, [5]> var_225 = const()[name = string("op_225"), val = tensor<int32, [5]>([1024, 32, 2, 32, 2])];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_226_cast_fp16 = reshape(shape = var_225, x = t_3_cast_fp16)[name = string("op_226_cast_fp16")];
+            tensor<fp16, [32, 1, 32, 1]> var_230_to_fp16 = const()[name = string("op_230_to_fp16"), val = tensor<fp16, [32, 1, 32, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63895808)))];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_231_cast_fp16 = mul(x = var_226_cast_fp16, y = var_230_to_fp16)[name = string("op_231_cast_fp16")];
+            tensor<int32, [1]> blocks_1_axes_0 = const()[name = string("blocks_1_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_1_keep_dims_0 = const()[name = string("blocks_1_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> blocks_1_cast_fp16 = reduce_sum(axes = blocks_1_axes_0, keep_dims = blocks_1_keep_dims_0, x = var_231_cast_fp16)[name = string("blocks_1_cast_fp16")];
+            tensor<int32, [4]> var_234_begin_0 = const()[name = string("op_234_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_234_end_0 = const()[name = string("op_234_end_0"), val = tensor<int32, [4]>([1024, 32, 2, 2])];
+            tensor<bool, [4]> var_234_end_mask_0 = const()[name = string("op_234_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 32, 1, 2]> var_234_cast_fp16 = slice_by_index(begin = var_234_begin_0, end = var_234_end_0, end_mask = var_234_end_mask_0, x = blocks_1_cast_fp16)[name = string("op_234_cast_fp16")];
+            tensor<int32, [4]> x_5_begin_0 = const()[name = string("x_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_5_end_0 = const()[name = string("x_5_end_0"), val = tensor<int32, [4]>([1024, 32, 1, 1])];
+            tensor<bool, [4]> x_5_end_mask_0 = const()[name = string("x_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 32, 1, 1]> x_5_cast_fp16 = slice_by_index(begin = x_5_begin_0, end = x_5_end_0, end_mask = x_5_end_mask_0, x = var_234_cast_fp16)[name = string("x_5_cast_fp16")];
+            bool var_244_transpose_x_0 = const()[name = string("op_244_transpose_x_0"), val = bool(false)];
+            bool var_244_transpose_y_0 = const()[name = string("op_244_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_238_to_fp16 = const()[name = string("op_238_to_fp16"), val = tensor<fp16, [1024, 32, 1, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63897920)))];
+            tensor<fp16, [1024, 32, 1, 1]> var_244_cast_fp16 = matmul(transpose_x = var_244_transpose_x_0, transpose_y = var_244_transpose_y_0, x = x_5_cast_fp16, y = var_238_to_fp16)[name = string("op_244_cast_fp16")];
+            bool var_245_transpose_x_0 = const()[name = string("op_245_transpose_x_0"), val = bool(false)];
+            bool var_245_transpose_y_0 = const()[name = string("op_245_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_245_cast_fp16 = matmul(transpose_x = var_245_transpose_x_0, transpose_y = var_245_transpose_y_0, x = var_238_to_fp16, y = var_244_cast_fp16)[name = string("op_245_cast_fp16")];
+            fp16 const_6_promoted_to_fp16 = const()[name = string("const_6_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 32, 1, 1]> lower_1_cast_fp16 = mul(x = var_245_cast_fp16, y = const_6_promoted_to_fp16)[name = string("lower_1_cast_fp16")];
+            bool bottom_1_interleave_0 = const()[name = string("bottom_1_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> bottom_1_cast_fp16 = concat(axis = var_66, interleave = bottom_1_interleave_0, values = (lower_1_cast_fp16, var_238_to_fp16))[name = string("bottom_1_cast_fp16")];
+            bool inv_3_interleave_0 = const()[name = string("inv_3_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> top_1_to_fp16 = const()[name = string("top_1_to_fp16"), val = tensor<fp16, [1024, 32, 1, 2]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(63963520)))];
+            tensor<fp16, [1024, 32, 2, 2]> inv_3_cast_fp16 = concat(axis = var_37, interleave = inv_3_interleave_0, values = (top_1_to_fp16, bottom_1_cast_fp16))[name = string("inv_3_cast_fp16")];
+            tensor<int32, [5]> var_254 = const()[name = string("op_254"), val = tensor<int32, [5]>([1024, 16, 4, 16, 4])];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_255_cast_fp16 = reshape(shape = var_254, x = t_3_cast_fp16)[name = string("op_255_cast_fp16")];
+            tensor<fp16, [16, 1, 16, 1]> var_259_to_fp16 = const()[name = string("op_259_to_fp16"), val = tensor<fp16, [16, 1, 16, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64094656)))];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_260_cast_fp16 = mul(x = var_255_cast_fp16, y = var_259_to_fp16)[name = string("op_260_cast_fp16")];
+            tensor<int32, [1]> blocks_3_axes_0 = const()[name = string("blocks_3_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_3_keep_dims_0 = const()[name = string("blocks_3_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> blocks_3_cast_fp16 = reduce_sum(axes = blocks_3_axes_0, keep_dims = blocks_3_keep_dims_0, x = var_260_cast_fp16)[name = string("blocks_3_cast_fp16")];
+            tensor<int32, [4]> var_263_begin_0 = const()[name = string("op_263_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_263_end_0 = const()[name = string("op_263_end_0"), val = tensor<int32, [4]>([1024, 16, 4, 4])];
+            tensor<bool, [4]> var_263_end_mask_0 = const()[name = string("op_263_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 16, 2, 4]> var_263_cast_fp16 = slice_by_index(begin = var_263_begin_0, end = var_263_end_0, end_mask = var_263_end_mask_0, x = blocks_3_cast_fp16)[name = string("op_263_cast_fp16")];
+            tensor<int32, [4]> x_7_begin_0 = const()[name = string("x_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_7_end_0 = const()[name = string("x_7_end_0"), val = tensor<int32, [4]>([1024, 16, 2, 2])];
+            tensor<bool, [4]> x_7_end_mask_0 = const()[name = string("x_7_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 16, 2, 2]> x_7_cast_fp16 = slice_by_index(begin = x_7_begin_0, end = x_7_end_0, end_mask = x_7_end_mask_0, x = var_263_cast_fp16)[name = string("x_7_cast_fp16")];
+            tensor<int32, [5]> var_265 = const()[name = string("op_265"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<fp16, [1024, 16, 2, 2, 2]> pair_5_cast_fp16 = reshape(shape = var_265, x = inv_3_cast_fp16)[name = string("pair_5_cast_fp16")];
+            tensor<int32, [5]> var_267_begin_0 = const()[name = string("op_267_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_267_end_0 = const()[name = string("op_267_end_0"), val = tensor<int32, [5]>([1024, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_267_end_mask_0 = const()[name = string("op_267_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_267_squeeze_mask_0 = const()[name = string("op_267_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_267_cast_fp16 = slice_by_index(begin = var_267_begin_0, end = var_267_end_0, end_mask = var_267_end_mask_0, squeeze_mask = var_267_squeeze_mask_0, x = pair_5_cast_fp16)[name = string("op_267_cast_fp16")];
+            tensor<int32, [5]> var_270_begin_0 = const()[name = string("op_270_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_270_end_0 = const()[name = string("op_270_end_0"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_270_end_mask_0 = const()[name = string("op_270_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_270_squeeze_mask_0 = const()[name = string("op_270_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_270_cast_fp16 = slice_by_index(begin = var_270_begin_0, end = var_270_end_0, end_mask = var_270_end_mask_0, squeeze_mask = var_270_squeeze_mask_0, x = pair_5_cast_fp16)[name = string("op_270_cast_fp16")];
+            bool var_273_transpose_x_0 = const()[name = string("op_273_transpose_x_0"), val = bool(false)];
+            bool var_273_transpose_y_0 = const()[name = string("op_273_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_273_cast_fp16 = matmul(transpose_x = var_273_transpose_x_0, transpose_y = var_273_transpose_y_0, x = x_7_cast_fp16, y = var_267_cast_fp16)[name = string("op_273_cast_fp16")];
+            bool var_274_transpose_x_0 = const()[name = string("op_274_transpose_x_0"), val = bool(false)];
+            bool var_274_transpose_y_0 = const()[name = string("op_274_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_274_cast_fp16 = matmul(transpose_x = var_274_transpose_x_0, transpose_y = var_274_transpose_y_0, x = var_270_cast_fp16, y = var_273_cast_fp16)[name = string("op_274_cast_fp16")];
+            fp16 const_7_promoted_to_fp16 = const()[name = string("const_7_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 16, 2, 2]> lower_3_cast_fp16 = mul(x = var_274_cast_fp16, y = const_7_promoted_to_fp16)[name = string("lower_3_cast_fp16")];
+            bool top_3_interleave_0 = const()[name = string("top_3_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_276_to_fp16 = const()[name = string("op_276_to_fp16"), val = tensor<fp16, [1024, 16, 2, 2]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64095232)))];
+            tensor<fp16, [1024, 16, 2, 4]> top_3_cast_fp16 = concat(axis = var_66, interleave = top_3_interleave_0, values = (var_267_cast_fp16, var_276_to_fp16))[name = string("top_3_cast_fp16")];
+            bool bottom_3_interleave_0 = const()[name = string("bottom_3_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> bottom_3_cast_fp16 = concat(axis = var_66, interleave = bottom_3_interleave_0, values = (lower_3_cast_fp16, var_270_cast_fp16))[name = string("bottom_3_cast_fp16")];
+            bool inv_5_interleave_0 = const()[name = string("inv_5_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> inv_5_cast_fp16 = concat(axis = var_37, interleave = inv_5_interleave_0, values = (top_3_cast_fp16, bottom_3_cast_fp16))[name = string("inv_5_cast_fp16")];
+            tensor<int32, [5]> var_283 = const()[name = string("op_283"), val = tensor<int32, [5]>([1024, 8, 8, 8, 8])];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_284_cast_fp16 = reshape(shape = var_283, x = t_3_cast_fp16)[name = string("op_284_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_288_to_fp16 = const()[name = string("op_288_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64226368)))];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_289_cast_fp16 = mul(x = var_284_cast_fp16, y = var_288_to_fp16)[name = string("op_289_cast_fp16")];
+            tensor<int32, [1]> blocks_5_axes_0 = const()[name = string("blocks_5_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_5_keep_dims_0 = const()[name = string("blocks_5_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> blocks_5_cast_fp16 = reduce_sum(axes = blocks_5_axes_0, keep_dims = blocks_5_keep_dims_0, x = var_289_cast_fp16)[name = string("blocks_5_cast_fp16")];
+            tensor<int32, [4]> var_292_begin_0 = const()[name = string("op_292_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_292_end_0 = const()[name = string("op_292_end_0"), val = tensor<int32, [4]>([1024, 8, 8, 8])];
+            tensor<bool, [4]> var_292_end_mask_0 = const()[name = string("op_292_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 8, 4, 8]> var_292_cast_fp16 = slice_by_index(begin = var_292_begin_0, end = var_292_end_0, end_mask = var_292_end_mask_0, x = blocks_5_cast_fp16)[name = string("op_292_cast_fp16")];
+            tensor<int32, [4]> x_9_begin_0 = const()[name = string("x_9_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_9_end_0 = const()[name = string("x_9_end_0"), val = tensor<int32, [4]>([1024, 8, 4, 4])];
+            tensor<bool, [4]> x_9_end_mask_0 = const()[name = string("x_9_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 8, 4, 4]> x_9_cast_fp16 = slice_by_index(begin = x_9_begin_0, end = x_9_end_0, end_mask = x_9_end_mask_0, x = var_292_cast_fp16)[name = string("x_9_cast_fp16")];
+            tensor<int32, [5]> var_294 = const()[name = string("op_294"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<fp16, [1024, 8, 2, 4, 4]> pair_7_cast_fp16 = reshape(shape = var_294, x = inv_5_cast_fp16)[name = string("pair_7_cast_fp16")];
+            tensor<int32, [5]> var_296_begin_0 = const()[name = string("op_296_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_296_end_0 = const()[name = string("op_296_end_0"), val = tensor<int32, [5]>([1024, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_296_end_mask_0 = const()[name = string("op_296_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_296_squeeze_mask_0 = const()[name = string("op_296_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_296_cast_fp16 = slice_by_index(begin = var_296_begin_0, end = var_296_end_0, end_mask = var_296_end_mask_0, squeeze_mask = var_296_squeeze_mask_0, x = pair_7_cast_fp16)[name = string("op_296_cast_fp16")];
+            tensor<int32, [5]> var_299_begin_0 = const()[name = string("op_299_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_299_end_0 = const()[name = string("op_299_end_0"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_299_end_mask_0 = const()[name = string("op_299_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_299_squeeze_mask_0 = const()[name = string("op_299_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_299_cast_fp16 = slice_by_index(begin = var_299_begin_0, end = var_299_end_0, end_mask = var_299_end_mask_0, squeeze_mask = var_299_squeeze_mask_0, x = pair_7_cast_fp16)[name = string("op_299_cast_fp16")];
+            bool var_302_transpose_x_0 = const()[name = string("op_302_transpose_x_0"), val = bool(false)];
+            bool var_302_transpose_y_0 = const()[name = string("op_302_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_302_cast_fp16 = matmul(transpose_x = var_302_transpose_x_0, transpose_y = var_302_transpose_y_0, x = x_9_cast_fp16, y = var_296_cast_fp16)[name = string("op_302_cast_fp16")];
+            bool var_303_transpose_x_0 = const()[name = string("op_303_transpose_x_0"), val = bool(false)];
+            bool var_303_transpose_y_0 = const()[name = string("op_303_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_303_cast_fp16 = matmul(transpose_x = var_303_transpose_x_0, transpose_y = var_303_transpose_y_0, x = var_299_cast_fp16, y = var_302_cast_fp16)[name = string("op_303_cast_fp16")];
+            fp16 const_8_promoted_to_fp16 = const()[name = string("const_8_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 8, 4, 4]> lower_5_cast_fp16 = mul(x = var_303_cast_fp16, y = const_8_promoted_to_fp16)[name = string("lower_5_cast_fp16")];
+            bool top_5_interleave_0 = const()[name = string("top_5_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_305_to_fp16 = const()[name = string("op_305_to_fp16"), val = tensor<fp16, [1024, 8, 4, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64226560)))];
+            tensor<fp16, [1024, 8, 4, 8]> top_5_cast_fp16 = concat(axis = var_66, interleave = top_5_interleave_0, values = (var_296_cast_fp16, var_305_to_fp16))[name = string("top_5_cast_fp16")];
+            bool bottom_5_interleave_0 = const()[name = string("bottom_5_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> bottom_5_cast_fp16 = concat(axis = var_66, interleave = bottom_5_interleave_0, values = (lower_5_cast_fp16, var_299_cast_fp16))[name = string("bottom_5_cast_fp16")];
+            bool inv_7_interleave_0 = const()[name = string("inv_7_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> inv_7_cast_fp16 = concat(axis = var_37, interleave = inv_7_interleave_0, values = (top_5_cast_fp16, bottom_5_cast_fp16))[name = string("inv_7_cast_fp16")];
+            tensor<int32, [5]> var_312 = const()[name = string("op_312"), val = tensor<int32, [5]>([1024, 4, 16, 4, 16])];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_313_cast_fp16 = reshape(shape = var_312, x = t_3_cast_fp16)[name = string("op_313_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_317_to_fp16 = const()[name = string("op_317_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64488768)))];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_318_cast_fp16 = mul(x = var_313_cast_fp16, y = var_317_to_fp16)[name = string("op_318_cast_fp16")];
+            tensor<int32, [1]> blocks_7_axes_0 = const()[name = string("blocks_7_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_7_keep_dims_0 = const()[name = string("blocks_7_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> blocks_7_cast_fp16 = reduce_sum(axes = blocks_7_axes_0, keep_dims = blocks_7_keep_dims_0, x = var_318_cast_fp16)[name = string("blocks_7_cast_fp16")];
+            tensor<int32, [4]> var_321_begin_0 = const()[name = string("op_321_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_321_end_0 = const()[name = string("op_321_end_0"), val = tensor<int32, [4]>([1024, 4, 16, 16])];
+            tensor<bool, [4]> var_321_end_mask_0 = const()[name = string("op_321_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 4, 8, 16]> var_321_cast_fp16 = slice_by_index(begin = var_321_begin_0, end = var_321_end_0, end_mask = var_321_end_mask_0, x = blocks_7_cast_fp16)[name = string("op_321_cast_fp16")];
+            tensor<int32, [4]> x_11_begin_0 = const()[name = string("x_11_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_11_end_0 = const()[name = string("x_11_end_0"), val = tensor<int32, [4]>([1024, 4, 8, 8])];
+            tensor<bool, [4]> x_11_end_mask_0 = const()[name = string("x_11_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 4, 8, 8]> x_11_cast_fp16 = slice_by_index(begin = x_11_begin_0, end = x_11_end_0, end_mask = x_11_end_mask_0, x = var_321_cast_fp16)[name = string("x_11_cast_fp16")];
+            tensor<int32, [5]> var_323 = const()[name = string("op_323"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<fp16, [1024, 4, 2, 8, 8]> pair_9_cast_fp16 = reshape(shape = var_323, x = inv_7_cast_fp16)[name = string("pair_9_cast_fp16")];
+            tensor<int32, [5]> var_325_begin_0 = const()[name = string("op_325_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_325_end_0 = const()[name = string("op_325_end_0"), val = tensor<int32, [5]>([1024, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_325_end_mask_0 = const()[name = string("op_325_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_325_squeeze_mask_0 = const()[name = string("op_325_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_325_cast_fp16 = slice_by_index(begin = var_325_begin_0, end = var_325_end_0, end_mask = var_325_end_mask_0, squeeze_mask = var_325_squeeze_mask_0, x = pair_9_cast_fp16)[name = string("op_325_cast_fp16")];
+            tensor<int32, [5]> var_328_begin_0 = const()[name = string("op_328_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_328_end_0 = const()[name = string("op_328_end_0"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_328_end_mask_0 = const()[name = string("op_328_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_328_squeeze_mask_0 = const()[name = string("op_328_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_328_cast_fp16 = slice_by_index(begin = var_328_begin_0, end = var_328_end_0, end_mask = var_328_end_mask_0, squeeze_mask = var_328_squeeze_mask_0, x = pair_9_cast_fp16)[name = string("op_328_cast_fp16")];
+            bool var_331_transpose_x_0 = const()[name = string("op_331_transpose_x_0"), val = bool(false)];
+            bool var_331_transpose_y_0 = const()[name = string("op_331_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_331_cast_fp16 = matmul(transpose_x = var_331_transpose_x_0, transpose_y = var_331_transpose_y_0, x = x_11_cast_fp16, y = var_325_cast_fp16)[name = string("op_331_cast_fp16")];
+            bool var_332_transpose_x_0 = const()[name = string("op_332_transpose_x_0"), val = bool(false)];
+            bool var_332_transpose_y_0 = const()[name = string("op_332_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_332_cast_fp16 = matmul(transpose_x = var_332_transpose_x_0, transpose_y = var_332_transpose_y_0, x = var_328_cast_fp16, y = var_331_cast_fp16)[name = string("op_332_cast_fp16")];
+            fp16 const_9_promoted_to_fp16 = const()[name = string("const_9_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 4, 8, 8]> lower_7_cast_fp16 = mul(x = var_332_cast_fp16, y = const_9_promoted_to_fp16)[name = string("lower_7_cast_fp16")];
+            bool top_7_interleave_0 = const()[name = string("top_7_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_334_to_fp16 = const()[name = string("op_334_to_fp16"), val = tensor<fp16, [1024, 4, 8, 8]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(64488896)))];
+            tensor<fp16, [1024, 4, 8, 16]> top_7_cast_fp16 = concat(axis = var_66, interleave = top_7_interleave_0, values = (var_325_cast_fp16, var_334_to_fp16))[name = string("top_7_cast_fp16")];
+            bool bottom_7_interleave_0 = const()[name = string("bottom_7_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> bottom_7_cast_fp16 = concat(axis = var_66, interleave = bottom_7_interleave_0, values = (lower_7_cast_fp16, var_328_cast_fp16))[name = string("bottom_7_cast_fp16")];
+            bool inv_9_interleave_0 = const()[name = string("inv_9_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> inv_9_cast_fp16 = concat(axis = var_37, interleave = inv_9_interleave_0, values = (top_7_cast_fp16, bottom_7_cast_fp16))[name = string("inv_9_cast_fp16")];
+            tensor<int32, [5]> var_341 = const()[name = string("op_341"), val = tensor<int32, [5]>([1024, 2, 32, 2, 32])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_342_cast_fp16 = reshape(shape = var_341, x = t_3_cast_fp16)[name = string("op_342_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_346_to_fp16 = const()[name = string("op_346_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_347_cast_fp16 = mul(x = var_342_cast_fp16, y = var_346_to_fp16)[name = string("op_347_cast_fp16")];
+            tensor<int32, [1]> blocks_9_axes_0 = const()[name = string("blocks_9_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_9_keep_dims_0 = const()[name = string("blocks_9_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> blocks_9_cast_fp16 = reduce_sum(axes = blocks_9_axes_0, keep_dims = blocks_9_keep_dims_0, x = var_347_cast_fp16)[name = string("blocks_9_cast_fp16")];
+            tensor<int32, [4]> var_350_begin_0 = const()[name = string("op_350_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_350_end_0 = const()[name = string("op_350_end_0"), val = tensor<int32, [4]>([1024, 2, 32, 32])];
+            tensor<bool, [4]> var_350_end_mask_0 = const()[name = string("op_350_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 2, 16, 32]> var_350_cast_fp16 = slice_by_index(begin = var_350_begin_0, end = var_350_end_0, end_mask = var_350_end_mask_0, x = blocks_9_cast_fp16)[name = string("op_350_cast_fp16")];
+            tensor<int32, [4]> x_13_begin_0 = const()[name = string("x_13_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_13_end_0 = const()[name = string("x_13_end_0"), val = tensor<int32, [4]>([1024, 2, 16, 16])];
+            tensor<bool, [4]> x_13_end_mask_0 = const()[name = string("x_13_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 2, 16, 16]> x_13_cast_fp16 = slice_by_index(begin = x_13_begin_0, end = x_13_end_0, end_mask = x_13_end_mask_0, x = var_350_cast_fp16)[name = string("x_13_cast_fp16")];
+            tensor<int32, [5]> var_352 = const()[name = string("op_352"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<fp16, [1024, 2, 2, 16, 16]> pair_11_cast_fp16 = reshape(shape = var_352, x = inv_9_cast_fp16)[name = string("pair_11_cast_fp16")];
+            tensor<int32, [5]> var_354_begin_0 = const()[name = string("op_354_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_354_end_0 = const()[name = string("op_354_end_0"), val = tensor<int32, [5]>([1024, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_354_end_mask_0 = const()[name = string("op_354_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_354_squeeze_mask_0 = const()[name = string("op_354_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_354_cast_fp16 = slice_by_index(begin = var_354_begin_0, end = var_354_end_0, end_mask = var_354_end_mask_0, squeeze_mask = var_354_squeeze_mask_0, x = pair_11_cast_fp16)[name = string("op_354_cast_fp16")];
+            tensor<int32, [5]> var_357_begin_0 = const()[name = string("op_357_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_357_end_0 = const()[name = string("op_357_end_0"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_357_end_mask_0 = const()[name = string("op_357_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_357_squeeze_mask_0 = const()[name = string("op_357_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_357_cast_fp16 = slice_by_index(begin = var_357_begin_0, end = var_357_end_0, end_mask = var_357_end_mask_0, squeeze_mask = var_357_squeeze_mask_0, x = pair_11_cast_fp16)[name = string("op_357_cast_fp16")];
+            bool var_360_transpose_x_0 = const()[name = string("op_360_transpose_x_0"), val = bool(false)];
+            bool var_360_transpose_y_0 = const()[name = string("op_360_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_360_cast_fp16 = matmul(transpose_x = var_360_transpose_x_0, transpose_y = var_360_transpose_y_0, x = x_13_cast_fp16, y = var_354_cast_fp16)[name = string("op_360_cast_fp16")];
+            bool var_361_transpose_x_0 = const()[name = string("op_361_transpose_x_0"), val = bool(false)];
+            bool var_361_transpose_y_0 = const()[name = string("op_361_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_361_cast_fp16 = matmul(transpose_x = var_361_transpose_x_0, transpose_y = var_361_transpose_y_0, x = var_357_cast_fp16, y = var_360_cast_fp16)[name = string("op_361_cast_fp16")];
+            fp16 const_10_promoted_to_fp16 = const()[name = string("const_10_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 2, 16, 16]> lower_9_cast_fp16 = mul(x = var_361_cast_fp16, y = const_10_promoted_to_fp16)[name = string("lower_9_cast_fp16")];
+            bool top_9_interleave_0 = const()[name = string("top_9_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_363_to_fp16 = const()[name = string("op_363_to_fp16"), val = tensor<fp16, [1024, 2, 16, 16]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(65013248)))];
+            tensor<fp16, [1024, 2, 16, 32]> top_9_cast_fp16 = concat(axis = var_66, interleave = top_9_interleave_0, values = (var_354_cast_fp16, var_363_to_fp16))[name = string("top_9_cast_fp16")];
+            bool bottom_9_interleave_0 = const()[name = string("bottom_9_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> bottom_9_cast_fp16 = concat(axis = var_66, interleave = bottom_9_interleave_0, values = (lower_9_cast_fp16, var_357_cast_fp16))[name = string("bottom_9_cast_fp16")];
+            bool inv_11_interleave_0 = const()[name = string("inv_11_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> inv_11_cast_fp16 = concat(axis = var_37, interleave = inv_11_interleave_0, values = (top_9_cast_fp16, bottom_9_cast_fp16))[name = string("inv_11_cast_fp16")];
+            tensor<int32, [5]> var_370 = const()[name = string("op_370"), val = tensor<int32, [5]>([1024, 1, 64, 1, 64])];
+            tensor<fp16, [1024, 1, 64, 1, 64]> var_371_cast_fp16 = reshape(shape = var_370, x = t_3_cast_fp16)[name = string("op_371_cast_fp16")];
+            tensor<int32, [1]> blocks_11_axes_0 = const()[name = string("blocks_11_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_11_keep_dims_0 = const()[name = string("blocks_11_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> blocks_11_cast_fp16 = reduce_sum(axes = blocks_11_axes_0, keep_dims = blocks_11_keep_dims_0, x = var_371_cast_fp16)[name = string("blocks_11_cast_fp16")];
+            tensor<int32, [4]> var_379_begin_0 = const()[name = string("op_379_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_379_end_0 = const()[name = string("op_379_end_0"), val = tensor<int32, [4]>([1024, 1, 64, 64])];
+            tensor<bool, [4]> var_379_end_mask_0 = const()[name = string("op_379_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 1, 32, 64]> var_379_cast_fp16 = slice_by_index(begin = var_379_begin_0, end = var_379_end_0, end_mask = var_379_end_mask_0, x = blocks_11_cast_fp16)[name = string("op_379_cast_fp16")];
+            tensor<int32, [4]> x_15_begin_0 = const()[name = string("x_15_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_15_end_0 = const()[name = string("x_15_end_0"), val = tensor<int32, [4]>([1024, 1, 32, 32])];
+            tensor<bool, [4]> x_15_end_mask_0 = const()[name = string("x_15_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 1, 32, 32]> x_15_cast_fp16 = slice_by_index(begin = x_15_begin_0, end = x_15_end_0, end_mask = x_15_end_mask_0, x = var_379_cast_fp16)[name = string("x_15_cast_fp16")];
+            tensor<int32, [5]> var_381 = const()[name = string("op_381"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<fp16, [1024, 1, 2, 32, 32]> pair_13_cast_fp16 = reshape(shape = var_381, x = inv_11_cast_fp16)[name = string("pair_13_cast_fp16")];
+            tensor<int32, [5]> var_383_begin_0 = const()[name = string("op_383_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_383_end_0 = const()[name = string("op_383_end_0"), val = tensor<int32, [5]>([1024, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_383_end_mask_0 = const()[name = string("op_383_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_383_squeeze_mask_0 = const()[name = string("op_383_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_383_cast_fp16 = slice_by_index(begin = var_383_begin_0, end = var_383_end_0, end_mask = var_383_end_mask_0, squeeze_mask = var_383_squeeze_mask_0, x = pair_13_cast_fp16)[name = string("op_383_cast_fp16")];
+            tensor<int32, [5]> var_386_begin_0 = const()[name = string("op_386_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_386_end_0 = const()[name = string("op_386_end_0"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_386_end_mask_0 = const()[name = string("op_386_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_386_squeeze_mask_0 = const()[name = string("op_386_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_386_cast_fp16 = slice_by_index(begin = var_386_begin_0, end = var_386_end_0, end_mask = var_386_end_mask_0, squeeze_mask = var_386_squeeze_mask_0, x = pair_13_cast_fp16)[name = string("op_386_cast_fp16")];
+            bool var_389_transpose_x_0 = const()[name = string("op_389_transpose_x_0"), val = bool(false)];
+            bool var_389_transpose_y_0 = const()[name = string("op_389_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_389_cast_fp16 = matmul(transpose_x = var_389_transpose_x_0, transpose_y = var_389_transpose_y_0, x = x_15_cast_fp16, y = var_383_cast_fp16)[name = string("op_389_cast_fp16")];
+            bool var_390_transpose_x_0 = const()[name = string("op_390_transpose_x_0"), val = bool(false)];
+            bool var_390_transpose_y_0 = const()[name = string("op_390_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_390_cast_fp16 = matmul(transpose_x = var_390_transpose_x_0, transpose_y = var_390_transpose_y_0, x = var_386_cast_fp16, y = var_389_cast_fp16)[name = string("op_390_cast_fp16")];
+            fp16 const_11_promoted_to_fp16 = const()[name = string("const_11_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 1, 32, 32]> lower_11_cast_fp16 = mul(x = var_390_cast_fp16, y = const_11_promoted_to_fp16)[name = string("lower_11_cast_fp16")];
+            bool top_11_interleave_0 = const()[name = string("top_11_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_392_to_fp16 = const()[name = string("op_392_to_fp16"), val = tensor<fp16, [1024, 1, 32, 32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(66061888)))];
+            tensor<fp16, [1024, 1, 32, 64]> top_11_cast_fp16 = concat(axis = var_66, interleave = top_11_interleave_0, values = (var_383_cast_fp16, var_392_to_fp16))[name = string("top_11_cast_fp16")];
+            bool bottom_11_interleave_0 = const()[name = string("bottom_11_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> bottom_11_cast_fp16 = concat(axis = var_66, interleave = bottom_11_interleave_0, values = (lower_11_cast_fp16, var_386_cast_fp16))[name = string("bottom_11_cast_fp16")];
+            bool inv_13_interleave_0 = const()[name = string("inv_13_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> inv_13_cast_fp16 = concat(axis = var_37, interleave = inv_13_interleave_0, values = (top_11_cast_fp16, bottom_11_cast_fp16))[name = string("inv_13_cast_fp16")];
+            tensor<int32, [4]> var_399 = const()[name = string("op_399"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> inv_15_cast_fp16 = reshape(shape = var_399, x = inv_13_cast_fp16)[name = string("inv_15_cast_fp16")];
+            bool new_v_1_transpose_x_0 = const()[name = string("new_v_1_transpose_x_0"), val = bool(false)];
+            bool new_v_1_transpose_y_0 = const()[name = string("new_v_1_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> new_v_1_cast_fp16 = matmul(transpose_x = new_v_1_transpose_x_0, transpose_y = new_v_1_transpose_y_0, x = inv_15_cast_fp16, y = v_beta_1_cast_fp16)[name = string("new_v_1_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_402_cast_fp16 = exp(x = cum_1_cast_fp16)[name = string("op_402_cast_fp16")];
+            tensor<int32, [1]> var_403_axes_0 = const()[name = string("op_403_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_403_cast_fp16 = expand_dims(axes = var_403_axes_0, x = var_402_cast_fp16)[name = string("op_403_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> var_404_cast_fp16 = mul(x = k_beta_1_cast_fp16, y = var_403_cast_fp16)[name = string("op_404_cast_fp16")];
+            bool k_cumdecay_1_transpose_x_0 = const()[name = string("k_cumdecay_1_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_1_transpose_y_0 = const()[name = string("k_cumdecay_1_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> k_cumdecay_1_cast_fp16 = matmul(transpose_x = k_cumdecay_1_transpose_x_0, transpose_y = k_cumdecay_1_transpose_y_0, x = inv_15_cast_fp16, y = var_404_cast_fp16)[name = string("k_cumdecay_1_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> q_dec_1_cast_fp16 = mul(x = q_9_cast_fp16, y = var_403_cast_fp16)[name = string("q_dec_1_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_409_cast_fp16 = exp(x = to_end_1_cast_fp16)[name = string("op_409_cast_fp16")];
+            tensor<int32, [1]> var_410_axes_0 = const()[name = string("op_410_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_410_cast_fp16 = expand_dims(axes = var_410_axes_0, x = var_409_cast_fp16)[name = string("op_410_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_dec_1_cast_fp16 = mul(x = k_7_cast_fp16, y = var_410_cast_fp16)[name = string("k_dec_1_cast_fp16")];
+            tensor<int32, [3]> var_412_begin_0 = const()[name = string("op_412_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_412_end_0 = const()[name = string("op_412_end_0"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<bool, [3]> var_412_end_mask_0 = const()[name = string("op_412_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 32, 1]> var_412_cast_fp16 = slice_by_index(begin = var_412_begin_0, end = var_412_end_0, end_mask = var_412_end_mask_0, x = cum_1_cast_fp16)[name = string("op_412_cast_fp16")];
+            tensor<fp16, [32, 32, 1]> var_413_cast_fp16 = exp(x = var_412_cast_fp16)[name = string("op_413_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_1_axes_0 = const()[name = string("chunk_decay_1_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 1, 1]> chunk_decay_1_cast_fp16 = expand_dims(axes = chunk_decay_1_axes_0, x = var_413_cast_fp16)[name = string("chunk_decay_1_cast_fp16")];
+            tensor<int32, [4]> var_418_begin_0 = const()[name = string("op_418_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_418_end_0 = const()[name = string("op_418_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_418_end_mask_0 = const()[name = string("op_418_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_418_squeeze_mask_0 = const()[name = string("op_418_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_418_cast_fp16 = slice_by_index(begin = var_418_begin_0, end = var_418_end_0, end_mask = var_418_end_mask_0, squeeze_mask = var_418_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_418_cast_fp16")];
+            tensor<int32, [4]> var_420_begin_0 = const()[name = string("op_420_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_420_end_0 = const()[name = string("op_420_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_420_end_mask_0 = const()[name = string("op_420_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_420_squeeze_mask_0 = const()[name = string("op_420_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_420_cast_fp16 = slice_by_index(begin = var_420_begin_0, end = var_420_end_0, end_mask = var_420_end_mask_0, squeeze_mask = var_420_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_420_cast_fp16")];
+            bool var_421_transpose_x_0 = const()[name = string("op_421_transpose_x_0"), val = bool(false)];
+            bool var_421_transpose_y_0 = const()[name = string("op_421_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> state_1_to_fp16 = const()[name = string("state_1_to_fp16"), val = tensor<fp16, [32, 128, 128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(68159104)))];
+            tensor<fp16, [32, 64, 128]> var_421_cast_fp16 = matmul(transpose_x = var_421_transpose_x_0, transpose_y = var_421_transpose_y_0, x = var_420_cast_fp16, y = state_1_to_fp16)[name = string("op_421_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_1_cast_fp16 = sub(x = var_418_cast_fp16, y = var_421_cast_fp16)[name = string("v_new_1_cast_fp16")];
+            tensor<int32, [4]> var_424_begin_0 = const()[name = string("op_424_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_424_end_0 = const()[name = string("op_424_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_424_end_mask_0 = const()[name = string("op_424_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_424_squeeze_mask_0 = const()[name = string("op_424_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_424_cast_fp16 = slice_by_index(begin = var_424_begin_0, end = var_424_end_0, end_mask = var_424_end_mask_0, squeeze_mask = var_424_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_424_cast_fp16")];
+            bool var_425_transpose_x_0 = const()[name = string("op_425_transpose_x_0"), val = bool(false)];
+            bool var_425_transpose_y_0 = const()[name = string("op_425_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_425_cast_fp16 = matmul(transpose_x = var_425_transpose_x_0, transpose_y = var_425_transpose_y_0, x = var_424_cast_fp16, y = state_1_to_fp16)[name = string("op_425_cast_fp16")];
+            tensor<int32, [4]> var_427_begin_0 = const()[name = string("op_427_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_427_end_0 = const()[name = string("op_427_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_427_end_mask_0 = const()[name = string("op_427_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_427_squeeze_mask_0 = const()[name = string("op_427_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_427_cast_fp16 = slice_by_index(begin = var_427_begin_0, end = var_427_end_0, end_mask = var_427_end_mask_0, squeeze_mask = var_427_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_427_cast_fp16")];
+            bool var_428_transpose_x_0 = const()[name = string("op_428_transpose_x_0"), val = bool(false)];
+            bool var_428_transpose_y_0 = const()[name = string("op_428_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_428_cast_fp16 = matmul(transpose_x = var_428_transpose_x_0, transpose_y = var_428_transpose_y_0, x = var_427_cast_fp16, y = v_new_1_cast_fp16)[name = string("op_428_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_429_cast_fp16 = add(x = var_425_cast_fp16, y = var_428_cast_fp16)[name = string("op_429_cast_fp16")];
+            tensor<int32, [4]> var_431_begin_0 = const()[name = string("op_431_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_431_end_0 = const()[name = string("op_431_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_431_end_mask_0 = const()[name = string("op_431_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_431_squeeze_mask_0 = const()[name = string("op_431_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_431_cast_fp16 = slice_by_index(begin = var_431_begin_0, end = var_431_end_0, end_mask = var_431_end_mask_0, squeeze_mask = var_431_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_431_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_432_cast_fp16 = mul(x = state_1_to_fp16, y = var_431_cast_fp16)[name = string("op_432_cast_fp16")];
+            tensor<int32, [4]> var_434_begin_0 = const()[name = string("op_434_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_434_end_0 = const()[name = string("op_434_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_434_end_mask_0 = const()[name = string("op_434_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_434_squeeze_mask_0 = const()[name = string("op_434_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_434_cast_fp16 = slice_by_index(begin = var_434_begin_0, end = var_434_end_0, end_mask = var_434_end_mask_0, squeeze_mask = var_434_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_434_cast_fp16")];
+            bool var_436_transpose_x_1 = const()[name = string("op_436_transpose_x_1"), val = bool(true)];
+            bool var_436_transpose_y_1 = const()[name = string("op_436_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_436_cast_fp16 = matmul(transpose_x = var_436_transpose_x_1, transpose_y = var_436_transpose_y_1, x = var_434_cast_fp16, y = v_new_1_cast_fp16)[name = string("op_436_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_3_cast_fp16 = add(x = var_432_cast_fp16, y = var_436_cast_fp16)[name = string("state_3_cast_fp16")];
+            tensor<int32, [4]> var_439_begin_0 = const()[name = string("op_439_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_439_end_0 = const()[name = string("op_439_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_439_end_mask_0 = const()[name = string("op_439_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_439_squeeze_mask_0 = const()[name = string("op_439_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_439_cast_fp16 = slice_by_index(begin = var_439_begin_0, end = var_439_end_0, end_mask = var_439_end_mask_0, squeeze_mask = var_439_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_439_cast_fp16")];
+            tensor<int32, [4]> var_441_begin_0 = const()[name = string("op_441_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_441_end_0 = const()[name = string("op_441_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_441_end_mask_0 = const()[name = string("op_441_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_441_squeeze_mask_0 = const()[name = string("op_441_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_441_cast_fp16 = slice_by_index(begin = var_441_begin_0, end = var_441_end_0, end_mask = var_441_end_mask_0, squeeze_mask = var_441_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_441_cast_fp16")];
+            bool var_442_transpose_x_0 = const()[name = string("op_442_transpose_x_0"), val = bool(false)];
+            bool var_442_transpose_y_0 = const()[name = string("op_442_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_442_cast_fp16 = matmul(transpose_x = var_442_transpose_x_0, transpose_y = var_442_transpose_y_0, x = var_441_cast_fp16, y = state_3_cast_fp16)[name = string("op_442_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_3_cast_fp16 = sub(x = var_439_cast_fp16, y = var_442_cast_fp16)[name = string("v_new_3_cast_fp16")];
+            tensor<int32, [4]> var_445_begin_0 = const()[name = string("op_445_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_445_end_0 = const()[name = string("op_445_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_445_end_mask_0 = const()[name = string("op_445_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_445_squeeze_mask_0 = const()[name = string("op_445_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_445_cast_fp16 = slice_by_index(begin = var_445_begin_0, end = var_445_end_0, end_mask = var_445_end_mask_0, squeeze_mask = var_445_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_445_cast_fp16")];
+            bool var_446_transpose_x_0 = const()[name = string("op_446_transpose_x_0"), val = bool(false)];
+            bool var_446_transpose_y_0 = const()[name = string("op_446_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_446_cast_fp16 = matmul(transpose_x = var_446_transpose_x_0, transpose_y = var_446_transpose_y_0, x = var_445_cast_fp16, y = state_3_cast_fp16)[name = string("op_446_cast_fp16")];
+            tensor<int32, [4]> var_448_begin_0 = const()[name = string("op_448_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_448_end_0 = const()[name = string("op_448_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_448_end_mask_0 = const()[name = string("op_448_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_448_squeeze_mask_0 = const()[name = string("op_448_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_448_cast_fp16 = slice_by_index(begin = var_448_begin_0, end = var_448_end_0, end_mask = var_448_end_mask_0, squeeze_mask = var_448_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_448_cast_fp16")];
+            bool var_449_transpose_x_0 = const()[name = string("op_449_transpose_x_0"), val = bool(false)];
+            bool var_449_transpose_y_0 = const()[name = string("op_449_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_449_cast_fp16 = matmul(transpose_x = var_449_transpose_x_0, transpose_y = var_449_transpose_y_0, x = var_448_cast_fp16, y = v_new_3_cast_fp16)[name = string("op_449_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_450_cast_fp16 = add(x = var_446_cast_fp16, y = var_449_cast_fp16)[name = string("op_450_cast_fp16")];
+            tensor<int32, [4]> var_452_begin_0 = const()[name = string("op_452_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_452_end_0 = const()[name = string("op_452_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_452_end_mask_0 = const()[name = string("op_452_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_452_squeeze_mask_0 = const()[name = string("op_452_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_452_cast_fp16 = slice_by_index(begin = var_452_begin_0, end = var_452_end_0, end_mask = var_452_end_mask_0, squeeze_mask = var_452_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_452_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_453_cast_fp16 = mul(x = state_3_cast_fp16, y = var_452_cast_fp16)[name = string("op_453_cast_fp16")];
+            tensor<int32, [4]> var_455_begin_0 = const()[name = string("op_455_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_455_end_0 = const()[name = string("op_455_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_455_end_mask_0 = const()[name = string("op_455_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_455_squeeze_mask_0 = const()[name = string("op_455_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_455_cast_fp16 = slice_by_index(begin = var_455_begin_0, end = var_455_end_0, end_mask = var_455_end_mask_0, squeeze_mask = var_455_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_455_cast_fp16")];
+            bool var_457_transpose_x_1 = const()[name = string("op_457_transpose_x_1"), val = bool(true)];
+            bool var_457_transpose_y_1 = const()[name = string("op_457_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_457_cast_fp16 = matmul(transpose_x = var_457_transpose_x_1, transpose_y = var_457_transpose_y_1, x = var_455_cast_fp16, y = v_new_3_cast_fp16)[name = string("op_457_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_5_cast_fp16 = add(x = var_453_cast_fp16, y = var_457_cast_fp16)[name = string("state_5_cast_fp16")];
+            tensor<int32, [4]> var_460_begin_0 = const()[name = string("op_460_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_460_end_0 = const()[name = string("op_460_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_460_end_mask_0 = const()[name = string("op_460_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_460_squeeze_mask_0 = const()[name = string("op_460_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_460_cast_fp16 = slice_by_index(begin = var_460_begin_0, end = var_460_end_0, end_mask = var_460_end_mask_0, squeeze_mask = var_460_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_460_cast_fp16")];
+            tensor<int32, [4]> var_462_begin_0 = const()[name = string("op_462_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_462_end_0 = const()[name = string("op_462_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_462_end_mask_0 = const()[name = string("op_462_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_462_squeeze_mask_0 = const()[name = string("op_462_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_462_cast_fp16 = slice_by_index(begin = var_462_begin_0, end = var_462_end_0, end_mask = var_462_end_mask_0, squeeze_mask = var_462_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_462_cast_fp16")];
+            bool var_463_transpose_x_0 = const()[name = string("op_463_transpose_x_0"), val = bool(false)];
+            bool var_463_transpose_y_0 = const()[name = string("op_463_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_463_cast_fp16 = matmul(transpose_x = var_463_transpose_x_0, transpose_y = var_463_transpose_y_0, x = var_462_cast_fp16, y = state_5_cast_fp16)[name = string("op_463_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_5_cast_fp16 = sub(x = var_460_cast_fp16, y = var_463_cast_fp16)[name = string("v_new_5_cast_fp16")];
+            tensor<int32, [4]> var_466_begin_0 = const()[name = string("op_466_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_466_end_0 = const()[name = string("op_466_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_466_end_mask_0 = const()[name = string("op_466_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_466_squeeze_mask_0 = const()[name = string("op_466_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_466_cast_fp16 = slice_by_index(begin = var_466_begin_0, end = var_466_end_0, end_mask = var_466_end_mask_0, squeeze_mask = var_466_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_466_cast_fp16")];
+            bool var_467_transpose_x_0 = const()[name = string("op_467_transpose_x_0"), val = bool(false)];
+            bool var_467_transpose_y_0 = const()[name = string("op_467_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_467_cast_fp16 = matmul(transpose_x = var_467_transpose_x_0, transpose_y = var_467_transpose_y_0, x = var_466_cast_fp16, y = state_5_cast_fp16)[name = string("op_467_cast_fp16")];
+            tensor<int32, [4]> var_469_begin_0 = const()[name = string("op_469_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_469_end_0 = const()[name = string("op_469_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_469_end_mask_0 = const()[name = string("op_469_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_469_squeeze_mask_0 = const()[name = string("op_469_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_469_cast_fp16 = slice_by_index(begin = var_469_begin_0, end = var_469_end_0, end_mask = var_469_end_mask_0, squeeze_mask = var_469_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_469_cast_fp16")];
+            bool var_470_transpose_x_0 = const()[name = string("op_470_transpose_x_0"), val = bool(false)];
+            bool var_470_transpose_y_0 = const()[name = string("op_470_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_470_cast_fp16 = matmul(transpose_x = var_470_transpose_x_0, transpose_y = var_470_transpose_y_0, x = var_469_cast_fp16, y = v_new_5_cast_fp16)[name = string("op_470_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_471_cast_fp16 = add(x = var_467_cast_fp16, y = var_470_cast_fp16)[name = string("op_471_cast_fp16")];
+            tensor<int32, [4]> var_473_begin_0 = const()[name = string("op_473_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_473_end_0 = const()[name = string("op_473_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_473_end_mask_0 = const()[name = string("op_473_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_473_squeeze_mask_0 = const()[name = string("op_473_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_473_cast_fp16 = slice_by_index(begin = var_473_begin_0, end = var_473_end_0, end_mask = var_473_end_mask_0, squeeze_mask = var_473_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_473_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_474_cast_fp16 = mul(x = state_5_cast_fp16, y = var_473_cast_fp16)[name = string("op_474_cast_fp16")];
+            tensor<int32, [4]> var_476_begin_0 = const()[name = string("op_476_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_476_end_0 = const()[name = string("op_476_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_476_end_mask_0 = const()[name = string("op_476_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_476_squeeze_mask_0 = const()[name = string("op_476_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_476_cast_fp16 = slice_by_index(begin = var_476_begin_0, end = var_476_end_0, end_mask = var_476_end_mask_0, squeeze_mask = var_476_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_476_cast_fp16")];
+            bool var_478_transpose_x_1 = const()[name = string("op_478_transpose_x_1"), val = bool(true)];
+            bool var_478_transpose_y_1 = const()[name = string("op_478_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_478_cast_fp16 = matmul(transpose_x = var_478_transpose_x_1, transpose_y = var_478_transpose_y_1, x = var_476_cast_fp16, y = v_new_5_cast_fp16)[name = string("op_478_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_7_cast_fp16 = add(x = var_474_cast_fp16, y = var_478_cast_fp16)[name = string("state_7_cast_fp16")];
+            tensor<int32, [4]> var_481_begin_0 = const()[name = string("op_481_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_481_end_0 = const()[name = string("op_481_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_481_end_mask_0 = const()[name = string("op_481_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_481_squeeze_mask_0 = const()[name = string("op_481_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_481_cast_fp16 = slice_by_index(begin = var_481_begin_0, end = var_481_end_0, end_mask = var_481_end_mask_0, squeeze_mask = var_481_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_481_cast_fp16")];
+            tensor<int32, [4]> var_483_begin_0 = const()[name = string("op_483_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_483_end_0 = const()[name = string("op_483_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_483_end_mask_0 = const()[name = string("op_483_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_483_squeeze_mask_0 = const()[name = string("op_483_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_483_cast_fp16 = slice_by_index(begin = var_483_begin_0, end = var_483_end_0, end_mask = var_483_end_mask_0, squeeze_mask = var_483_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_483_cast_fp16")];
+            bool var_484_transpose_x_0 = const()[name = string("op_484_transpose_x_0"), val = bool(false)];
+            bool var_484_transpose_y_0 = const()[name = string("op_484_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_484_cast_fp16 = matmul(transpose_x = var_484_transpose_x_0, transpose_y = var_484_transpose_y_0, x = var_483_cast_fp16, y = state_7_cast_fp16)[name = string("op_484_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_7_cast_fp16 = sub(x = var_481_cast_fp16, y = var_484_cast_fp16)[name = string("v_new_7_cast_fp16")];
+            tensor<int32, [4]> var_487_begin_0 = const()[name = string("op_487_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_487_end_0 = const()[name = string("op_487_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_487_end_mask_0 = const()[name = string("op_487_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_487_squeeze_mask_0 = const()[name = string("op_487_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_487_cast_fp16 = slice_by_index(begin = var_487_begin_0, end = var_487_end_0, end_mask = var_487_end_mask_0, squeeze_mask = var_487_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_487_cast_fp16")];
+            bool var_488_transpose_x_0 = const()[name = string("op_488_transpose_x_0"), val = bool(false)];
+            bool var_488_transpose_y_0 = const()[name = string("op_488_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_488_cast_fp16 = matmul(transpose_x = var_488_transpose_x_0, transpose_y = var_488_transpose_y_0, x = var_487_cast_fp16, y = state_7_cast_fp16)[name = string("op_488_cast_fp16")];
+            tensor<int32, [4]> var_490_begin_0 = const()[name = string("op_490_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_490_end_0 = const()[name = string("op_490_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_490_end_mask_0 = const()[name = string("op_490_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_490_squeeze_mask_0 = const()[name = string("op_490_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_490_cast_fp16 = slice_by_index(begin = var_490_begin_0, end = var_490_end_0, end_mask = var_490_end_mask_0, squeeze_mask = var_490_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_490_cast_fp16")];
+            bool var_491_transpose_x_0 = const()[name = string("op_491_transpose_x_0"), val = bool(false)];
+            bool var_491_transpose_y_0 = const()[name = string("op_491_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_491_cast_fp16 = matmul(transpose_x = var_491_transpose_x_0, transpose_y = var_491_transpose_y_0, x = var_490_cast_fp16, y = v_new_7_cast_fp16)[name = string("op_491_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_492_cast_fp16 = add(x = var_488_cast_fp16, y = var_491_cast_fp16)[name = string("op_492_cast_fp16")];
+            tensor<int32, [4]> var_494_begin_0 = const()[name = string("op_494_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_494_end_0 = const()[name = string("op_494_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_494_end_mask_0 = const()[name = string("op_494_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_494_squeeze_mask_0 = const()[name = string("op_494_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_494_cast_fp16 = slice_by_index(begin = var_494_begin_0, end = var_494_end_0, end_mask = var_494_end_mask_0, squeeze_mask = var_494_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_494_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_495_cast_fp16 = mul(x = state_7_cast_fp16, y = var_494_cast_fp16)[name = string("op_495_cast_fp16")];
+            tensor<int32, [4]> var_497_begin_0 = const()[name = string("op_497_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_497_end_0 = const()[name = string("op_497_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_497_end_mask_0 = const()[name = string("op_497_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_497_squeeze_mask_0 = const()[name = string("op_497_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_497_cast_fp16 = slice_by_index(begin = var_497_begin_0, end = var_497_end_0, end_mask = var_497_end_mask_0, squeeze_mask = var_497_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_497_cast_fp16")];
+            bool var_499_transpose_x_1 = const()[name = string("op_499_transpose_x_1"), val = bool(true)];
+            bool var_499_transpose_y_1 = const()[name = string("op_499_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_499_cast_fp16 = matmul(transpose_x = var_499_transpose_x_1, transpose_y = var_499_transpose_y_1, x = var_497_cast_fp16, y = v_new_7_cast_fp16)[name = string("op_499_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_9_cast_fp16 = add(x = var_495_cast_fp16, y = var_499_cast_fp16)[name = string("state_9_cast_fp16")];
+            tensor<int32, [4]> var_502_begin_0 = const()[name = string("op_502_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_502_end_0 = const()[name = string("op_502_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_502_end_mask_0 = const()[name = string("op_502_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_502_squeeze_mask_0 = const()[name = string("op_502_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_502_cast_fp16 = slice_by_index(begin = var_502_begin_0, end = var_502_end_0, end_mask = var_502_end_mask_0, squeeze_mask = var_502_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_502_cast_fp16")];
+            tensor<int32, [4]> var_504_begin_0 = const()[name = string("op_504_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_504_end_0 = const()[name = string("op_504_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_504_end_mask_0 = const()[name = string("op_504_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_504_squeeze_mask_0 = const()[name = string("op_504_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_504_cast_fp16 = slice_by_index(begin = var_504_begin_0, end = var_504_end_0, end_mask = var_504_end_mask_0, squeeze_mask = var_504_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_504_cast_fp16")];
+            bool var_505_transpose_x_0 = const()[name = string("op_505_transpose_x_0"), val = bool(false)];
+            bool var_505_transpose_y_0 = const()[name = string("op_505_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_505_cast_fp16 = matmul(transpose_x = var_505_transpose_x_0, transpose_y = var_505_transpose_y_0, x = var_504_cast_fp16, y = state_9_cast_fp16)[name = string("op_505_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_9_cast_fp16 = sub(x = var_502_cast_fp16, y = var_505_cast_fp16)[name = string("v_new_9_cast_fp16")];
+            tensor<int32, [4]> var_508_begin_0 = const()[name = string("op_508_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_508_end_0 = const()[name = string("op_508_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_508_end_mask_0 = const()[name = string("op_508_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_508_squeeze_mask_0 = const()[name = string("op_508_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_508_cast_fp16 = slice_by_index(begin = var_508_begin_0, end = var_508_end_0, end_mask = var_508_end_mask_0, squeeze_mask = var_508_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_508_cast_fp16")];
+            bool var_509_transpose_x_0 = const()[name = string("op_509_transpose_x_0"), val = bool(false)];
+            bool var_509_transpose_y_0 = const()[name = string("op_509_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_509_cast_fp16 = matmul(transpose_x = var_509_transpose_x_0, transpose_y = var_509_transpose_y_0, x = var_508_cast_fp16, y = state_9_cast_fp16)[name = string("op_509_cast_fp16")];
+            tensor<int32, [4]> var_511_begin_0 = const()[name = string("op_511_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_511_end_0 = const()[name = string("op_511_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_511_end_mask_0 = const()[name = string("op_511_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_511_squeeze_mask_0 = const()[name = string("op_511_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_511_cast_fp16 = slice_by_index(begin = var_511_begin_0, end = var_511_end_0, end_mask = var_511_end_mask_0, squeeze_mask = var_511_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_511_cast_fp16")];
+            bool var_512_transpose_x_0 = const()[name = string("op_512_transpose_x_0"), val = bool(false)];
+            bool var_512_transpose_y_0 = const()[name = string("op_512_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_512_cast_fp16 = matmul(transpose_x = var_512_transpose_x_0, transpose_y = var_512_transpose_y_0, x = var_511_cast_fp16, y = v_new_9_cast_fp16)[name = string("op_512_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_513_cast_fp16 = add(x = var_509_cast_fp16, y = var_512_cast_fp16)[name = string("op_513_cast_fp16")];
+            tensor<int32, [4]> var_515_begin_0 = const()[name = string("op_515_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_515_end_0 = const()[name = string("op_515_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_515_end_mask_0 = const()[name = string("op_515_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_515_squeeze_mask_0 = const()[name = string("op_515_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_515_cast_fp16 = slice_by_index(begin = var_515_begin_0, end = var_515_end_0, end_mask = var_515_end_mask_0, squeeze_mask = var_515_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_515_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_516_cast_fp16 = mul(x = state_9_cast_fp16, y = var_515_cast_fp16)[name = string("op_516_cast_fp16")];
+            tensor<int32, [4]> var_518_begin_0 = const()[name = string("op_518_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_518_end_0 = const()[name = string("op_518_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_518_end_mask_0 = const()[name = string("op_518_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_518_squeeze_mask_0 = const()[name = string("op_518_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_518_cast_fp16 = slice_by_index(begin = var_518_begin_0, end = var_518_end_0, end_mask = var_518_end_mask_0, squeeze_mask = var_518_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_518_cast_fp16")];
+            bool var_520_transpose_x_1 = const()[name = string("op_520_transpose_x_1"), val = bool(true)];
+            bool var_520_transpose_y_1 = const()[name = string("op_520_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_520_cast_fp16 = matmul(transpose_x = var_520_transpose_x_1, transpose_y = var_520_transpose_y_1, x = var_518_cast_fp16, y = v_new_9_cast_fp16)[name = string("op_520_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_11_cast_fp16 = add(x = var_516_cast_fp16, y = var_520_cast_fp16)[name = string("state_11_cast_fp16")];
+            tensor<int32, [4]> var_523_begin_0 = const()[name = string("op_523_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_523_end_0 = const()[name = string("op_523_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_523_end_mask_0 = const()[name = string("op_523_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_523_squeeze_mask_0 = const()[name = string("op_523_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_523_cast_fp16 = slice_by_index(begin = var_523_begin_0, end = var_523_end_0, end_mask = var_523_end_mask_0, squeeze_mask = var_523_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_523_cast_fp16")];
+            tensor<int32, [4]> var_525_begin_0 = const()[name = string("op_525_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_525_end_0 = const()[name = string("op_525_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_525_end_mask_0 = const()[name = string("op_525_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_525_squeeze_mask_0 = const()[name = string("op_525_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_525_cast_fp16 = slice_by_index(begin = var_525_begin_0, end = var_525_end_0, end_mask = var_525_end_mask_0, squeeze_mask = var_525_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_525_cast_fp16")];
+            bool var_526_transpose_x_0 = const()[name = string("op_526_transpose_x_0"), val = bool(false)];
+            bool var_526_transpose_y_0 = const()[name = string("op_526_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_526_cast_fp16 = matmul(transpose_x = var_526_transpose_x_0, transpose_y = var_526_transpose_y_0, x = var_525_cast_fp16, y = state_11_cast_fp16)[name = string("op_526_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_11_cast_fp16 = sub(x = var_523_cast_fp16, y = var_526_cast_fp16)[name = string("v_new_11_cast_fp16")];
+            tensor<int32, [4]> var_529_begin_0 = const()[name = string("op_529_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_529_end_0 = const()[name = string("op_529_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_529_end_mask_0 = const()[name = string("op_529_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_529_squeeze_mask_0 = const()[name = string("op_529_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_529_cast_fp16 = slice_by_index(begin = var_529_begin_0, end = var_529_end_0, end_mask = var_529_end_mask_0, squeeze_mask = var_529_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_529_cast_fp16")];
+            bool var_530_transpose_x_0 = const()[name = string("op_530_transpose_x_0"), val = bool(false)];
+            bool var_530_transpose_y_0 = const()[name = string("op_530_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_530_cast_fp16 = matmul(transpose_x = var_530_transpose_x_0, transpose_y = var_530_transpose_y_0, x = var_529_cast_fp16, y = state_11_cast_fp16)[name = string("op_530_cast_fp16")];
+            tensor<int32, [4]> var_532_begin_0 = const()[name = string("op_532_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_532_end_0 = const()[name = string("op_532_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_532_end_mask_0 = const()[name = string("op_532_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_532_squeeze_mask_0 = const()[name = string("op_532_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_532_cast_fp16 = slice_by_index(begin = var_532_begin_0, end = var_532_end_0, end_mask = var_532_end_mask_0, squeeze_mask = var_532_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_532_cast_fp16")];
+            bool var_533_transpose_x_0 = const()[name = string("op_533_transpose_x_0"), val = bool(false)];
+            bool var_533_transpose_y_0 = const()[name = string("op_533_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_533_cast_fp16 = matmul(transpose_x = var_533_transpose_x_0, transpose_y = var_533_transpose_y_0, x = var_532_cast_fp16, y = v_new_11_cast_fp16)[name = string("op_533_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_534_cast_fp16 = add(x = var_530_cast_fp16, y = var_533_cast_fp16)[name = string("op_534_cast_fp16")];
+            tensor<int32, [4]> var_536_begin_0 = const()[name = string("op_536_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_536_end_0 = const()[name = string("op_536_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_536_end_mask_0 = const()[name = string("op_536_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_536_squeeze_mask_0 = const()[name = string("op_536_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_536_cast_fp16 = slice_by_index(begin = var_536_begin_0, end = var_536_end_0, end_mask = var_536_end_mask_0, squeeze_mask = var_536_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_536_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_537_cast_fp16 = mul(x = state_11_cast_fp16, y = var_536_cast_fp16)[name = string("op_537_cast_fp16")];
+            tensor<int32, [4]> var_539_begin_0 = const()[name = string("op_539_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_539_end_0 = const()[name = string("op_539_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_539_end_mask_0 = const()[name = string("op_539_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_539_squeeze_mask_0 = const()[name = string("op_539_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_539_cast_fp16 = slice_by_index(begin = var_539_begin_0, end = var_539_end_0, end_mask = var_539_end_mask_0, squeeze_mask = var_539_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_539_cast_fp16")];
+            bool var_541_transpose_x_1 = const()[name = string("op_541_transpose_x_1"), val = bool(true)];
+            bool var_541_transpose_y_1 = const()[name = string("op_541_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_541_cast_fp16 = matmul(transpose_x = var_541_transpose_x_1, transpose_y = var_541_transpose_y_1, x = var_539_cast_fp16, y = v_new_11_cast_fp16)[name = string("op_541_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_13_cast_fp16 = add(x = var_537_cast_fp16, y = var_541_cast_fp16)[name = string("state_13_cast_fp16")];
+            tensor<int32, [4]> var_544_begin_0 = const()[name = string("op_544_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_544_end_0 = const()[name = string("op_544_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_544_end_mask_0 = const()[name = string("op_544_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_544_squeeze_mask_0 = const()[name = string("op_544_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_544_cast_fp16 = slice_by_index(begin = var_544_begin_0, end = var_544_end_0, end_mask = var_544_end_mask_0, squeeze_mask = var_544_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_544_cast_fp16")];
+            tensor<int32, [4]> var_546_begin_0 = const()[name = string("op_546_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_546_end_0 = const()[name = string("op_546_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_546_end_mask_0 = const()[name = string("op_546_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_546_squeeze_mask_0 = const()[name = string("op_546_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_546_cast_fp16 = slice_by_index(begin = var_546_begin_0, end = var_546_end_0, end_mask = var_546_end_mask_0, squeeze_mask = var_546_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_546_cast_fp16")];
+            bool var_547_transpose_x_0 = const()[name = string("op_547_transpose_x_0"), val = bool(false)];
+            bool var_547_transpose_y_0 = const()[name = string("op_547_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_547_cast_fp16 = matmul(transpose_x = var_547_transpose_x_0, transpose_y = var_547_transpose_y_0, x = var_546_cast_fp16, y = state_13_cast_fp16)[name = string("op_547_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_13_cast_fp16 = sub(x = var_544_cast_fp16, y = var_547_cast_fp16)[name = string("v_new_13_cast_fp16")];
+            tensor<int32, [4]> var_550_begin_0 = const()[name = string("op_550_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_550_end_0 = const()[name = string("op_550_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_550_end_mask_0 = const()[name = string("op_550_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_550_squeeze_mask_0 = const()[name = string("op_550_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_550_cast_fp16 = slice_by_index(begin = var_550_begin_0, end = var_550_end_0, end_mask = var_550_end_mask_0, squeeze_mask = var_550_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_550_cast_fp16")];
+            bool var_551_transpose_x_0 = const()[name = string("op_551_transpose_x_0"), val = bool(false)];
+            bool var_551_transpose_y_0 = const()[name = string("op_551_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_551_cast_fp16 = matmul(transpose_x = var_551_transpose_x_0, transpose_y = var_551_transpose_y_0, x = var_550_cast_fp16, y = state_13_cast_fp16)[name = string("op_551_cast_fp16")];
+            tensor<int32, [4]> var_553_begin_0 = const()[name = string("op_553_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_553_end_0 = const()[name = string("op_553_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_553_end_mask_0 = const()[name = string("op_553_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_553_squeeze_mask_0 = const()[name = string("op_553_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_553_cast_fp16 = slice_by_index(begin = var_553_begin_0, end = var_553_end_0, end_mask = var_553_end_mask_0, squeeze_mask = var_553_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_553_cast_fp16")];
+            bool var_554_transpose_x_0 = const()[name = string("op_554_transpose_x_0"), val = bool(false)];
+            bool var_554_transpose_y_0 = const()[name = string("op_554_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_554_cast_fp16 = matmul(transpose_x = var_554_transpose_x_0, transpose_y = var_554_transpose_y_0, x = var_553_cast_fp16, y = v_new_13_cast_fp16)[name = string("op_554_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_555_cast_fp16 = add(x = var_551_cast_fp16, y = var_554_cast_fp16)[name = string("op_555_cast_fp16")];
+            tensor<int32, [4]> var_557_begin_0 = const()[name = string("op_557_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_557_end_0 = const()[name = string("op_557_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_557_end_mask_0 = const()[name = string("op_557_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_557_squeeze_mask_0 = const()[name = string("op_557_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_557_cast_fp16 = slice_by_index(begin = var_557_begin_0, end = var_557_end_0, end_mask = var_557_end_mask_0, squeeze_mask = var_557_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_557_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_558_cast_fp16 = mul(x = state_13_cast_fp16, y = var_557_cast_fp16)[name = string("op_558_cast_fp16")];
+            tensor<int32, [4]> var_560_begin_0 = const()[name = string("op_560_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_560_end_0 = const()[name = string("op_560_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_560_end_mask_0 = const()[name = string("op_560_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_560_squeeze_mask_0 = const()[name = string("op_560_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_560_cast_fp16 = slice_by_index(begin = var_560_begin_0, end = var_560_end_0, end_mask = var_560_end_mask_0, squeeze_mask = var_560_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_560_cast_fp16")];
+            bool var_562_transpose_x_1 = const()[name = string("op_562_transpose_x_1"), val = bool(true)];
+            bool var_562_transpose_y_1 = const()[name = string("op_562_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_562_cast_fp16 = matmul(transpose_x = var_562_transpose_x_1, transpose_y = var_562_transpose_y_1, x = var_560_cast_fp16, y = v_new_13_cast_fp16)[name = string("op_562_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_15_cast_fp16 = add(x = var_558_cast_fp16, y = var_562_cast_fp16)[name = string("state_15_cast_fp16")];
+            tensor<int32, [4]> var_565_begin_0 = const()[name = string("op_565_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_565_end_0 = const()[name = string("op_565_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_565_end_mask_0 = const()[name = string("op_565_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_565_squeeze_mask_0 = const()[name = string("op_565_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_565_cast_fp16 = slice_by_index(begin = var_565_begin_0, end = var_565_end_0, end_mask = var_565_end_mask_0, squeeze_mask = var_565_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_565_cast_fp16")];
+            tensor<int32, [4]> var_567_begin_0 = const()[name = string("op_567_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_567_end_0 = const()[name = string("op_567_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_567_end_mask_0 = const()[name = string("op_567_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_567_squeeze_mask_0 = const()[name = string("op_567_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_567_cast_fp16 = slice_by_index(begin = var_567_begin_0, end = var_567_end_0, end_mask = var_567_end_mask_0, squeeze_mask = var_567_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_567_cast_fp16")];
+            bool var_568_transpose_x_0 = const()[name = string("op_568_transpose_x_0"), val = bool(false)];
+            bool var_568_transpose_y_0 = const()[name = string("op_568_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_568_cast_fp16 = matmul(transpose_x = var_568_transpose_x_0, transpose_y = var_568_transpose_y_0, x = var_567_cast_fp16, y = state_15_cast_fp16)[name = string("op_568_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_15_cast_fp16 = sub(x = var_565_cast_fp16, y = var_568_cast_fp16)[name = string("v_new_15_cast_fp16")];
+            tensor<int32, [4]> var_571_begin_0 = const()[name = string("op_571_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_571_end_0 = const()[name = string("op_571_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_571_end_mask_0 = const()[name = string("op_571_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_571_squeeze_mask_0 = const()[name = string("op_571_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_571_cast_fp16 = slice_by_index(begin = var_571_begin_0, end = var_571_end_0, end_mask = var_571_end_mask_0, squeeze_mask = var_571_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_571_cast_fp16")];
+            bool var_572_transpose_x_0 = const()[name = string("op_572_transpose_x_0"), val = bool(false)];
+            bool var_572_transpose_y_0 = const()[name = string("op_572_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_572_cast_fp16 = matmul(transpose_x = var_572_transpose_x_0, transpose_y = var_572_transpose_y_0, x = var_571_cast_fp16, y = state_15_cast_fp16)[name = string("op_572_cast_fp16")];
+            tensor<int32, [4]> var_574_begin_0 = const()[name = string("op_574_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_574_end_0 = const()[name = string("op_574_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_574_end_mask_0 = const()[name = string("op_574_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_574_squeeze_mask_0 = const()[name = string("op_574_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_574_cast_fp16 = slice_by_index(begin = var_574_begin_0, end = var_574_end_0, end_mask = var_574_end_mask_0, squeeze_mask = var_574_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_574_cast_fp16")];
+            bool var_575_transpose_x_0 = const()[name = string("op_575_transpose_x_0"), val = bool(false)];
+            bool var_575_transpose_y_0 = const()[name = string("op_575_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_575_cast_fp16 = matmul(transpose_x = var_575_transpose_x_0, transpose_y = var_575_transpose_y_0, x = var_574_cast_fp16, y = v_new_15_cast_fp16)[name = string("op_575_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_576_cast_fp16 = add(x = var_572_cast_fp16, y = var_575_cast_fp16)[name = string("op_576_cast_fp16")];
+            tensor<int32, [4]> var_578_begin_0 = const()[name = string("op_578_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_578_end_0 = const()[name = string("op_578_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_578_end_mask_0 = const()[name = string("op_578_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_578_squeeze_mask_0 = const()[name = string("op_578_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_578_cast_fp16 = slice_by_index(begin = var_578_begin_0, end = var_578_end_0, end_mask = var_578_end_mask_0, squeeze_mask = var_578_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_578_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_579_cast_fp16 = mul(x = state_15_cast_fp16, y = var_578_cast_fp16)[name = string("op_579_cast_fp16")];
+            tensor<int32, [4]> var_581_begin_0 = const()[name = string("op_581_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_581_end_0 = const()[name = string("op_581_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_581_end_mask_0 = const()[name = string("op_581_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_581_squeeze_mask_0 = const()[name = string("op_581_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_581_cast_fp16 = slice_by_index(begin = var_581_begin_0, end = var_581_end_0, end_mask = var_581_end_mask_0, squeeze_mask = var_581_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_581_cast_fp16")];
+            bool var_583_transpose_x_1 = const()[name = string("op_583_transpose_x_1"), val = bool(true)];
+            bool var_583_transpose_y_1 = const()[name = string("op_583_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_583_cast_fp16 = matmul(transpose_x = var_583_transpose_x_1, transpose_y = var_583_transpose_y_1, x = var_581_cast_fp16, y = v_new_15_cast_fp16)[name = string("op_583_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_17_cast_fp16 = add(x = var_579_cast_fp16, y = var_583_cast_fp16)[name = string("state_17_cast_fp16")];
+            tensor<int32, [4]> var_586_begin_0 = const()[name = string("op_586_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_586_end_0 = const()[name = string("op_586_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_586_end_mask_0 = const()[name = string("op_586_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_586_squeeze_mask_0 = const()[name = string("op_586_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_586_cast_fp16 = slice_by_index(begin = var_586_begin_0, end = var_586_end_0, end_mask = var_586_end_mask_0, squeeze_mask = var_586_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_586_cast_fp16")];
+            tensor<int32, [4]> var_588_begin_0 = const()[name = string("op_588_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_588_end_0 = const()[name = string("op_588_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_588_end_mask_0 = const()[name = string("op_588_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_588_squeeze_mask_0 = const()[name = string("op_588_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_588_cast_fp16 = slice_by_index(begin = var_588_begin_0, end = var_588_end_0, end_mask = var_588_end_mask_0, squeeze_mask = var_588_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_588_cast_fp16")];
+            bool var_589_transpose_x_0 = const()[name = string("op_589_transpose_x_0"), val = bool(false)];
+            bool var_589_transpose_y_0 = const()[name = string("op_589_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_589_cast_fp16 = matmul(transpose_x = var_589_transpose_x_0, transpose_y = var_589_transpose_y_0, x = var_588_cast_fp16, y = state_17_cast_fp16)[name = string("op_589_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_17_cast_fp16 = sub(x = var_586_cast_fp16, y = var_589_cast_fp16)[name = string("v_new_17_cast_fp16")];
+            tensor<int32, [4]> var_592_begin_0 = const()[name = string("op_592_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_592_end_0 = const()[name = string("op_592_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_592_end_mask_0 = const()[name = string("op_592_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_592_squeeze_mask_0 = const()[name = string("op_592_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_592_cast_fp16 = slice_by_index(begin = var_592_begin_0, end = var_592_end_0, end_mask = var_592_end_mask_0, squeeze_mask = var_592_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_592_cast_fp16")];
+            bool var_593_transpose_x_0 = const()[name = string("op_593_transpose_x_0"), val = bool(false)];
+            bool var_593_transpose_y_0 = const()[name = string("op_593_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_593_cast_fp16 = matmul(transpose_x = var_593_transpose_x_0, transpose_y = var_593_transpose_y_0, x = var_592_cast_fp16, y = state_17_cast_fp16)[name = string("op_593_cast_fp16")];
+            tensor<int32, [4]> var_595_begin_0 = const()[name = string("op_595_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_595_end_0 = const()[name = string("op_595_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_595_end_mask_0 = const()[name = string("op_595_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_595_squeeze_mask_0 = const()[name = string("op_595_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_595_cast_fp16 = slice_by_index(begin = var_595_begin_0, end = var_595_end_0, end_mask = var_595_end_mask_0, squeeze_mask = var_595_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_595_cast_fp16")];
+            bool var_596_transpose_x_0 = const()[name = string("op_596_transpose_x_0"), val = bool(false)];
+            bool var_596_transpose_y_0 = const()[name = string("op_596_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_596_cast_fp16 = matmul(transpose_x = var_596_transpose_x_0, transpose_y = var_596_transpose_y_0, x = var_595_cast_fp16, y = v_new_17_cast_fp16)[name = string("op_596_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_597_cast_fp16 = add(x = var_593_cast_fp16, y = var_596_cast_fp16)[name = string("op_597_cast_fp16")];
+            tensor<int32, [4]> var_599_begin_0 = const()[name = string("op_599_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_599_end_0 = const()[name = string("op_599_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_599_end_mask_0 = const()[name = string("op_599_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_599_squeeze_mask_0 = const()[name = string("op_599_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_599_cast_fp16 = slice_by_index(begin = var_599_begin_0, end = var_599_end_0, end_mask = var_599_end_mask_0, squeeze_mask = var_599_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_599_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_600_cast_fp16 = mul(x = state_17_cast_fp16, y = var_599_cast_fp16)[name = string("op_600_cast_fp16")];
+            tensor<int32, [4]> var_602_begin_0 = const()[name = string("op_602_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_602_end_0 = const()[name = string("op_602_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_602_end_mask_0 = const()[name = string("op_602_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_602_squeeze_mask_0 = const()[name = string("op_602_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_602_cast_fp16 = slice_by_index(begin = var_602_begin_0, end = var_602_end_0, end_mask = var_602_end_mask_0, squeeze_mask = var_602_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_602_cast_fp16")];
+            bool var_604_transpose_x_1 = const()[name = string("op_604_transpose_x_1"), val = bool(true)];
+            bool var_604_transpose_y_1 = const()[name = string("op_604_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_604_cast_fp16 = matmul(transpose_x = var_604_transpose_x_1, transpose_y = var_604_transpose_y_1, x = var_602_cast_fp16, y = v_new_17_cast_fp16)[name = string("op_604_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_19_cast_fp16 = add(x = var_600_cast_fp16, y = var_604_cast_fp16)[name = string("state_19_cast_fp16")];
+            tensor<int32, [4]> var_607_begin_0 = const()[name = string("op_607_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_607_end_0 = const()[name = string("op_607_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_607_end_mask_0 = const()[name = string("op_607_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_607_squeeze_mask_0 = const()[name = string("op_607_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_607_cast_fp16 = slice_by_index(begin = var_607_begin_0, end = var_607_end_0, end_mask = var_607_end_mask_0, squeeze_mask = var_607_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_607_cast_fp16")];
+            tensor<int32, [4]> var_609_begin_0 = const()[name = string("op_609_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_609_end_0 = const()[name = string("op_609_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_609_end_mask_0 = const()[name = string("op_609_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_609_squeeze_mask_0 = const()[name = string("op_609_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_609_cast_fp16 = slice_by_index(begin = var_609_begin_0, end = var_609_end_0, end_mask = var_609_end_mask_0, squeeze_mask = var_609_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_609_cast_fp16")];
+            bool var_610_transpose_x_0 = const()[name = string("op_610_transpose_x_0"), val = bool(false)];
+            bool var_610_transpose_y_0 = const()[name = string("op_610_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_610_cast_fp16 = matmul(transpose_x = var_610_transpose_x_0, transpose_y = var_610_transpose_y_0, x = var_609_cast_fp16, y = state_19_cast_fp16)[name = string("op_610_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_19_cast_fp16 = sub(x = var_607_cast_fp16, y = var_610_cast_fp16)[name = string("v_new_19_cast_fp16")];
+            tensor<int32, [4]> var_613_begin_0 = const()[name = string("op_613_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_613_end_0 = const()[name = string("op_613_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_613_end_mask_0 = const()[name = string("op_613_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_613_squeeze_mask_0 = const()[name = string("op_613_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_613_cast_fp16 = slice_by_index(begin = var_613_begin_0, end = var_613_end_0, end_mask = var_613_end_mask_0, squeeze_mask = var_613_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_613_cast_fp16")];
+            bool var_614_transpose_x_0 = const()[name = string("op_614_transpose_x_0"), val = bool(false)];
+            bool var_614_transpose_y_0 = const()[name = string("op_614_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_614_cast_fp16 = matmul(transpose_x = var_614_transpose_x_0, transpose_y = var_614_transpose_y_0, x = var_613_cast_fp16, y = state_19_cast_fp16)[name = string("op_614_cast_fp16")];
+            tensor<int32, [4]> var_616_begin_0 = const()[name = string("op_616_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_616_end_0 = const()[name = string("op_616_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_616_end_mask_0 = const()[name = string("op_616_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_616_squeeze_mask_0 = const()[name = string("op_616_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_616_cast_fp16 = slice_by_index(begin = var_616_begin_0, end = var_616_end_0, end_mask = var_616_end_mask_0, squeeze_mask = var_616_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_616_cast_fp16")];
+            bool var_617_transpose_x_0 = const()[name = string("op_617_transpose_x_0"), val = bool(false)];
+            bool var_617_transpose_y_0 = const()[name = string("op_617_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_617_cast_fp16 = matmul(transpose_x = var_617_transpose_x_0, transpose_y = var_617_transpose_y_0, x = var_616_cast_fp16, y = v_new_19_cast_fp16)[name = string("op_617_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_618_cast_fp16 = add(x = var_614_cast_fp16, y = var_617_cast_fp16)[name = string("op_618_cast_fp16")];
+            tensor<int32, [4]> var_620_begin_0 = const()[name = string("op_620_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_620_end_0 = const()[name = string("op_620_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_620_end_mask_0 = const()[name = string("op_620_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_620_squeeze_mask_0 = const()[name = string("op_620_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_620_cast_fp16 = slice_by_index(begin = var_620_begin_0, end = var_620_end_0, end_mask = var_620_end_mask_0, squeeze_mask = var_620_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_620_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_621_cast_fp16 = mul(x = state_19_cast_fp16, y = var_620_cast_fp16)[name = string("op_621_cast_fp16")];
+            tensor<int32, [4]> var_623_begin_0 = const()[name = string("op_623_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_623_end_0 = const()[name = string("op_623_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_623_end_mask_0 = const()[name = string("op_623_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_623_squeeze_mask_0 = const()[name = string("op_623_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_623_cast_fp16 = slice_by_index(begin = var_623_begin_0, end = var_623_end_0, end_mask = var_623_end_mask_0, squeeze_mask = var_623_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_623_cast_fp16")];
+            bool var_625_transpose_x_1 = const()[name = string("op_625_transpose_x_1"), val = bool(true)];
+            bool var_625_transpose_y_1 = const()[name = string("op_625_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_625_cast_fp16 = matmul(transpose_x = var_625_transpose_x_1, transpose_y = var_625_transpose_y_1, x = var_623_cast_fp16, y = v_new_19_cast_fp16)[name = string("op_625_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_21_cast_fp16 = add(x = var_621_cast_fp16, y = var_625_cast_fp16)[name = string("state_21_cast_fp16")];
+            tensor<int32, [4]> var_628_begin_0 = const()[name = string("op_628_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_628_end_0 = const()[name = string("op_628_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_628_end_mask_0 = const()[name = string("op_628_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_628_squeeze_mask_0 = const()[name = string("op_628_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_628_cast_fp16 = slice_by_index(begin = var_628_begin_0, end = var_628_end_0, end_mask = var_628_end_mask_0, squeeze_mask = var_628_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_628_cast_fp16")];
+            tensor<int32, [4]> var_630_begin_0 = const()[name = string("op_630_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_630_end_0 = const()[name = string("op_630_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_630_end_mask_0 = const()[name = string("op_630_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_630_squeeze_mask_0 = const()[name = string("op_630_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_630_cast_fp16 = slice_by_index(begin = var_630_begin_0, end = var_630_end_0, end_mask = var_630_end_mask_0, squeeze_mask = var_630_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_630_cast_fp16")];
+            bool var_631_transpose_x_0 = const()[name = string("op_631_transpose_x_0"), val = bool(false)];
+            bool var_631_transpose_y_0 = const()[name = string("op_631_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_631_cast_fp16 = matmul(transpose_x = var_631_transpose_x_0, transpose_y = var_631_transpose_y_0, x = var_630_cast_fp16, y = state_21_cast_fp16)[name = string("op_631_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_21_cast_fp16 = sub(x = var_628_cast_fp16, y = var_631_cast_fp16)[name = string("v_new_21_cast_fp16")];
+            tensor<int32, [4]> var_634_begin_0 = const()[name = string("op_634_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_634_end_0 = const()[name = string("op_634_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_634_end_mask_0 = const()[name = string("op_634_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_634_squeeze_mask_0 = const()[name = string("op_634_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_634_cast_fp16 = slice_by_index(begin = var_634_begin_0, end = var_634_end_0, end_mask = var_634_end_mask_0, squeeze_mask = var_634_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_634_cast_fp16")];
+            bool var_635_transpose_x_0 = const()[name = string("op_635_transpose_x_0"), val = bool(false)];
+            bool var_635_transpose_y_0 = const()[name = string("op_635_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_635_cast_fp16 = matmul(transpose_x = var_635_transpose_x_0, transpose_y = var_635_transpose_y_0, x = var_634_cast_fp16, y = state_21_cast_fp16)[name = string("op_635_cast_fp16")];
+            tensor<int32, [4]> var_637_begin_0 = const()[name = string("op_637_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_637_end_0 = const()[name = string("op_637_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_637_end_mask_0 = const()[name = string("op_637_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_637_squeeze_mask_0 = const()[name = string("op_637_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_637_cast_fp16 = slice_by_index(begin = var_637_begin_0, end = var_637_end_0, end_mask = var_637_end_mask_0, squeeze_mask = var_637_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_637_cast_fp16")];
+            bool var_638_transpose_x_0 = const()[name = string("op_638_transpose_x_0"), val = bool(false)];
+            bool var_638_transpose_y_0 = const()[name = string("op_638_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_638_cast_fp16 = matmul(transpose_x = var_638_transpose_x_0, transpose_y = var_638_transpose_y_0, x = var_637_cast_fp16, y = v_new_21_cast_fp16)[name = string("op_638_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_639_cast_fp16 = add(x = var_635_cast_fp16, y = var_638_cast_fp16)[name = string("op_639_cast_fp16")];
+            tensor<int32, [4]> var_641_begin_0 = const()[name = string("op_641_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_641_end_0 = const()[name = string("op_641_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_641_end_mask_0 = const()[name = string("op_641_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_641_squeeze_mask_0 = const()[name = string("op_641_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_641_cast_fp16 = slice_by_index(begin = var_641_begin_0, end = var_641_end_0, end_mask = var_641_end_mask_0, squeeze_mask = var_641_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_641_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_642_cast_fp16 = mul(x = state_21_cast_fp16, y = var_641_cast_fp16)[name = string("op_642_cast_fp16")];
+            tensor<int32, [4]> var_644_begin_0 = const()[name = string("op_644_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_644_end_0 = const()[name = string("op_644_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_644_end_mask_0 = const()[name = string("op_644_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_644_squeeze_mask_0 = const()[name = string("op_644_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_644_cast_fp16 = slice_by_index(begin = var_644_begin_0, end = var_644_end_0, end_mask = var_644_end_mask_0, squeeze_mask = var_644_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_644_cast_fp16")];
+            bool var_646_transpose_x_1 = const()[name = string("op_646_transpose_x_1"), val = bool(true)];
+            bool var_646_transpose_y_1 = const()[name = string("op_646_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_646_cast_fp16 = matmul(transpose_x = var_646_transpose_x_1, transpose_y = var_646_transpose_y_1, x = var_644_cast_fp16, y = v_new_21_cast_fp16)[name = string("op_646_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_23_cast_fp16 = add(x = var_642_cast_fp16, y = var_646_cast_fp16)[name = string("state_23_cast_fp16")];
+            tensor<int32, [4]> var_649_begin_0 = const()[name = string("op_649_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_649_end_0 = const()[name = string("op_649_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_649_end_mask_0 = const()[name = string("op_649_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_649_squeeze_mask_0 = const()[name = string("op_649_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_649_cast_fp16 = slice_by_index(begin = var_649_begin_0, end = var_649_end_0, end_mask = var_649_end_mask_0, squeeze_mask = var_649_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_649_cast_fp16")];
+            tensor<int32, [4]> var_651_begin_0 = const()[name = string("op_651_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_651_end_0 = const()[name = string("op_651_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_651_end_mask_0 = const()[name = string("op_651_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_651_squeeze_mask_0 = const()[name = string("op_651_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_651_cast_fp16 = slice_by_index(begin = var_651_begin_0, end = var_651_end_0, end_mask = var_651_end_mask_0, squeeze_mask = var_651_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_651_cast_fp16")];
+            bool var_652_transpose_x_0 = const()[name = string("op_652_transpose_x_0"), val = bool(false)];
+            bool var_652_transpose_y_0 = const()[name = string("op_652_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_652_cast_fp16 = matmul(transpose_x = var_652_transpose_x_0, transpose_y = var_652_transpose_y_0, x = var_651_cast_fp16, y = state_23_cast_fp16)[name = string("op_652_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_23_cast_fp16 = sub(x = var_649_cast_fp16, y = var_652_cast_fp16)[name = string("v_new_23_cast_fp16")];
+            tensor<int32, [4]> var_655_begin_0 = const()[name = string("op_655_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_655_end_0 = const()[name = string("op_655_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_655_end_mask_0 = const()[name = string("op_655_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_655_squeeze_mask_0 = const()[name = string("op_655_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_655_cast_fp16 = slice_by_index(begin = var_655_begin_0, end = var_655_end_0, end_mask = var_655_end_mask_0, squeeze_mask = var_655_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_655_cast_fp16")];
+            bool var_656_transpose_x_0 = const()[name = string("op_656_transpose_x_0"), val = bool(false)];
+            bool var_656_transpose_y_0 = const()[name = string("op_656_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_656_cast_fp16 = matmul(transpose_x = var_656_transpose_x_0, transpose_y = var_656_transpose_y_0, x = var_655_cast_fp16, y = state_23_cast_fp16)[name = string("op_656_cast_fp16")];
+            tensor<int32, [4]> var_658_begin_0 = const()[name = string("op_658_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_658_end_0 = const()[name = string("op_658_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_658_end_mask_0 = const()[name = string("op_658_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_658_squeeze_mask_0 = const()[name = string("op_658_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_658_cast_fp16 = slice_by_index(begin = var_658_begin_0, end = var_658_end_0, end_mask = var_658_end_mask_0, squeeze_mask = var_658_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_658_cast_fp16")];
+            bool var_659_transpose_x_0 = const()[name = string("op_659_transpose_x_0"), val = bool(false)];
+            bool var_659_transpose_y_0 = const()[name = string("op_659_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_659_cast_fp16 = matmul(transpose_x = var_659_transpose_x_0, transpose_y = var_659_transpose_y_0, x = var_658_cast_fp16, y = v_new_23_cast_fp16)[name = string("op_659_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_660_cast_fp16 = add(x = var_656_cast_fp16, y = var_659_cast_fp16)[name = string("op_660_cast_fp16")];
+            tensor<int32, [4]> var_662_begin_0 = const()[name = string("op_662_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_662_end_0 = const()[name = string("op_662_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_662_end_mask_0 = const()[name = string("op_662_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_662_squeeze_mask_0 = const()[name = string("op_662_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_662_cast_fp16 = slice_by_index(begin = var_662_begin_0, end = var_662_end_0, end_mask = var_662_end_mask_0, squeeze_mask = var_662_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_662_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_663_cast_fp16 = mul(x = state_23_cast_fp16, y = var_662_cast_fp16)[name = string("op_663_cast_fp16")];
+            tensor<int32, [4]> var_665_begin_0 = const()[name = string("op_665_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_665_end_0 = const()[name = string("op_665_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_665_end_mask_0 = const()[name = string("op_665_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_665_squeeze_mask_0 = const()[name = string("op_665_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_665_cast_fp16 = slice_by_index(begin = var_665_begin_0, end = var_665_end_0, end_mask = var_665_end_mask_0, squeeze_mask = var_665_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_665_cast_fp16")];
+            bool var_667_transpose_x_1 = const()[name = string("op_667_transpose_x_1"), val = bool(true)];
+            bool var_667_transpose_y_1 = const()[name = string("op_667_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_667_cast_fp16 = matmul(transpose_x = var_667_transpose_x_1, transpose_y = var_667_transpose_y_1, x = var_665_cast_fp16, y = v_new_23_cast_fp16)[name = string("op_667_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_25_cast_fp16 = add(x = var_663_cast_fp16, y = var_667_cast_fp16)[name = string("state_25_cast_fp16")];
+            tensor<int32, [4]> var_670_begin_0 = const()[name = string("op_670_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_670_end_0 = const()[name = string("op_670_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_670_end_mask_0 = const()[name = string("op_670_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_670_squeeze_mask_0 = const()[name = string("op_670_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_670_cast_fp16 = slice_by_index(begin = var_670_begin_0, end = var_670_end_0, end_mask = var_670_end_mask_0, squeeze_mask = var_670_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_670_cast_fp16")];
+            tensor<int32, [4]> var_672_begin_0 = const()[name = string("op_672_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_672_end_0 = const()[name = string("op_672_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_672_end_mask_0 = const()[name = string("op_672_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_672_squeeze_mask_0 = const()[name = string("op_672_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_672_cast_fp16 = slice_by_index(begin = var_672_begin_0, end = var_672_end_0, end_mask = var_672_end_mask_0, squeeze_mask = var_672_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_672_cast_fp16")];
+            bool var_673_transpose_x_0 = const()[name = string("op_673_transpose_x_0"), val = bool(false)];
+            bool var_673_transpose_y_0 = const()[name = string("op_673_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_673_cast_fp16 = matmul(transpose_x = var_673_transpose_x_0, transpose_y = var_673_transpose_y_0, x = var_672_cast_fp16, y = state_25_cast_fp16)[name = string("op_673_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_25_cast_fp16 = sub(x = var_670_cast_fp16, y = var_673_cast_fp16)[name = string("v_new_25_cast_fp16")];
+            tensor<int32, [4]> var_676_begin_0 = const()[name = string("op_676_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_676_end_0 = const()[name = string("op_676_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_676_end_mask_0 = const()[name = string("op_676_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_676_squeeze_mask_0 = const()[name = string("op_676_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_676_cast_fp16 = slice_by_index(begin = var_676_begin_0, end = var_676_end_0, end_mask = var_676_end_mask_0, squeeze_mask = var_676_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_676_cast_fp16")];
+            bool var_677_transpose_x_0 = const()[name = string("op_677_transpose_x_0"), val = bool(false)];
+            bool var_677_transpose_y_0 = const()[name = string("op_677_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_677_cast_fp16 = matmul(transpose_x = var_677_transpose_x_0, transpose_y = var_677_transpose_y_0, x = var_676_cast_fp16, y = state_25_cast_fp16)[name = string("op_677_cast_fp16")];
+            tensor<int32, [4]> var_679_begin_0 = const()[name = string("op_679_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_679_end_0 = const()[name = string("op_679_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_679_end_mask_0 = const()[name = string("op_679_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_679_squeeze_mask_0 = const()[name = string("op_679_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_679_cast_fp16 = slice_by_index(begin = var_679_begin_0, end = var_679_end_0, end_mask = var_679_end_mask_0, squeeze_mask = var_679_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_679_cast_fp16")];
+            bool var_680_transpose_x_0 = const()[name = string("op_680_transpose_x_0"), val = bool(false)];
+            bool var_680_transpose_y_0 = const()[name = string("op_680_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_680_cast_fp16 = matmul(transpose_x = var_680_transpose_x_0, transpose_y = var_680_transpose_y_0, x = var_679_cast_fp16, y = v_new_25_cast_fp16)[name = string("op_680_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_681_cast_fp16 = add(x = var_677_cast_fp16, y = var_680_cast_fp16)[name = string("op_681_cast_fp16")];
+            tensor<int32, [4]> var_683_begin_0 = const()[name = string("op_683_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_683_end_0 = const()[name = string("op_683_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_683_end_mask_0 = const()[name = string("op_683_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_683_squeeze_mask_0 = const()[name = string("op_683_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_683_cast_fp16 = slice_by_index(begin = var_683_begin_0, end = var_683_end_0, end_mask = var_683_end_mask_0, squeeze_mask = var_683_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_683_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_684_cast_fp16 = mul(x = state_25_cast_fp16, y = var_683_cast_fp16)[name = string("op_684_cast_fp16")];
+            tensor<int32, [4]> var_686_begin_0 = const()[name = string("op_686_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_686_end_0 = const()[name = string("op_686_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_686_end_mask_0 = const()[name = string("op_686_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_686_squeeze_mask_0 = const()[name = string("op_686_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_686_cast_fp16 = slice_by_index(begin = var_686_begin_0, end = var_686_end_0, end_mask = var_686_end_mask_0, squeeze_mask = var_686_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_686_cast_fp16")];
+            bool var_688_transpose_x_1 = const()[name = string("op_688_transpose_x_1"), val = bool(true)];
+            bool var_688_transpose_y_1 = const()[name = string("op_688_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_688_cast_fp16 = matmul(transpose_x = var_688_transpose_x_1, transpose_y = var_688_transpose_y_1, x = var_686_cast_fp16, y = v_new_25_cast_fp16)[name = string("op_688_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_27_cast_fp16 = add(x = var_684_cast_fp16, y = var_688_cast_fp16)[name = string("state_27_cast_fp16")];
+            tensor<int32, [4]> var_691_begin_0 = const()[name = string("op_691_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_691_end_0 = const()[name = string("op_691_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_691_end_mask_0 = const()[name = string("op_691_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_691_squeeze_mask_0 = const()[name = string("op_691_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_691_cast_fp16 = slice_by_index(begin = var_691_begin_0, end = var_691_end_0, end_mask = var_691_end_mask_0, squeeze_mask = var_691_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_691_cast_fp16")];
+            tensor<int32, [4]> var_693_begin_0 = const()[name = string("op_693_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_693_end_0 = const()[name = string("op_693_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_693_end_mask_0 = const()[name = string("op_693_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_693_squeeze_mask_0 = const()[name = string("op_693_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_693_cast_fp16 = slice_by_index(begin = var_693_begin_0, end = var_693_end_0, end_mask = var_693_end_mask_0, squeeze_mask = var_693_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_693_cast_fp16")];
+            bool var_694_transpose_x_0 = const()[name = string("op_694_transpose_x_0"), val = bool(false)];
+            bool var_694_transpose_y_0 = const()[name = string("op_694_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_694_cast_fp16 = matmul(transpose_x = var_694_transpose_x_0, transpose_y = var_694_transpose_y_0, x = var_693_cast_fp16, y = state_27_cast_fp16)[name = string("op_694_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_27_cast_fp16 = sub(x = var_691_cast_fp16, y = var_694_cast_fp16)[name = string("v_new_27_cast_fp16")];
+            tensor<int32, [4]> var_697_begin_0 = const()[name = string("op_697_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_697_end_0 = const()[name = string("op_697_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_697_end_mask_0 = const()[name = string("op_697_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_697_squeeze_mask_0 = const()[name = string("op_697_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_697_cast_fp16 = slice_by_index(begin = var_697_begin_0, end = var_697_end_0, end_mask = var_697_end_mask_0, squeeze_mask = var_697_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_697_cast_fp16")];
+            bool var_698_transpose_x_0 = const()[name = string("op_698_transpose_x_0"), val = bool(false)];
+            bool var_698_transpose_y_0 = const()[name = string("op_698_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_698_cast_fp16 = matmul(transpose_x = var_698_transpose_x_0, transpose_y = var_698_transpose_y_0, x = var_697_cast_fp16, y = state_27_cast_fp16)[name = string("op_698_cast_fp16")];
+            tensor<int32, [4]> var_700_begin_0 = const()[name = string("op_700_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_700_end_0 = const()[name = string("op_700_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_700_end_mask_0 = const()[name = string("op_700_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_700_squeeze_mask_0 = const()[name = string("op_700_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_700_cast_fp16 = slice_by_index(begin = var_700_begin_0, end = var_700_end_0, end_mask = var_700_end_mask_0, squeeze_mask = var_700_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_700_cast_fp16")];
+            bool var_701_transpose_x_0 = const()[name = string("op_701_transpose_x_0"), val = bool(false)];
+            bool var_701_transpose_y_0 = const()[name = string("op_701_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_701_cast_fp16 = matmul(transpose_x = var_701_transpose_x_0, transpose_y = var_701_transpose_y_0, x = var_700_cast_fp16, y = v_new_27_cast_fp16)[name = string("op_701_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_702_cast_fp16 = add(x = var_698_cast_fp16, y = var_701_cast_fp16)[name = string("op_702_cast_fp16")];
+            tensor<int32, [4]> var_704_begin_0 = const()[name = string("op_704_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_704_end_0 = const()[name = string("op_704_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_704_end_mask_0 = const()[name = string("op_704_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_704_squeeze_mask_0 = const()[name = string("op_704_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_704_cast_fp16 = slice_by_index(begin = var_704_begin_0, end = var_704_end_0, end_mask = var_704_end_mask_0, squeeze_mask = var_704_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_704_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_705_cast_fp16 = mul(x = state_27_cast_fp16, y = var_704_cast_fp16)[name = string("op_705_cast_fp16")];
+            tensor<int32, [4]> var_707_begin_0 = const()[name = string("op_707_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_707_end_0 = const()[name = string("op_707_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_707_end_mask_0 = const()[name = string("op_707_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_707_squeeze_mask_0 = const()[name = string("op_707_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_707_cast_fp16 = slice_by_index(begin = var_707_begin_0, end = var_707_end_0, end_mask = var_707_end_mask_0, squeeze_mask = var_707_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_707_cast_fp16")];
+            bool var_709_transpose_x_1 = const()[name = string("op_709_transpose_x_1"), val = bool(true)];
+            bool var_709_transpose_y_1 = const()[name = string("op_709_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_709_cast_fp16 = matmul(transpose_x = var_709_transpose_x_1, transpose_y = var_709_transpose_y_1, x = var_707_cast_fp16, y = v_new_27_cast_fp16)[name = string("op_709_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_29_cast_fp16 = add(x = var_705_cast_fp16, y = var_709_cast_fp16)[name = string("state_29_cast_fp16")];
+            tensor<int32, [4]> var_712_begin_0 = const()[name = string("op_712_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_712_end_0 = const()[name = string("op_712_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_712_end_mask_0 = const()[name = string("op_712_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_712_squeeze_mask_0 = const()[name = string("op_712_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_712_cast_fp16 = slice_by_index(begin = var_712_begin_0, end = var_712_end_0, end_mask = var_712_end_mask_0, squeeze_mask = var_712_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_712_cast_fp16")];
+            tensor<int32, [4]> var_714_begin_0 = const()[name = string("op_714_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_714_end_0 = const()[name = string("op_714_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_714_end_mask_0 = const()[name = string("op_714_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_714_squeeze_mask_0 = const()[name = string("op_714_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_714_cast_fp16 = slice_by_index(begin = var_714_begin_0, end = var_714_end_0, end_mask = var_714_end_mask_0, squeeze_mask = var_714_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_714_cast_fp16")];
+            bool var_715_transpose_x_0 = const()[name = string("op_715_transpose_x_0"), val = bool(false)];
+            bool var_715_transpose_y_0 = const()[name = string("op_715_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_715_cast_fp16 = matmul(transpose_x = var_715_transpose_x_0, transpose_y = var_715_transpose_y_0, x = var_714_cast_fp16, y = state_29_cast_fp16)[name = string("op_715_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_29_cast_fp16 = sub(x = var_712_cast_fp16, y = var_715_cast_fp16)[name = string("v_new_29_cast_fp16")];
+            tensor<int32, [4]> var_718_begin_0 = const()[name = string("op_718_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_718_end_0 = const()[name = string("op_718_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_718_end_mask_0 = const()[name = string("op_718_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_718_squeeze_mask_0 = const()[name = string("op_718_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_718_cast_fp16 = slice_by_index(begin = var_718_begin_0, end = var_718_end_0, end_mask = var_718_end_mask_0, squeeze_mask = var_718_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_718_cast_fp16")];
+            bool var_719_transpose_x_0 = const()[name = string("op_719_transpose_x_0"), val = bool(false)];
+            bool var_719_transpose_y_0 = const()[name = string("op_719_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_719_cast_fp16 = matmul(transpose_x = var_719_transpose_x_0, transpose_y = var_719_transpose_y_0, x = var_718_cast_fp16, y = state_29_cast_fp16)[name = string("op_719_cast_fp16")];
+            tensor<int32, [4]> var_721_begin_0 = const()[name = string("op_721_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_721_end_0 = const()[name = string("op_721_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_721_end_mask_0 = const()[name = string("op_721_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_721_squeeze_mask_0 = const()[name = string("op_721_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_721_cast_fp16 = slice_by_index(begin = var_721_begin_0, end = var_721_end_0, end_mask = var_721_end_mask_0, squeeze_mask = var_721_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_721_cast_fp16")];
+            bool var_722_transpose_x_0 = const()[name = string("op_722_transpose_x_0"), val = bool(false)];
+            bool var_722_transpose_y_0 = const()[name = string("op_722_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_722_cast_fp16 = matmul(transpose_x = var_722_transpose_x_0, transpose_y = var_722_transpose_y_0, x = var_721_cast_fp16, y = v_new_29_cast_fp16)[name = string("op_722_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_723_cast_fp16 = add(x = var_719_cast_fp16, y = var_722_cast_fp16)[name = string("op_723_cast_fp16")];
+            tensor<int32, [4]> var_725_begin_0 = const()[name = string("op_725_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_725_end_0 = const()[name = string("op_725_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_725_end_mask_0 = const()[name = string("op_725_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_725_squeeze_mask_0 = const()[name = string("op_725_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_725_cast_fp16 = slice_by_index(begin = var_725_begin_0, end = var_725_end_0, end_mask = var_725_end_mask_0, squeeze_mask = var_725_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_725_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_726_cast_fp16 = mul(x = state_29_cast_fp16, y = var_725_cast_fp16)[name = string("op_726_cast_fp16")];
+            tensor<int32, [4]> var_728_begin_0 = const()[name = string("op_728_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_728_end_0 = const()[name = string("op_728_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_728_end_mask_0 = const()[name = string("op_728_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_728_squeeze_mask_0 = const()[name = string("op_728_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_728_cast_fp16 = slice_by_index(begin = var_728_begin_0, end = var_728_end_0, end_mask = var_728_end_mask_0, squeeze_mask = var_728_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_728_cast_fp16")];
+            bool var_730_transpose_x_1 = const()[name = string("op_730_transpose_x_1"), val = bool(true)];
+            bool var_730_transpose_y_1 = const()[name = string("op_730_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_730_cast_fp16 = matmul(transpose_x = var_730_transpose_x_1, transpose_y = var_730_transpose_y_1, x = var_728_cast_fp16, y = v_new_29_cast_fp16)[name = string("op_730_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_31_cast_fp16 = add(x = var_726_cast_fp16, y = var_730_cast_fp16)[name = string("state_31_cast_fp16")];
+            tensor<int32, [4]> var_733_begin_0 = const()[name = string("op_733_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_733_end_0 = const()[name = string("op_733_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_733_end_mask_0 = const()[name = string("op_733_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_733_squeeze_mask_0 = const()[name = string("op_733_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_733_cast_fp16 = slice_by_index(begin = var_733_begin_0, end = var_733_end_0, end_mask = var_733_end_mask_0, squeeze_mask = var_733_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_733_cast_fp16")];
+            tensor<int32, [4]> var_735_begin_0 = const()[name = string("op_735_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_735_end_0 = const()[name = string("op_735_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_735_end_mask_0 = const()[name = string("op_735_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_735_squeeze_mask_0 = const()[name = string("op_735_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_735_cast_fp16 = slice_by_index(begin = var_735_begin_0, end = var_735_end_0, end_mask = var_735_end_mask_0, squeeze_mask = var_735_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_735_cast_fp16")];
+            bool var_736_transpose_x_0 = const()[name = string("op_736_transpose_x_0"), val = bool(false)];
+            bool var_736_transpose_y_0 = const()[name = string("op_736_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_736_cast_fp16 = matmul(transpose_x = var_736_transpose_x_0, transpose_y = var_736_transpose_y_0, x = var_735_cast_fp16, y = state_31_cast_fp16)[name = string("op_736_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_31_cast_fp16 = sub(x = var_733_cast_fp16, y = var_736_cast_fp16)[name = string("v_new_31_cast_fp16")];
+            tensor<int32, [4]> var_739_begin_0 = const()[name = string("op_739_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_739_end_0 = const()[name = string("op_739_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_739_end_mask_0 = const()[name = string("op_739_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_739_squeeze_mask_0 = const()[name = string("op_739_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_739_cast_fp16 = slice_by_index(begin = var_739_begin_0, end = var_739_end_0, end_mask = var_739_end_mask_0, squeeze_mask = var_739_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_739_cast_fp16")];
+            bool var_740_transpose_x_0 = const()[name = string("op_740_transpose_x_0"), val = bool(false)];
+            bool var_740_transpose_y_0 = const()[name = string("op_740_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_740_cast_fp16 = matmul(transpose_x = var_740_transpose_x_0, transpose_y = var_740_transpose_y_0, x = var_739_cast_fp16, y = state_31_cast_fp16)[name = string("op_740_cast_fp16")];
+            tensor<int32, [4]> var_742_begin_0 = const()[name = string("op_742_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_742_end_0 = const()[name = string("op_742_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_742_end_mask_0 = const()[name = string("op_742_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_742_squeeze_mask_0 = const()[name = string("op_742_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_742_cast_fp16 = slice_by_index(begin = var_742_begin_0, end = var_742_end_0, end_mask = var_742_end_mask_0, squeeze_mask = var_742_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_742_cast_fp16")];
+            bool var_743_transpose_x_0 = const()[name = string("op_743_transpose_x_0"), val = bool(false)];
+            bool var_743_transpose_y_0 = const()[name = string("op_743_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_743_cast_fp16 = matmul(transpose_x = var_743_transpose_x_0, transpose_y = var_743_transpose_y_0, x = var_742_cast_fp16, y = v_new_31_cast_fp16)[name = string("op_743_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_744_cast_fp16 = add(x = var_740_cast_fp16, y = var_743_cast_fp16)[name = string("op_744_cast_fp16")];
+            tensor<int32, [4]> var_746_begin_0 = const()[name = string("op_746_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_746_end_0 = const()[name = string("op_746_end_0"), val = tensor<int32, [4]>([32, 16, 1, 1])];
+            tensor<bool, [4]> var_746_end_mask_0 = const()[name = string("op_746_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_746_squeeze_mask_0 = const()[name = string("op_746_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_746_cast_fp16 = slice_by_index(begin = var_746_begin_0, end = var_746_end_0, end_mask = var_746_end_mask_0, squeeze_mask = var_746_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_746_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_747_cast_fp16 = mul(x = state_31_cast_fp16, y = var_746_cast_fp16)[name = string("op_747_cast_fp16")];
+            tensor<int32, [4]> var_749_begin_0 = const()[name = string("op_749_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_749_end_0 = const()[name = string("op_749_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_749_end_mask_0 = const()[name = string("op_749_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_749_squeeze_mask_0 = const()[name = string("op_749_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_749_cast_fp16 = slice_by_index(begin = var_749_begin_0, end = var_749_end_0, end_mask = var_749_end_mask_0, squeeze_mask = var_749_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_749_cast_fp16")];
+            bool var_751_transpose_x_1 = const()[name = string("op_751_transpose_x_1"), val = bool(true)];
+            bool var_751_transpose_y_1 = const()[name = string("op_751_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_751_cast_fp16 = matmul(transpose_x = var_751_transpose_x_1, transpose_y = var_751_transpose_y_1, x = var_749_cast_fp16, y = v_new_31_cast_fp16)[name = string("op_751_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_33_cast_fp16 = add(x = var_747_cast_fp16, y = var_751_cast_fp16)[name = string("state_33_cast_fp16")];
+            tensor<int32, [4]> var_754_begin_0 = const()[name = string("op_754_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_754_end_0 = const()[name = string("op_754_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_754_end_mask_0 = const()[name = string("op_754_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_754_squeeze_mask_0 = const()[name = string("op_754_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_754_cast_fp16 = slice_by_index(begin = var_754_begin_0, end = var_754_end_0, end_mask = var_754_end_mask_0, squeeze_mask = var_754_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_754_cast_fp16")];
+            tensor<int32, [4]> var_756_begin_0 = const()[name = string("op_756_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_756_end_0 = const()[name = string("op_756_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_756_end_mask_0 = const()[name = string("op_756_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_756_squeeze_mask_0 = const()[name = string("op_756_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_756_cast_fp16 = slice_by_index(begin = var_756_begin_0, end = var_756_end_0, end_mask = var_756_end_mask_0, squeeze_mask = var_756_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_756_cast_fp16")];
+            bool var_757_transpose_x_0 = const()[name = string("op_757_transpose_x_0"), val = bool(false)];
+            bool var_757_transpose_y_0 = const()[name = string("op_757_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_757_cast_fp16 = matmul(transpose_x = var_757_transpose_x_0, transpose_y = var_757_transpose_y_0, x = var_756_cast_fp16, y = state_33_cast_fp16)[name = string("op_757_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_33_cast_fp16 = sub(x = var_754_cast_fp16, y = var_757_cast_fp16)[name = string("v_new_33_cast_fp16")];
+            tensor<int32, [4]> var_760_begin_0 = const()[name = string("op_760_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_760_end_0 = const()[name = string("op_760_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_760_end_mask_0 = const()[name = string("op_760_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_760_squeeze_mask_0 = const()[name = string("op_760_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_760_cast_fp16 = slice_by_index(begin = var_760_begin_0, end = var_760_end_0, end_mask = var_760_end_mask_0, squeeze_mask = var_760_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_760_cast_fp16")];
+            bool var_761_transpose_x_0 = const()[name = string("op_761_transpose_x_0"), val = bool(false)];
+            bool var_761_transpose_y_0 = const()[name = string("op_761_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_761_cast_fp16 = matmul(transpose_x = var_761_transpose_x_0, transpose_y = var_761_transpose_y_0, x = var_760_cast_fp16, y = state_33_cast_fp16)[name = string("op_761_cast_fp16")];
+            tensor<int32, [4]> var_763_begin_0 = const()[name = string("op_763_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_763_end_0 = const()[name = string("op_763_end_0"), val = tensor<int32, [4]>([32, 17, 64, 64])];
+            tensor<bool, [4]> var_763_end_mask_0 = const()[name = string("op_763_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_763_squeeze_mask_0 = const()[name = string("op_763_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_763_cast_fp16 = slice_by_index(begin = var_763_begin_0, end = var_763_end_0, end_mask = var_763_end_mask_0, squeeze_mask = var_763_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_763_cast_fp16")];
+            bool var_764_transpose_x_0 = const()[name = string("op_764_transpose_x_0"), val = bool(false)];
+            bool var_764_transpose_y_0 = const()[name = string("op_764_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_764_cast_fp16 = matmul(transpose_x = var_764_transpose_x_0, transpose_y = var_764_transpose_y_0, x = var_763_cast_fp16, y = v_new_33_cast_fp16)[name = string("op_764_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_765_cast_fp16 = add(x = var_761_cast_fp16, y = var_764_cast_fp16)[name = string("op_765_cast_fp16")];
+            tensor<int32, [4]> var_767_begin_0 = const()[name = string("op_767_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_767_end_0 = const()[name = string("op_767_end_0"), val = tensor<int32, [4]>([32, 17, 1, 1])];
+            tensor<bool, [4]> var_767_end_mask_0 = const()[name = string("op_767_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_767_squeeze_mask_0 = const()[name = string("op_767_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_767_cast_fp16 = slice_by_index(begin = var_767_begin_0, end = var_767_end_0, end_mask = var_767_end_mask_0, squeeze_mask = var_767_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_767_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_768_cast_fp16 = mul(x = state_33_cast_fp16, y = var_767_cast_fp16)[name = string("op_768_cast_fp16")];
+            tensor<int32, [4]> var_770_begin_0 = const()[name = string("op_770_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_770_end_0 = const()[name = string("op_770_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_770_end_mask_0 = const()[name = string("op_770_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_770_squeeze_mask_0 = const()[name = string("op_770_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_770_cast_fp16 = slice_by_index(begin = var_770_begin_0, end = var_770_end_0, end_mask = var_770_end_mask_0, squeeze_mask = var_770_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_770_cast_fp16")];
+            bool var_772_transpose_x_1 = const()[name = string("op_772_transpose_x_1"), val = bool(true)];
+            bool var_772_transpose_y_1 = const()[name = string("op_772_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_772_cast_fp16 = matmul(transpose_x = var_772_transpose_x_1, transpose_y = var_772_transpose_y_1, x = var_770_cast_fp16, y = v_new_33_cast_fp16)[name = string("op_772_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_35_cast_fp16 = add(x = var_768_cast_fp16, y = var_772_cast_fp16)[name = string("state_35_cast_fp16")];
+            tensor<int32, [4]> var_775_begin_0 = const()[name = string("op_775_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_775_end_0 = const()[name = string("op_775_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_775_end_mask_0 = const()[name = string("op_775_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_775_squeeze_mask_0 = const()[name = string("op_775_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_775_cast_fp16 = slice_by_index(begin = var_775_begin_0, end = var_775_end_0, end_mask = var_775_end_mask_0, squeeze_mask = var_775_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_775_cast_fp16")];
+            tensor<int32, [4]> var_777_begin_0 = const()[name = string("op_777_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_777_end_0 = const()[name = string("op_777_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_777_end_mask_0 = const()[name = string("op_777_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_777_squeeze_mask_0 = const()[name = string("op_777_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_777_cast_fp16 = slice_by_index(begin = var_777_begin_0, end = var_777_end_0, end_mask = var_777_end_mask_0, squeeze_mask = var_777_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_777_cast_fp16")];
+            bool var_778_transpose_x_0 = const()[name = string("op_778_transpose_x_0"), val = bool(false)];
+            bool var_778_transpose_y_0 = const()[name = string("op_778_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_778_cast_fp16 = matmul(transpose_x = var_778_transpose_x_0, transpose_y = var_778_transpose_y_0, x = var_777_cast_fp16, y = state_35_cast_fp16)[name = string("op_778_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_35_cast_fp16 = sub(x = var_775_cast_fp16, y = var_778_cast_fp16)[name = string("v_new_35_cast_fp16")];
+            tensor<int32, [4]> var_781_begin_0 = const()[name = string("op_781_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_781_end_0 = const()[name = string("op_781_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_781_end_mask_0 = const()[name = string("op_781_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_781_squeeze_mask_0 = const()[name = string("op_781_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_781_cast_fp16 = slice_by_index(begin = var_781_begin_0, end = var_781_end_0, end_mask = var_781_end_mask_0, squeeze_mask = var_781_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_781_cast_fp16")];
+            bool var_782_transpose_x_0 = const()[name = string("op_782_transpose_x_0"), val = bool(false)];
+            bool var_782_transpose_y_0 = const()[name = string("op_782_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_782_cast_fp16 = matmul(transpose_x = var_782_transpose_x_0, transpose_y = var_782_transpose_y_0, x = var_781_cast_fp16, y = state_35_cast_fp16)[name = string("op_782_cast_fp16")];
+            tensor<int32, [4]> var_784_begin_0 = const()[name = string("op_784_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_784_end_0 = const()[name = string("op_784_end_0"), val = tensor<int32, [4]>([32, 18, 64, 64])];
+            tensor<bool, [4]> var_784_end_mask_0 = const()[name = string("op_784_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_784_squeeze_mask_0 = const()[name = string("op_784_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_784_cast_fp16 = slice_by_index(begin = var_784_begin_0, end = var_784_end_0, end_mask = var_784_end_mask_0, squeeze_mask = var_784_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_784_cast_fp16")];
+            bool var_785_transpose_x_0 = const()[name = string("op_785_transpose_x_0"), val = bool(false)];
+            bool var_785_transpose_y_0 = const()[name = string("op_785_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_785_cast_fp16 = matmul(transpose_x = var_785_transpose_x_0, transpose_y = var_785_transpose_y_0, x = var_784_cast_fp16, y = v_new_35_cast_fp16)[name = string("op_785_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_786_cast_fp16 = add(x = var_782_cast_fp16, y = var_785_cast_fp16)[name = string("op_786_cast_fp16")];
+            tensor<int32, [4]> var_788_begin_0 = const()[name = string("op_788_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_788_end_0 = const()[name = string("op_788_end_0"), val = tensor<int32, [4]>([32, 18, 1, 1])];
+            tensor<bool, [4]> var_788_end_mask_0 = const()[name = string("op_788_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_788_squeeze_mask_0 = const()[name = string("op_788_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_788_cast_fp16 = slice_by_index(begin = var_788_begin_0, end = var_788_end_0, end_mask = var_788_end_mask_0, squeeze_mask = var_788_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_788_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_789_cast_fp16 = mul(x = state_35_cast_fp16, y = var_788_cast_fp16)[name = string("op_789_cast_fp16")];
+            tensor<int32, [4]> var_791_begin_0 = const()[name = string("op_791_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_791_end_0 = const()[name = string("op_791_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_791_end_mask_0 = const()[name = string("op_791_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_791_squeeze_mask_0 = const()[name = string("op_791_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_791_cast_fp16 = slice_by_index(begin = var_791_begin_0, end = var_791_end_0, end_mask = var_791_end_mask_0, squeeze_mask = var_791_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_791_cast_fp16")];
+            bool var_793_transpose_x_1 = const()[name = string("op_793_transpose_x_1"), val = bool(true)];
+            bool var_793_transpose_y_1 = const()[name = string("op_793_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_793_cast_fp16 = matmul(transpose_x = var_793_transpose_x_1, transpose_y = var_793_transpose_y_1, x = var_791_cast_fp16, y = v_new_35_cast_fp16)[name = string("op_793_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_37_cast_fp16 = add(x = var_789_cast_fp16, y = var_793_cast_fp16)[name = string("state_37_cast_fp16")];
+            tensor<int32, [4]> var_796_begin_0 = const()[name = string("op_796_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_796_end_0 = const()[name = string("op_796_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_796_end_mask_0 = const()[name = string("op_796_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_796_squeeze_mask_0 = const()[name = string("op_796_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_796_cast_fp16 = slice_by_index(begin = var_796_begin_0, end = var_796_end_0, end_mask = var_796_end_mask_0, squeeze_mask = var_796_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_796_cast_fp16")];
+            tensor<int32, [4]> var_798_begin_0 = const()[name = string("op_798_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_798_end_0 = const()[name = string("op_798_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_798_end_mask_0 = const()[name = string("op_798_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_798_squeeze_mask_0 = const()[name = string("op_798_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_798_cast_fp16 = slice_by_index(begin = var_798_begin_0, end = var_798_end_0, end_mask = var_798_end_mask_0, squeeze_mask = var_798_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_798_cast_fp16")];
+            bool var_799_transpose_x_0 = const()[name = string("op_799_transpose_x_0"), val = bool(false)];
+            bool var_799_transpose_y_0 = const()[name = string("op_799_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_799_cast_fp16 = matmul(transpose_x = var_799_transpose_x_0, transpose_y = var_799_transpose_y_0, x = var_798_cast_fp16, y = state_37_cast_fp16)[name = string("op_799_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_37_cast_fp16 = sub(x = var_796_cast_fp16, y = var_799_cast_fp16)[name = string("v_new_37_cast_fp16")];
+            tensor<int32, [4]> var_802_begin_0 = const()[name = string("op_802_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_802_end_0 = const()[name = string("op_802_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_802_end_mask_0 = const()[name = string("op_802_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_802_squeeze_mask_0 = const()[name = string("op_802_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_802_cast_fp16 = slice_by_index(begin = var_802_begin_0, end = var_802_end_0, end_mask = var_802_end_mask_0, squeeze_mask = var_802_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_802_cast_fp16")];
+            bool var_803_transpose_x_0 = const()[name = string("op_803_transpose_x_0"), val = bool(false)];
+            bool var_803_transpose_y_0 = const()[name = string("op_803_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_803_cast_fp16 = matmul(transpose_x = var_803_transpose_x_0, transpose_y = var_803_transpose_y_0, x = var_802_cast_fp16, y = state_37_cast_fp16)[name = string("op_803_cast_fp16")];
+            tensor<int32, [4]> var_805_begin_0 = const()[name = string("op_805_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_805_end_0 = const()[name = string("op_805_end_0"), val = tensor<int32, [4]>([32, 19, 64, 64])];
+            tensor<bool, [4]> var_805_end_mask_0 = const()[name = string("op_805_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_805_squeeze_mask_0 = const()[name = string("op_805_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_805_cast_fp16 = slice_by_index(begin = var_805_begin_0, end = var_805_end_0, end_mask = var_805_end_mask_0, squeeze_mask = var_805_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_805_cast_fp16")];
+            bool var_806_transpose_x_0 = const()[name = string("op_806_transpose_x_0"), val = bool(false)];
+            bool var_806_transpose_y_0 = const()[name = string("op_806_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_806_cast_fp16 = matmul(transpose_x = var_806_transpose_x_0, transpose_y = var_806_transpose_y_0, x = var_805_cast_fp16, y = v_new_37_cast_fp16)[name = string("op_806_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_807_cast_fp16 = add(x = var_803_cast_fp16, y = var_806_cast_fp16)[name = string("op_807_cast_fp16")];
+            tensor<int32, [4]> var_809_begin_0 = const()[name = string("op_809_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_809_end_0 = const()[name = string("op_809_end_0"), val = tensor<int32, [4]>([32, 19, 1, 1])];
+            tensor<bool, [4]> var_809_end_mask_0 = const()[name = string("op_809_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_809_squeeze_mask_0 = const()[name = string("op_809_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_809_cast_fp16 = slice_by_index(begin = var_809_begin_0, end = var_809_end_0, end_mask = var_809_end_mask_0, squeeze_mask = var_809_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_809_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_810_cast_fp16 = mul(x = state_37_cast_fp16, y = var_809_cast_fp16)[name = string("op_810_cast_fp16")];
+            tensor<int32, [4]> var_812_begin_0 = const()[name = string("op_812_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_812_end_0 = const()[name = string("op_812_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_812_end_mask_0 = const()[name = string("op_812_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_812_squeeze_mask_0 = const()[name = string("op_812_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_812_cast_fp16 = slice_by_index(begin = var_812_begin_0, end = var_812_end_0, end_mask = var_812_end_mask_0, squeeze_mask = var_812_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_812_cast_fp16")];
+            bool var_814_transpose_x_1 = const()[name = string("op_814_transpose_x_1"), val = bool(true)];
+            bool var_814_transpose_y_1 = const()[name = string("op_814_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_814_cast_fp16 = matmul(transpose_x = var_814_transpose_x_1, transpose_y = var_814_transpose_y_1, x = var_812_cast_fp16, y = v_new_37_cast_fp16)[name = string("op_814_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_39_cast_fp16 = add(x = var_810_cast_fp16, y = var_814_cast_fp16)[name = string("state_39_cast_fp16")];
+            tensor<int32, [4]> var_817_begin_0 = const()[name = string("op_817_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_817_end_0 = const()[name = string("op_817_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_817_end_mask_0 = const()[name = string("op_817_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_817_squeeze_mask_0 = const()[name = string("op_817_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_817_cast_fp16 = slice_by_index(begin = var_817_begin_0, end = var_817_end_0, end_mask = var_817_end_mask_0, squeeze_mask = var_817_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_817_cast_fp16")];
+            tensor<int32, [4]> var_819_begin_0 = const()[name = string("op_819_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_819_end_0 = const()[name = string("op_819_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_819_end_mask_0 = const()[name = string("op_819_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_819_squeeze_mask_0 = const()[name = string("op_819_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_819_cast_fp16 = slice_by_index(begin = var_819_begin_0, end = var_819_end_0, end_mask = var_819_end_mask_0, squeeze_mask = var_819_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_819_cast_fp16")];
+            bool var_820_transpose_x_0 = const()[name = string("op_820_transpose_x_0"), val = bool(false)];
+            bool var_820_transpose_y_0 = const()[name = string("op_820_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_820_cast_fp16 = matmul(transpose_x = var_820_transpose_x_0, transpose_y = var_820_transpose_y_0, x = var_819_cast_fp16, y = state_39_cast_fp16)[name = string("op_820_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_39_cast_fp16 = sub(x = var_817_cast_fp16, y = var_820_cast_fp16)[name = string("v_new_39_cast_fp16")];
+            tensor<int32, [4]> var_823_begin_0 = const()[name = string("op_823_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_823_end_0 = const()[name = string("op_823_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_823_end_mask_0 = const()[name = string("op_823_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_823_squeeze_mask_0 = const()[name = string("op_823_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_823_cast_fp16 = slice_by_index(begin = var_823_begin_0, end = var_823_end_0, end_mask = var_823_end_mask_0, squeeze_mask = var_823_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_823_cast_fp16")];
+            bool var_824_transpose_x_0 = const()[name = string("op_824_transpose_x_0"), val = bool(false)];
+            bool var_824_transpose_y_0 = const()[name = string("op_824_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_824_cast_fp16 = matmul(transpose_x = var_824_transpose_x_0, transpose_y = var_824_transpose_y_0, x = var_823_cast_fp16, y = state_39_cast_fp16)[name = string("op_824_cast_fp16")];
+            tensor<int32, [4]> var_826_begin_0 = const()[name = string("op_826_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_826_end_0 = const()[name = string("op_826_end_0"), val = tensor<int32, [4]>([32, 20, 64, 64])];
+            tensor<bool, [4]> var_826_end_mask_0 = const()[name = string("op_826_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_826_squeeze_mask_0 = const()[name = string("op_826_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_826_cast_fp16 = slice_by_index(begin = var_826_begin_0, end = var_826_end_0, end_mask = var_826_end_mask_0, squeeze_mask = var_826_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_826_cast_fp16")];
+            bool var_827_transpose_x_0 = const()[name = string("op_827_transpose_x_0"), val = bool(false)];
+            bool var_827_transpose_y_0 = const()[name = string("op_827_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_827_cast_fp16 = matmul(transpose_x = var_827_transpose_x_0, transpose_y = var_827_transpose_y_0, x = var_826_cast_fp16, y = v_new_39_cast_fp16)[name = string("op_827_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_828_cast_fp16 = add(x = var_824_cast_fp16, y = var_827_cast_fp16)[name = string("op_828_cast_fp16")];
+            tensor<int32, [4]> var_830_begin_0 = const()[name = string("op_830_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_830_end_0 = const()[name = string("op_830_end_0"), val = tensor<int32, [4]>([32, 20, 1, 1])];
+            tensor<bool, [4]> var_830_end_mask_0 = const()[name = string("op_830_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_830_squeeze_mask_0 = const()[name = string("op_830_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_830_cast_fp16 = slice_by_index(begin = var_830_begin_0, end = var_830_end_0, end_mask = var_830_end_mask_0, squeeze_mask = var_830_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_830_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_831_cast_fp16 = mul(x = state_39_cast_fp16, y = var_830_cast_fp16)[name = string("op_831_cast_fp16")];
+            tensor<int32, [4]> var_833_begin_0 = const()[name = string("op_833_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_833_end_0 = const()[name = string("op_833_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_833_end_mask_0 = const()[name = string("op_833_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_833_squeeze_mask_0 = const()[name = string("op_833_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_833_cast_fp16 = slice_by_index(begin = var_833_begin_0, end = var_833_end_0, end_mask = var_833_end_mask_0, squeeze_mask = var_833_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_833_cast_fp16")];
+            bool var_835_transpose_x_1 = const()[name = string("op_835_transpose_x_1"), val = bool(true)];
+            bool var_835_transpose_y_1 = const()[name = string("op_835_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_835_cast_fp16 = matmul(transpose_x = var_835_transpose_x_1, transpose_y = var_835_transpose_y_1, x = var_833_cast_fp16, y = v_new_39_cast_fp16)[name = string("op_835_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_41_cast_fp16 = add(x = var_831_cast_fp16, y = var_835_cast_fp16)[name = string("state_41_cast_fp16")];
+            tensor<int32, [4]> var_838_begin_0 = const()[name = string("op_838_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_838_end_0 = const()[name = string("op_838_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_838_end_mask_0 = const()[name = string("op_838_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_838_squeeze_mask_0 = const()[name = string("op_838_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_838_cast_fp16 = slice_by_index(begin = var_838_begin_0, end = var_838_end_0, end_mask = var_838_end_mask_0, squeeze_mask = var_838_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_838_cast_fp16")];
+            tensor<int32, [4]> var_840_begin_0 = const()[name = string("op_840_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_840_end_0 = const()[name = string("op_840_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_840_end_mask_0 = const()[name = string("op_840_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_840_squeeze_mask_0 = const()[name = string("op_840_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_840_cast_fp16 = slice_by_index(begin = var_840_begin_0, end = var_840_end_0, end_mask = var_840_end_mask_0, squeeze_mask = var_840_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_840_cast_fp16")];
+            bool var_841_transpose_x_0 = const()[name = string("op_841_transpose_x_0"), val = bool(false)];
+            bool var_841_transpose_y_0 = const()[name = string("op_841_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_841_cast_fp16 = matmul(transpose_x = var_841_transpose_x_0, transpose_y = var_841_transpose_y_0, x = var_840_cast_fp16, y = state_41_cast_fp16)[name = string("op_841_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_41_cast_fp16 = sub(x = var_838_cast_fp16, y = var_841_cast_fp16)[name = string("v_new_41_cast_fp16")];
+            tensor<int32, [4]> var_844_begin_0 = const()[name = string("op_844_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_844_end_0 = const()[name = string("op_844_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_844_end_mask_0 = const()[name = string("op_844_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_844_squeeze_mask_0 = const()[name = string("op_844_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_844_cast_fp16 = slice_by_index(begin = var_844_begin_0, end = var_844_end_0, end_mask = var_844_end_mask_0, squeeze_mask = var_844_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_844_cast_fp16")];
+            bool var_845_transpose_x_0 = const()[name = string("op_845_transpose_x_0"), val = bool(false)];
+            bool var_845_transpose_y_0 = const()[name = string("op_845_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_845_cast_fp16 = matmul(transpose_x = var_845_transpose_x_0, transpose_y = var_845_transpose_y_0, x = var_844_cast_fp16, y = state_41_cast_fp16)[name = string("op_845_cast_fp16")];
+            tensor<int32, [4]> var_847_begin_0 = const()[name = string("op_847_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_847_end_0 = const()[name = string("op_847_end_0"), val = tensor<int32, [4]>([32, 21, 64, 64])];
+            tensor<bool, [4]> var_847_end_mask_0 = const()[name = string("op_847_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_847_squeeze_mask_0 = const()[name = string("op_847_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_847_cast_fp16 = slice_by_index(begin = var_847_begin_0, end = var_847_end_0, end_mask = var_847_end_mask_0, squeeze_mask = var_847_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_847_cast_fp16")];
+            bool var_848_transpose_x_0 = const()[name = string("op_848_transpose_x_0"), val = bool(false)];
+            bool var_848_transpose_y_0 = const()[name = string("op_848_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_848_cast_fp16 = matmul(transpose_x = var_848_transpose_x_0, transpose_y = var_848_transpose_y_0, x = var_847_cast_fp16, y = v_new_41_cast_fp16)[name = string("op_848_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_849_cast_fp16 = add(x = var_845_cast_fp16, y = var_848_cast_fp16)[name = string("op_849_cast_fp16")];
+            tensor<int32, [4]> var_851_begin_0 = const()[name = string("op_851_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_851_end_0 = const()[name = string("op_851_end_0"), val = tensor<int32, [4]>([32, 21, 1, 1])];
+            tensor<bool, [4]> var_851_end_mask_0 = const()[name = string("op_851_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_851_squeeze_mask_0 = const()[name = string("op_851_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_851_cast_fp16 = slice_by_index(begin = var_851_begin_0, end = var_851_end_0, end_mask = var_851_end_mask_0, squeeze_mask = var_851_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_851_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_852_cast_fp16 = mul(x = state_41_cast_fp16, y = var_851_cast_fp16)[name = string("op_852_cast_fp16")];
+            tensor<int32, [4]> var_854_begin_0 = const()[name = string("op_854_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_854_end_0 = const()[name = string("op_854_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_854_end_mask_0 = const()[name = string("op_854_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_854_squeeze_mask_0 = const()[name = string("op_854_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_854_cast_fp16 = slice_by_index(begin = var_854_begin_0, end = var_854_end_0, end_mask = var_854_end_mask_0, squeeze_mask = var_854_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_854_cast_fp16")];
+            bool var_856_transpose_x_1 = const()[name = string("op_856_transpose_x_1"), val = bool(true)];
+            bool var_856_transpose_y_1 = const()[name = string("op_856_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_856_cast_fp16 = matmul(transpose_x = var_856_transpose_x_1, transpose_y = var_856_transpose_y_1, x = var_854_cast_fp16, y = v_new_41_cast_fp16)[name = string("op_856_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_43_cast_fp16 = add(x = var_852_cast_fp16, y = var_856_cast_fp16)[name = string("state_43_cast_fp16")];
+            tensor<int32, [4]> var_859_begin_0 = const()[name = string("op_859_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_859_end_0 = const()[name = string("op_859_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_859_end_mask_0 = const()[name = string("op_859_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_859_squeeze_mask_0 = const()[name = string("op_859_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_859_cast_fp16 = slice_by_index(begin = var_859_begin_0, end = var_859_end_0, end_mask = var_859_end_mask_0, squeeze_mask = var_859_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_859_cast_fp16")];
+            tensor<int32, [4]> var_861_begin_0 = const()[name = string("op_861_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_861_end_0 = const()[name = string("op_861_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_861_end_mask_0 = const()[name = string("op_861_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_861_squeeze_mask_0 = const()[name = string("op_861_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_861_cast_fp16 = slice_by_index(begin = var_861_begin_0, end = var_861_end_0, end_mask = var_861_end_mask_0, squeeze_mask = var_861_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_861_cast_fp16")];
+            bool var_862_transpose_x_0 = const()[name = string("op_862_transpose_x_0"), val = bool(false)];
+            bool var_862_transpose_y_0 = const()[name = string("op_862_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_862_cast_fp16 = matmul(transpose_x = var_862_transpose_x_0, transpose_y = var_862_transpose_y_0, x = var_861_cast_fp16, y = state_43_cast_fp16)[name = string("op_862_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_43_cast_fp16 = sub(x = var_859_cast_fp16, y = var_862_cast_fp16)[name = string("v_new_43_cast_fp16")];
+            tensor<int32, [4]> var_865_begin_0 = const()[name = string("op_865_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_865_end_0 = const()[name = string("op_865_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_865_end_mask_0 = const()[name = string("op_865_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_865_squeeze_mask_0 = const()[name = string("op_865_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_865_cast_fp16 = slice_by_index(begin = var_865_begin_0, end = var_865_end_0, end_mask = var_865_end_mask_0, squeeze_mask = var_865_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_865_cast_fp16")];
+            bool var_866_transpose_x_0 = const()[name = string("op_866_transpose_x_0"), val = bool(false)];
+            bool var_866_transpose_y_0 = const()[name = string("op_866_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_866_cast_fp16 = matmul(transpose_x = var_866_transpose_x_0, transpose_y = var_866_transpose_y_0, x = var_865_cast_fp16, y = state_43_cast_fp16)[name = string("op_866_cast_fp16")];
+            tensor<int32, [4]> var_868_begin_0 = const()[name = string("op_868_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_868_end_0 = const()[name = string("op_868_end_0"), val = tensor<int32, [4]>([32, 22, 64, 64])];
+            tensor<bool, [4]> var_868_end_mask_0 = const()[name = string("op_868_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_868_squeeze_mask_0 = const()[name = string("op_868_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_868_cast_fp16 = slice_by_index(begin = var_868_begin_0, end = var_868_end_0, end_mask = var_868_end_mask_0, squeeze_mask = var_868_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_868_cast_fp16")];
+            bool var_869_transpose_x_0 = const()[name = string("op_869_transpose_x_0"), val = bool(false)];
+            bool var_869_transpose_y_0 = const()[name = string("op_869_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_869_cast_fp16 = matmul(transpose_x = var_869_transpose_x_0, transpose_y = var_869_transpose_y_0, x = var_868_cast_fp16, y = v_new_43_cast_fp16)[name = string("op_869_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_870_cast_fp16 = add(x = var_866_cast_fp16, y = var_869_cast_fp16)[name = string("op_870_cast_fp16")];
+            tensor<int32, [4]> var_872_begin_0 = const()[name = string("op_872_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_872_end_0 = const()[name = string("op_872_end_0"), val = tensor<int32, [4]>([32, 22, 1, 1])];
+            tensor<bool, [4]> var_872_end_mask_0 = const()[name = string("op_872_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_872_squeeze_mask_0 = const()[name = string("op_872_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_872_cast_fp16 = slice_by_index(begin = var_872_begin_0, end = var_872_end_0, end_mask = var_872_end_mask_0, squeeze_mask = var_872_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_872_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_873_cast_fp16 = mul(x = state_43_cast_fp16, y = var_872_cast_fp16)[name = string("op_873_cast_fp16")];
+            tensor<int32, [4]> var_875_begin_0 = const()[name = string("op_875_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_875_end_0 = const()[name = string("op_875_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_875_end_mask_0 = const()[name = string("op_875_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_875_squeeze_mask_0 = const()[name = string("op_875_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_875_cast_fp16 = slice_by_index(begin = var_875_begin_0, end = var_875_end_0, end_mask = var_875_end_mask_0, squeeze_mask = var_875_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_875_cast_fp16")];
+            bool var_877_transpose_x_1 = const()[name = string("op_877_transpose_x_1"), val = bool(true)];
+            bool var_877_transpose_y_1 = const()[name = string("op_877_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_877_cast_fp16 = matmul(transpose_x = var_877_transpose_x_1, transpose_y = var_877_transpose_y_1, x = var_875_cast_fp16, y = v_new_43_cast_fp16)[name = string("op_877_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_45_cast_fp16 = add(x = var_873_cast_fp16, y = var_877_cast_fp16)[name = string("state_45_cast_fp16")];
+            tensor<int32, [4]> var_880_begin_0 = const()[name = string("op_880_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_880_end_0 = const()[name = string("op_880_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_880_end_mask_0 = const()[name = string("op_880_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_880_squeeze_mask_0 = const()[name = string("op_880_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_880_cast_fp16 = slice_by_index(begin = var_880_begin_0, end = var_880_end_0, end_mask = var_880_end_mask_0, squeeze_mask = var_880_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_880_cast_fp16")];
+            tensor<int32, [4]> var_882_begin_0 = const()[name = string("op_882_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_882_end_0 = const()[name = string("op_882_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_882_end_mask_0 = const()[name = string("op_882_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_882_squeeze_mask_0 = const()[name = string("op_882_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_882_cast_fp16 = slice_by_index(begin = var_882_begin_0, end = var_882_end_0, end_mask = var_882_end_mask_0, squeeze_mask = var_882_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_882_cast_fp16")];
+            bool var_883_transpose_x_0 = const()[name = string("op_883_transpose_x_0"), val = bool(false)];
+            bool var_883_transpose_y_0 = const()[name = string("op_883_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_883_cast_fp16 = matmul(transpose_x = var_883_transpose_x_0, transpose_y = var_883_transpose_y_0, x = var_882_cast_fp16, y = state_45_cast_fp16)[name = string("op_883_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_45_cast_fp16 = sub(x = var_880_cast_fp16, y = var_883_cast_fp16)[name = string("v_new_45_cast_fp16")];
+            tensor<int32, [4]> var_886_begin_0 = const()[name = string("op_886_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_886_end_0 = const()[name = string("op_886_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_886_end_mask_0 = const()[name = string("op_886_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_886_squeeze_mask_0 = const()[name = string("op_886_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_886_cast_fp16 = slice_by_index(begin = var_886_begin_0, end = var_886_end_0, end_mask = var_886_end_mask_0, squeeze_mask = var_886_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_886_cast_fp16")];
+            bool var_887_transpose_x_0 = const()[name = string("op_887_transpose_x_0"), val = bool(false)];
+            bool var_887_transpose_y_0 = const()[name = string("op_887_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_887_cast_fp16 = matmul(transpose_x = var_887_transpose_x_0, transpose_y = var_887_transpose_y_0, x = var_886_cast_fp16, y = state_45_cast_fp16)[name = string("op_887_cast_fp16")];
+            tensor<int32, [4]> var_889_begin_0 = const()[name = string("op_889_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_889_end_0 = const()[name = string("op_889_end_0"), val = tensor<int32, [4]>([32, 23, 64, 64])];
+            tensor<bool, [4]> var_889_end_mask_0 = const()[name = string("op_889_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_889_squeeze_mask_0 = const()[name = string("op_889_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_889_cast_fp16 = slice_by_index(begin = var_889_begin_0, end = var_889_end_0, end_mask = var_889_end_mask_0, squeeze_mask = var_889_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_889_cast_fp16")];
+            bool var_890_transpose_x_0 = const()[name = string("op_890_transpose_x_0"), val = bool(false)];
+            bool var_890_transpose_y_0 = const()[name = string("op_890_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_890_cast_fp16 = matmul(transpose_x = var_890_transpose_x_0, transpose_y = var_890_transpose_y_0, x = var_889_cast_fp16, y = v_new_45_cast_fp16)[name = string("op_890_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_891_cast_fp16 = add(x = var_887_cast_fp16, y = var_890_cast_fp16)[name = string("op_891_cast_fp16")];
+            tensor<int32, [4]> var_893_begin_0 = const()[name = string("op_893_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_893_end_0 = const()[name = string("op_893_end_0"), val = tensor<int32, [4]>([32, 23, 1, 1])];
+            tensor<bool, [4]> var_893_end_mask_0 = const()[name = string("op_893_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_893_squeeze_mask_0 = const()[name = string("op_893_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_893_cast_fp16 = slice_by_index(begin = var_893_begin_0, end = var_893_end_0, end_mask = var_893_end_mask_0, squeeze_mask = var_893_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_893_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_894_cast_fp16 = mul(x = state_45_cast_fp16, y = var_893_cast_fp16)[name = string("op_894_cast_fp16")];
+            tensor<int32, [4]> var_896_begin_0 = const()[name = string("op_896_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_896_end_0 = const()[name = string("op_896_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_896_end_mask_0 = const()[name = string("op_896_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_896_squeeze_mask_0 = const()[name = string("op_896_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_896_cast_fp16 = slice_by_index(begin = var_896_begin_0, end = var_896_end_0, end_mask = var_896_end_mask_0, squeeze_mask = var_896_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_896_cast_fp16")];
+            bool var_898_transpose_x_1 = const()[name = string("op_898_transpose_x_1"), val = bool(true)];
+            bool var_898_transpose_y_1 = const()[name = string("op_898_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_898_cast_fp16 = matmul(transpose_x = var_898_transpose_x_1, transpose_y = var_898_transpose_y_1, x = var_896_cast_fp16, y = v_new_45_cast_fp16)[name = string("op_898_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_47_cast_fp16 = add(x = var_894_cast_fp16, y = var_898_cast_fp16)[name = string("state_47_cast_fp16")];
+            tensor<int32, [4]> var_901_begin_0 = const()[name = string("op_901_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_901_end_0 = const()[name = string("op_901_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_901_end_mask_0 = const()[name = string("op_901_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_901_squeeze_mask_0 = const()[name = string("op_901_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_901_cast_fp16 = slice_by_index(begin = var_901_begin_0, end = var_901_end_0, end_mask = var_901_end_mask_0, squeeze_mask = var_901_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_901_cast_fp16")];
+            tensor<int32, [4]> var_903_begin_0 = const()[name = string("op_903_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_903_end_0 = const()[name = string("op_903_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_903_end_mask_0 = const()[name = string("op_903_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_903_squeeze_mask_0 = const()[name = string("op_903_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_903_cast_fp16 = slice_by_index(begin = var_903_begin_0, end = var_903_end_0, end_mask = var_903_end_mask_0, squeeze_mask = var_903_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_903_cast_fp16")];
+            bool var_904_transpose_x_0 = const()[name = string("op_904_transpose_x_0"), val = bool(false)];
+            bool var_904_transpose_y_0 = const()[name = string("op_904_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_904_cast_fp16 = matmul(transpose_x = var_904_transpose_x_0, transpose_y = var_904_transpose_y_0, x = var_903_cast_fp16, y = state_47_cast_fp16)[name = string("op_904_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_47_cast_fp16 = sub(x = var_901_cast_fp16, y = var_904_cast_fp16)[name = string("v_new_47_cast_fp16")];
+            tensor<int32, [4]> var_907_begin_0 = const()[name = string("op_907_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_907_end_0 = const()[name = string("op_907_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_907_end_mask_0 = const()[name = string("op_907_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_907_squeeze_mask_0 = const()[name = string("op_907_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_907_cast_fp16 = slice_by_index(begin = var_907_begin_0, end = var_907_end_0, end_mask = var_907_end_mask_0, squeeze_mask = var_907_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_907_cast_fp16")];
+            bool var_908_transpose_x_0 = const()[name = string("op_908_transpose_x_0"), val = bool(false)];
+            bool var_908_transpose_y_0 = const()[name = string("op_908_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_908_cast_fp16 = matmul(transpose_x = var_908_transpose_x_0, transpose_y = var_908_transpose_y_0, x = var_907_cast_fp16, y = state_47_cast_fp16)[name = string("op_908_cast_fp16")];
+            tensor<int32, [4]> var_910_begin_0 = const()[name = string("op_910_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_910_end_0 = const()[name = string("op_910_end_0"), val = tensor<int32, [4]>([32, 24, 64, 64])];
+            tensor<bool, [4]> var_910_end_mask_0 = const()[name = string("op_910_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_910_squeeze_mask_0 = const()[name = string("op_910_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_910_cast_fp16 = slice_by_index(begin = var_910_begin_0, end = var_910_end_0, end_mask = var_910_end_mask_0, squeeze_mask = var_910_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_910_cast_fp16")];
+            bool var_911_transpose_x_0 = const()[name = string("op_911_transpose_x_0"), val = bool(false)];
+            bool var_911_transpose_y_0 = const()[name = string("op_911_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_911_cast_fp16 = matmul(transpose_x = var_911_transpose_x_0, transpose_y = var_911_transpose_y_0, x = var_910_cast_fp16, y = v_new_47_cast_fp16)[name = string("op_911_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_912_cast_fp16 = add(x = var_908_cast_fp16, y = var_911_cast_fp16)[name = string("op_912_cast_fp16")];
+            tensor<int32, [4]> var_914_begin_0 = const()[name = string("op_914_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_914_end_0 = const()[name = string("op_914_end_0"), val = tensor<int32, [4]>([32, 24, 1, 1])];
+            tensor<bool, [4]> var_914_end_mask_0 = const()[name = string("op_914_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_914_squeeze_mask_0 = const()[name = string("op_914_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_914_cast_fp16 = slice_by_index(begin = var_914_begin_0, end = var_914_end_0, end_mask = var_914_end_mask_0, squeeze_mask = var_914_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_914_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_915_cast_fp16 = mul(x = state_47_cast_fp16, y = var_914_cast_fp16)[name = string("op_915_cast_fp16")];
+            tensor<int32, [4]> var_917_begin_0 = const()[name = string("op_917_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_917_end_0 = const()[name = string("op_917_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_917_end_mask_0 = const()[name = string("op_917_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_917_squeeze_mask_0 = const()[name = string("op_917_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_917_cast_fp16 = slice_by_index(begin = var_917_begin_0, end = var_917_end_0, end_mask = var_917_end_mask_0, squeeze_mask = var_917_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_917_cast_fp16")];
+            bool var_919_transpose_x_1 = const()[name = string("op_919_transpose_x_1"), val = bool(true)];
+            bool var_919_transpose_y_1 = const()[name = string("op_919_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_919_cast_fp16 = matmul(transpose_x = var_919_transpose_x_1, transpose_y = var_919_transpose_y_1, x = var_917_cast_fp16, y = v_new_47_cast_fp16)[name = string("op_919_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_49_cast_fp16 = add(x = var_915_cast_fp16, y = var_919_cast_fp16)[name = string("state_49_cast_fp16")];
+            tensor<int32, [4]> var_922_begin_0 = const()[name = string("op_922_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_922_end_0 = const()[name = string("op_922_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_922_end_mask_0 = const()[name = string("op_922_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_922_squeeze_mask_0 = const()[name = string("op_922_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_922_cast_fp16 = slice_by_index(begin = var_922_begin_0, end = var_922_end_0, end_mask = var_922_end_mask_0, squeeze_mask = var_922_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_922_cast_fp16")];
+            tensor<int32, [4]> var_924_begin_0 = const()[name = string("op_924_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_924_end_0 = const()[name = string("op_924_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_924_end_mask_0 = const()[name = string("op_924_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_924_squeeze_mask_0 = const()[name = string("op_924_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_924_cast_fp16 = slice_by_index(begin = var_924_begin_0, end = var_924_end_0, end_mask = var_924_end_mask_0, squeeze_mask = var_924_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_924_cast_fp16")];
+            bool var_925_transpose_x_0 = const()[name = string("op_925_transpose_x_0"), val = bool(false)];
+            bool var_925_transpose_y_0 = const()[name = string("op_925_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_925_cast_fp16 = matmul(transpose_x = var_925_transpose_x_0, transpose_y = var_925_transpose_y_0, x = var_924_cast_fp16, y = state_49_cast_fp16)[name = string("op_925_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_49_cast_fp16 = sub(x = var_922_cast_fp16, y = var_925_cast_fp16)[name = string("v_new_49_cast_fp16")];
+            tensor<int32, [4]> var_928_begin_0 = const()[name = string("op_928_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_928_end_0 = const()[name = string("op_928_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_928_end_mask_0 = const()[name = string("op_928_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_928_squeeze_mask_0 = const()[name = string("op_928_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_928_cast_fp16 = slice_by_index(begin = var_928_begin_0, end = var_928_end_0, end_mask = var_928_end_mask_0, squeeze_mask = var_928_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_928_cast_fp16")];
+            bool var_929_transpose_x_0 = const()[name = string("op_929_transpose_x_0"), val = bool(false)];
+            bool var_929_transpose_y_0 = const()[name = string("op_929_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_929_cast_fp16 = matmul(transpose_x = var_929_transpose_x_0, transpose_y = var_929_transpose_y_0, x = var_928_cast_fp16, y = state_49_cast_fp16)[name = string("op_929_cast_fp16")];
+            tensor<int32, [4]> var_931_begin_0 = const()[name = string("op_931_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_931_end_0 = const()[name = string("op_931_end_0"), val = tensor<int32, [4]>([32, 25, 64, 64])];
+            tensor<bool, [4]> var_931_end_mask_0 = const()[name = string("op_931_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_931_squeeze_mask_0 = const()[name = string("op_931_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_931_cast_fp16 = slice_by_index(begin = var_931_begin_0, end = var_931_end_0, end_mask = var_931_end_mask_0, squeeze_mask = var_931_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_931_cast_fp16")];
+            bool var_932_transpose_x_0 = const()[name = string("op_932_transpose_x_0"), val = bool(false)];
+            bool var_932_transpose_y_0 = const()[name = string("op_932_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_932_cast_fp16 = matmul(transpose_x = var_932_transpose_x_0, transpose_y = var_932_transpose_y_0, x = var_931_cast_fp16, y = v_new_49_cast_fp16)[name = string("op_932_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_933_cast_fp16 = add(x = var_929_cast_fp16, y = var_932_cast_fp16)[name = string("op_933_cast_fp16")];
+            tensor<int32, [4]> var_935_begin_0 = const()[name = string("op_935_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_935_end_0 = const()[name = string("op_935_end_0"), val = tensor<int32, [4]>([32, 25, 1, 1])];
+            tensor<bool, [4]> var_935_end_mask_0 = const()[name = string("op_935_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_935_squeeze_mask_0 = const()[name = string("op_935_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_935_cast_fp16 = slice_by_index(begin = var_935_begin_0, end = var_935_end_0, end_mask = var_935_end_mask_0, squeeze_mask = var_935_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_935_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_936_cast_fp16 = mul(x = state_49_cast_fp16, y = var_935_cast_fp16)[name = string("op_936_cast_fp16")];
+            tensor<int32, [4]> var_938_begin_0 = const()[name = string("op_938_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_938_end_0 = const()[name = string("op_938_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_938_end_mask_0 = const()[name = string("op_938_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_938_squeeze_mask_0 = const()[name = string("op_938_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_938_cast_fp16 = slice_by_index(begin = var_938_begin_0, end = var_938_end_0, end_mask = var_938_end_mask_0, squeeze_mask = var_938_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_938_cast_fp16")];
+            bool var_940_transpose_x_1 = const()[name = string("op_940_transpose_x_1"), val = bool(true)];
+            bool var_940_transpose_y_1 = const()[name = string("op_940_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_940_cast_fp16 = matmul(transpose_x = var_940_transpose_x_1, transpose_y = var_940_transpose_y_1, x = var_938_cast_fp16, y = v_new_49_cast_fp16)[name = string("op_940_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_51_cast_fp16 = add(x = var_936_cast_fp16, y = var_940_cast_fp16)[name = string("state_51_cast_fp16")];
+            tensor<int32, [4]> var_943_begin_0 = const()[name = string("op_943_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_943_end_0 = const()[name = string("op_943_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_943_end_mask_0 = const()[name = string("op_943_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_943_squeeze_mask_0 = const()[name = string("op_943_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_943_cast_fp16 = slice_by_index(begin = var_943_begin_0, end = var_943_end_0, end_mask = var_943_end_mask_0, squeeze_mask = var_943_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_943_cast_fp16")];
+            tensor<int32, [4]> var_945_begin_0 = const()[name = string("op_945_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_945_end_0 = const()[name = string("op_945_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_945_end_mask_0 = const()[name = string("op_945_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_945_squeeze_mask_0 = const()[name = string("op_945_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_945_cast_fp16 = slice_by_index(begin = var_945_begin_0, end = var_945_end_0, end_mask = var_945_end_mask_0, squeeze_mask = var_945_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_945_cast_fp16")];
+            bool var_946_transpose_x_0 = const()[name = string("op_946_transpose_x_0"), val = bool(false)];
+            bool var_946_transpose_y_0 = const()[name = string("op_946_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_946_cast_fp16 = matmul(transpose_x = var_946_transpose_x_0, transpose_y = var_946_transpose_y_0, x = var_945_cast_fp16, y = state_51_cast_fp16)[name = string("op_946_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_51_cast_fp16 = sub(x = var_943_cast_fp16, y = var_946_cast_fp16)[name = string("v_new_51_cast_fp16")];
+            tensor<int32, [4]> var_949_begin_0 = const()[name = string("op_949_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_949_end_0 = const()[name = string("op_949_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_949_end_mask_0 = const()[name = string("op_949_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_949_squeeze_mask_0 = const()[name = string("op_949_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_949_cast_fp16 = slice_by_index(begin = var_949_begin_0, end = var_949_end_0, end_mask = var_949_end_mask_0, squeeze_mask = var_949_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_949_cast_fp16")];
+            bool var_950_transpose_x_0 = const()[name = string("op_950_transpose_x_0"), val = bool(false)];
+            bool var_950_transpose_y_0 = const()[name = string("op_950_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_950_cast_fp16 = matmul(transpose_x = var_950_transpose_x_0, transpose_y = var_950_transpose_y_0, x = var_949_cast_fp16, y = state_51_cast_fp16)[name = string("op_950_cast_fp16")];
+            tensor<int32, [4]> var_952_begin_0 = const()[name = string("op_952_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_952_end_0 = const()[name = string("op_952_end_0"), val = tensor<int32, [4]>([32, 26, 64, 64])];
+            tensor<bool, [4]> var_952_end_mask_0 = const()[name = string("op_952_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_952_squeeze_mask_0 = const()[name = string("op_952_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_952_cast_fp16 = slice_by_index(begin = var_952_begin_0, end = var_952_end_0, end_mask = var_952_end_mask_0, squeeze_mask = var_952_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_952_cast_fp16")];
+            bool var_953_transpose_x_0 = const()[name = string("op_953_transpose_x_0"), val = bool(false)];
+            bool var_953_transpose_y_0 = const()[name = string("op_953_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_953_cast_fp16 = matmul(transpose_x = var_953_transpose_x_0, transpose_y = var_953_transpose_y_0, x = var_952_cast_fp16, y = v_new_51_cast_fp16)[name = string("op_953_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_954_cast_fp16 = add(x = var_950_cast_fp16, y = var_953_cast_fp16)[name = string("op_954_cast_fp16")];
+            tensor<int32, [4]> var_956_begin_0 = const()[name = string("op_956_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_956_end_0 = const()[name = string("op_956_end_0"), val = tensor<int32, [4]>([32, 26, 1, 1])];
+            tensor<bool, [4]> var_956_end_mask_0 = const()[name = string("op_956_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_956_squeeze_mask_0 = const()[name = string("op_956_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_956_cast_fp16 = slice_by_index(begin = var_956_begin_0, end = var_956_end_0, end_mask = var_956_end_mask_0, squeeze_mask = var_956_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_956_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_957_cast_fp16 = mul(x = state_51_cast_fp16, y = var_956_cast_fp16)[name = string("op_957_cast_fp16")];
+            tensor<int32, [4]> var_959_begin_0 = const()[name = string("op_959_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_959_end_0 = const()[name = string("op_959_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_959_end_mask_0 = const()[name = string("op_959_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_959_squeeze_mask_0 = const()[name = string("op_959_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_959_cast_fp16 = slice_by_index(begin = var_959_begin_0, end = var_959_end_0, end_mask = var_959_end_mask_0, squeeze_mask = var_959_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_959_cast_fp16")];
+            bool var_961_transpose_x_1 = const()[name = string("op_961_transpose_x_1"), val = bool(true)];
+            bool var_961_transpose_y_1 = const()[name = string("op_961_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_961_cast_fp16 = matmul(transpose_x = var_961_transpose_x_1, transpose_y = var_961_transpose_y_1, x = var_959_cast_fp16, y = v_new_51_cast_fp16)[name = string("op_961_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_53_cast_fp16 = add(x = var_957_cast_fp16, y = var_961_cast_fp16)[name = string("state_53_cast_fp16")];
+            tensor<int32, [4]> var_964_begin_0 = const()[name = string("op_964_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_964_end_0 = const()[name = string("op_964_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_964_end_mask_0 = const()[name = string("op_964_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_964_squeeze_mask_0 = const()[name = string("op_964_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_964_cast_fp16 = slice_by_index(begin = var_964_begin_0, end = var_964_end_0, end_mask = var_964_end_mask_0, squeeze_mask = var_964_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_964_cast_fp16")];
+            tensor<int32, [4]> var_966_begin_0 = const()[name = string("op_966_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_966_end_0 = const()[name = string("op_966_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_966_end_mask_0 = const()[name = string("op_966_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_966_squeeze_mask_0 = const()[name = string("op_966_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_966_cast_fp16 = slice_by_index(begin = var_966_begin_0, end = var_966_end_0, end_mask = var_966_end_mask_0, squeeze_mask = var_966_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_966_cast_fp16")];
+            bool var_967_transpose_x_0 = const()[name = string("op_967_transpose_x_0"), val = bool(false)];
+            bool var_967_transpose_y_0 = const()[name = string("op_967_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_967_cast_fp16 = matmul(transpose_x = var_967_transpose_x_0, transpose_y = var_967_transpose_y_0, x = var_966_cast_fp16, y = state_53_cast_fp16)[name = string("op_967_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_53_cast_fp16 = sub(x = var_964_cast_fp16, y = var_967_cast_fp16)[name = string("v_new_53_cast_fp16")];
+            tensor<int32, [4]> var_970_begin_0 = const()[name = string("op_970_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_970_end_0 = const()[name = string("op_970_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_970_end_mask_0 = const()[name = string("op_970_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_970_squeeze_mask_0 = const()[name = string("op_970_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_970_cast_fp16 = slice_by_index(begin = var_970_begin_0, end = var_970_end_0, end_mask = var_970_end_mask_0, squeeze_mask = var_970_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_970_cast_fp16")];
+            bool var_971_transpose_x_0 = const()[name = string("op_971_transpose_x_0"), val = bool(false)];
+            bool var_971_transpose_y_0 = const()[name = string("op_971_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_971_cast_fp16 = matmul(transpose_x = var_971_transpose_x_0, transpose_y = var_971_transpose_y_0, x = var_970_cast_fp16, y = state_53_cast_fp16)[name = string("op_971_cast_fp16")];
+            tensor<int32, [4]> var_973_begin_0 = const()[name = string("op_973_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_973_end_0 = const()[name = string("op_973_end_0"), val = tensor<int32, [4]>([32, 27, 64, 64])];
+            tensor<bool, [4]> var_973_end_mask_0 = const()[name = string("op_973_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_973_squeeze_mask_0 = const()[name = string("op_973_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_973_cast_fp16 = slice_by_index(begin = var_973_begin_0, end = var_973_end_0, end_mask = var_973_end_mask_0, squeeze_mask = var_973_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_973_cast_fp16")];
+            bool var_974_transpose_x_0 = const()[name = string("op_974_transpose_x_0"), val = bool(false)];
+            bool var_974_transpose_y_0 = const()[name = string("op_974_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_974_cast_fp16 = matmul(transpose_x = var_974_transpose_x_0, transpose_y = var_974_transpose_y_0, x = var_973_cast_fp16, y = v_new_53_cast_fp16)[name = string("op_974_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_975_cast_fp16 = add(x = var_971_cast_fp16, y = var_974_cast_fp16)[name = string("op_975_cast_fp16")];
+            tensor<int32, [4]> var_977_begin_0 = const()[name = string("op_977_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_977_end_0 = const()[name = string("op_977_end_0"), val = tensor<int32, [4]>([32, 27, 1, 1])];
+            tensor<bool, [4]> var_977_end_mask_0 = const()[name = string("op_977_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_977_squeeze_mask_0 = const()[name = string("op_977_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_977_cast_fp16 = slice_by_index(begin = var_977_begin_0, end = var_977_end_0, end_mask = var_977_end_mask_0, squeeze_mask = var_977_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_977_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_978_cast_fp16 = mul(x = state_53_cast_fp16, y = var_977_cast_fp16)[name = string("op_978_cast_fp16")];
+            tensor<int32, [4]> var_980_begin_0 = const()[name = string("op_980_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_980_end_0 = const()[name = string("op_980_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_980_end_mask_0 = const()[name = string("op_980_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_980_squeeze_mask_0 = const()[name = string("op_980_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_980_cast_fp16 = slice_by_index(begin = var_980_begin_0, end = var_980_end_0, end_mask = var_980_end_mask_0, squeeze_mask = var_980_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_980_cast_fp16")];
+            bool var_982_transpose_x_1 = const()[name = string("op_982_transpose_x_1"), val = bool(true)];
+            bool var_982_transpose_y_1 = const()[name = string("op_982_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_982_cast_fp16 = matmul(transpose_x = var_982_transpose_x_1, transpose_y = var_982_transpose_y_1, x = var_980_cast_fp16, y = v_new_53_cast_fp16)[name = string("op_982_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_55_cast_fp16 = add(x = var_978_cast_fp16, y = var_982_cast_fp16)[name = string("state_55_cast_fp16")];
+            tensor<int32, [4]> var_985_begin_0 = const()[name = string("op_985_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_985_end_0 = const()[name = string("op_985_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_985_end_mask_0 = const()[name = string("op_985_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_985_squeeze_mask_0 = const()[name = string("op_985_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_985_cast_fp16 = slice_by_index(begin = var_985_begin_0, end = var_985_end_0, end_mask = var_985_end_mask_0, squeeze_mask = var_985_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_985_cast_fp16")];
+            tensor<int32, [4]> var_987_begin_0 = const()[name = string("op_987_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_987_end_0 = const()[name = string("op_987_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_987_end_mask_0 = const()[name = string("op_987_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_987_squeeze_mask_0 = const()[name = string("op_987_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_987_cast_fp16 = slice_by_index(begin = var_987_begin_0, end = var_987_end_0, end_mask = var_987_end_mask_0, squeeze_mask = var_987_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_987_cast_fp16")];
+            bool var_988_transpose_x_0 = const()[name = string("op_988_transpose_x_0"), val = bool(false)];
+            bool var_988_transpose_y_0 = const()[name = string("op_988_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_988_cast_fp16 = matmul(transpose_x = var_988_transpose_x_0, transpose_y = var_988_transpose_y_0, x = var_987_cast_fp16, y = state_55_cast_fp16)[name = string("op_988_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_55_cast_fp16 = sub(x = var_985_cast_fp16, y = var_988_cast_fp16)[name = string("v_new_55_cast_fp16")];
+            tensor<int32, [4]> var_991_begin_0 = const()[name = string("op_991_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_991_end_0 = const()[name = string("op_991_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_991_end_mask_0 = const()[name = string("op_991_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_991_squeeze_mask_0 = const()[name = string("op_991_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_991_cast_fp16 = slice_by_index(begin = var_991_begin_0, end = var_991_end_0, end_mask = var_991_end_mask_0, squeeze_mask = var_991_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_991_cast_fp16")];
+            bool var_992_transpose_x_0 = const()[name = string("op_992_transpose_x_0"), val = bool(false)];
+            bool var_992_transpose_y_0 = const()[name = string("op_992_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_992_cast_fp16 = matmul(transpose_x = var_992_transpose_x_0, transpose_y = var_992_transpose_y_0, x = var_991_cast_fp16, y = state_55_cast_fp16)[name = string("op_992_cast_fp16")];
+            tensor<int32, [4]> var_994_begin_0 = const()[name = string("op_994_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_994_end_0 = const()[name = string("op_994_end_0"), val = tensor<int32, [4]>([32, 28, 64, 64])];
+            tensor<bool, [4]> var_994_end_mask_0 = const()[name = string("op_994_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_994_squeeze_mask_0 = const()[name = string("op_994_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_994_cast_fp16 = slice_by_index(begin = var_994_begin_0, end = var_994_end_0, end_mask = var_994_end_mask_0, squeeze_mask = var_994_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_994_cast_fp16")];
+            bool var_995_transpose_x_0 = const()[name = string("op_995_transpose_x_0"), val = bool(false)];
+            bool var_995_transpose_y_0 = const()[name = string("op_995_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_995_cast_fp16 = matmul(transpose_x = var_995_transpose_x_0, transpose_y = var_995_transpose_y_0, x = var_994_cast_fp16, y = v_new_55_cast_fp16)[name = string("op_995_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_996_cast_fp16 = add(x = var_992_cast_fp16, y = var_995_cast_fp16)[name = string("op_996_cast_fp16")];
+            tensor<int32, [4]> var_998_begin_0 = const()[name = string("op_998_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_998_end_0 = const()[name = string("op_998_end_0"), val = tensor<int32, [4]>([32, 28, 1, 1])];
+            tensor<bool, [4]> var_998_end_mask_0 = const()[name = string("op_998_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_998_squeeze_mask_0 = const()[name = string("op_998_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_998_cast_fp16 = slice_by_index(begin = var_998_begin_0, end = var_998_end_0, end_mask = var_998_end_mask_0, squeeze_mask = var_998_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_998_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_999_cast_fp16 = mul(x = state_55_cast_fp16, y = var_998_cast_fp16)[name = string("op_999_cast_fp16")];
+            tensor<int32, [4]> var_1001_begin_0 = const()[name = string("op_1001_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_1001_end_0 = const()[name = string("op_1001_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_1001_end_mask_0 = const()[name = string("op_1001_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1001_squeeze_mask_0 = const()[name = string("op_1001_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1001_cast_fp16 = slice_by_index(begin = var_1001_begin_0, end = var_1001_end_0, end_mask = var_1001_end_mask_0, squeeze_mask = var_1001_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_1001_cast_fp16")];
+            bool var_1003_transpose_x_1 = const()[name = string("op_1003_transpose_x_1"), val = bool(true)];
+            bool var_1003_transpose_y_1 = const()[name = string("op_1003_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1003_cast_fp16 = matmul(transpose_x = var_1003_transpose_x_1, transpose_y = var_1003_transpose_y_1, x = var_1001_cast_fp16, y = v_new_55_cast_fp16)[name = string("op_1003_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_57_cast_fp16 = add(x = var_999_cast_fp16, y = var_1003_cast_fp16)[name = string("state_57_cast_fp16")];
+            tensor<int32, [4]> var_1006_begin_0 = const()[name = string("op_1006_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_1006_end_0 = const()[name = string("op_1006_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_1006_end_mask_0 = const()[name = string("op_1006_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1006_squeeze_mask_0 = const()[name = string("op_1006_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1006_cast_fp16 = slice_by_index(begin = var_1006_begin_0, end = var_1006_end_0, end_mask = var_1006_end_mask_0, squeeze_mask = var_1006_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_1006_cast_fp16")];
+            tensor<int32, [4]> var_1008_begin_0 = const()[name = string("op_1008_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_1008_end_0 = const()[name = string("op_1008_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_1008_end_mask_0 = const()[name = string("op_1008_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1008_squeeze_mask_0 = const()[name = string("op_1008_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1008_cast_fp16 = slice_by_index(begin = var_1008_begin_0, end = var_1008_end_0, end_mask = var_1008_end_mask_0, squeeze_mask = var_1008_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_1008_cast_fp16")];
+            bool var_1009_transpose_x_0 = const()[name = string("op_1009_transpose_x_0"), val = bool(false)];
+            bool var_1009_transpose_y_0 = const()[name = string("op_1009_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1009_cast_fp16 = matmul(transpose_x = var_1009_transpose_x_0, transpose_y = var_1009_transpose_y_0, x = var_1008_cast_fp16, y = state_57_cast_fp16)[name = string("op_1009_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_57_cast_fp16 = sub(x = var_1006_cast_fp16, y = var_1009_cast_fp16)[name = string("v_new_57_cast_fp16")];
+            tensor<int32, [4]> var_1012_begin_0 = const()[name = string("op_1012_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_1012_end_0 = const()[name = string("op_1012_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_1012_end_mask_0 = const()[name = string("op_1012_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1012_squeeze_mask_0 = const()[name = string("op_1012_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1012_cast_fp16 = slice_by_index(begin = var_1012_begin_0, end = var_1012_end_0, end_mask = var_1012_end_mask_0, squeeze_mask = var_1012_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_1012_cast_fp16")];
+            bool var_1013_transpose_x_0 = const()[name = string("op_1013_transpose_x_0"), val = bool(false)];
+            bool var_1013_transpose_y_0 = const()[name = string("op_1013_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1013_cast_fp16 = matmul(transpose_x = var_1013_transpose_x_0, transpose_y = var_1013_transpose_y_0, x = var_1012_cast_fp16, y = state_57_cast_fp16)[name = string("op_1013_cast_fp16")];
+            tensor<int32, [4]> var_1015_begin_0 = const()[name = string("op_1015_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_1015_end_0 = const()[name = string("op_1015_end_0"), val = tensor<int32, [4]>([32, 29, 64, 64])];
+            tensor<bool, [4]> var_1015_end_mask_0 = const()[name = string("op_1015_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1015_squeeze_mask_0 = const()[name = string("op_1015_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1015_cast_fp16 = slice_by_index(begin = var_1015_begin_0, end = var_1015_end_0, end_mask = var_1015_end_mask_0, squeeze_mask = var_1015_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_1015_cast_fp16")];
+            bool var_1016_transpose_x_0 = const()[name = string("op_1016_transpose_x_0"), val = bool(false)];
+            bool var_1016_transpose_y_0 = const()[name = string("op_1016_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1016_cast_fp16 = matmul(transpose_x = var_1016_transpose_x_0, transpose_y = var_1016_transpose_y_0, x = var_1015_cast_fp16, y = v_new_57_cast_fp16)[name = string("op_1016_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1017_cast_fp16 = add(x = var_1013_cast_fp16, y = var_1016_cast_fp16)[name = string("op_1017_cast_fp16")];
+            tensor<int32, [4]> var_1019_begin_0 = const()[name = string("op_1019_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_1019_end_0 = const()[name = string("op_1019_end_0"), val = tensor<int32, [4]>([32, 29, 1, 1])];
+            tensor<bool, [4]> var_1019_end_mask_0 = const()[name = string("op_1019_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1019_squeeze_mask_0 = const()[name = string("op_1019_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1019_cast_fp16 = slice_by_index(begin = var_1019_begin_0, end = var_1019_end_0, end_mask = var_1019_end_mask_0, squeeze_mask = var_1019_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_1019_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1020_cast_fp16 = mul(x = state_57_cast_fp16, y = var_1019_cast_fp16)[name = string("op_1020_cast_fp16")];
+            tensor<int32, [4]> var_1022_begin_0 = const()[name = string("op_1022_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_1022_end_0 = const()[name = string("op_1022_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_1022_end_mask_0 = const()[name = string("op_1022_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1022_squeeze_mask_0 = const()[name = string("op_1022_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1022_cast_fp16 = slice_by_index(begin = var_1022_begin_0, end = var_1022_end_0, end_mask = var_1022_end_mask_0, squeeze_mask = var_1022_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_1022_cast_fp16")];
+            bool var_1024_transpose_x_1 = const()[name = string("op_1024_transpose_x_1"), val = bool(true)];
+            bool var_1024_transpose_y_1 = const()[name = string("op_1024_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1024_cast_fp16 = matmul(transpose_x = var_1024_transpose_x_1, transpose_y = var_1024_transpose_y_1, x = var_1022_cast_fp16, y = v_new_57_cast_fp16)[name = string("op_1024_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_59_cast_fp16 = add(x = var_1020_cast_fp16, y = var_1024_cast_fp16)[name = string("state_59_cast_fp16")];
+            tensor<int32, [4]> var_1027_begin_0 = const()[name = string("op_1027_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_1027_end_0 = const()[name = string("op_1027_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_1027_end_mask_0 = const()[name = string("op_1027_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1027_squeeze_mask_0 = const()[name = string("op_1027_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1027_cast_fp16 = slice_by_index(begin = var_1027_begin_0, end = var_1027_end_0, end_mask = var_1027_end_mask_0, squeeze_mask = var_1027_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_1027_cast_fp16")];
+            tensor<int32, [4]> var_1029_begin_0 = const()[name = string("op_1029_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_1029_end_0 = const()[name = string("op_1029_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_1029_end_mask_0 = const()[name = string("op_1029_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1029_squeeze_mask_0 = const()[name = string("op_1029_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1029_cast_fp16 = slice_by_index(begin = var_1029_begin_0, end = var_1029_end_0, end_mask = var_1029_end_mask_0, squeeze_mask = var_1029_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_1029_cast_fp16")];
+            bool var_1030_transpose_x_0 = const()[name = string("op_1030_transpose_x_0"), val = bool(false)];
+            bool var_1030_transpose_y_0 = const()[name = string("op_1030_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1030_cast_fp16 = matmul(transpose_x = var_1030_transpose_x_0, transpose_y = var_1030_transpose_y_0, x = var_1029_cast_fp16, y = state_59_cast_fp16)[name = string("op_1030_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_59_cast_fp16 = sub(x = var_1027_cast_fp16, y = var_1030_cast_fp16)[name = string("v_new_59_cast_fp16")];
+            tensor<int32, [4]> var_1033_begin_0 = const()[name = string("op_1033_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_1033_end_0 = const()[name = string("op_1033_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_1033_end_mask_0 = const()[name = string("op_1033_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1033_squeeze_mask_0 = const()[name = string("op_1033_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1033_cast_fp16 = slice_by_index(begin = var_1033_begin_0, end = var_1033_end_0, end_mask = var_1033_end_mask_0, squeeze_mask = var_1033_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_1033_cast_fp16")];
+            bool var_1034_transpose_x_0 = const()[name = string("op_1034_transpose_x_0"), val = bool(false)];
+            bool var_1034_transpose_y_0 = const()[name = string("op_1034_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1034_cast_fp16 = matmul(transpose_x = var_1034_transpose_x_0, transpose_y = var_1034_transpose_y_0, x = var_1033_cast_fp16, y = state_59_cast_fp16)[name = string("op_1034_cast_fp16")];
+            tensor<int32, [4]> var_1036_begin_0 = const()[name = string("op_1036_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_1036_end_0 = const()[name = string("op_1036_end_0"), val = tensor<int32, [4]>([32, 30, 64, 64])];
+            tensor<bool, [4]> var_1036_end_mask_0 = const()[name = string("op_1036_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1036_squeeze_mask_0 = const()[name = string("op_1036_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1036_cast_fp16 = slice_by_index(begin = var_1036_begin_0, end = var_1036_end_0, end_mask = var_1036_end_mask_0, squeeze_mask = var_1036_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_1036_cast_fp16")];
+            bool var_1037_transpose_x_0 = const()[name = string("op_1037_transpose_x_0"), val = bool(false)];
+            bool var_1037_transpose_y_0 = const()[name = string("op_1037_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1037_cast_fp16 = matmul(transpose_x = var_1037_transpose_x_0, transpose_y = var_1037_transpose_y_0, x = var_1036_cast_fp16, y = v_new_59_cast_fp16)[name = string("op_1037_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1038_cast_fp16 = add(x = var_1034_cast_fp16, y = var_1037_cast_fp16)[name = string("op_1038_cast_fp16")];
+            tensor<int32, [4]> var_1040_begin_0 = const()[name = string("op_1040_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_1040_end_0 = const()[name = string("op_1040_end_0"), val = tensor<int32, [4]>([32, 30, 1, 1])];
+            tensor<bool, [4]> var_1040_end_mask_0 = const()[name = string("op_1040_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1040_squeeze_mask_0 = const()[name = string("op_1040_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1040_cast_fp16 = slice_by_index(begin = var_1040_begin_0, end = var_1040_end_0, end_mask = var_1040_end_mask_0, squeeze_mask = var_1040_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_1040_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1041_cast_fp16 = mul(x = state_59_cast_fp16, y = var_1040_cast_fp16)[name = string("op_1041_cast_fp16")];
+            tensor<int32, [4]> var_1043_begin_0 = const()[name = string("op_1043_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_1043_end_0 = const()[name = string("op_1043_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_1043_end_mask_0 = const()[name = string("op_1043_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1043_squeeze_mask_0 = const()[name = string("op_1043_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1043_cast_fp16 = slice_by_index(begin = var_1043_begin_0, end = var_1043_end_0, end_mask = var_1043_end_mask_0, squeeze_mask = var_1043_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_1043_cast_fp16")];
+            bool var_1045_transpose_x_1 = const()[name = string("op_1045_transpose_x_1"), val = bool(true)];
+            bool var_1045_transpose_y_1 = const()[name = string("op_1045_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1045_cast_fp16 = matmul(transpose_x = var_1045_transpose_x_1, transpose_y = var_1045_transpose_y_1, x = var_1043_cast_fp16, y = v_new_59_cast_fp16)[name = string("op_1045_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_61_cast_fp16 = add(x = var_1041_cast_fp16, y = var_1045_cast_fp16)[name = string("state_61_cast_fp16")];
+            tensor<int32, [4]> var_1048_begin_0 = const()[name = string("op_1048_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_1048_end_0 = const()[name = string("op_1048_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_1048_end_mask_0 = const()[name = string("op_1048_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1048_squeeze_mask_0 = const()[name = string("op_1048_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1048_cast_fp16 = slice_by_index(begin = var_1048_begin_0, end = var_1048_end_0, end_mask = var_1048_end_mask_0, squeeze_mask = var_1048_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_1048_cast_fp16")];
+            tensor<int32, [4]> var_1050_begin_0 = const()[name = string("op_1050_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_1050_end_0 = const()[name = string("op_1050_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_1050_end_mask_0 = const()[name = string("op_1050_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1050_squeeze_mask_0 = const()[name = string("op_1050_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1050_cast_fp16 = slice_by_index(begin = var_1050_begin_0, end = var_1050_end_0, end_mask = var_1050_end_mask_0, squeeze_mask = var_1050_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_1050_cast_fp16")];
+            bool var_1051_transpose_x_0 = const()[name = string("op_1051_transpose_x_0"), val = bool(false)];
+            bool var_1051_transpose_y_0 = const()[name = string("op_1051_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1051_cast_fp16 = matmul(transpose_x = var_1051_transpose_x_0, transpose_y = var_1051_transpose_y_0, x = var_1050_cast_fp16, y = state_61_cast_fp16)[name = string("op_1051_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_61_cast_fp16 = sub(x = var_1048_cast_fp16, y = var_1051_cast_fp16)[name = string("v_new_61_cast_fp16")];
+            tensor<int32, [4]> var_1054_begin_0 = const()[name = string("op_1054_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_1054_end_0 = const()[name = string("op_1054_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_1054_end_mask_0 = const()[name = string("op_1054_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1054_squeeze_mask_0 = const()[name = string("op_1054_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1054_cast_fp16 = slice_by_index(begin = var_1054_begin_0, end = var_1054_end_0, end_mask = var_1054_end_mask_0, squeeze_mask = var_1054_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_1054_cast_fp16")];
+            bool var_1055_transpose_x_0 = const()[name = string("op_1055_transpose_x_0"), val = bool(false)];
+            bool var_1055_transpose_y_0 = const()[name = string("op_1055_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1055_cast_fp16 = matmul(transpose_x = var_1055_transpose_x_0, transpose_y = var_1055_transpose_y_0, x = var_1054_cast_fp16, y = state_61_cast_fp16)[name = string("op_1055_cast_fp16")];
+            tensor<int32, [4]> var_1057_begin_0 = const()[name = string("op_1057_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_1057_end_0 = const()[name = string("op_1057_end_0"), val = tensor<int32, [4]>([32, 31, 64, 64])];
+            tensor<bool, [4]> var_1057_end_mask_0 = const()[name = string("op_1057_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1057_squeeze_mask_0 = const()[name = string("op_1057_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1057_cast_fp16 = slice_by_index(begin = var_1057_begin_0, end = var_1057_end_0, end_mask = var_1057_end_mask_0, squeeze_mask = var_1057_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_1057_cast_fp16")];
+            bool var_1058_transpose_x_0 = const()[name = string("op_1058_transpose_x_0"), val = bool(false)];
+            bool var_1058_transpose_y_0 = const()[name = string("op_1058_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1058_cast_fp16 = matmul(transpose_x = var_1058_transpose_x_0, transpose_y = var_1058_transpose_y_0, x = var_1057_cast_fp16, y = v_new_61_cast_fp16)[name = string("op_1058_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1059_cast_fp16 = add(x = var_1055_cast_fp16, y = var_1058_cast_fp16)[name = string("op_1059_cast_fp16")];
+            tensor<int32, [4]> var_1061_begin_0 = const()[name = string("op_1061_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_1061_end_0 = const()[name = string("op_1061_end_0"), val = tensor<int32, [4]>([32, 31, 1, 1])];
+            tensor<bool, [4]> var_1061_end_mask_0 = const()[name = string("op_1061_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1061_squeeze_mask_0 = const()[name = string("op_1061_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1061_cast_fp16 = slice_by_index(begin = var_1061_begin_0, end = var_1061_end_0, end_mask = var_1061_end_mask_0, squeeze_mask = var_1061_squeeze_mask_0, x = chunk_decay_1_cast_fp16)[name = string("op_1061_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1062_cast_fp16 = mul(x = state_61_cast_fp16, y = var_1061_cast_fp16)[name = string("op_1062_cast_fp16")];
+            tensor<int32, [4]> var_1064_begin_0 = const()[name = string("op_1064_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_1064_end_0 = const()[name = string("op_1064_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_1064_end_mask_0 = const()[name = string("op_1064_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1064_squeeze_mask_0 = const()[name = string("op_1064_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1064_cast_fp16 = slice_by_index(begin = var_1064_begin_0, end = var_1064_end_0, end_mask = var_1064_end_mask_0, squeeze_mask = var_1064_squeeze_mask_0, x = k_dec_1_cast_fp16)[name = string("op_1064_cast_fp16")];
+            bool var_1066_transpose_x_1 = const()[name = string("op_1066_transpose_x_1"), val = bool(true)];
+            bool var_1066_transpose_y_1 = const()[name = string("op_1066_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1066_cast_fp16 = matmul(transpose_x = var_1066_transpose_x_1, transpose_y = var_1066_transpose_y_1, x = var_1064_cast_fp16, y = v_new_61_cast_fp16)[name = string("op_1066_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_63_cast_fp16 = add(x = var_1062_cast_fp16, y = var_1066_cast_fp16)[name = string("state_63_cast_fp16")];
+            tensor<int32, [4]> var_1069_begin_0 = const()[name = string("op_1069_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_1069_end_0 = const()[name = string("op_1069_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_1069_end_mask_0 = const()[name = string("op_1069_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1069_squeeze_mask_0 = const()[name = string("op_1069_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1069_cast_fp16 = slice_by_index(begin = var_1069_begin_0, end = var_1069_end_0, end_mask = var_1069_end_mask_0, squeeze_mask = var_1069_squeeze_mask_0, x = new_v_1_cast_fp16)[name = string("op_1069_cast_fp16")];
+            tensor<int32, [4]> var_1071_begin_0 = const()[name = string("op_1071_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_1071_end_0 = const()[name = string("op_1071_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_1071_end_mask_0 = const()[name = string("op_1071_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1071_squeeze_mask_0 = const()[name = string("op_1071_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1071_cast_fp16 = slice_by_index(begin = var_1071_begin_0, end = var_1071_end_0, end_mask = var_1071_end_mask_0, squeeze_mask = var_1071_squeeze_mask_0, x = k_cumdecay_1_cast_fp16)[name = string("op_1071_cast_fp16")];
+            bool var_1072_transpose_x_0 = const()[name = string("op_1072_transpose_x_0"), val = bool(false)];
+            bool var_1072_transpose_y_0 = const()[name = string("op_1072_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1072_cast_fp16 = matmul(transpose_x = var_1072_transpose_x_0, transpose_y = var_1072_transpose_y_0, x = var_1071_cast_fp16, y = state_63_cast_fp16)[name = string("op_1072_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_63_cast_fp16 = sub(x = var_1069_cast_fp16, y = var_1072_cast_fp16)[name = string("v_new_63_cast_fp16")];
+            tensor<int32, [4]> var_1075_begin_0 = const()[name = string("op_1075_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_1075_end_0 = const()[name = string("op_1075_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_1075_end_mask_0 = const()[name = string("op_1075_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1075_squeeze_mask_0 = const()[name = string("op_1075_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1075_cast_fp16 = slice_by_index(begin = var_1075_begin_0, end = var_1075_end_0, end_mask = var_1075_end_mask_0, squeeze_mask = var_1075_squeeze_mask_0, x = q_dec_1_cast_fp16)[name = string("op_1075_cast_fp16")];
+            bool var_1076_transpose_x_0 = const()[name = string("op_1076_transpose_x_0"), val = bool(false)];
+            bool var_1076_transpose_y_0 = const()[name = string("op_1076_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1076_cast_fp16 = matmul(transpose_x = var_1076_transpose_x_0, transpose_y = var_1076_transpose_y_0, x = var_1075_cast_fp16, y = state_63_cast_fp16)[name = string("op_1076_cast_fp16")];
+            tensor<int32, [4]> var_1078_begin_0 = const()[name = string("op_1078_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_1078_end_0 = const()[name = string("op_1078_end_0"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<bool, [4]> var_1078_end_mask_0 = const()[name = string("op_1078_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1078_squeeze_mask_0 = const()[name = string("op_1078_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1078_cast_fp16 = slice_by_index(begin = var_1078_begin_0, end = var_1078_end_0, end_mask = var_1078_end_mask_0, squeeze_mask = var_1078_squeeze_mask_0, x = attn_intra_1_cast_fp16)[name = string("op_1078_cast_fp16")];
+            bool var_1079_transpose_x_0 = const()[name = string("op_1079_transpose_x_0"), val = bool(false)];
+            bool var_1079_transpose_y_0 = const()[name = string("op_1079_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1079_cast_fp16 = matmul(transpose_x = var_1079_transpose_x_0, transpose_y = var_1079_transpose_y_0, x = var_1078_cast_fp16, y = v_new_63_cast_fp16)[name = string("op_1079_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1080_cast_fp16 = add(x = var_1076_cast_fp16, y = var_1079_cast_fp16)[name = string("op_1080_cast_fp16")];
+            int32 var_1082_axis_0 = const()[name = string("op_1082_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 32, 64, 128]> var_1082_cast_fp16 = stack(axis = var_1082_axis_0, values = (var_429_cast_fp16, var_450_cast_fp16, var_471_cast_fp16, var_492_cast_fp16, var_513_cast_fp16, var_534_cast_fp16, var_555_cast_fp16, var_576_cast_fp16, var_597_cast_fp16, var_618_cast_fp16, var_639_cast_fp16, var_660_cast_fp16, var_681_cast_fp16, var_702_cast_fp16, var_723_cast_fp16, var_744_cast_fp16, var_765_cast_fp16, var_786_cast_fp16, var_807_cast_fp16, var_828_cast_fp16, var_849_cast_fp16, var_870_cast_fp16, var_891_cast_fp16, var_912_cast_fp16, var_933_cast_fp16, var_954_cast_fp16, var_975_cast_fp16, var_996_cast_fp16, var_1017_cast_fp16, var_1038_cast_fp16, var_1059_cast_fp16, var_1080_cast_fp16))[name = string("op_1082_cast_fp16")];
+            tensor<int32, [3]> var_1083 = const()[name = string("op_1083"), val = tensor<int32, [3]>([32, 2048, 128])];
+            tensor<fp16, [32, 2048, 128]> var_1084_cast_fp16 = reshape(shape = var_1083, x = var_1082_cast_fp16)[name = string("op_1084_cast_fp16")];
+            tensor<int32, [3]> var_1085 = const()[name = string("op_1085"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_7_begin_0 = const()[name = string("input_7_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_7_end_0 = const()[name = string("input_7_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> input_7_end_mask_0 = const()[name = string("input_7_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_7_squeeze_mask_0 = const()[name = string("input_7_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> input_7_cast_fp16 = slice_by_index(begin = input_7_begin_0, end = input_7_end_0, end_mask = input_7_end_mask_0, squeeze_mask = input_7_squeeze_mask_0, x = z_1_cast_fp16)[name = string("input_7_cast_fp16")];
+            fp16 var_65_promoted_1_to_fp16 = const()[name = string("op_65_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [2048, 32, 128]> x_17_cast_fp16 = transpose(perm = var_1085, x = var_1084_cast_fp16)[name = string("transpose_48")];
+            tensor<fp16, [2048, 32, 128]> var_1089_cast_fp16 = pow(x = x_17_cast_fp16, y = var_65_promoted_1_to_fp16)[name = string("op_1089_cast_fp16")];
+            tensor<int32, [1]> var_1091_axes_0 = const()[name = string("op_1091_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1091_keep_dims_0 = const()[name = string("op_1091_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [2048, 32, 1]> var_1091_cast_fp16 = reduce_mean(axes = var_1091_axes_0, keep_dims = var_1091_keep_dims_0, x = var_1089_cast_fp16)[name = string("op_1091_cast_fp16")];
+            fp16 var_1092_to_fp16 = const()[name = string("op_1092_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [2048, 32, 1]> var_1093_cast_fp16 = add(x = var_1091_cast_fp16, y = var_1092_to_fp16)[name = string("op_1093_cast_fp16")];
+            fp32 var_1094_epsilon_0 = const()[name = string("op_1094_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [2048, 32, 1]> var_1094_cast_fp16 = rsqrt(epsilon = var_1094_epsilon_0, x = var_1093_cast_fp16)[name = string("op_1094_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> x_19_cast_fp16 = mul(x = x_17_cast_fp16, y = var_1094_cast_fp16)[name = string("x_19_cast_fp16")];
+            tensor<fp16, [128]> layers_0_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_0_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69207744)))];
+            tensor<fp16, [2048, 32, 128]> var_1096_cast_fp16 = mul(x = layers_0_linear_attn_norm_weight_to_fp16, y = x_19_cast_fp16)[name = string("op_1096_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> var_1097_cast_fp16 = silu(x = input_7_cast_fp16)[name = string("op_1097_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> core_1_cast_fp16 = mul(x = var_1096_cast_fp16, y = var_1097_cast_fp16)[name = string("core_1_cast_fp16")];
+            tensor<int32, [3]> var_1099 = const()[name = string("op_1099"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> input_9_cast_fp16 = reshape(shape = var_1099, x = core_1_cast_fp16)[name = string("input_9_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_0_linear_attn_out_proj_weight_to_fp16 = const()[name = string("layers_0_linear_attn_out_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(69208064)))];
+            tensor<fp16, [2560]> linear_4_bias_0_to_fp16 = const()[name = string("linear_4_bias_0_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90179648)))];
+            tensor<fp16, [1, 2048, 2560]> linear_4_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_0_linear_attn_out_proj_weight_to_fp16, x = input_9_cast_fp16)[name = string("linear_4_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_21_cast_fp16 = add(x = hidden, y = linear_4_cast_fp16)[name = string("x_21_cast_fp16")];
+            fp16 var_65_promoted_2_to_fp16 = const()[name = string("op_65_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_1105_cast_fp16 = pow(x = x_21_cast_fp16, y = var_65_promoted_2_to_fp16)[name = string("op_1105_cast_fp16")];
+            tensor<int32, [1]> var_1107_axes_0 = const()[name = string("op_1107_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1107_keep_dims_0 = const()[name = string("op_1107_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_1107_cast_fp16 = reduce_mean(axes = var_1107_axes_0, keep_dims = var_1107_keep_dims_0, x = var_1105_cast_fp16)[name = string("op_1107_cast_fp16")];
+            fp16 var_1108_to_fp16 = const()[name = string("op_1108_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_1109_cast_fp16 = add(x = var_1107_cast_fp16, y = var_1108_to_fp16)[name = string("op_1109_cast_fp16")];
+            fp32 var_1110_epsilon_0 = const()[name = string("op_1110_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_1110_cast_fp16 = rsqrt(epsilon = var_1110_epsilon_0, x = var_1109_cast_fp16)[name = string("op_1110_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_23_cast_fp16 = mul(x = x_21_cast_fp16, y = var_1110_cast_fp16)[name = string("x_23_cast_fp16")];
+            tensor<fp16, [2560]> var_1113_to_fp16 = const()[name = string("op_1113_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90184832)))];
+            tensor<fp16, [1, 2048, 2560]> input_11_cast_fp16 = mul(x = x_23_cast_fp16, y = var_1113_to_fp16)[name = string("input_11_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_0_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_0_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(90190016)))];
+            tensor<fp16, [9216]> linear_5_bias_0_to_fp16 = const()[name = string("linear_5_bias_0_to_fp16"), val = tensor<fp16, [9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(137376000)))];
+            tensor<fp16, [1, 2048, 9216]> linear_5_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_0_mlp_gate_proj_weight_to_fp16, x = input_11_cast_fp16)[name = string("linear_5_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_1120_cast_fp16 = silu(x = linear_5_cast_fp16)[name = string("op_1120_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_0_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_0_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(137394496)))];
+            tensor<fp16, [1, 2048, 9216]> linear_6_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_0_mlp_up_proj_weight_to_fp16, x = input_11_cast_fp16)[name = string("linear_6_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_15_cast_fp16 = mul(x = var_1120_cast_fp16, y = linear_6_cast_fp16)[name = string("input_15_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_0_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_0_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(184580480)))];
+            tensor<fp16, [1, 2048, 2560]> linear_7_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_0_mlp_down_proj_weight_to_fp16, x = input_15_cast_fp16)[name = string("linear_7_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_25_cast_fp16 = add(x = x_21_cast_fp16, y = linear_7_cast_fp16)[name = string("x_25_cast_fp16")];
+            int32 var_1140 = const()[name = string("op_1140"), val = int32(-2)];
+            int32 var_1169 = const()[name = string("op_1169"), val = int32(-1)];
+            fp16 var_1168_promoted_to_fp16 = const()[name = string("op_1168_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_1178_cast_fp16 = pow(x = x_25_cast_fp16, y = var_1168_promoted_to_fp16)[name = string("op_1178_cast_fp16")];
+            tensor<int32, [1]> var_1180_axes_0 = const()[name = string("op_1180_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1180_keep_dims_0 = const()[name = string("op_1180_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_1180_cast_fp16 = reduce_mean(axes = var_1180_axes_0, keep_dims = var_1180_keep_dims_0, x = var_1178_cast_fp16)[name = string("op_1180_cast_fp16")];
+            fp16 var_1181_to_fp16 = const()[name = string("op_1181_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_1182_cast_fp16 = add(x = var_1180_cast_fp16, y = var_1181_to_fp16)[name = string("op_1182_cast_fp16")];
+            fp32 var_1183_epsilon_0 = const()[name = string("op_1183_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_1183_cast_fp16 = rsqrt(epsilon = var_1183_epsilon_0, x = var_1182_cast_fp16)[name = string("op_1183_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_27_cast_fp16 = mul(x = x_25_cast_fp16, y = var_1183_cast_fp16)[name = string("x_27_cast_fp16")];
+            tensor<fp16, [2560]> var_1186_to_fp16 = const()[name = string("op_1186_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(231766464)))];
+            tensor<fp16, [1, 2048, 2560]> x_29_cast_fp16 = mul(x = x_27_cast_fp16, y = var_1186_to_fp16)[name = string("x_29_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_1_linear_attn_in_proj_qkv_weight_to_fp16 = const()[name = string("layers_1_linear_attn_in_proj_qkv_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(231771648)))];
+            tensor<fp16, [1, 2048, 8192]> linear_8_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_qkv_weight_to_fp16, x = x_29_cast_fp16)[name = string("linear_8_cast_fp16")];
+            tensor<int32, [3]> input_17_perm_0 = const()[name = string("input_17_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_19_pad_0 = const()[name = string("input_19_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_19_mode_0 = const()[name = string("input_19_mode_0"), val = string("constant")];
+            fp16 const_13_to_fp16 = const()[name = string("const_13_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 2048]> input_17_cast_fp16 = transpose(perm = input_17_perm_0, x = linear_8_cast_fp16)[name = string("transpose_47")];
+            tensor<fp16, [1, 8192, 2051]> input_19_cast_fp16 = pad(constant_val = const_13_to_fp16, mode = input_19_mode_0, pad = input_19_pad_0, x = input_17_cast_fp16)[name = string("input_19_cast_fp16")];
+            string input_21_pad_type_0 = const()[name = string("input_21_pad_type_0"), val = string("valid")];
+            int32 input_21_groups_0 = const()[name = string("input_21_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_21_strides_0 = const()[name = string("input_21_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_21_pad_0 = const()[name = string("input_21_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_21_dilations_0 = const()[name = string("input_21_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_1_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_1_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273714752)))];
+            tensor<fp16, [1, 8192, 2048]> input_21_cast_fp16 = conv(dilations = input_21_dilations_0, groups = input_21_groups_0, pad = input_21_pad_0, pad_type = input_21_pad_type_0, strides = input_21_strides_0, weight = layers_1_linear_attn_conv1d_weight_to_fp16, x = input_19_cast_fp16)[name = string("input_21_cast_fp16")];
+            tensor<fp16, [1, 8192, 2048]> var_1221_cast_fp16 = silu(x = input_21_cast_fp16)[name = string("op_1221_cast_fp16")];
+            tensor<int32, [3]> qkv_3_perm_0 = const()[name = string("qkv_3_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_1223_begin_0 = const()[name = string("op_1223_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_1223_end_0 = const()[name = string("op_1223_end_0"), val = tensor<int32, [3]>([1, 2048, 2048])];
+            tensor<bool, [3]> var_1223_end_mask_0 = const()[name = string("op_1223_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 8192]> qkv_3_cast_fp16 = transpose(perm = qkv_3_perm_0, x = var_1221_cast_fp16)[name = string("transpose_46")];
+            tensor<fp16, [1, 2048, 2048]> var_1223_cast_fp16 = slice_by_index(begin = var_1223_begin_0, end = var_1223_end_0, end_mask = var_1223_end_mask_0, x = qkv_3_cast_fp16)[name = string("op_1223_cast_fp16")];
+            tensor<int32, [4]> var_1224 = const()[name = string("op_1224"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> q_11_cast_fp16 = reshape(shape = var_1224, x = var_1223_cast_fp16)[name = string("q_11_cast_fp16")];
+            tensor<int32, [3]> var_1226_begin_0 = const()[name = string("op_1226_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_1226_end_0 = const()[name = string("op_1226_end_0"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<bool, [3]> var_1226_end_mask_0 = const()[name = string("op_1226_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 2048]> var_1226_cast_fp16 = slice_by_index(begin = var_1226_begin_0, end = var_1226_end_0, end_mask = var_1226_end_mask_0, x = qkv_3_cast_fp16)[name = string("op_1226_cast_fp16")];
+            tensor<int32, [4]> var_1227 = const()[name = string("op_1227"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> k_9_cast_fp16 = reshape(shape = var_1227, x = var_1226_cast_fp16)[name = string("k_9_cast_fp16")];
+            tensor<int32, [3]> var_1229_begin_0 = const()[name = string("op_1229_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_1229_end_0 = const()[name = string("op_1229_end_0"), val = tensor<int32, [3]>([1, 2048, 8192])];
+            tensor<bool, [3]> var_1229_end_mask_0 = const()[name = string("op_1229_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 2048, 4096]> var_1229_cast_fp16 = slice_by_index(begin = var_1229_begin_0, end = var_1229_end_0, end_mask = var_1229_end_mask_0, x = qkv_3_cast_fp16)[name = string("op_1229_cast_fp16")];
+            tensor<int32, [4]> var_1230 = const()[name = string("op_1230"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> v_5_cast_fp16 = reshape(shape = var_1230, x = var_1229_cast_fp16)[name = string("v_5_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_1_linear_attn_in_proj_z_weight_to_fp16 = const()[name = string("layers_1_linear_attn_in_proj_z_weight_to_fp16"), val = tensor<fp16, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(273780352)))];
+            tensor<fp16, [1, 2048, 4096]> linear_9_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_z_weight_to_fp16, x = x_29_cast_fp16)[name = string("linear_9_cast_fp16")];
+            tensor<int32, [4]> var_1234 = const()[name = string("op_1234"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> z_3_cast_fp16 = reshape(shape = var_1234, x = linear_9_cast_fp16)[name = string("z_3_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_1_linear_attn_in_proj_b_weight_to_fp16 = const()[name = string("layers_1_linear_attn_in_proj_b_weight_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(294751936)))];
+            tensor<fp16, [32]> linear_10_bias_0_to_fp16 = const()[name = string("linear_10_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(294915840)))];
+            tensor<fp16, [1, 2048, 32]> linear_10_cast_fp16 = linear(bias = linear_10_bias_0_to_fp16, weight = layers_1_linear_attn_in_proj_b_weight_to_fp16, x = x_29_cast_fp16)[name = string("linear_10_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> beta_5_cast_fp16 = sigmoid(x = linear_10_cast_fp16)[name = string("beta_5_cast_fp16")];
+            tensor<fp16, [32, 2560]> var_1243_weight_0_to_fp16 = const()[name = string("op_1243_weight_0_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(294915968)))];
+            tensor<fp16, [32]> var_1243_bias_0_to_fp16 = const()[name = string("op_1243_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295079872)))];
+            tensor<fp16, [1, 2048, 32]> var_1243_cast_fp16 = linear(bias = var_1243_bias_0_to_fp16, weight = var_1243_weight_0_to_fp16, x = x_29_cast_fp16)[name = string("op_1243_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> var_1244_cast_fp16 = softplus(x = var_1243_cast_fp16)[name = string("op_1244_cast_fp16")];
+            tensor<fp16, [32]> var_1240_to_fp16 = const()[name = string("op_1240_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295080000)))];
+            tensor<fp16, [1, 2048, 32]> g_5_cast_fp16 = mul(x = var_1240_to_fp16, y = var_1244_cast_fp16)[name = string("g_5_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_1246_cast_fp16 = mul(x = q_11_cast_fp16, y = q_11_cast_fp16)[name = string("op_1246_cast_fp16")];
+            tensor<int32, [1]> var_1248_axes_0 = const()[name = string("op_1248_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1248_keep_dims_0 = const()[name = string("op_1248_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_1248_cast_fp16 = reduce_sum(axes = var_1248_axes_0, keep_dims = var_1248_keep_dims_0, x = var_1246_cast_fp16)[name = string("op_1248_cast_fp16")];
+            fp16 var_1249_to_fp16 = const()[name = string("op_1249_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_1250_cast_fp16 = add(x = var_1248_cast_fp16, y = var_1249_to_fp16)[name = string("op_1250_cast_fp16")];
+            fp32 var_1251_epsilon_0 = const()[name = string("op_1251_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_1251_cast_fp16 = rsqrt(epsilon = var_1251_epsilon_0, x = var_1250_cast_fp16)[name = string("op_1251_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> q_13_cast_fp16 = mul(x = q_11_cast_fp16, y = var_1251_cast_fp16)[name = string("q_13_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_1253_cast_fp16 = mul(x = k_9_cast_fp16, y = k_9_cast_fp16)[name = string("op_1253_cast_fp16")];
+            tensor<int32, [1]> var_1255_axes_0 = const()[name = string("op_1255_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_1255_keep_dims_0 = const()[name = string("op_1255_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_1255_cast_fp16 = reduce_sum(axes = var_1255_axes_0, keep_dims = var_1255_keep_dims_0, x = var_1253_cast_fp16)[name = string("op_1255_cast_fp16")];
+            fp16 var_1256_to_fp16 = const()[name = string("op_1256_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_1257_cast_fp16 = add(x = var_1255_cast_fp16, y = var_1256_to_fp16)[name = string("op_1257_cast_fp16")];
+            fp32 var_1258_epsilon_0 = const()[name = string("op_1258_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_1258_cast_fp16 = rsqrt(epsilon = var_1258_epsilon_0, x = var_1257_cast_fp16)[name = string("op_1258_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> k_11_cast_fp16 = mul(x = k_9_cast_fp16, y = var_1258_cast_fp16)[name = string("k_11_cast_fp16")];
+            tensor<int32, [1]> var_1263_axes_0 = const()[name = string("op_1263_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_1263_cast_fp16 = expand_dims(axes = var_1263_axes_0, x = q_13_cast_fp16)[name = string("op_1263_cast_fp16")];
+            tensor<int32, [5]> var_1265_reps_0 = const()[name = string("op_1265_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_1265_cast_fp16 = tile(reps = var_1265_reps_0, x = var_1263_cast_fp16)[name = string("op_1265_cast_fp16")];
+            tensor<int32, [4]> var_1266 = const()[name = string("op_1266"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> q_15_cast_fp16 = reshape(shape = var_1266, x = var_1265_cast_fp16)[name = string("q_15_cast_fp16")];
+            tensor<int32, [1]> var_1271_axes_0 = const()[name = string("op_1271_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_1271_cast_fp16 = expand_dims(axes = var_1271_axes_0, x = k_11_cast_fp16)[name = string("op_1271_cast_fp16")];
+            tensor<int32, [5]> var_1273_reps_0 = const()[name = string("op_1273_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_1273_cast_fp16 = tile(reps = var_1273_reps_0, x = var_1271_cast_fp16)[name = string("op_1273_cast_fp16")];
+            tensor<int32, [4]> var_1274 = const()[name = string("op_1274"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> k_13_cast_fp16 = reshape(shape = var_1274, x = var_1273_cast_fp16)[name = string("k_13_cast_fp16")];
+            fp16 var_1276_to_fp16 = const()[name = string("op_1276_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 2048, 32, 128]> q_17_cast_fp16 = mul(x = q_15_cast_fp16, y = var_1276_to_fp16)[name = string("q_17_cast_fp16")];
+            tensor<int32, [4]> var_1278_begin_0 = const()[name = string("op_1278_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1278_end_0 = const()[name = string("op_1278_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_1278_end_mask_0 = const()[name = string("op_1278_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_1278_squeeze_mask_0 = const()[name = string("op_1278_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_1278_cast_fp16 = slice_by_index(begin = var_1278_begin_0, end = var_1278_end_0, end_mask = var_1278_end_mask_0, squeeze_mask = var_1278_squeeze_mask_0, x = q_17_cast_fp16)[name = string("op_1278_cast_fp16")];
+            tensor<int32, [3]> var_1279 = const()[name = string("op_1279"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_1281 = const()[name = string("op_1281"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_1280_cast_fp16 = transpose(perm = var_1279, x = var_1278_cast_fp16)[name = string("transpose_45")];
+            tensor<fp16, [32, 32, 64, 128]> q_19_cast_fp16 = reshape(shape = var_1281, x = var_1280_cast_fp16)[name = string("q_19_cast_fp16")];
+            tensor<int32, [4]> var_1283_begin_0 = const()[name = string("op_1283_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1283_end_0 = const()[name = string("op_1283_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_1283_end_mask_0 = const()[name = string("op_1283_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_1283_squeeze_mask_0 = const()[name = string("op_1283_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_1283_cast_fp16 = slice_by_index(begin = var_1283_begin_0, end = var_1283_end_0, end_mask = var_1283_end_mask_0, squeeze_mask = var_1283_squeeze_mask_0, x = k_13_cast_fp16)[name = string("op_1283_cast_fp16")];
+            tensor<int32, [3]> var_1284 = const()[name = string("op_1284"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_1286 = const()[name = string("op_1286"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_1285_cast_fp16 = transpose(perm = var_1284, x = var_1283_cast_fp16)[name = string("transpose_44")];
+            tensor<fp16, [32, 32, 64, 128]> k_15_cast_fp16 = reshape(shape = var_1286, x = var_1285_cast_fp16)[name = string("k_15_cast_fp16")];
+            tensor<int32, [4]> var_1288_begin_0 = const()[name = string("op_1288_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1288_end_0 = const()[name = string("op_1288_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_1288_end_mask_0 = const()[name = string("op_1288_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_1288_squeeze_mask_0 = const()[name = string("op_1288_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_1288_cast_fp16 = slice_by_index(begin = var_1288_begin_0, end = var_1288_end_0, end_mask = var_1288_end_mask_0, squeeze_mask = var_1288_squeeze_mask_0, x = v_5_cast_fp16)[name = string("op_1288_cast_fp16")];
+            tensor<int32, [3]> var_1289 = const()[name = string("op_1289"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_1291 = const()[name = string("op_1291"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_1290_cast_fp16 = transpose(perm = var_1289, x = var_1288_cast_fp16)[name = string("transpose_43")];
+            tensor<fp16, [32, 32, 64, 128]> v_7_cast_fp16 = reshape(shape = var_1291, x = var_1290_cast_fp16)[name = string("v_7_cast_fp16")];
+            tensor<int32, [3]> var_1293_begin_0 = const()[name = string("op_1293_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_1293_end_0 = const()[name = string("op_1293_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_1293_end_mask_0 = const()[name = string("op_1293_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_1293_squeeze_mask_0 = const()[name = string("op_1293_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_1293_cast_fp16 = slice_by_index(begin = var_1293_begin_0, end = var_1293_end_0, end_mask = var_1293_end_mask_0, squeeze_mask = var_1293_squeeze_mask_0, x = beta_5_cast_fp16)[name = string("op_1293_cast_fp16")];
+            tensor<int32, [2]> var_1294_perm_0 = const()[name = string("op_1294_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_1295 = const()[name = string("op_1295"), val = tensor<int32, [4]>([32, 32, 64, 1])];
+            tensor<fp16, [32, 2048]> var_1294_cast_fp16 = transpose(perm = var_1294_perm_0, x = var_1293_cast_fp16)[name = string("transpose_42")];
+            tensor<fp16, [32, 32, 64, 1]> beta_7_cast_fp16 = reshape(shape = var_1295, x = var_1294_cast_fp16)[name = string("beta_7_cast_fp16")];
+            tensor<int32, [3]> var_1297_begin_0 = const()[name = string("op_1297_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_1297_end_0 = const()[name = string("op_1297_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_1297_end_mask_0 = const()[name = string("op_1297_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_1297_squeeze_mask_0 = const()[name = string("op_1297_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_1297_cast_fp16 = slice_by_index(begin = var_1297_begin_0, end = var_1297_end_0, end_mask = var_1297_end_mask_0, squeeze_mask = var_1297_squeeze_mask_0, x = g_5_cast_fp16)[name = string("op_1297_cast_fp16")];
+            tensor<int32, [2]> var_1298_perm_0 = const()[name = string("op_1298_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_1299 = const()[name = string("op_1299"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<fp16, [32, 2048]> var_1298_cast_fp16 = transpose(perm = var_1298_perm_0, x = var_1297_cast_fp16)[name = string("transpose_41")];
+            tensor<fp16, [32, 32, 64]> g_7_cast_fp16 = reshape(shape = var_1299, x = var_1298_cast_fp16)[name = string("g_7_cast_fp16")];
+            tensor<int32, [1]> g_col_3_axes_0 = const()[name = string("g_col_3_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64, 1]> g_col_3_cast_fp16 = expand_dims(axes = g_col_3_axes_0, x = g_7_cast_fp16)[name = string("g_col_3_cast_fp16")];
+            bool var_1302_transpose_x_0 = const()[name = string("op_1302_transpose_x_0"), val = bool(false)];
+            bool var_1302_transpose_y_0 = const()[name = string("op_1302_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_1302_cast_fp16 = matmul(transpose_x = var_1302_transpose_x_0, transpose_y = var_1302_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_3_cast_fp16)[name = string("op_1302_cast_fp16")];
+            tensor<int32, [1]> cum_3_axes_0 = const()[name = string("cum_3_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> cum_3_cast_fp16 = squeeze(axes = cum_3_axes_0, x = var_1302_cast_fp16)[name = string("cum_3_cast_fp16")];
+            bool var_1304_transpose_x_0 = const()[name = string("op_1304_transpose_x_0"), val = bool(false)];
+            bool var_1304_transpose_y_0 = const()[name = string("op_1304_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_1304_cast_fp16 = matmul(transpose_x = var_1304_transpose_x_0, transpose_y = var_1304_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_3_cast_fp16)[name = string("op_1304_cast_fp16")];
+            tensor<int32, [1]> to_end_3_axes_0 = const()[name = string("to_end_3_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> to_end_3_cast_fp16 = squeeze(axes = to_end_3_axes_0, x = var_1304_cast_fp16)[name = string("to_end_3_cast_fp16")];
+            bool var_1306_transpose_x_0 = const()[name = string("op_1306_transpose_x_0"), val = bool(false)];
+            bool var_1306_transpose_y_0 = const()[name = string("op_1306_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 4096, 1]> var_1306_cast_fp16 = matmul(transpose_x = var_1306_transpose_x_0, transpose_y = var_1306_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_3_cast_fp16)[name = string("op_1306_cast_fp16")];
+            tensor<int32, [4]> var_1307 = const()[name = string("op_1307"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> pair_15_cast_fp16 = reshape(shape = var_1307, x = var_1306_cast_fp16)[name = string("pair_15_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_1309_cast_fp16 = exp(x = pair_15_cast_fp16)[name = string("op_1309_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> pair_decay_3_cast_fp16 = mul(x = var_1309_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_3_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_beta_3_cast_fp16 = mul(x = k_15_cast_fp16, y = beta_7_cast_fp16)[name = string("k_beta_3_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> v_beta_3_cast_fp16 = mul(x = v_7_cast_fp16, y = beta_7_cast_fp16)[name = string("v_beta_3_cast_fp16")];
+            bool var_1314_transpose_x_1 = const()[name = string("op_1314_transpose_x_1"), val = bool(false)];
+            bool var_1314_transpose_y_1 = const()[name = string("op_1314_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_1314_cast_fp16 = matmul(transpose_x = var_1314_transpose_x_1, transpose_y = var_1314_transpose_y_1, x = k_beta_3_cast_fp16, y = k_15_cast_fp16)[name = string("op_1314_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> kkt_3_cast_fp16 = mul(x = var_1314_cast_fp16, y = pair_decay_3_cast_fp16)[name = string("kkt_3_cast_fp16")];
+            bool var_1317_transpose_x_1 = const()[name = string("op_1317_transpose_x_1"), val = bool(false)];
+            bool var_1317_transpose_y_1 = const()[name = string("op_1317_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_1317_cast_fp16 = matmul(transpose_x = var_1317_transpose_x_1, transpose_y = var_1317_transpose_y_1, x = q_19_cast_fp16, y = k_15_cast_fp16)[name = string("op_1317_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> attn_intra_3_cast_fp16 = mul(x = var_1317_cast_fp16, y = pair_decay_3_cast_fp16)[name = string("attn_intra_3_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_1319_cast_fp16 = mul(x = kkt_3_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_1319_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> t_5_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_1319_cast_fp16)[name = string("t_5_cast_fp16")];
+            tensor<int32, [3]> var_1323 = const()[name = string("op_1323"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [1024, 64, 64]> t_7_cast_fp16 = reshape(shape = var_1323, x = t_5_cast_fp16)[name = string("t_7_cast_fp16")];
+            tensor<int32, [5]> var_1328 = const()[name = string("op_1328"), val = tensor<int32, [5]>([1024, 32, 2, 32, 2])];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_1329_cast_fp16 = reshape(shape = var_1328, x = t_7_cast_fp16)[name = string("op_1329_cast_fp16")];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_1334_cast_fp16 = mul(x = var_1329_cast_fp16, y = var_230_to_fp16)[name = string("op_1334_cast_fp16")];
+            tensor<int32, [1]> blocks_13_axes_0 = const()[name = string("blocks_13_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_13_keep_dims_0 = const()[name = string("blocks_13_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> blocks_13_cast_fp16 = reduce_sum(axes = blocks_13_axes_0, keep_dims = blocks_13_keep_dims_0, x = var_1334_cast_fp16)[name = string("blocks_13_cast_fp16")];
+            tensor<int32, [4]> var_1337_begin_0 = const()[name = string("op_1337_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_1337_end_0 = const()[name = string("op_1337_end_0"), val = tensor<int32, [4]>([1024, 32, 2, 2])];
+            tensor<bool, [4]> var_1337_end_mask_0 = const()[name = string("op_1337_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 32, 1, 2]> var_1337_cast_fp16 = slice_by_index(begin = var_1337_begin_0, end = var_1337_end_0, end_mask = var_1337_end_mask_0, x = blocks_13_cast_fp16)[name = string("op_1337_cast_fp16")];
+            tensor<int32, [4]> x_31_begin_0 = const()[name = string("x_31_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_31_end_0 = const()[name = string("x_31_end_0"), val = tensor<int32, [4]>([1024, 32, 1, 1])];
+            tensor<bool, [4]> x_31_end_mask_0 = const()[name = string("x_31_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 32, 1, 1]> x_31_cast_fp16 = slice_by_index(begin = x_31_begin_0, end = x_31_end_0, end_mask = x_31_end_mask_0, x = var_1337_cast_fp16)[name = string("x_31_cast_fp16")];
+            bool var_1347_transpose_x_0 = const()[name = string("op_1347_transpose_x_0"), val = bool(false)];
+            bool var_1347_transpose_y_0 = const()[name = string("op_1347_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_1347_cast_fp16 = matmul(transpose_x = var_1347_transpose_x_0, transpose_y = var_1347_transpose_y_0, x = x_31_cast_fp16, y = var_238_to_fp16)[name = string("op_1347_cast_fp16")];
+            bool var_1348_transpose_x_0 = const()[name = string("op_1348_transpose_x_0"), val = bool(false)];
+            bool var_1348_transpose_y_0 = const()[name = string("op_1348_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_1348_cast_fp16 = matmul(transpose_x = var_1348_transpose_x_0, transpose_y = var_1348_transpose_y_0, x = var_238_to_fp16, y = var_1347_cast_fp16)[name = string("op_1348_cast_fp16")];
+            fp16 const_18_promoted_to_fp16 = const()[name = string("const_18_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 32, 1, 1]> lower_13_cast_fp16 = mul(x = var_1348_cast_fp16, y = const_18_promoted_to_fp16)[name = string("lower_13_cast_fp16")];
+            bool bottom_13_interleave_0 = const()[name = string("bottom_13_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> bottom_13_cast_fp16 = concat(axis = var_1169, interleave = bottom_13_interleave_0, values = (lower_13_cast_fp16, var_238_to_fp16))[name = string("bottom_13_cast_fp16")];
+            bool inv_19_interleave_0 = const()[name = string("inv_19_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> inv_19_cast_fp16 = concat(axis = var_1140, interleave = inv_19_interleave_0, values = (top_1_to_fp16, bottom_13_cast_fp16))[name = string("inv_19_cast_fp16")];
+            tensor<int32, [5]> var_1357 = const()[name = string("op_1357"), val = tensor<int32, [5]>([1024, 16, 4, 16, 4])];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_1358_cast_fp16 = reshape(shape = var_1357, x = t_7_cast_fp16)[name = string("op_1358_cast_fp16")];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_1363_cast_fp16 = mul(x = var_1358_cast_fp16, y = var_259_to_fp16)[name = string("op_1363_cast_fp16")];
+            tensor<int32, [1]> blocks_15_axes_0 = const()[name = string("blocks_15_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_15_keep_dims_0 = const()[name = string("blocks_15_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> blocks_15_cast_fp16 = reduce_sum(axes = blocks_15_axes_0, keep_dims = blocks_15_keep_dims_0, x = var_1363_cast_fp16)[name = string("blocks_15_cast_fp16")];
+            tensor<int32, [4]> var_1366_begin_0 = const()[name = string("op_1366_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_1366_end_0 = const()[name = string("op_1366_end_0"), val = tensor<int32, [4]>([1024, 16, 4, 4])];
+            tensor<bool, [4]> var_1366_end_mask_0 = const()[name = string("op_1366_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 16, 2, 4]> var_1366_cast_fp16 = slice_by_index(begin = var_1366_begin_0, end = var_1366_end_0, end_mask = var_1366_end_mask_0, x = blocks_15_cast_fp16)[name = string("op_1366_cast_fp16")];
+            tensor<int32, [4]> x_33_begin_0 = const()[name = string("x_33_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_33_end_0 = const()[name = string("x_33_end_0"), val = tensor<int32, [4]>([1024, 16, 2, 2])];
+            tensor<bool, [4]> x_33_end_mask_0 = const()[name = string("x_33_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 16, 2, 2]> x_33_cast_fp16 = slice_by_index(begin = x_33_begin_0, end = x_33_end_0, end_mask = x_33_end_mask_0, x = var_1366_cast_fp16)[name = string("x_33_cast_fp16")];
+            tensor<int32, [5]> var_1368 = const()[name = string("op_1368"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<fp16, [1024, 16, 2, 2, 2]> pair_19_cast_fp16 = reshape(shape = var_1368, x = inv_19_cast_fp16)[name = string("pair_19_cast_fp16")];
+            tensor<int32, [5]> var_1370_begin_0 = const()[name = string("op_1370_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1370_end_0 = const()[name = string("op_1370_end_0"), val = tensor<int32, [5]>([1024, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_1370_end_mask_0 = const()[name = string("op_1370_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1370_squeeze_mask_0 = const()[name = string("op_1370_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_1370_cast_fp16 = slice_by_index(begin = var_1370_begin_0, end = var_1370_end_0, end_mask = var_1370_end_mask_0, squeeze_mask = var_1370_squeeze_mask_0, x = pair_19_cast_fp16)[name = string("op_1370_cast_fp16")];
+            tensor<int32, [5]> var_1373_begin_0 = const()[name = string("op_1373_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1373_end_0 = const()[name = string("op_1373_end_0"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_1373_end_mask_0 = const()[name = string("op_1373_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1373_squeeze_mask_0 = const()[name = string("op_1373_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_1373_cast_fp16 = slice_by_index(begin = var_1373_begin_0, end = var_1373_end_0, end_mask = var_1373_end_mask_0, squeeze_mask = var_1373_squeeze_mask_0, x = pair_19_cast_fp16)[name = string("op_1373_cast_fp16")];
+            bool var_1376_transpose_x_0 = const()[name = string("op_1376_transpose_x_0"), val = bool(false)];
+            bool var_1376_transpose_y_0 = const()[name = string("op_1376_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_1376_cast_fp16 = matmul(transpose_x = var_1376_transpose_x_0, transpose_y = var_1376_transpose_y_0, x = x_33_cast_fp16, y = var_1370_cast_fp16)[name = string("op_1376_cast_fp16")];
+            bool var_1377_transpose_x_0 = const()[name = string("op_1377_transpose_x_0"), val = bool(false)];
+            bool var_1377_transpose_y_0 = const()[name = string("op_1377_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_1377_cast_fp16 = matmul(transpose_x = var_1377_transpose_x_0, transpose_y = var_1377_transpose_y_0, x = var_1373_cast_fp16, y = var_1376_cast_fp16)[name = string("op_1377_cast_fp16")];
+            fp16 const_19_promoted_to_fp16 = const()[name = string("const_19_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 16, 2, 2]> lower_15_cast_fp16 = mul(x = var_1377_cast_fp16, y = const_19_promoted_to_fp16)[name = string("lower_15_cast_fp16")];
+            bool top_15_interleave_0 = const()[name = string("top_15_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> top_15_cast_fp16 = concat(axis = var_1169, interleave = top_15_interleave_0, values = (var_1370_cast_fp16, var_276_to_fp16))[name = string("top_15_cast_fp16")];
+            bool bottom_15_interleave_0 = const()[name = string("bottom_15_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> bottom_15_cast_fp16 = concat(axis = var_1169, interleave = bottom_15_interleave_0, values = (lower_15_cast_fp16, var_1373_cast_fp16))[name = string("bottom_15_cast_fp16")];
+            bool inv_21_interleave_0 = const()[name = string("inv_21_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> inv_21_cast_fp16 = concat(axis = var_1140, interleave = inv_21_interleave_0, values = (top_15_cast_fp16, bottom_15_cast_fp16))[name = string("inv_21_cast_fp16")];
+            tensor<int32, [5]> var_1386 = const()[name = string("op_1386"), val = tensor<int32, [5]>([1024, 8, 8, 8, 8])];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_1387_cast_fp16 = reshape(shape = var_1386, x = t_7_cast_fp16)[name = string("op_1387_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_1391_to_fp16 = const()[name = string("op_1391_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295080128)))];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_1392_cast_fp16 = mul(x = var_1387_cast_fp16, y = var_1391_to_fp16)[name = string("op_1392_cast_fp16")];
+            tensor<int32, [1]> blocks_17_axes_0 = const()[name = string("blocks_17_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_17_keep_dims_0 = const()[name = string("blocks_17_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> blocks_17_cast_fp16 = reduce_sum(axes = blocks_17_axes_0, keep_dims = blocks_17_keep_dims_0, x = var_1392_cast_fp16)[name = string("blocks_17_cast_fp16")];
+            tensor<int32, [4]> var_1395_begin_0 = const()[name = string("op_1395_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_1395_end_0 = const()[name = string("op_1395_end_0"), val = tensor<int32, [4]>([1024, 8, 8, 8])];
+            tensor<bool, [4]> var_1395_end_mask_0 = const()[name = string("op_1395_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 8, 4, 8]> var_1395_cast_fp16 = slice_by_index(begin = var_1395_begin_0, end = var_1395_end_0, end_mask = var_1395_end_mask_0, x = blocks_17_cast_fp16)[name = string("op_1395_cast_fp16")];
+            tensor<int32, [4]> x_35_begin_0 = const()[name = string("x_35_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_35_end_0 = const()[name = string("x_35_end_0"), val = tensor<int32, [4]>([1024, 8, 4, 4])];
+            tensor<bool, [4]> x_35_end_mask_0 = const()[name = string("x_35_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 8, 4, 4]> x_35_cast_fp16 = slice_by_index(begin = x_35_begin_0, end = x_35_end_0, end_mask = x_35_end_mask_0, x = var_1395_cast_fp16)[name = string("x_35_cast_fp16")];
+            tensor<int32, [5]> var_1397 = const()[name = string("op_1397"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<fp16, [1024, 8, 2, 4, 4]> pair_21_cast_fp16 = reshape(shape = var_1397, x = inv_21_cast_fp16)[name = string("pair_21_cast_fp16")];
+            tensor<int32, [5]> var_1399_begin_0 = const()[name = string("op_1399_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1399_end_0 = const()[name = string("op_1399_end_0"), val = tensor<int32, [5]>([1024, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_1399_end_mask_0 = const()[name = string("op_1399_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1399_squeeze_mask_0 = const()[name = string("op_1399_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_1399_cast_fp16 = slice_by_index(begin = var_1399_begin_0, end = var_1399_end_0, end_mask = var_1399_end_mask_0, squeeze_mask = var_1399_squeeze_mask_0, x = pair_21_cast_fp16)[name = string("op_1399_cast_fp16")];
+            tensor<int32, [5]> var_1402_begin_0 = const()[name = string("op_1402_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1402_end_0 = const()[name = string("op_1402_end_0"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_1402_end_mask_0 = const()[name = string("op_1402_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1402_squeeze_mask_0 = const()[name = string("op_1402_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_1402_cast_fp16 = slice_by_index(begin = var_1402_begin_0, end = var_1402_end_0, end_mask = var_1402_end_mask_0, squeeze_mask = var_1402_squeeze_mask_0, x = pair_21_cast_fp16)[name = string("op_1402_cast_fp16")];
+            bool var_1405_transpose_x_0 = const()[name = string("op_1405_transpose_x_0"), val = bool(false)];
+            bool var_1405_transpose_y_0 = const()[name = string("op_1405_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_1405_cast_fp16 = matmul(transpose_x = var_1405_transpose_x_0, transpose_y = var_1405_transpose_y_0, x = x_35_cast_fp16, y = var_1399_cast_fp16)[name = string("op_1405_cast_fp16")];
+            bool var_1406_transpose_x_0 = const()[name = string("op_1406_transpose_x_0"), val = bool(false)];
+            bool var_1406_transpose_y_0 = const()[name = string("op_1406_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_1406_cast_fp16 = matmul(transpose_x = var_1406_transpose_x_0, transpose_y = var_1406_transpose_y_0, x = var_1402_cast_fp16, y = var_1405_cast_fp16)[name = string("op_1406_cast_fp16")];
+            fp16 const_20_promoted_to_fp16 = const()[name = string("const_20_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 8, 4, 4]> lower_17_cast_fp16 = mul(x = var_1406_cast_fp16, y = const_20_promoted_to_fp16)[name = string("lower_17_cast_fp16")];
+            bool top_17_interleave_0 = const()[name = string("top_17_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> top_17_cast_fp16 = concat(axis = var_1169, interleave = top_17_interleave_0, values = (var_1399_cast_fp16, var_305_to_fp16))[name = string("top_17_cast_fp16")];
+            bool bottom_17_interleave_0 = const()[name = string("bottom_17_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> bottom_17_cast_fp16 = concat(axis = var_1169, interleave = bottom_17_interleave_0, values = (lower_17_cast_fp16, var_1402_cast_fp16))[name = string("bottom_17_cast_fp16")];
+            bool inv_23_interleave_0 = const()[name = string("inv_23_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> inv_23_cast_fp16 = concat(axis = var_1140, interleave = inv_23_interleave_0, values = (top_17_cast_fp16, bottom_17_cast_fp16))[name = string("inv_23_cast_fp16")];
+            tensor<int32, [5]> var_1415 = const()[name = string("op_1415"), val = tensor<int32, [5]>([1024, 4, 16, 4, 16])];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_1416_cast_fp16 = reshape(shape = var_1415, x = t_7_cast_fp16)[name = string("op_1416_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_1420_to_fp16 = const()[name = string("op_1420_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295080320)))];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_1421_cast_fp16 = mul(x = var_1416_cast_fp16, y = var_1420_to_fp16)[name = string("op_1421_cast_fp16")];
+            tensor<int32, [1]> blocks_19_axes_0 = const()[name = string("blocks_19_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_19_keep_dims_0 = const()[name = string("blocks_19_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> blocks_19_cast_fp16 = reduce_sum(axes = blocks_19_axes_0, keep_dims = blocks_19_keep_dims_0, x = var_1421_cast_fp16)[name = string("blocks_19_cast_fp16")];
+            tensor<int32, [4]> var_1424_begin_0 = const()[name = string("op_1424_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_1424_end_0 = const()[name = string("op_1424_end_0"), val = tensor<int32, [4]>([1024, 4, 16, 16])];
+            tensor<bool, [4]> var_1424_end_mask_0 = const()[name = string("op_1424_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 4, 8, 16]> var_1424_cast_fp16 = slice_by_index(begin = var_1424_begin_0, end = var_1424_end_0, end_mask = var_1424_end_mask_0, x = blocks_19_cast_fp16)[name = string("op_1424_cast_fp16")];
+            tensor<int32, [4]> x_37_begin_0 = const()[name = string("x_37_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_37_end_0 = const()[name = string("x_37_end_0"), val = tensor<int32, [4]>([1024, 4, 8, 8])];
+            tensor<bool, [4]> x_37_end_mask_0 = const()[name = string("x_37_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 4, 8, 8]> x_37_cast_fp16 = slice_by_index(begin = x_37_begin_0, end = x_37_end_0, end_mask = x_37_end_mask_0, x = var_1424_cast_fp16)[name = string("x_37_cast_fp16")];
+            tensor<int32, [5]> var_1426 = const()[name = string("op_1426"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<fp16, [1024, 4, 2, 8, 8]> pair_23_cast_fp16 = reshape(shape = var_1426, x = inv_23_cast_fp16)[name = string("pair_23_cast_fp16")];
+            tensor<int32, [5]> var_1428_begin_0 = const()[name = string("op_1428_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1428_end_0 = const()[name = string("op_1428_end_0"), val = tensor<int32, [5]>([1024, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_1428_end_mask_0 = const()[name = string("op_1428_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1428_squeeze_mask_0 = const()[name = string("op_1428_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_1428_cast_fp16 = slice_by_index(begin = var_1428_begin_0, end = var_1428_end_0, end_mask = var_1428_end_mask_0, squeeze_mask = var_1428_squeeze_mask_0, x = pair_23_cast_fp16)[name = string("op_1428_cast_fp16")];
+            tensor<int32, [5]> var_1431_begin_0 = const()[name = string("op_1431_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1431_end_0 = const()[name = string("op_1431_end_0"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_1431_end_mask_0 = const()[name = string("op_1431_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1431_squeeze_mask_0 = const()[name = string("op_1431_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_1431_cast_fp16 = slice_by_index(begin = var_1431_begin_0, end = var_1431_end_0, end_mask = var_1431_end_mask_0, squeeze_mask = var_1431_squeeze_mask_0, x = pair_23_cast_fp16)[name = string("op_1431_cast_fp16")];
+            bool var_1434_transpose_x_0 = const()[name = string("op_1434_transpose_x_0"), val = bool(false)];
+            bool var_1434_transpose_y_0 = const()[name = string("op_1434_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_1434_cast_fp16 = matmul(transpose_x = var_1434_transpose_x_0, transpose_y = var_1434_transpose_y_0, x = x_37_cast_fp16, y = var_1428_cast_fp16)[name = string("op_1434_cast_fp16")];
+            bool var_1435_transpose_x_0 = const()[name = string("op_1435_transpose_x_0"), val = bool(false)];
+            bool var_1435_transpose_y_0 = const()[name = string("op_1435_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_1435_cast_fp16 = matmul(transpose_x = var_1435_transpose_x_0, transpose_y = var_1435_transpose_y_0, x = var_1431_cast_fp16, y = var_1434_cast_fp16)[name = string("op_1435_cast_fp16")];
+            fp16 const_21_promoted_to_fp16 = const()[name = string("const_21_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 4, 8, 8]> lower_19_cast_fp16 = mul(x = var_1435_cast_fp16, y = const_21_promoted_to_fp16)[name = string("lower_19_cast_fp16")];
+            bool top_19_interleave_0 = const()[name = string("top_19_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> top_19_cast_fp16 = concat(axis = var_1169, interleave = top_19_interleave_0, values = (var_1428_cast_fp16, var_334_to_fp16))[name = string("top_19_cast_fp16")];
+            bool bottom_19_interleave_0 = const()[name = string("bottom_19_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> bottom_19_cast_fp16 = concat(axis = var_1169, interleave = bottom_19_interleave_0, values = (lower_19_cast_fp16, var_1431_cast_fp16))[name = string("bottom_19_cast_fp16")];
+            bool inv_25_interleave_0 = const()[name = string("inv_25_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> inv_25_cast_fp16 = concat(axis = var_1140, interleave = inv_25_interleave_0, values = (top_19_cast_fp16, bottom_19_cast_fp16))[name = string("inv_25_cast_fp16")];
+            tensor<int32, [5]> var_1444 = const()[name = string("op_1444"), val = tensor<int32, [5]>([1024, 2, 32, 2, 32])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_1445_cast_fp16 = reshape(shape = var_1444, x = t_7_cast_fp16)[name = string("op_1445_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_1449_to_fp16 = const()[name = string("op_1449_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_1450_cast_fp16 = mul(x = var_1445_cast_fp16, y = var_1449_to_fp16)[name = string("op_1450_cast_fp16")];
+            tensor<int32, [1]> blocks_21_axes_0 = const()[name = string("blocks_21_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_21_keep_dims_0 = const()[name = string("blocks_21_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> blocks_21_cast_fp16 = reduce_sum(axes = blocks_21_axes_0, keep_dims = blocks_21_keep_dims_0, x = var_1450_cast_fp16)[name = string("blocks_21_cast_fp16")];
+            tensor<int32, [4]> var_1453_begin_0 = const()[name = string("op_1453_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_1453_end_0 = const()[name = string("op_1453_end_0"), val = tensor<int32, [4]>([1024, 2, 32, 32])];
+            tensor<bool, [4]> var_1453_end_mask_0 = const()[name = string("op_1453_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 2, 16, 32]> var_1453_cast_fp16 = slice_by_index(begin = var_1453_begin_0, end = var_1453_end_0, end_mask = var_1453_end_mask_0, x = blocks_21_cast_fp16)[name = string("op_1453_cast_fp16")];
+            tensor<int32, [4]> x_39_begin_0 = const()[name = string("x_39_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_39_end_0 = const()[name = string("x_39_end_0"), val = tensor<int32, [4]>([1024, 2, 16, 16])];
+            tensor<bool, [4]> x_39_end_mask_0 = const()[name = string("x_39_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 2, 16, 16]> x_39_cast_fp16 = slice_by_index(begin = x_39_begin_0, end = x_39_end_0, end_mask = x_39_end_mask_0, x = var_1453_cast_fp16)[name = string("x_39_cast_fp16")];
+            tensor<int32, [5]> var_1455 = const()[name = string("op_1455"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<fp16, [1024, 2, 2, 16, 16]> pair_25_cast_fp16 = reshape(shape = var_1455, x = inv_25_cast_fp16)[name = string("pair_25_cast_fp16")];
+            tensor<int32, [5]> var_1457_begin_0 = const()[name = string("op_1457_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1457_end_0 = const()[name = string("op_1457_end_0"), val = tensor<int32, [5]>([1024, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_1457_end_mask_0 = const()[name = string("op_1457_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1457_squeeze_mask_0 = const()[name = string("op_1457_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_1457_cast_fp16 = slice_by_index(begin = var_1457_begin_0, end = var_1457_end_0, end_mask = var_1457_end_mask_0, squeeze_mask = var_1457_squeeze_mask_0, x = pair_25_cast_fp16)[name = string("op_1457_cast_fp16")];
+            tensor<int32, [5]> var_1460_begin_0 = const()[name = string("op_1460_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1460_end_0 = const()[name = string("op_1460_end_0"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_1460_end_mask_0 = const()[name = string("op_1460_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1460_squeeze_mask_0 = const()[name = string("op_1460_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_1460_cast_fp16 = slice_by_index(begin = var_1460_begin_0, end = var_1460_end_0, end_mask = var_1460_end_mask_0, squeeze_mask = var_1460_squeeze_mask_0, x = pair_25_cast_fp16)[name = string("op_1460_cast_fp16")];
+            bool var_1463_transpose_x_0 = const()[name = string("op_1463_transpose_x_0"), val = bool(false)];
+            bool var_1463_transpose_y_0 = const()[name = string("op_1463_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_1463_cast_fp16 = matmul(transpose_x = var_1463_transpose_x_0, transpose_y = var_1463_transpose_y_0, x = x_39_cast_fp16, y = var_1457_cast_fp16)[name = string("op_1463_cast_fp16")];
+            bool var_1464_transpose_x_0 = const()[name = string("op_1464_transpose_x_0"), val = bool(false)];
+            bool var_1464_transpose_y_0 = const()[name = string("op_1464_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_1464_cast_fp16 = matmul(transpose_x = var_1464_transpose_x_0, transpose_y = var_1464_transpose_y_0, x = var_1460_cast_fp16, y = var_1463_cast_fp16)[name = string("op_1464_cast_fp16")];
+            fp16 const_22_promoted_to_fp16 = const()[name = string("const_22_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 2, 16, 16]> lower_21_cast_fp16 = mul(x = var_1464_cast_fp16, y = const_22_promoted_to_fp16)[name = string("lower_21_cast_fp16")];
+            bool top_21_interleave_0 = const()[name = string("top_21_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> top_21_cast_fp16 = concat(axis = var_1169, interleave = top_21_interleave_0, values = (var_1457_cast_fp16, var_363_to_fp16))[name = string("top_21_cast_fp16")];
+            bool bottom_21_interleave_0 = const()[name = string("bottom_21_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> bottom_21_cast_fp16 = concat(axis = var_1169, interleave = bottom_21_interleave_0, values = (lower_21_cast_fp16, var_1460_cast_fp16))[name = string("bottom_21_cast_fp16")];
+            bool inv_27_interleave_0 = const()[name = string("inv_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> inv_27_cast_fp16 = concat(axis = var_1140, interleave = inv_27_interleave_0, values = (top_21_cast_fp16, bottom_21_cast_fp16))[name = string("inv_27_cast_fp16")];
+            tensor<int32, [5]> var_1473 = const()[name = string("op_1473"), val = tensor<int32, [5]>([1024, 1, 64, 1, 64])];
+            tensor<fp16, [1024, 1, 64, 1, 64]> var_1474_cast_fp16 = reshape(shape = var_1473, x = t_7_cast_fp16)[name = string("op_1474_cast_fp16")];
+            tensor<int32, [1]> blocks_23_axes_0 = const()[name = string("blocks_23_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_23_keep_dims_0 = const()[name = string("blocks_23_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> blocks_23_cast_fp16 = reduce_sum(axes = blocks_23_axes_0, keep_dims = blocks_23_keep_dims_0, x = var_1474_cast_fp16)[name = string("blocks_23_cast_fp16")];
+            tensor<int32, [4]> var_1482_begin_0 = const()[name = string("op_1482_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_1482_end_0 = const()[name = string("op_1482_end_0"), val = tensor<int32, [4]>([1024, 1, 64, 64])];
+            tensor<bool, [4]> var_1482_end_mask_0 = const()[name = string("op_1482_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 1, 32, 64]> var_1482_cast_fp16 = slice_by_index(begin = var_1482_begin_0, end = var_1482_end_0, end_mask = var_1482_end_mask_0, x = blocks_23_cast_fp16)[name = string("op_1482_cast_fp16")];
+            tensor<int32, [4]> x_41_begin_0 = const()[name = string("x_41_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_41_end_0 = const()[name = string("x_41_end_0"), val = tensor<int32, [4]>([1024, 1, 32, 32])];
+            tensor<bool, [4]> x_41_end_mask_0 = const()[name = string("x_41_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 1, 32, 32]> x_41_cast_fp16 = slice_by_index(begin = x_41_begin_0, end = x_41_end_0, end_mask = x_41_end_mask_0, x = var_1482_cast_fp16)[name = string("x_41_cast_fp16")];
+            tensor<int32, [5]> var_1484 = const()[name = string("op_1484"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<fp16, [1024, 1, 2, 32, 32]> pair_27_cast_fp16 = reshape(shape = var_1484, x = inv_27_cast_fp16)[name = string("pair_27_cast_fp16")];
+            tensor<int32, [5]> var_1486_begin_0 = const()[name = string("op_1486_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_1486_end_0 = const()[name = string("op_1486_end_0"), val = tensor<int32, [5]>([1024, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_1486_end_mask_0 = const()[name = string("op_1486_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1486_squeeze_mask_0 = const()[name = string("op_1486_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_1486_cast_fp16 = slice_by_index(begin = var_1486_begin_0, end = var_1486_end_0, end_mask = var_1486_end_mask_0, squeeze_mask = var_1486_squeeze_mask_0, x = pair_27_cast_fp16)[name = string("op_1486_cast_fp16")];
+            tensor<int32, [5]> var_1489_begin_0 = const()[name = string("op_1489_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_1489_end_0 = const()[name = string("op_1489_end_0"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_1489_end_mask_0 = const()[name = string("op_1489_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_1489_squeeze_mask_0 = const()[name = string("op_1489_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_1489_cast_fp16 = slice_by_index(begin = var_1489_begin_0, end = var_1489_end_0, end_mask = var_1489_end_mask_0, squeeze_mask = var_1489_squeeze_mask_0, x = pair_27_cast_fp16)[name = string("op_1489_cast_fp16")];
+            bool var_1492_transpose_x_0 = const()[name = string("op_1492_transpose_x_0"), val = bool(false)];
+            bool var_1492_transpose_y_0 = const()[name = string("op_1492_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_1492_cast_fp16 = matmul(transpose_x = var_1492_transpose_x_0, transpose_y = var_1492_transpose_y_0, x = x_41_cast_fp16, y = var_1486_cast_fp16)[name = string("op_1492_cast_fp16")];
+            bool var_1493_transpose_x_0 = const()[name = string("op_1493_transpose_x_0"), val = bool(false)];
+            bool var_1493_transpose_y_0 = const()[name = string("op_1493_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_1493_cast_fp16 = matmul(transpose_x = var_1493_transpose_x_0, transpose_y = var_1493_transpose_y_0, x = var_1489_cast_fp16, y = var_1492_cast_fp16)[name = string("op_1493_cast_fp16")];
+            fp16 const_23_promoted_to_fp16 = const()[name = string("const_23_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 1, 32, 32]> lower_23_cast_fp16 = mul(x = var_1493_cast_fp16, y = const_23_promoted_to_fp16)[name = string("lower_23_cast_fp16")];
+            bool top_23_interleave_0 = const()[name = string("top_23_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> top_23_cast_fp16 = concat(axis = var_1169, interleave = top_23_interleave_0, values = (var_1486_cast_fp16, var_392_to_fp16))[name = string("top_23_cast_fp16")];
+            bool bottom_23_interleave_0 = const()[name = string("bottom_23_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> bottom_23_cast_fp16 = concat(axis = var_1169, interleave = bottom_23_interleave_0, values = (lower_23_cast_fp16, var_1489_cast_fp16))[name = string("bottom_23_cast_fp16")];
+            bool inv_29_interleave_0 = const()[name = string("inv_29_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> inv_29_cast_fp16 = concat(axis = var_1140, interleave = inv_29_interleave_0, values = (top_23_cast_fp16, bottom_23_cast_fp16))[name = string("inv_29_cast_fp16")];
+            tensor<int32, [4]> var_1502 = const()[name = string("op_1502"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> inv_31_cast_fp16 = reshape(shape = var_1502, x = inv_29_cast_fp16)[name = string("inv_31_cast_fp16")];
+            bool new_v_3_transpose_x_0 = const()[name = string("new_v_3_transpose_x_0"), val = bool(false)];
+            bool new_v_3_transpose_y_0 = const()[name = string("new_v_3_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> new_v_3_cast_fp16 = matmul(transpose_x = new_v_3_transpose_x_0, transpose_y = new_v_3_transpose_y_0, x = inv_31_cast_fp16, y = v_beta_3_cast_fp16)[name = string("new_v_3_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_1505_cast_fp16 = exp(x = cum_3_cast_fp16)[name = string("op_1505_cast_fp16")];
+            tensor<int32, [1]> var_1506_axes_0 = const()[name = string("op_1506_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_1506_cast_fp16 = expand_dims(axes = var_1506_axes_0, x = var_1505_cast_fp16)[name = string("op_1506_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> var_1507_cast_fp16 = mul(x = k_beta_3_cast_fp16, y = var_1506_cast_fp16)[name = string("op_1507_cast_fp16")];
+            bool k_cumdecay_3_transpose_x_0 = const()[name = string("k_cumdecay_3_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_3_transpose_y_0 = const()[name = string("k_cumdecay_3_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> k_cumdecay_3_cast_fp16 = matmul(transpose_x = k_cumdecay_3_transpose_x_0, transpose_y = k_cumdecay_3_transpose_y_0, x = inv_31_cast_fp16, y = var_1507_cast_fp16)[name = string("k_cumdecay_3_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> q_dec_3_cast_fp16 = mul(x = q_19_cast_fp16, y = var_1506_cast_fp16)[name = string("q_dec_3_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_1512_cast_fp16 = exp(x = to_end_3_cast_fp16)[name = string("op_1512_cast_fp16")];
+            tensor<int32, [1]> var_1513_axes_0 = const()[name = string("op_1513_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_1513_cast_fp16 = expand_dims(axes = var_1513_axes_0, x = var_1512_cast_fp16)[name = string("op_1513_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_dec_3_cast_fp16 = mul(x = k_15_cast_fp16, y = var_1513_cast_fp16)[name = string("k_dec_3_cast_fp16")];
+            tensor<int32, [3]> var_1515_begin_0 = const()[name = string("op_1515_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_1515_end_0 = const()[name = string("op_1515_end_0"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<bool, [3]> var_1515_end_mask_0 = const()[name = string("op_1515_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 32, 1]> var_1515_cast_fp16 = slice_by_index(begin = var_1515_begin_0, end = var_1515_end_0, end_mask = var_1515_end_mask_0, x = cum_3_cast_fp16)[name = string("op_1515_cast_fp16")];
+            tensor<fp16, [32, 32, 1]> var_1516_cast_fp16 = exp(x = var_1515_cast_fp16)[name = string("op_1516_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_3_axes_0 = const()[name = string("chunk_decay_3_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 1, 1]> chunk_decay_3_cast_fp16 = expand_dims(axes = chunk_decay_3_axes_0, x = var_1516_cast_fp16)[name = string("chunk_decay_3_cast_fp16")];
+            tensor<int32, [4]> var_1521_begin_0 = const()[name = string("op_1521_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1521_end_0 = const()[name = string("op_1521_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1521_end_mask_0 = const()[name = string("op_1521_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1521_squeeze_mask_0 = const()[name = string("op_1521_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1521_cast_fp16 = slice_by_index(begin = var_1521_begin_0, end = var_1521_end_0, end_mask = var_1521_end_mask_0, squeeze_mask = var_1521_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1521_cast_fp16")];
+            tensor<int32, [4]> var_1523_begin_0 = const()[name = string("op_1523_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1523_end_0 = const()[name = string("op_1523_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1523_end_mask_0 = const()[name = string("op_1523_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1523_squeeze_mask_0 = const()[name = string("op_1523_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1523_cast_fp16 = slice_by_index(begin = var_1523_begin_0, end = var_1523_end_0, end_mask = var_1523_end_mask_0, squeeze_mask = var_1523_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1523_cast_fp16")];
+            bool var_1524_transpose_x_0 = const()[name = string("op_1524_transpose_x_0"), val = bool(false)];
+            bool var_1524_transpose_y_0 = const()[name = string("op_1524_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1524_cast_fp16 = matmul(transpose_x = var_1524_transpose_x_0, transpose_y = var_1524_transpose_y_0, x = var_1523_cast_fp16, y = state_1_to_fp16)[name = string("op_1524_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_65_cast_fp16 = sub(x = var_1521_cast_fp16, y = var_1524_cast_fp16)[name = string("v_new_65_cast_fp16")];
+            tensor<int32, [4]> var_1527_begin_0 = const()[name = string("op_1527_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1527_end_0 = const()[name = string("op_1527_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1527_end_mask_0 = const()[name = string("op_1527_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1527_squeeze_mask_0 = const()[name = string("op_1527_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1527_cast_fp16 = slice_by_index(begin = var_1527_begin_0, end = var_1527_end_0, end_mask = var_1527_end_mask_0, squeeze_mask = var_1527_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1527_cast_fp16")];
+            bool var_1528_transpose_x_0 = const()[name = string("op_1528_transpose_x_0"), val = bool(false)];
+            bool var_1528_transpose_y_0 = const()[name = string("op_1528_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1528_cast_fp16 = matmul(transpose_x = var_1528_transpose_x_0, transpose_y = var_1528_transpose_y_0, x = var_1527_cast_fp16, y = state_1_to_fp16)[name = string("op_1528_cast_fp16")];
+            tensor<int32, [4]> var_1530_begin_0 = const()[name = string("op_1530_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1530_end_0 = const()[name = string("op_1530_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_1530_end_mask_0 = const()[name = string("op_1530_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1530_squeeze_mask_0 = const()[name = string("op_1530_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1530_cast_fp16 = slice_by_index(begin = var_1530_begin_0, end = var_1530_end_0, end_mask = var_1530_end_mask_0, squeeze_mask = var_1530_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1530_cast_fp16")];
+            bool var_1531_transpose_x_0 = const()[name = string("op_1531_transpose_x_0"), val = bool(false)];
+            bool var_1531_transpose_y_0 = const()[name = string("op_1531_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1531_cast_fp16 = matmul(transpose_x = var_1531_transpose_x_0, transpose_y = var_1531_transpose_y_0, x = var_1530_cast_fp16, y = v_new_65_cast_fp16)[name = string("op_1531_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1532_cast_fp16 = add(x = var_1528_cast_fp16, y = var_1531_cast_fp16)[name = string("op_1532_cast_fp16")];
+            tensor<int32, [4]> var_1534_begin_0 = const()[name = string("op_1534_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1534_end_0 = const()[name = string("op_1534_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_1534_end_mask_0 = const()[name = string("op_1534_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1534_squeeze_mask_0 = const()[name = string("op_1534_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1534_cast_fp16 = slice_by_index(begin = var_1534_begin_0, end = var_1534_end_0, end_mask = var_1534_end_mask_0, squeeze_mask = var_1534_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1534_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1535_cast_fp16 = mul(x = state_1_to_fp16, y = var_1534_cast_fp16)[name = string("op_1535_cast_fp16")];
+            tensor<int32, [4]> var_1537_begin_0 = const()[name = string("op_1537_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_1537_end_0 = const()[name = string("op_1537_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_1537_end_mask_0 = const()[name = string("op_1537_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1537_squeeze_mask_0 = const()[name = string("op_1537_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1537_cast_fp16 = slice_by_index(begin = var_1537_begin_0, end = var_1537_end_0, end_mask = var_1537_end_mask_0, squeeze_mask = var_1537_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1537_cast_fp16")];
+            bool var_1539_transpose_x_1 = const()[name = string("op_1539_transpose_x_1"), val = bool(true)];
+            bool var_1539_transpose_y_1 = const()[name = string("op_1539_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1539_cast_fp16 = matmul(transpose_x = var_1539_transpose_x_1, transpose_y = var_1539_transpose_y_1, x = var_1537_cast_fp16, y = v_new_65_cast_fp16)[name = string("op_1539_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_67_cast_fp16 = add(x = var_1535_cast_fp16, y = var_1539_cast_fp16)[name = string("state_67_cast_fp16")];
+            tensor<int32, [4]> var_1542_begin_0 = const()[name = string("op_1542_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1542_end_0 = const()[name = string("op_1542_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1542_end_mask_0 = const()[name = string("op_1542_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1542_squeeze_mask_0 = const()[name = string("op_1542_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1542_cast_fp16 = slice_by_index(begin = var_1542_begin_0, end = var_1542_end_0, end_mask = var_1542_end_mask_0, squeeze_mask = var_1542_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1542_cast_fp16")];
+            tensor<int32, [4]> var_1544_begin_0 = const()[name = string("op_1544_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1544_end_0 = const()[name = string("op_1544_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1544_end_mask_0 = const()[name = string("op_1544_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1544_squeeze_mask_0 = const()[name = string("op_1544_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1544_cast_fp16 = slice_by_index(begin = var_1544_begin_0, end = var_1544_end_0, end_mask = var_1544_end_mask_0, squeeze_mask = var_1544_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1544_cast_fp16")];
+            bool var_1545_transpose_x_0 = const()[name = string("op_1545_transpose_x_0"), val = bool(false)];
+            bool var_1545_transpose_y_0 = const()[name = string("op_1545_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1545_cast_fp16 = matmul(transpose_x = var_1545_transpose_x_0, transpose_y = var_1545_transpose_y_0, x = var_1544_cast_fp16, y = state_67_cast_fp16)[name = string("op_1545_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_67_cast_fp16 = sub(x = var_1542_cast_fp16, y = var_1545_cast_fp16)[name = string("v_new_67_cast_fp16")];
+            tensor<int32, [4]> var_1548_begin_0 = const()[name = string("op_1548_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1548_end_0 = const()[name = string("op_1548_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1548_end_mask_0 = const()[name = string("op_1548_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1548_squeeze_mask_0 = const()[name = string("op_1548_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1548_cast_fp16 = slice_by_index(begin = var_1548_begin_0, end = var_1548_end_0, end_mask = var_1548_end_mask_0, squeeze_mask = var_1548_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1548_cast_fp16")];
+            bool var_1549_transpose_x_0 = const()[name = string("op_1549_transpose_x_0"), val = bool(false)];
+            bool var_1549_transpose_y_0 = const()[name = string("op_1549_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1549_cast_fp16 = matmul(transpose_x = var_1549_transpose_x_0, transpose_y = var_1549_transpose_y_0, x = var_1548_cast_fp16, y = state_67_cast_fp16)[name = string("op_1549_cast_fp16")];
+            tensor<int32, [4]> var_1551_begin_0 = const()[name = string("op_1551_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1551_end_0 = const()[name = string("op_1551_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_1551_end_mask_0 = const()[name = string("op_1551_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1551_squeeze_mask_0 = const()[name = string("op_1551_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1551_cast_fp16 = slice_by_index(begin = var_1551_begin_0, end = var_1551_end_0, end_mask = var_1551_end_mask_0, squeeze_mask = var_1551_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1551_cast_fp16")];
+            bool var_1552_transpose_x_0 = const()[name = string("op_1552_transpose_x_0"), val = bool(false)];
+            bool var_1552_transpose_y_0 = const()[name = string("op_1552_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1552_cast_fp16 = matmul(transpose_x = var_1552_transpose_x_0, transpose_y = var_1552_transpose_y_0, x = var_1551_cast_fp16, y = v_new_67_cast_fp16)[name = string("op_1552_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1553_cast_fp16 = add(x = var_1549_cast_fp16, y = var_1552_cast_fp16)[name = string("op_1553_cast_fp16")];
+            tensor<int32, [4]> var_1555_begin_0 = const()[name = string("op_1555_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1555_end_0 = const()[name = string("op_1555_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_1555_end_mask_0 = const()[name = string("op_1555_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1555_squeeze_mask_0 = const()[name = string("op_1555_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1555_cast_fp16 = slice_by_index(begin = var_1555_begin_0, end = var_1555_end_0, end_mask = var_1555_end_mask_0, squeeze_mask = var_1555_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1555_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1556_cast_fp16 = mul(x = state_67_cast_fp16, y = var_1555_cast_fp16)[name = string("op_1556_cast_fp16")];
+            tensor<int32, [4]> var_1558_begin_0 = const()[name = string("op_1558_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_1558_end_0 = const()[name = string("op_1558_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_1558_end_mask_0 = const()[name = string("op_1558_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1558_squeeze_mask_0 = const()[name = string("op_1558_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1558_cast_fp16 = slice_by_index(begin = var_1558_begin_0, end = var_1558_end_0, end_mask = var_1558_end_mask_0, squeeze_mask = var_1558_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1558_cast_fp16")];
+            bool var_1560_transpose_x_1 = const()[name = string("op_1560_transpose_x_1"), val = bool(true)];
+            bool var_1560_transpose_y_1 = const()[name = string("op_1560_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1560_cast_fp16 = matmul(transpose_x = var_1560_transpose_x_1, transpose_y = var_1560_transpose_y_1, x = var_1558_cast_fp16, y = v_new_67_cast_fp16)[name = string("op_1560_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_69_cast_fp16 = add(x = var_1556_cast_fp16, y = var_1560_cast_fp16)[name = string("state_69_cast_fp16")];
+            tensor<int32, [4]> var_1563_begin_0 = const()[name = string("op_1563_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1563_end_0 = const()[name = string("op_1563_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1563_end_mask_0 = const()[name = string("op_1563_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1563_squeeze_mask_0 = const()[name = string("op_1563_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1563_cast_fp16 = slice_by_index(begin = var_1563_begin_0, end = var_1563_end_0, end_mask = var_1563_end_mask_0, squeeze_mask = var_1563_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1563_cast_fp16")];
+            tensor<int32, [4]> var_1565_begin_0 = const()[name = string("op_1565_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1565_end_0 = const()[name = string("op_1565_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1565_end_mask_0 = const()[name = string("op_1565_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1565_squeeze_mask_0 = const()[name = string("op_1565_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1565_cast_fp16 = slice_by_index(begin = var_1565_begin_0, end = var_1565_end_0, end_mask = var_1565_end_mask_0, squeeze_mask = var_1565_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1565_cast_fp16")];
+            bool var_1566_transpose_x_0 = const()[name = string("op_1566_transpose_x_0"), val = bool(false)];
+            bool var_1566_transpose_y_0 = const()[name = string("op_1566_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1566_cast_fp16 = matmul(transpose_x = var_1566_transpose_x_0, transpose_y = var_1566_transpose_y_0, x = var_1565_cast_fp16, y = state_69_cast_fp16)[name = string("op_1566_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_69_cast_fp16 = sub(x = var_1563_cast_fp16, y = var_1566_cast_fp16)[name = string("v_new_69_cast_fp16")];
+            tensor<int32, [4]> var_1569_begin_0 = const()[name = string("op_1569_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1569_end_0 = const()[name = string("op_1569_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1569_end_mask_0 = const()[name = string("op_1569_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1569_squeeze_mask_0 = const()[name = string("op_1569_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1569_cast_fp16 = slice_by_index(begin = var_1569_begin_0, end = var_1569_end_0, end_mask = var_1569_end_mask_0, squeeze_mask = var_1569_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1569_cast_fp16")];
+            bool var_1570_transpose_x_0 = const()[name = string("op_1570_transpose_x_0"), val = bool(false)];
+            bool var_1570_transpose_y_0 = const()[name = string("op_1570_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1570_cast_fp16 = matmul(transpose_x = var_1570_transpose_x_0, transpose_y = var_1570_transpose_y_0, x = var_1569_cast_fp16, y = state_69_cast_fp16)[name = string("op_1570_cast_fp16")];
+            tensor<int32, [4]> var_1572_begin_0 = const()[name = string("op_1572_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1572_end_0 = const()[name = string("op_1572_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_1572_end_mask_0 = const()[name = string("op_1572_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1572_squeeze_mask_0 = const()[name = string("op_1572_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1572_cast_fp16 = slice_by_index(begin = var_1572_begin_0, end = var_1572_end_0, end_mask = var_1572_end_mask_0, squeeze_mask = var_1572_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1572_cast_fp16")];
+            bool var_1573_transpose_x_0 = const()[name = string("op_1573_transpose_x_0"), val = bool(false)];
+            bool var_1573_transpose_y_0 = const()[name = string("op_1573_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1573_cast_fp16 = matmul(transpose_x = var_1573_transpose_x_0, transpose_y = var_1573_transpose_y_0, x = var_1572_cast_fp16, y = v_new_69_cast_fp16)[name = string("op_1573_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1574_cast_fp16 = add(x = var_1570_cast_fp16, y = var_1573_cast_fp16)[name = string("op_1574_cast_fp16")];
+            tensor<int32, [4]> var_1576_begin_0 = const()[name = string("op_1576_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1576_end_0 = const()[name = string("op_1576_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_1576_end_mask_0 = const()[name = string("op_1576_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1576_squeeze_mask_0 = const()[name = string("op_1576_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1576_cast_fp16 = slice_by_index(begin = var_1576_begin_0, end = var_1576_end_0, end_mask = var_1576_end_mask_0, squeeze_mask = var_1576_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1576_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1577_cast_fp16 = mul(x = state_69_cast_fp16, y = var_1576_cast_fp16)[name = string("op_1577_cast_fp16")];
+            tensor<int32, [4]> var_1579_begin_0 = const()[name = string("op_1579_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_1579_end_0 = const()[name = string("op_1579_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_1579_end_mask_0 = const()[name = string("op_1579_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1579_squeeze_mask_0 = const()[name = string("op_1579_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1579_cast_fp16 = slice_by_index(begin = var_1579_begin_0, end = var_1579_end_0, end_mask = var_1579_end_mask_0, squeeze_mask = var_1579_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1579_cast_fp16")];
+            bool var_1581_transpose_x_1 = const()[name = string("op_1581_transpose_x_1"), val = bool(true)];
+            bool var_1581_transpose_y_1 = const()[name = string("op_1581_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1581_cast_fp16 = matmul(transpose_x = var_1581_transpose_x_1, transpose_y = var_1581_transpose_y_1, x = var_1579_cast_fp16, y = v_new_69_cast_fp16)[name = string("op_1581_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_71_cast_fp16 = add(x = var_1577_cast_fp16, y = var_1581_cast_fp16)[name = string("state_71_cast_fp16")];
+            tensor<int32, [4]> var_1584_begin_0 = const()[name = string("op_1584_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1584_end_0 = const()[name = string("op_1584_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1584_end_mask_0 = const()[name = string("op_1584_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1584_squeeze_mask_0 = const()[name = string("op_1584_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1584_cast_fp16 = slice_by_index(begin = var_1584_begin_0, end = var_1584_end_0, end_mask = var_1584_end_mask_0, squeeze_mask = var_1584_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1584_cast_fp16")];
+            tensor<int32, [4]> var_1586_begin_0 = const()[name = string("op_1586_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1586_end_0 = const()[name = string("op_1586_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1586_end_mask_0 = const()[name = string("op_1586_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1586_squeeze_mask_0 = const()[name = string("op_1586_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1586_cast_fp16 = slice_by_index(begin = var_1586_begin_0, end = var_1586_end_0, end_mask = var_1586_end_mask_0, squeeze_mask = var_1586_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1586_cast_fp16")];
+            bool var_1587_transpose_x_0 = const()[name = string("op_1587_transpose_x_0"), val = bool(false)];
+            bool var_1587_transpose_y_0 = const()[name = string("op_1587_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1587_cast_fp16 = matmul(transpose_x = var_1587_transpose_x_0, transpose_y = var_1587_transpose_y_0, x = var_1586_cast_fp16, y = state_71_cast_fp16)[name = string("op_1587_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_71_cast_fp16 = sub(x = var_1584_cast_fp16, y = var_1587_cast_fp16)[name = string("v_new_71_cast_fp16")];
+            tensor<int32, [4]> var_1590_begin_0 = const()[name = string("op_1590_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1590_end_0 = const()[name = string("op_1590_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1590_end_mask_0 = const()[name = string("op_1590_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1590_squeeze_mask_0 = const()[name = string("op_1590_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1590_cast_fp16 = slice_by_index(begin = var_1590_begin_0, end = var_1590_end_0, end_mask = var_1590_end_mask_0, squeeze_mask = var_1590_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1590_cast_fp16")];
+            bool var_1591_transpose_x_0 = const()[name = string("op_1591_transpose_x_0"), val = bool(false)];
+            bool var_1591_transpose_y_0 = const()[name = string("op_1591_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1591_cast_fp16 = matmul(transpose_x = var_1591_transpose_x_0, transpose_y = var_1591_transpose_y_0, x = var_1590_cast_fp16, y = state_71_cast_fp16)[name = string("op_1591_cast_fp16")];
+            tensor<int32, [4]> var_1593_begin_0 = const()[name = string("op_1593_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1593_end_0 = const()[name = string("op_1593_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_1593_end_mask_0 = const()[name = string("op_1593_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1593_squeeze_mask_0 = const()[name = string("op_1593_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1593_cast_fp16 = slice_by_index(begin = var_1593_begin_0, end = var_1593_end_0, end_mask = var_1593_end_mask_0, squeeze_mask = var_1593_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1593_cast_fp16")];
+            bool var_1594_transpose_x_0 = const()[name = string("op_1594_transpose_x_0"), val = bool(false)];
+            bool var_1594_transpose_y_0 = const()[name = string("op_1594_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1594_cast_fp16 = matmul(transpose_x = var_1594_transpose_x_0, transpose_y = var_1594_transpose_y_0, x = var_1593_cast_fp16, y = v_new_71_cast_fp16)[name = string("op_1594_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1595_cast_fp16 = add(x = var_1591_cast_fp16, y = var_1594_cast_fp16)[name = string("op_1595_cast_fp16")];
+            tensor<int32, [4]> var_1597_begin_0 = const()[name = string("op_1597_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1597_end_0 = const()[name = string("op_1597_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_1597_end_mask_0 = const()[name = string("op_1597_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1597_squeeze_mask_0 = const()[name = string("op_1597_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1597_cast_fp16 = slice_by_index(begin = var_1597_begin_0, end = var_1597_end_0, end_mask = var_1597_end_mask_0, squeeze_mask = var_1597_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1597_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1598_cast_fp16 = mul(x = state_71_cast_fp16, y = var_1597_cast_fp16)[name = string("op_1598_cast_fp16")];
+            tensor<int32, [4]> var_1600_begin_0 = const()[name = string("op_1600_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_1600_end_0 = const()[name = string("op_1600_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_1600_end_mask_0 = const()[name = string("op_1600_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1600_squeeze_mask_0 = const()[name = string("op_1600_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1600_cast_fp16 = slice_by_index(begin = var_1600_begin_0, end = var_1600_end_0, end_mask = var_1600_end_mask_0, squeeze_mask = var_1600_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1600_cast_fp16")];
+            bool var_1602_transpose_x_1 = const()[name = string("op_1602_transpose_x_1"), val = bool(true)];
+            bool var_1602_transpose_y_1 = const()[name = string("op_1602_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1602_cast_fp16 = matmul(transpose_x = var_1602_transpose_x_1, transpose_y = var_1602_transpose_y_1, x = var_1600_cast_fp16, y = v_new_71_cast_fp16)[name = string("op_1602_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_73_cast_fp16 = add(x = var_1598_cast_fp16, y = var_1602_cast_fp16)[name = string("state_73_cast_fp16")];
+            tensor<int32, [4]> var_1605_begin_0 = const()[name = string("op_1605_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1605_end_0 = const()[name = string("op_1605_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1605_end_mask_0 = const()[name = string("op_1605_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1605_squeeze_mask_0 = const()[name = string("op_1605_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1605_cast_fp16 = slice_by_index(begin = var_1605_begin_0, end = var_1605_end_0, end_mask = var_1605_end_mask_0, squeeze_mask = var_1605_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1605_cast_fp16")];
+            tensor<int32, [4]> var_1607_begin_0 = const()[name = string("op_1607_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1607_end_0 = const()[name = string("op_1607_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1607_end_mask_0 = const()[name = string("op_1607_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1607_squeeze_mask_0 = const()[name = string("op_1607_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1607_cast_fp16 = slice_by_index(begin = var_1607_begin_0, end = var_1607_end_0, end_mask = var_1607_end_mask_0, squeeze_mask = var_1607_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1607_cast_fp16")];
+            bool var_1608_transpose_x_0 = const()[name = string("op_1608_transpose_x_0"), val = bool(false)];
+            bool var_1608_transpose_y_0 = const()[name = string("op_1608_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1608_cast_fp16 = matmul(transpose_x = var_1608_transpose_x_0, transpose_y = var_1608_transpose_y_0, x = var_1607_cast_fp16, y = state_73_cast_fp16)[name = string("op_1608_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_73_cast_fp16 = sub(x = var_1605_cast_fp16, y = var_1608_cast_fp16)[name = string("v_new_73_cast_fp16")];
+            tensor<int32, [4]> var_1611_begin_0 = const()[name = string("op_1611_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1611_end_0 = const()[name = string("op_1611_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1611_end_mask_0 = const()[name = string("op_1611_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1611_squeeze_mask_0 = const()[name = string("op_1611_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1611_cast_fp16 = slice_by_index(begin = var_1611_begin_0, end = var_1611_end_0, end_mask = var_1611_end_mask_0, squeeze_mask = var_1611_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1611_cast_fp16")];
+            bool var_1612_transpose_x_0 = const()[name = string("op_1612_transpose_x_0"), val = bool(false)];
+            bool var_1612_transpose_y_0 = const()[name = string("op_1612_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1612_cast_fp16 = matmul(transpose_x = var_1612_transpose_x_0, transpose_y = var_1612_transpose_y_0, x = var_1611_cast_fp16, y = state_73_cast_fp16)[name = string("op_1612_cast_fp16")];
+            tensor<int32, [4]> var_1614_begin_0 = const()[name = string("op_1614_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1614_end_0 = const()[name = string("op_1614_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_1614_end_mask_0 = const()[name = string("op_1614_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1614_squeeze_mask_0 = const()[name = string("op_1614_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1614_cast_fp16 = slice_by_index(begin = var_1614_begin_0, end = var_1614_end_0, end_mask = var_1614_end_mask_0, squeeze_mask = var_1614_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1614_cast_fp16")];
+            bool var_1615_transpose_x_0 = const()[name = string("op_1615_transpose_x_0"), val = bool(false)];
+            bool var_1615_transpose_y_0 = const()[name = string("op_1615_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1615_cast_fp16 = matmul(transpose_x = var_1615_transpose_x_0, transpose_y = var_1615_transpose_y_0, x = var_1614_cast_fp16, y = v_new_73_cast_fp16)[name = string("op_1615_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1616_cast_fp16 = add(x = var_1612_cast_fp16, y = var_1615_cast_fp16)[name = string("op_1616_cast_fp16")];
+            tensor<int32, [4]> var_1618_begin_0 = const()[name = string("op_1618_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1618_end_0 = const()[name = string("op_1618_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_1618_end_mask_0 = const()[name = string("op_1618_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1618_squeeze_mask_0 = const()[name = string("op_1618_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1618_cast_fp16 = slice_by_index(begin = var_1618_begin_0, end = var_1618_end_0, end_mask = var_1618_end_mask_0, squeeze_mask = var_1618_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1618_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1619_cast_fp16 = mul(x = state_73_cast_fp16, y = var_1618_cast_fp16)[name = string("op_1619_cast_fp16")];
+            tensor<int32, [4]> var_1621_begin_0 = const()[name = string("op_1621_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_1621_end_0 = const()[name = string("op_1621_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_1621_end_mask_0 = const()[name = string("op_1621_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1621_squeeze_mask_0 = const()[name = string("op_1621_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1621_cast_fp16 = slice_by_index(begin = var_1621_begin_0, end = var_1621_end_0, end_mask = var_1621_end_mask_0, squeeze_mask = var_1621_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1621_cast_fp16")];
+            bool var_1623_transpose_x_1 = const()[name = string("op_1623_transpose_x_1"), val = bool(true)];
+            bool var_1623_transpose_y_1 = const()[name = string("op_1623_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1623_cast_fp16 = matmul(transpose_x = var_1623_transpose_x_1, transpose_y = var_1623_transpose_y_1, x = var_1621_cast_fp16, y = v_new_73_cast_fp16)[name = string("op_1623_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_75_cast_fp16 = add(x = var_1619_cast_fp16, y = var_1623_cast_fp16)[name = string("state_75_cast_fp16")];
+            tensor<int32, [4]> var_1626_begin_0 = const()[name = string("op_1626_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1626_end_0 = const()[name = string("op_1626_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1626_end_mask_0 = const()[name = string("op_1626_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1626_squeeze_mask_0 = const()[name = string("op_1626_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1626_cast_fp16 = slice_by_index(begin = var_1626_begin_0, end = var_1626_end_0, end_mask = var_1626_end_mask_0, squeeze_mask = var_1626_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1626_cast_fp16")];
+            tensor<int32, [4]> var_1628_begin_0 = const()[name = string("op_1628_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1628_end_0 = const()[name = string("op_1628_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1628_end_mask_0 = const()[name = string("op_1628_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1628_squeeze_mask_0 = const()[name = string("op_1628_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1628_cast_fp16 = slice_by_index(begin = var_1628_begin_0, end = var_1628_end_0, end_mask = var_1628_end_mask_0, squeeze_mask = var_1628_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1628_cast_fp16")];
+            bool var_1629_transpose_x_0 = const()[name = string("op_1629_transpose_x_0"), val = bool(false)];
+            bool var_1629_transpose_y_0 = const()[name = string("op_1629_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1629_cast_fp16 = matmul(transpose_x = var_1629_transpose_x_0, transpose_y = var_1629_transpose_y_0, x = var_1628_cast_fp16, y = state_75_cast_fp16)[name = string("op_1629_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_75_cast_fp16 = sub(x = var_1626_cast_fp16, y = var_1629_cast_fp16)[name = string("v_new_75_cast_fp16")];
+            tensor<int32, [4]> var_1632_begin_0 = const()[name = string("op_1632_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1632_end_0 = const()[name = string("op_1632_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1632_end_mask_0 = const()[name = string("op_1632_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1632_squeeze_mask_0 = const()[name = string("op_1632_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1632_cast_fp16 = slice_by_index(begin = var_1632_begin_0, end = var_1632_end_0, end_mask = var_1632_end_mask_0, squeeze_mask = var_1632_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1632_cast_fp16")];
+            bool var_1633_transpose_x_0 = const()[name = string("op_1633_transpose_x_0"), val = bool(false)];
+            bool var_1633_transpose_y_0 = const()[name = string("op_1633_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1633_cast_fp16 = matmul(transpose_x = var_1633_transpose_x_0, transpose_y = var_1633_transpose_y_0, x = var_1632_cast_fp16, y = state_75_cast_fp16)[name = string("op_1633_cast_fp16")];
+            tensor<int32, [4]> var_1635_begin_0 = const()[name = string("op_1635_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1635_end_0 = const()[name = string("op_1635_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_1635_end_mask_0 = const()[name = string("op_1635_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1635_squeeze_mask_0 = const()[name = string("op_1635_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1635_cast_fp16 = slice_by_index(begin = var_1635_begin_0, end = var_1635_end_0, end_mask = var_1635_end_mask_0, squeeze_mask = var_1635_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1635_cast_fp16")];
+            bool var_1636_transpose_x_0 = const()[name = string("op_1636_transpose_x_0"), val = bool(false)];
+            bool var_1636_transpose_y_0 = const()[name = string("op_1636_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1636_cast_fp16 = matmul(transpose_x = var_1636_transpose_x_0, transpose_y = var_1636_transpose_y_0, x = var_1635_cast_fp16, y = v_new_75_cast_fp16)[name = string("op_1636_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1637_cast_fp16 = add(x = var_1633_cast_fp16, y = var_1636_cast_fp16)[name = string("op_1637_cast_fp16")];
+            tensor<int32, [4]> var_1639_begin_0 = const()[name = string("op_1639_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1639_end_0 = const()[name = string("op_1639_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_1639_end_mask_0 = const()[name = string("op_1639_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1639_squeeze_mask_0 = const()[name = string("op_1639_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1639_cast_fp16 = slice_by_index(begin = var_1639_begin_0, end = var_1639_end_0, end_mask = var_1639_end_mask_0, squeeze_mask = var_1639_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1639_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1640_cast_fp16 = mul(x = state_75_cast_fp16, y = var_1639_cast_fp16)[name = string("op_1640_cast_fp16")];
+            tensor<int32, [4]> var_1642_begin_0 = const()[name = string("op_1642_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_1642_end_0 = const()[name = string("op_1642_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_1642_end_mask_0 = const()[name = string("op_1642_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1642_squeeze_mask_0 = const()[name = string("op_1642_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1642_cast_fp16 = slice_by_index(begin = var_1642_begin_0, end = var_1642_end_0, end_mask = var_1642_end_mask_0, squeeze_mask = var_1642_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1642_cast_fp16")];
+            bool var_1644_transpose_x_1 = const()[name = string("op_1644_transpose_x_1"), val = bool(true)];
+            bool var_1644_transpose_y_1 = const()[name = string("op_1644_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1644_cast_fp16 = matmul(transpose_x = var_1644_transpose_x_1, transpose_y = var_1644_transpose_y_1, x = var_1642_cast_fp16, y = v_new_75_cast_fp16)[name = string("op_1644_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_77_cast_fp16 = add(x = var_1640_cast_fp16, y = var_1644_cast_fp16)[name = string("state_77_cast_fp16")];
+            tensor<int32, [4]> var_1647_begin_0 = const()[name = string("op_1647_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1647_end_0 = const()[name = string("op_1647_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1647_end_mask_0 = const()[name = string("op_1647_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1647_squeeze_mask_0 = const()[name = string("op_1647_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1647_cast_fp16 = slice_by_index(begin = var_1647_begin_0, end = var_1647_end_0, end_mask = var_1647_end_mask_0, squeeze_mask = var_1647_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1647_cast_fp16")];
+            tensor<int32, [4]> var_1649_begin_0 = const()[name = string("op_1649_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1649_end_0 = const()[name = string("op_1649_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1649_end_mask_0 = const()[name = string("op_1649_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1649_squeeze_mask_0 = const()[name = string("op_1649_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1649_cast_fp16 = slice_by_index(begin = var_1649_begin_0, end = var_1649_end_0, end_mask = var_1649_end_mask_0, squeeze_mask = var_1649_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1649_cast_fp16")];
+            bool var_1650_transpose_x_0 = const()[name = string("op_1650_transpose_x_0"), val = bool(false)];
+            bool var_1650_transpose_y_0 = const()[name = string("op_1650_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1650_cast_fp16 = matmul(transpose_x = var_1650_transpose_x_0, transpose_y = var_1650_transpose_y_0, x = var_1649_cast_fp16, y = state_77_cast_fp16)[name = string("op_1650_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_77_cast_fp16 = sub(x = var_1647_cast_fp16, y = var_1650_cast_fp16)[name = string("v_new_77_cast_fp16")];
+            tensor<int32, [4]> var_1653_begin_0 = const()[name = string("op_1653_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1653_end_0 = const()[name = string("op_1653_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1653_end_mask_0 = const()[name = string("op_1653_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1653_squeeze_mask_0 = const()[name = string("op_1653_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1653_cast_fp16 = slice_by_index(begin = var_1653_begin_0, end = var_1653_end_0, end_mask = var_1653_end_mask_0, squeeze_mask = var_1653_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1653_cast_fp16")];
+            bool var_1654_transpose_x_0 = const()[name = string("op_1654_transpose_x_0"), val = bool(false)];
+            bool var_1654_transpose_y_0 = const()[name = string("op_1654_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1654_cast_fp16 = matmul(transpose_x = var_1654_transpose_x_0, transpose_y = var_1654_transpose_y_0, x = var_1653_cast_fp16, y = state_77_cast_fp16)[name = string("op_1654_cast_fp16")];
+            tensor<int32, [4]> var_1656_begin_0 = const()[name = string("op_1656_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1656_end_0 = const()[name = string("op_1656_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_1656_end_mask_0 = const()[name = string("op_1656_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1656_squeeze_mask_0 = const()[name = string("op_1656_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1656_cast_fp16 = slice_by_index(begin = var_1656_begin_0, end = var_1656_end_0, end_mask = var_1656_end_mask_0, squeeze_mask = var_1656_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1656_cast_fp16")];
+            bool var_1657_transpose_x_0 = const()[name = string("op_1657_transpose_x_0"), val = bool(false)];
+            bool var_1657_transpose_y_0 = const()[name = string("op_1657_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1657_cast_fp16 = matmul(transpose_x = var_1657_transpose_x_0, transpose_y = var_1657_transpose_y_0, x = var_1656_cast_fp16, y = v_new_77_cast_fp16)[name = string("op_1657_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1658_cast_fp16 = add(x = var_1654_cast_fp16, y = var_1657_cast_fp16)[name = string("op_1658_cast_fp16")];
+            tensor<int32, [4]> var_1660_begin_0 = const()[name = string("op_1660_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1660_end_0 = const()[name = string("op_1660_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_1660_end_mask_0 = const()[name = string("op_1660_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1660_squeeze_mask_0 = const()[name = string("op_1660_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1660_cast_fp16 = slice_by_index(begin = var_1660_begin_0, end = var_1660_end_0, end_mask = var_1660_end_mask_0, squeeze_mask = var_1660_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1660_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1661_cast_fp16 = mul(x = state_77_cast_fp16, y = var_1660_cast_fp16)[name = string("op_1661_cast_fp16")];
+            tensor<int32, [4]> var_1663_begin_0 = const()[name = string("op_1663_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_1663_end_0 = const()[name = string("op_1663_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_1663_end_mask_0 = const()[name = string("op_1663_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1663_squeeze_mask_0 = const()[name = string("op_1663_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1663_cast_fp16 = slice_by_index(begin = var_1663_begin_0, end = var_1663_end_0, end_mask = var_1663_end_mask_0, squeeze_mask = var_1663_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1663_cast_fp16")];
+            bool var_1665_transpose_x_1 = const()[name = string("op_1665_transpose_x_1"), val = bool(true)];
+            bool var_1665_transpose_y_1 = const()[name = string("op_1665_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1665_cast_fp16 = matmul(transpose_x = var_1665_transpose_x_1, transpose_y = var_1665_transpose_y_1, x = var_1663_cast_fp16, y = v_new_77_cast_fp16)[name = string("op_1665_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_79_cast_fp16 = add(x = var_1661_cast_fp16, y = var_1665_cast_fp16)[name = string("state_79_cast_fp16")];
+            tensor<int32, [4]> var_1668_begin_0 = const()[name = string("op_1668_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1668_end_0 = const()[name = string("op_1668_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1668_end_mask_0 = const()[name = string("op_1668_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1668_squeeze_mask_0 = const()[name = string("op_1668_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1668_cast_fp16 = slice_by_index(begin = var_1668_begin_0, end = var_1668_end_0, end_mask = var_1668_end_mask_0, squeeze_mask = var_1668_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1668_cast_fp16")];
+            tensor<int32, [4]> var_1670_begin_0 = const()[name = string("op_1670_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1670_end_0 = const()[name = string("op_1670_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1670_end_mask_0 = const()[name = string("op_1670_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1670_squeeze_mask_0 = const()[name = string("op_1670_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1670_cast_fp16 = slice_by_index(begin = var_1670_begin_0, end = var_1670_end_0, end_mask = var_1670_end_mask_0, squeeze_mask = var_1670_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1670_cast_fp16")];
+            bool var_1671_transpose_x_0 = const()[name = string("op_1671_transpose_x_0"), val = bool(false)];
+            bool var_1671_transpose_y_0 = const()[name = string("op_1671_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1671_cast_fp16 = matmul(transpose_x = var_1671_transpose_x_0, transpose_y = var_1671_transpose_y_0, x = var_1670_cast_fp16, y = state_79_cast_fp16)[name = string("op_1671_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_79_cast_fp16 = sub(x = var_1668_cast_fp16, y = var_1671_cast_fp16)[name = string("v_new_79_cast_fp16")];
+            tensor<int32, [4]> var_1674_begin_0 = const()[name = string("op_1674_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1674_end_0 = const()[name = string("op_1674_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1674_end_mask_0 = const()[name = string("op_1674_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1674_squeeze_mask_0 = const()[name = string("op_1674_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1674_cast_fp16 = slice_by_index(begin = var_1674_begin_0, end = var_1674_end_0, end_mask = var_1674_end_mask_0, squeeze_mask = var_1674_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1674_cast_fp16")];
+            bool var_1675_transpose_x_0 = const()[name = string("op_1675_transpose_x_0"), val = bool(false)];
+            bool var_1675_transpose_y_0 = const()[name = string("op_1675_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1675_cast_fp16 = matmul(transpose_x = var_1675_transpose_x_0, transpose_y = var_1675_transpose_y_0, x = var_1674_cast_fp16, y = state_79_cast_fp16)[name = string("op_1675_cast_fp16")];
+            tensor<int32, [4]> var_1677_begin_0 = const()[name = string("op_1677_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1677_end_0 = const()[name = string("op_1677_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_1677_end_mask_0 = const()[name = string("op_1677_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1677_squeeze_mask_0 = const()[name = string("op_1677_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1677_cast_fp16 = slice_by_index(begin = var_1677_begin_0, end = var_1677_end_0, end_mask = var_1677_end_mask_0, squeeze_mask = var_1677_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1677_cast_fp16")];
+            bool var_1678_transpose_x_0 = const()[name = string("op_1678_transpose_x_0"), val = bool(false)];
+            bool var_1678_transpose_y_0 = const()[name = string("op_1678_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1678_cast_fp16 = matmul(transpose_x = var_1678_transpose_x_0, transpose_y = var_1678_transpose_y_0, x = var_1677_cast_fp16, y = v_new_79_cast_fp16)[name = string("op_1678_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1679_cast_fp16 = add(x = var_1675_cast_fp16, y = var_1678_cast_fp16)[name = string("op_1679_cast_fp16")];
+            tensor<int32, [4]> var_1681_begin_0 = const()[name = string("op_1681_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1681_end_0 = const()[name = string("op_1681_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_1681_end_mask_0 = const()[name = string("op_1681_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1681_squeeze_mask_0 = const()[name = string("op_1681_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1681_cast_fp16 = slice_by_index(begin = var_1681_begin_0, end = var_1681_end_0, end_mask = var_1681_end_mask_0, squeeze_mask = var_1681_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1681_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1682_cast_fp16 = mul(x = state_79_cast_fp16, y = var_1681_cast_fp16)[name = string("op_1682_cast_fp16")];
+            tensor<int32, [4]> var_1684_begin_0 = const()[name = string("op_1684_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_1684_end_0 = const()[name = string("op_1684_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_1684_end_mask_0 = const()[name = string("op_1684_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1684_squeeze_mask_0 = const()[name = string("op_1684_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1684_cast_fp16 = slice_by_index(begin = var_1684_begin_0, end = var_1684_end_0, end_mask = var_1684_end_mask_0, squeeze_mask = var_1684_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1684_cast_fp16")];
+            bool var_1686_transpose_x_1 = const()[name = string("op_1686_transpose_x_1"), val = bool(true)];
+            bool var_1686_transpose_y_1 = const()[name = string("op_1686_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1686_cast_fp16 = matmul(transpose_x = var_1686_transpose_x_1, transpose_y = var_1686_transpose_y_1, x = var_1684_cast_fp16, y = v_new_79_cast_fp16)[name = string("op_1686_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_81_cast_fp16 = add(x = var_1682_cast_fp16, y = var_1686_cast_fp16)[name = string("state_81_cast_fp16")];
+            tensor<int32, [4]> var_1689_begin_0 = const()[name = string("op_1689_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1689_end_0 = const()[name = string("op_1689_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1689_end_mask_0 = const()[name = string("op_1689_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1689_squeeze_mask_0 = const()[name = string("op_1689_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1689_cast_fp16 = slice_by_index(begin = var_1689_begin_0, end = var_1689_end_0, end_mask = var_1689_end_mask_0, squeeze_mask = var_1689_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1689_cast_fp16")];
+            tensor<int32, [4]> var_1691_begin_0 = const()[name = string("op_1691_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1691_end_0 = const()[name = string("op_1691_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1691_end_mask_0 = const()[name = string("op_1691_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1691_squeeze_mask_0 = const()[name = string("op_1691_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1691_cast_fp16 = slice_by_index(begin = var_1691_begin_0, end = var_1691_end_0, end_mask = var_1691_end_mask_0, squeeze_mask = var_1691_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1691_cast_fp16")];
+            bool var_1692_transpose_x_0 = const()[name = string("op_1692_transpose_x_0"), val = bool(false)];
+            bool var_1692_transpose_y_0 = const()[name = string("op_1692_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1692_cast_fp16 = matmul(transpose_x = var_1692_transpose_x_0, transpose_y = var_1692_transpose_y_0, x = var_1691_cast_fp16, y = state_81_cast_fp16)[name = string("op_1692_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_81_cast_fp16 = sub(x = var_1689_cast_fp16, y = var_1692_cast_fp16)[name = string("v_new_81_cast_fp16")];
+            tensor<int32, [4]> var_1695_begin_0 = const()[name = string("op_1695_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1695_end_0 = const()[name = string("op_1695_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1695_end_mask_0 = const()[name = string("op_1695_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1695_squeeze_mask_0 = const()[name = string("op_1695_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1695_cast_fp16 = slice_by_index(begin = var_1695_begin_0, end = var_1695_end_0, end_mask = var_1695_end_mask_0, squeeze_mask = var_1695_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1695_cast_fp16")];
+            bool var_1696_transpose_x_0 = const()[name = string("op_1696_transpose_x_0"), val = bool(false)];
+            bool var_1696_transpose_y_0 = const()[name = string("op_1696_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1696_cast_fp16 = matmul(transpose_x = var_1696_transpose_x_0, transpose_y = var_1696_transpose_y_0, x = var_1695_cast_fp16, y = state_81_cast_fp16)[name = string("op_1696_cast_fp16")];
+            tensor<int32, [4]> var_1698_begin_0 = const()[name = string("op_1698_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1698_end_0 = const()[name = string("op_1698_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_1698_end_mask_0 = const()[name = string("op_1698_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1698_squeeze_mask_0 = const()[name = string("op_1698_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1698_cast_fp16 = slice_by_index(begin = var_1698_begin_0, end = var_1698_end_0, end_mask = var_1698_end_mask_0, squeeze_mask = var_1698_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1698_cast_fp16")];
+            bool var_1699_transpose_x_0 = const()[name = string("op_1699_transpose_x_0"), val = bool(false)];
+            bool var_1699_transpose_y_0 = const()[name = string("op_1699_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1699_cast_fp16 = matmul(transpose_x = var_1699_transpose_x_0, transpose_y = var_1699_transpose_y_0, x = var_1698_cast_fp16, y = v_new_81_cast_fp16)[name = string("op_1699_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1700_cast_fp16 = add(x = var_1696_cast_fp16, y = var_1699_cast_fp16)[name = string("op_1700_cast_fp16")];
+            tensor<int32, [4]> var_1702_begin_0 = const()[name = string("op_1702_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1702_end_0 = const()[name = string("op_1702_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_1702_end_mask_0 = const()[name = string("op_1702_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1702_squeeze_mask_0 = const()[name = string("op_1702_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1702_cast_fp16 = slice_by_index(begin = var_1702_begin_0, end = var_1702_end_0, end_mask = var_1702_end_mask_0, squeeze_mask = var_1702_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1702_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1703_cast_fp16 = mul(x = state_81_cast_fp16, y = var_1702_cast_fp16)[name = string("op_1703_cast_fp16")];
+            tensor<int32, [4]> var_1705_begin_0 = const()[name = string("op_1705_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_1705_end_0 = const()[name = string("op_1705_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_1705_end_mask_0 = const()[name = string("op_1705_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1705_squeeze_mask_0 = const()[name = string("op_1705_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1705_cast_fp16 = slice_by_index(begin = var_1705_begin_0, end = var_1705_end_0, end_mask = var_1705_end_mask_0, squeeze_mask = var_1705_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1705_cast_fp16")];
+            bool var_1707_transpose_x_1 = const()[name = string("op_1707_transpose_x_1"), val = bool(true)];
+            bool var_1707_transpose_y_1 = const()[name = string("op_1707_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1707_cast_fp16 = matmul(transpose_x = var_1707_transpose_x_1, transpose_y = var_1707_transpose_y_1, x = var_1705_cast_fp16, y = v_new_81_cast_fp16)[name = string("op_1707_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_83_cast_fp16 = add(x = var_1703_cast_fp16, y = var_1707_cast_fp16)[name = string("state_83_cast_fp16")];
+            tensor<int32, [4]> var_1710_begin_0 = const()[name = string("op_1710_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1710_end_0 = const()[name = string("op_1710_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1710_end_mask_0 = const()[name = string("op_1710_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1710_squeeze_mask_0 = const()[name = string("op_1710_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1710_cast_fp16 = slice_by_index(begin = var_1710_begin_0, end = var_1710_end_0, end_mask = var_1710_end_mask_0, squeeze_mask = var_1710_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1710_cast_fp16")];
+            tensor<int32, [4]> var_1712_begin_0 = const()[name = string("op_1712_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1712_end_0 = const()[name = string("op_1712_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1712_end_mask_0 = const()[name = string("op_1712_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1712_squeeze_mask_0 = const()[name = string("op_1712_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1712_cast_fp16 = slice_by_index(begin = var_1712_begin_0, end = var_1712_end_0, end_mask = var_1712_end_mask_0, squeeze_mask = var_1712_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1712_cast_fp16")];
+            bool var_1713_transpose_x_0 = const()[name = string("op_1713_transpose_x_0"), val = bool(false)];
+            bool var_1713_transpose_y_0 = const()[name = string("op_1713_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1713_cast_fp16 = matmul(transpose_x = var_1713_transpose_x_0, transpose_y = var_1713_transpose_y_0, x = var_1712_cast_fp16, y = state_83_cast_fp16)[name = string("op_1713_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_83_cast_fp16 = sub(x = var_1710_cast_fp16, y = var_1713_cast_fp16)[name = string("v_new_83_cast_fp16")];
+            tensor<int32, [4]> var_1716_begin_0 = const()[name = string("op_1716_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1716_end_0 = const()[name = string("op_1716_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1716_end_mask_0 = const()[name = string("op_1716_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1716_squeeze_mask_0 = const()[name = string("op_1716_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1716_cast_fp16 = slice_by_index(begin = var_1716_begin_0, end = var_1716_end_0, end_mask = var_1716_end_mask_0, squeeze_mask = var_1716_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1716_cast_fp16")];
+            bool var_1717_transpose_x_0 = const()[name = string("op_1717_transpose_x_0"), val = bool(false)];
+            bool var_1717_transpose_y_0 = const()[name = string("op_1717_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1717_cast_fp16 = matmul(transpose_x = var_1717_transpose_x_0, transpose_y = var_1717_transpose_y_0, x = var_1716_cast_fp16, y = state_83_cast_fp16)[name = string("op_1717_cast_fp16")];
+            tensor<int32, [4]> var_1719_begin_0 = const()[name = string("op_1719_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1719_end_0 = const()[name = string("op_1719_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_1719_end_mask_0 = const()[name = string("op_1719_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1719_squeeze_mask_0 = const()[name = string("op_1719_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1719_cast_fp16 = slice_by_index(begin = var_1719_begin_0, end = var_1719_end_0, end_mask = var_1719_end_mask_0, squeeze_mask = var_1719_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1719_cast_fp16")];
+            bool var_1720_transpose_x_0 = const()[name = string("op_1720_transpose_x_0"), val = bool(false)];
+            bool var_1720_transpose_y_0 = const()[name = string("op_1720_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1720_cast_fp16 = matmul(transpose_x = var_1720_transpose_x_0, transpose_y = var_1720_transpose_y_0, x = var_1719_cast_fp16, y = v_new_83_cast_fp16)[name = string("op_1720_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1721_cast_fp16 = add(x = var_1717_cast_fp16, y = var_1720_cast_fp16)[name = string("op_1721_cast_fp16")];
+            tensor<int32, [4]> var_1723_begin_0 = const()[name = string("op_1723_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1723_end_0 = const()[name = string("op_1723_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_1723_end_mask_0 = const()[name = string("op_1723_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1723_squeeze_mask_0 = const()[name = string("op_1723_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1723_cast_fp16 = slice_by_index(begin = var_1723_begin_0, end = var_1723_end_0, end_mask = var_1723_end_mask_0, squeeze_mask = var_1723_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1723_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1724_cast_fp16 = mul(x = state_83_cast_fp16, y = var_1723_cast_fp16)[name = string("op_1724_cast_fp16")];
+            tensor<int32, [4]> var_1726_begin_0 = const()[name = string("op_1726_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_1726_end_0 = const()[name = string("op_1726_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_1726_end_mask_0 = const()[name = string("op_1726_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1726_squeeze_mask_0 = const()[name = string("op_1726_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1726_cast_fp16 = slice_by_index(begin = var_1726_begin_0, end = var_1726_end_0, end_mask = var_1726_end_mask_0, squeeze_mask = var_1726_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1726_cast_fp16")];
+            bool var_1728_transpose_x_1 = const()[name = string("op_1728_transpose_x_1"), val = bool(true)];
+            bool var_1728_transpose_y_1 = const()[name = string("op_1728_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1728_cast_fp16 = matmul(transpose_x = var_1728_transpose_x_1, transpose_y = var_1728_transpose_y_1, x = var_1726_cast_fp16, y = v_new_83_cast_fp16)[name = string("op_1728_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_85_cast_fp16 = add(x = var_1724_cast_fp16, y = var_1728_cast_fp16)[name = string("state_85_cast_fp16")];
+            tensor<int32, [4]> var_1731_begin_0 = const()[name = string("op_1731_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1731_end_0 = const()[name = string("op_1731_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1731_end_mask_0 = const()[name = string("op_1731_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1731_squeeze_mask_0 = const()[name = string("op_1731_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1731_cast_fp16 = slice_by_index(begin = var_1731_begin_0, end = var_1731_end_0, end_mask = var_1731_end_mask_0, squeeze_mask = var_1731_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1731_cast_fp16")];
+            tensor<int32, [4]> var_1733_begin_0 = const()[name = string("op_1733_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1733_end_0 = const()[name = string("op_1733_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1733_end_mask_0 = const()[name = string("op_1733_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1733_squeeze_mask_0 = const()[name = string("op_1733_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1733_cast_fp16 = slice_by_index(begin = var_1733_begin_0, end = var_1733_end_0, end_mask = var_1733_end_mask_0, squeeze_mask = var_1733_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1733_cast_fp16")];
+            bool var_1734_transpose_x_0 = const()[name = string("op_1734_transpose_x_0"), val = bool(false)];
+            bool var_1734_transpose_y_0 = const()[name = string("op_1734_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1734_cast_fp16 = matmul(transpose_x = var_1734_transpose_x_0, transpose_y = var_1734_transpose_y_0, x = var_1733_cast_fp16, y = state_85_cast_fp16)[name = string("op_1734_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_85_cast_fp16 = sub(x = var_1731_cast_fp16, y = var_1734_cast_fp16)[name = string("v_new_85_cast_fp16")];
+            tensor<int32, [4]> var_1737_begin_0 = const()[name = string("op_1737_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1737_end_0 = const()[name = string("op_1737_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1737_end_mask_0 = const()[name = string("op_1737_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1737_squeeze_mask_0 = const()[name = string("op_1737_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1737_cast_fp16 = slice_by_index(begin = var_1737_begin_0, end = var_1737_end_0, end_mask = var_1737_end_mask_0, squeeze_mask = var_1737_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1737_cast_fp16")];
+            bool var_1738_transpose_x_0 = const()[name = string("op_1738_transpose_x_0"), val = bool(false)];
+            bool var_1738_transpose_y_0 = const()[name = string("op_1738_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1738_cast_fp16 = matmul(transpose_x = var_1738_transpose_x_0, transpose_y = var_1738_transpose_y_0, x = var_1737_cast_fp16, y = state_85_cast_fp16)[name = string("op_1738_cast_fp16")];
+            tensor<int32, [4]> var_1740_begin_0 = const()[name = string("op_1740_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1740_end_0 = const()[name = string("op_1740_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_1740_end_mask_0 = const()[name = string("op_1740_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1740_squeeze_mask_0 = const()[name = string("op_1740_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1740_cast_fp16 = slice_by_index(begin = var_1740_begin_0, end = var_1740_end_0, end_mask = var_1740_end_mask_0, squeeze_mask = var_1740_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1740_cast_fp16")];
+            bool var_1741_transpose_x_0 = const()[name = string("op_1741_transpose_x_0"), val = bool(false)];
+            bool var_1741_transpose_y_0 = const()[name = string("op_1741_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1741_cast_fp16 = matmul(transpose_x = var_1741_transpose_x_0, transpose_y = var_1741_transpose_y_0, x = var_1740_cast_fp16, y = v_new_85_cast_fp16)[name = string("op_1741_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1742_cast_fp16 = add(x = var_1738_cast_fp16, y = var_1741_cast_fp16)[name = string("op_1742_cast_fp16")];
+            tensor<int32, [4]> var_1744_begin_0 = const()[name = string("op_1744_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1744_end_0 = const()[name = string("op_1744_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_1744_end_mask_0 = const()[name = string("op_1744_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1744_squeeze_mask_0 = const()[name = string("op_1744_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1744_cast_fp16 = slice_by_index(begin = var_1744_begin_0, end = var_1744_end_0, end_mask = var_1744_end_mask_0, squeeze_mask = var_1744_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1744_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1745_cast_fp16 = mul(x = state_85_cast_fp16, y = var_1744_cast_fp16)[name = string("op_1745_cast_fp16")];
+            tensor<int32, [4]> var_1747_begin_0 = const()[name = string("op_1747_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_1747_end_0 = const()[name = string("op_1747_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_1747_end_mask_0 = const()[name = string("op_1747_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1747_squeeze_mask_0 = const()[name = string("op_1747_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1747_cast_fp16 = slice_by_index(begin = var_1747_begin_0, end = var_1747_end_0, end_mask = var_1747_end_mask_0, squeeze_mask = var_1747_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1747_cast_fp16")];
+            bool var_1749_transpose_x_1 = const()[name = string("op_1749_transpose_x_1"), val = bool(true)];
+            bool var_1749_transpose_y_1 = const()[name = string("op_1749_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1749_cast_fp16 = matmul(transpose_x = var_1749_transpose_x_1, transpose_y = var_1749_transpose_y_1, x = var_1747_cast_fp16, y = v_new_85_cast_fp16)[name = string("op_1749_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_87_cast_fp16 = add(x = var_1745_cast_fp16, y = var_1749_cast_fp16)[name = string("state_87_cast_fp16")];
+            tensor<int32, [4]> var_1752_begin_0 = const()[name = string("op_1752_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1752_end_0 = const()[name = string("op_1752_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1752_end_mask_0 = const()[name = string("op_1752_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1752_squeeze_mask_0 = const()[name = string("op_1752_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1752_cast_fp16 = slice_by_index(begin = var_1752_begin_0, end = var_1752_end_0, end_mask = var_1752_end_mask_0, squeeze_mask = var_1752_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1752_cast_fp16")];
+            tensor<int32, [4]> var_1754_begin_0 = const()[name = string("op_1754_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1754_end_0 = const()[name = string("op_1754_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1754_end_mask_0 = const()[name = string("op_1754_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1754_squeeze_mask_0 = const()[name = string("op_1754_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1754_cast_fp16 = slice_by_index(begin = var_1754_begin_0, end = var_1754_end_0, end_mask = var_1754_end_mask_0, squeeze_mask = var_1754_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1754_cast_fp16")];
+            bool var_1755_transpose_x_0 = const()[name = string("op_1755_transpose_x_0"), val = bool(false)];
+            bool var_1755_transpose_y_0 = const()[name = string("op_1755_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1755_cast_fp16 = matmul(transpose_x = var_1755_transpose_x_0, transpose_y = var_1755_transpose_y_0, x = var_1754_cast_fp16, y = state_87_cast_fp16)[name = string("op_1755_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_87_cast_fp16 = sub(x = var_1752_cast_fp16, y = var_1755_cast_fp16)[name = string("v_new_87_cast_fp16")];
+            tensor<int32, [4]> var_1758_begin_0 = const()[name = string("op_1758_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1758_end_0 = const()[name = string("op_1758_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1758_end_mask_0 = const()[name = string("op_1758_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1758_squeeze_mask_0 = const()[name = string("op_1758_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1758_cast_fp16 = slice_by_index(begin = var_1758_begin_0, end = var_1758_end_0, end_mask = var_1758_end_mask_0, squeeze_mask = var_1758_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1758_cast_fp16")];
+            bool var_1759_transpose_x_0 = const()[name = string("op_1759_transpose_x_0"), val = bool(false)];
+            bool var_1759_transpose_y_0 = const()[name = string("op_1759_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1759_cast_fp16 = matmul(transpose_x = var_1759_transpose_x_0, transpose_y = var_1759_transpose_y_0, x = var_1758_cast_fp16, y = state_87_cast_fp16)[name = string("op_1759_cast_fp16")];
+            tensor<int32, [4]> var_1761_begin_0 = const()[name = string("op_1761_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1761_end_0 = const()[name = string("op_1761_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_1761_end_mask_0 = const()[name = string("op_1761_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1761_squeeze_mask_0 = const()[name = string("op_1761_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1761_cast_fp16 = slice_by_index(begin = var_1761_begin_0, end = var_1761_end_0, end_mask = var_1761_end_mask_0, squeeze_mask = var_1761_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1761_cast_fp16")];
+            bool var_1762_transpose_x_0 = const()[name = string("op_1762_transpose_x_0"), val = bool(false)];
+            bool var_1762_transpose_y_0 = const()[name = string("op_1762_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1762_cast_fp16 = matmul(transpose_x = var_1762_transpose_x_0, transpose_y = var_1762_transpose_y_0, x = var_1761_cast_fp16, y = v_new_87_cast_fp16)[name = string("op_1762_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1763_cast_fp16 = add(x = var_1759_cast_fp16, y = var_1762_cast_fp16)[name = string("op_1763_cast_fp16")];
+            tensor<int32, [4]> var_1765_begin_0 = const()[name = string("op_1765_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1765_end_0 = const()[name = string("op_1765_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_1765_end_mask_0 = const()[name = string("op_1765_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1765_squeeze_mask_0 = const()[name = string("op_1765_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1765_cast_fp16 = slice_by_index(begin = var_1765_begin_0, end = var_1765_end_0, end_mask = var_1765_end_mask_0, squeeze_mask = var_1765_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1765_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1766_cast_fp16 = mul(x = state_87_cast_fp16, y = var_1765_cast_fp16)[name = string("op_1766_cast_fp16")];
+            tensor<int32, [4]> var_1768_begin_0 = const()[name = string("op_1768_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_1768_end_0 = const()[name = string("op_1768_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_1768_end_mask_0 = const()[name = string("op_1768_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1768_squeeze_mask_0 = const()[name = string("op_1768_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1768_cast_fp16 = slice_by_index(begin = var_1768_begin_0, end = var_1768_end_0, end_mask = var_1768_end_mask_0, squeeze_mask = var_1768_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1768_cast_fp16")];
+            bool var_1770_transpose_x_1 = const()[name = string("op_1770_transpose_x_1"), val = bool(true)];
+            bool var_1770_transpose_y_1 = const()[name = string("op_1770_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1770_cast_fp16 = matmul(transpose_x = var_1770_transpose_x_1, transpose_y = var_1770_transpose_y_1, x = var_1768_cast_fp16, y = v_new_87_cast_fp16)[name = string("op_1770_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_89_cast_fp16 = add(x = var_1766_cast_fp16, y = var_1770_cast_fp16)[name = string("state_89_cast_fp16")];
+            tensor<int32, [4]> var_1773_begin_0 = const()[name = string("op_1773_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1773_end_0 = const()[name = string("op_1773_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1773_end_mask_0 = const()[name = string("op_1773_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1773_squeeze_mask_0 = const()[name = string("op_1773_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1773_cast_fp16 = slice_by_index(begin = var_1773_begin_0, end = var_1773_end_0, end_mask = var_1773_end_mask_0, squeeze_mask = var_1773_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1773_cast_fp16")];
+            tensor<int32, [4]> var_1775_begin_0 = const()[name = string("op_1775_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1775_end_0 = const()[name = string("op_1775_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1775_end_mask_0 = const()[name = string("op_1775_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1775_squeeze_mask_0 = const()[name = string("op_1775_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1775_cast_fp16 = slice_by_index(begin = var_1775_begin_0, end = var_1775_end_0, end_mask = var_1775_end_mask_0, squeeze_mask = var_1775_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1775_cast_fp16")];
+            bool var_1776_transpose_x_0 = const()[name = string("op_1776_transpose_x_0"), val = bool(false)];
+            bool var_1776_transpose_y_0 = const()[name = string("op_1776_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1776_cast_fp16 = matmul(transpose_x = var_1776_transpose_x_0, transpose_y = var_1776_transpose_y_0, x = var_1775_cast_fp16, y = state_89_cast_fp16)[name = string("op_1776_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_89_cast_fp16 = sub(x = var_1773_cast_fp16, y = var_1776_cast_fp16)[name = string("v_new_89_cast_fp16")];
+            tensor<int32, [4]> var_1779_begin_0 = const()[name = string("op_1779_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1779_end_0 = const()[name = string("op_1779_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1779_end_mask_0 = const()[name = string("op_1779_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1779_squeeze_mask_0 = const()[name = string("op_1779_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1779_cast_fp16 = slice_by_index(begin = var_1779_begin_0, end = var_1779_end_0, end_mask = var_1779_end_mask_0, squeeze_mask = var_1779_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1779_cast_fp16")];
+            bool var_1780_transpose_x_0 = const()[name = string("op_1780_transpose_x_0"), val = bool(false)];
+            bool var_1780_transpose_y_0 = const()[name = string("op_1780_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1780_cast_fp16 = matmul(transpose_x = var_1780_transpose_x_0, transpose_y = var_1780_transpose_y_0, x = var_1779_cast_fp16, y = state_89_cast_fp16)[name = string("op_1780_cast_fp16")];
+            tensor<int32, [4]> var_1782_begin_0 = const()[name = string("op_1782_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1782_end_0 = const()[name = string("op_1782_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_1782_end_mask_0 = const()[name = string("op_1782_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1782_squeeze_mask_0 = const()[name = string("op_1782_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1782_cast_fp16 = slice_by_index(begin = var_1782_begin_0, end = var_1782_end_0, end_mask = var_1782_end_mask_0, squeeze_mask = var_1782_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1782_cast_fp16")];
+            bool var_1783_transpose_x_0 = const()[name = string("op_1783_transpose_x_0"), val = bool(false)];
+            bool var_1783_transpose_y_0 = const()[name = string("op_1783_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1783_cast_fp16 = matmul(transpose_x = var_1783_transpose_x_0, transpose_y = var_1783_transpose_y_0, x = var_1782_cast_fp16, y = v_new_89_cast_fp16)[name = string("op_1783_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1784_cast_fp16 = add(x = var_1780_cast_fp16, y = var_1783_cast_fp16)[name = string("op_1784_cast_fp16")];
+            tensor<int32, [4]> var_1786_begin_0 = const()[name = string("op_1786_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1786_end_0 = const()[name = string("op_1786_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_1786_end_mask_0 = const()[name = string("op_1786_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1786_squeeze_mask_0 = const()[name = string("op_1786_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1786_cast_fp16 = slice_by_index(begin = var_1786_begin_0, end = var_1786_end_0, end_mask = var_1786_end_mask_0, squeeze_mask = var_1786_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1786_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1787_cast_fp16 = mul(x = state_89_cast_fp16, y = var_1786_cast_fp16)[name = string("op_1787_cast_fp16")];
+            tensor<int32, [4]> var_1789_begin_0 = const()[name = string("op_1789_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_1789_end_0 = const()[name = string("op_1789_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_1789_end_mask_0 = const()[name = string("op_1789_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1789_squeeze_mask_0 = const()[name = string("op_1789_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1789_cast_fp16 = slice_by_index(begin = var_1789_begin_0, end = var_1789_end_0, end_mask = var_1789_end_mask_0, squeeze_mask = var_1789_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1789_cast_fp16")];
+            bool var_1791_transpose_x_1 = const()[name = string("op_1791_transpose_x_1"), val = bool(true)];
+            bool var_1791_transpose_y_1 = const()[name = string("op_1791_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1791_cast_fp16 = matmul(transpose_x = var_1791_transpose_x_1, transpose_y = var_1791_transpose_y_1, x = var_1789_cast_fp16, y = v_new_89_cast_fp16)[name = string("op_1791_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_91_cast_fp16 = add(x = var_1787_cast_fp16, y = var_1791_cast_fp16)[name = string("state_91_cast_fp16")];
+            tensor<int32, [4]> var_1794_begin_0 = const()[name = string("op_1794_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1794_end_0 = const()[name = string("op_1794_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1794_end_mask_0 = const()[name = string("op_1794_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1794_squeeze_mask_0 = const()[name = string("op_1794_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1794_cast_fp16 = slice_by_index(begin = var_1794_begin_0, end = var_1794_end_0, end_mask = var_1794_end_mask_0, squeeze_mask = var_1794_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1794_cast_fp16")];
+            tensor<int32, [4]> var_1796_begin_0 = const()[name = string("op_1796_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1796_end_0 = const()[name = string("op_1796_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1796_end_mask_0 = const()[name = string("op_1796_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1796_squeeze_mask_0 = const()[name = string("op_1796_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1796_cast_fp16 = slice_by_index(begin = var_1796_begin_0, end = var_1796_end_0, end_mask = var_1796_end_mask_0, squeeze_mask = var_1796_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1796_cast_fp16")];
+            bool var_1797_transpose_x_0 = const()[name = string("op_1797_transpose_x_0"), val = bool(false)];
+            bool var_1797_transpose_y_0 = const()[name = string("op_1797_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1797_cast_fp16 = matmul(transpose_x = var_1797_transpose_x_0, transpose_y = var_1797_transpose_y_0, x = var_1796_cast_fp16, y = state_91_cast_fp16)[name = string("op_1797_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_91_cast_fp16 = sub(x = var_1794_cast_fp16, y = var_1797_cast_fp16)[name = string("v_new_91_cast_fp16")];
+            tensor<int32, [4]> var_1800_begin_0 = const()[name = string("op_1800_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1800_end_0 = const()[name = string("op_1800_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1800_end_mask_0 = const()[name = string("op_1800_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1800_squeeze_mask_0 = const()[name = string("op_1800_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1800_cast_fp16 = slice_by_index(begin = var_1800_begin_0, end = var_1800_end_0, end_mask = var_1800_end_mask_0, squeeze_mask = var_1800_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1800_cast_fp16")];
+            bool var_1801_transpose_x_0 = const()[name = string("op_1801_transpose_x_0"), val = bool(false)];
+            bool var_1801_transpose_y_0 = const()[name = string("op_1801_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1801_cast_fp16 = matmul(transpose_x = var_1801_transpose_x_0, transpose_y = var_1801_transpose_y_0, x = var_1800_cast_fp16, y = state_91_cast_fp16)[name = string("op_1801_cast_fp16")];
+            tensor<int32, [4]> var_1803_begin_0 = const()[name = string("op_1803_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1803_end_0 = const()[name = string("op_1803_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_1803_end_mask_0 = const()[name = string("op_1803_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1803_squeeze_mask_0 = const()[name = string("op_1803_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1803_cast_fp16 = slice_by_index(begin = var_1803_begin_0, end = var_1803_end_0, end_mask = var_1803_end_mask_0, squeeze_mask = var_1803_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1803_cast_fp16")];
+            bool var_1804_transpose_x_0 = const()[name = string("op_1804_transpose_x_0"), val = bool(false)];
+            bool var_1804_transpose_y_0 = const()[name = string("op_1804_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1804_cast_fp16 = matmul(transpose_x = var_1804_transpose_x_0, transpose_y = var_1804_transpose_y_0, x = var_1803_cast_fp16, y = v_new_91_cast_fp16)[name = string("op_1804_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1805_cast_fp16 = add(x = var_1801_cast_fp16, y = var_1804_cast_fp16)[name = string("op_1805_cast_fp16")];
+            tensor<int32, [4]> var_1807_begin_0 = const()[name = string("op_1807_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1807_end_0 = const()[name = string("op_1807_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_1807_end_mask_0 = const()[name = string("op_1807_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1807_squeeze_mask_0 = const()[name = string("op_1807_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1807_cast_fp16 = slice_by_index(begin = var_1807_begin_0, end = var_1807_end_0, end_mask = var_1807_end_mask_0, squeeze_mask = var_1807_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1807_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1808_cast_fp16 = mul(x = state_91_cast_fp16, y = var_1807_cast_fp16)[name = string("op_1808_cast_fp16")];
+            tensor<int32, [4]> var_1810_begin_0 = const()[name = string("op_1810_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_1810_end_0 = const()[name = string("op_1810_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_1810_end_mask_0 = const()[name = string("op_1810_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1810_squeeze_mask_0 = const()[name = string("op_1810_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1810_cast_fp16 = slice_by_index(begin = var_1810_begin_0, end = var_1810_end_0, end_mask = var_1810_end_mask_0, squeeze_mask = var_1810_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1810_cast_fp16")];
+            bool var_1812_transpose_x_1 = const()[name = string("op_1812_transpose_x_1"), val = bool(true)];
+            bool var_1812_transpose_y_1 = const()[name = string("op_1812_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1812_cast_fp16 = matmul(transpose_x = var_1812_transpose_x_1, transpose_y = var_1812_transpose_y_1, x = var_1810_cast_fp16, y = v_new_91_cast_fp16)[name = string("op_1812_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_93_cast_fp16 = add(x = var_1808_cast_fp16, y = var_1812_cast_fp16)[name = string("state_93_cast_fp16")];
+            tensor<int32, [4]> var_1815_begin_0 = const()[name = string("op_1815_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1815_end_0 = const()[name = string("op_1815_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1815_end_mask_0 = const()[name = string("op_1815_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1815_squeeze_mask_0 = const()[name = string("op_1815_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1815_cast_fp16 = slice_by_index(begin = var_1815_begin_0, end = var_1815_end_0, end_mask = var_1815_end_mask_0, squeeze_mask = var_1815_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1815_cast_fp16")];
+            tensor<int32, [4]> var_1817_begin_0 = const()[name = string("op_1817_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1817_end_0 = const()[name = string("op_1817_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1817_end_mask_0 = const()[name = string("op_1817_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1817_squeeze_mask_0 = const()[name = string("op_1817_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1817_cast_fp16 = slice_by_index(begin = var_1817_begin_0, end = var_1817_end_0, end_mask = var_1817_end_mask_0, squeeze_mask = var_1817_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1817_cast_fp16")];
+            bool var_1818_transpose_x_0 = const()[name = string("op_1818_transpose_x_0"), val = bool(false)];
+            bool var_1818_transpose_y_0 = const()[name = string("op_1818_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1818_cast_fp16 = matmul(transpose_x = var_1818_transpose_x_0, transpose_y = var_1818_transpose_y_0, x = var_1817_cast_fp16, y = state_93_cast_fp16)[name = string("op_1818_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_93_cast_fp16 = sub(x = var_1815_cast_fp16, y = var_1818_cast_fp16)[name = string("v_new_93_cast_fp16")];
+            tensor<int32, [4]> var_1821_begin_0 = const()[name = string("op_1821_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1821_end_0 = const()[name = string("op_1821_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1821_end_mask_0 = const()[name = string("op_1821_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1821_squeeze_mask_0 = const()[name = string("op_1821_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1821_cast_fp16 = slice_by_index(begin = var_1821_begin_0, end = var_1821_end_0, end_mask = var_1821_end_mask_0, squeeze_mask = var_1821_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1821_cast_fp16")];
+            bool var_1822_transpose_x_0 = const()[name = string("op_1822_transpose_x_0"), val = bool(false)];
+            bool var_1822_transpose_y_0 = const()[name = string("op_1822_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1822_cast_fp16 = matmul(transpose_x = var_1822_transpose_x_0, transpose_y = var_1822_transpose_y_0, x = var_1821_cast_fp16, y = state_93_cast_fp16)[name = string("op_1822_cast_fp16")];
+            tensor<int32, [4]> var_1824_begin_0 = const()[name = string("op_1824_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1824_end_0 = const()[name = string("op_1824_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_1824_end_mask_0 = const()[name = string("op_1824_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1824_squeeze_mask_0 = const()[name = string("op_1824_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1824_cast_fp16 = slice_by_index(begin = var_1824_begin_0, end = var_1824_end_0, end_mask = var_1824_end_mask_0, squeeze_mask = var_1824_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1824_cast_fp16")];
+            bool var_1825_transpose_x_0 = const()[name = string("op_1825_transpose_x_0"), val = bool(false)];
+            bool var_1825_transpose_y_0 = const()[name = string("op_1825_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1825_cast_fp16 = matmul(transpose_x = var_1825_transpose_x_0, transpose_y = var_1825_transpose_y_0, x = var_1824_cast_fp16, y = v_new_93_cast_fp16)[name = string("op_1825_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1826_cast_fp16 = add(x = var_1822_cast_fp16, y = var_1825_cast_fp16)[name = string("op_1826_cast_fp16")];
+            tensor<int32, [4]> var_1828_begin_0 = const()[name = string("op_1828_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1828_end_0 = const()[name = string("op_1828_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_1828_end_mask_0 = const()[name = string("op_1828_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1828_squeeze_mask_0 = const()[name = string("op_1828_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1828_cast_fp16 = slice_by_index(begin = var_1828_begin_0, end = var_1828_end_0, end_mask = var_1828_end_mask_0, squeeze_mask = var_1828_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1828_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1829_cast_fp16 = mul(x = state_93_cast_fp16, y = var_1828_cast_fp16)[name = string("op_1829_cast_fp16")];
+            tensor<int32, [4]> var_1831_begin_0 = const()[name = string("op_1831_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_1831_end_0 = const()[name = string("op_1831_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_1831_end_mask_0 = const()[name = string("op_1831_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1831_squeeze_mask_0 = const()[name = string("op_1831_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1831_cast_fp16 = slice_by_index(begin = var_1831_begin_0, end = var_1831_end_0, end_mask = var_1831_end_mask_0, squeeze_mask = var_1831_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1831_cast_fp16")];
+            bool var_1833_transpose_x_1 = const()[name = string("op_1833_transpose_x_1"), val = bool(true)];
+            bool var_1833_transpose_y_1 = const()[name = string("op_1833_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1833_cast_fp16 = matmul(transpose_x = var_1833_transpose_x_1, transpose_y = var_1833_transpose_y_1, x = var_1831_cast_fp16, y = v_new_93_cast_fp16)[name = string("op_1833_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_95_cast_fp16 = add(x = var_1829_cast_fp16, y = var_1833_cast_fp16)[name = string("state_95_cast_fp16")];
+            tensor<int32, [4]> var_1836_begin_0 = const()[name = string("op_1836_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1836_end_0 = const()[name = string("op_1836_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1836_end_mask_0 = const()[name = string("op_1836_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1836_squeeze_mask_0 = const()[name = string("op_1836_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1836_cast_fp16 = slice_by_index(begin = var_1836_begin_0, end = var_1836_end_0, end_mask = var_1836_end_mask_0, squeeze_mask = var_1836_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1836_cast_fp16")];
+            tensor<int32, [4]> var_1838_begin_0 = const()[name = string("op_1838_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1838_end_0 = const()[name = string("op_1838_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1838_end_mask_0 = const()[name = string("op_1838_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1838_squeeze_mask_0 = const()[name = string("op_1838_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1838_cast_fp16 = slice_by_index(begin = var_1838_begin_0, end = var_1838_end_0, end_mask = var_1838_end_mask_0, squeeze_mask = var_1838_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1838_cast_fp16")];
+            bool var_1839_transpose_x_0 = const()[name = string("op_1839_transpose_x_0"), val = bool(false)];
+            bool var_1839_transpose_y_0 = const()[name = string("op_1839_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1839_cast_fp16 = matmul(transpose_x = var_1839_transpose_x_0, transpose_y = var_1839_transpose_y_0, x = var_1838_cast_fp16, y = state_95_cast_fp16)[name = string("op_1839_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_95_cast_fp16 = sub(x = var_1836_cast_fp16, y = var_1839_cast_fp16)[name = string("v_new_95_cast_fp16")];
+            tensor<int32, [4]> var_1842_begin_0 = const()[name = string("op_1842_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1842_end_0 = const()[name = string("op_1842_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1842_end_mask_0 = const()[name = string("op_1842_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1842_squeeze_mask_0 = const()[name = string("op_1842_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1842_cast_fp16 = slice_by_index(begin = var_1842_begin_0, end = var_1842_end_0, end_mask = var_1842_end_mask_0, squeeze_mask = var_1842_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1842_cast_fp16")];
+            bool var_1843_transpose_x_0 = const()[name = string("op_1843_transpose_x_0"), val = bool(false)];
+            bool var_1843_transpose_y_0 = const()[name = string("op_1843_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1843_cast_fp16 = matmul(transpose_x = var_1843_transpose_x_0, transpose_y = var_1843_transpose_y_0, x = var_1842_cast_fp16, y = state_95_cast_fp16)[name = string("op_1843_cast_fp16")];
+            tensor<int32, [4]> var_1845_begin_0 = const()[name = string("op_1845_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1845_end_0 = const()[name = string("op_1845_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_1845_end_mask_0 = const()[name = string("op_1845_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1845_squeeze_mask_0 = const()[name = string("op_1845_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1845_cast_fp16 = slice_by_index(begin = var_1845_begin_0, end = var_1845_end_0, end_mask = var_1845_end_mask_0, squeeze_mask = var_1845_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1845_cast_fp16")];
+            bool var_1846_transpose_x_0 = const()[name = string("op_1846_transpose_x_0"), val = bool(false)];
+            bool var_1846_transpose_y_0 = const()[name = string("op_1846_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1846_cast_fp16 = matmul(transpose_x = var_1846_transpose_x_0, transpose_y = var_1846_transpose_y_0, x = var_1845_cast_fp16, y = v_new_95_cast_fp16)[name = string("op_1846_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1847_cast_fp16 = add(x = var_1843_cast_fp16, y = var_1846_cast_fp16)[name = string("op_1847_cast_fp16")];
+            tensor<int32, [4]> var_1849_begin_0 = const()[name = string("op_1849_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1849_end_0 = const()[name = string("op_1849_end_0"), val = tensor<int32, [4]>([32, 16, 1, 1])];
+            tensor<bool, [4]> var_1849_end_mask_0 = const()[name = string("op_1849_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1849_squeeze_mask_0 = const()[name = string("op_1849_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1849_cast_fp16 = slice_by_index(begin = var_1849_begin_0, end = var_1849_end_0, end_mask = var_1849_end_mask_0, squeeze_mask = var_1849_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1849_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1850_cast_fp16 = mul(x = state_95_cast_fp16, y = var_1849_cast_fp16)[name = string("op_1850_cast_fp16")];
+            tensor<int32, [4]> var_1852_begin_0 = const()[name = string("op_1852_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_1852_end_0 = const()[name = string("op_1852_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_1852_end_mask_0 = const()[name = string("op_1852_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1852_squeeze_mask_0 = const()[name = string("op_1852_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1852_cast_fp16 = slice_by_index(begin = var_1852_begin_0, end = var_1852_end_0, end_mask = var_1852_end_mask_0, squeeze_mask = var_1852_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1852_cast_fp16")];
+            bool var_1854_transpose_x_1 = const()[name = string("op_1854_transpose_x_1"), val = bool(true)];
+            bool var_1854_transpose_y_1 = const()[name = string("op_1854_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1854_cast_fp16 = matmul(transpose_x = var_1854_transpose_x_1, transpose_y = var_1854_transpose_y_1, x = var_1852_cast_fp16, y = v_new_95_cast_fp16)[name = string("op_1854_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_97_cast_fp16 = add(x = var_1850_cast_fp16, y = var_1854_cast_fp16)[name = string("state_97_cast_fp16")];
+            tensor<int32, [4]> var_1857_begin_0 = const()[name = string("op_1857_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_1857_end_0 = const()[name = string("op_1857_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_1857_end_mask_0 = const()[name = string("op_1857_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1857_squeeze_mask_0 = const()[name = string("op_1857_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1857_cast_fp16 = slice_by_index(begin = var_1857_begin_0, end = var_1857_end_0, end_mask = var_1857_end_mask_0, squeeze_mask = var_1857_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1857_cast_fp16")];
+            tensor<int32, [4]> var_1859_begin_0 = const()[name = string("op_1859_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_1859_end_0 = const()[name = string("op_1859_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_1859_end_mask_0 = const()[name = string("op_1859_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1859_squeeze_mask_0 = const()[name = string("op_1859_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1859_cast_fp16 = slice_by_index(begin = var_1859_begin_0, end = var_1859_end_0, end_mask = var_1859_end_mask_0, squeeze_mask = var_1859_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1859_cast_fp16")];
+            bool var_1860_transpose_x_0 = const()[name = string("op_1860_transpose_x_0"), val = bool(false)];
+            bool var_1860_transpose_y_0 = const()[name = string("op_1860_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1860_cast_fp16 = matmul(transpose_x = var_1860_transpose_x_0, transpose_y = var_1860_transpose_y_0, x = var_1859_cast_fp16, y = state_97_cast_fp16)[name = string("op_1860_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_97_cast_fp16 = sub(x = var_1857_cast_fp16, y = var_1860_cast_fp16)[name = string("v_new_97_cast_fp16")];
+            tensor<int32, [4]> var_1863_begin_0 = const()[name = string("op_1863_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_1863_end_0 = const()[name = string("op_1863_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_1863_end_mask_0 = const()[name = string("op_1863_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1863_squeeze_mask_0 = const()[name = string("op_1863_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1863_cast_fp16 = slice_by_index(begin = var_1863_begin_0, end = var_1863_end_0, end_mask = var_1863_end_mask_0, squeeze_mask = var_1863_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1863_cast_fp16")];
+            bool var_1864_transpose_x_0 = const()[name = string("op_1864_transpose_x_0"), val = bool(false)];
+            bool var_1864_transpose_y_0 = const()[name = string("op_1864_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1864_cast_fp16 = matmul(transpose_x = var_1864_transpose_x_0, transpose_y = var_1864_transpose_y_0, x = var_1863_cast_fp16, y = state_97_cast_fp16)[name = string("op_1864_cast_fp16")];
+            tensor<int32, [4]> var_1866_begin_0 = const()[name = string("op_1866_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_1866_end_0 = const()[name = string("op_1866_end_0"), val = tensor<int32, [4]>([32, 17, 64, 64])];
+            tensor<bool, [4]> var_1866_end_mask_0 = const()[name = string("op_1866_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1866_squeeze_mask_0 = const()[name = string("op_1866_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1866_cast_fp16 = slice_by_index(begin = var_1866_begin_0, end = var_1866_end_0, end_mask = var_1866_end_mask_0, squeeze_mask = var_1866_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1866_cast_fp16")];
+            bool var_1867_transpose_x_0 = const()[name = string("op_1867_transpose_x_0"), val = bool(false)];
+            bool var_1867_transpose_y_0 = const()[name = string("op_1867_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1867_cast_fp16 = matmul(transpose_x = var_1867_transpose_x_0, transpose_y = var_1867_transpose_y_0, x = var_1866_cast_fp16, y = v_new_97_cast_fp16)[name = string("op_1867_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1868_cast_fp16 = add(x = var_1864_cast_fp16, y = var_1867_cast_fp16)[name = string("op_1868_cast_fp16")];
+            tensor<int32, [4]> var_1870_begin_0 = const()[name = string("op_1870_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_1870_end_0 = const()[name = string("op_1870_end_0"), val = tensor<int32, [4]>([32, 17, 1, 1])];
+            tensor<bool, [4]> var_1870_end_mask_0 = const()[name = string("op_1870_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1870_squeeze_mask_0 = const()[name = string("op_1870_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1870_cast_fp16 = slice_by_index(begin = var_1870_begin_0, end = var_1870_end_0, end_mask = var_1870_end_mask_0, squeeze_mask = var_1870_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1870_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1871_cast_fp16 = mul(x = state_97_cast_fp16, y = var_1870_cast_fp16)[name = string("op_1871_cast_fp16")];
+            tensor<int32, [4]> var_1873_begin_0 = const()[name = string("op_1873_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_1873_end_0 = const()[name = string("op_1873_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_1873_end_mask_0 = const()[name = string("op_1873_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1873_squeeze_mask_0 = const()[name = string("op_1873_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1873_cast_fp16 = slice_by_index(begin = var_1873_begin_0, end = var_1873_end_0, end_mask = var_1873_end_mask_0, squeeze_mask = var_1873_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1873_cast_fp16")];
+            bool var_1875_transpose_x_1 = const()[name = string("op_1875_transpose_x_1"), val = bool(true)];
+            bool var_1875_transpose_y_1 = const()[name = string("op_1875_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1875_cast_fp16 = matmul(transpose_x = var_1875_transpose_x_1, transpose_y = var_1875_transpose_y_1, x = var_1873_cast_fp16, y = v_new_97_cast_fp16)[name = string("op_1875_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_99_cast_fp16 = add(x = var_1871_cast_fp16, y = var_1875_cast_fp16)[name = string("state_99_cast_fp16")];
+            tensor<int32, [4]> var_1878_begin_0 = const()[name = string("op_1878_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_1878_end_0 = const()[name = string("op_1878_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_1878_end_mask_0 = const()[name = string("op_1878_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1878_squeeze_mask_0 = const()[name = string("op_1878_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1878_cast_fp16 = slice_by_index(begin = var_1878_begin_0, end = var_1878_end_0, end_mask = var_1878_end_mask_0, squeeze_mask = var_1878_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1878_cast_fp16")];
+            tensor<int32, [4]> var_1880_begin_0 = const()[name = string("op_1880_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_1880_end_0 = const()[name = string("op_1880_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_1880_end_mask_0 = const()[name = string("op_1880_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1880_squeeze_mask_0 = const()[name = string("op_1880_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1880_cast_fp16 = slice_by_index(begin = var_1880_begin_0, end = var_1880_end_0, end_mask = var_1880_end_mask_0, squeeze_mask = var_1880_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1880_cast_fp16")];
+            bool var_1881_transpose_x_0 = const()[name = string("op_1881_transpose_x_0"), val = bool(false)];
+            bool var_1881_transpose_y_0 = const()[name = string("op_1881_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1881_cast_fp16 = matmul(transpose_x = var_1881_transpose_x_0, transpose_y = var_1881_transpose_y_0, x = var_1880_cast_fp16, y = state_99_cast_fp16)[name = string("op_1881_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_99_cast_fp16 = sub(x = var_1878_cast_fp16, y = var_1881_cast_fp16)[name = string("v_new_99_cast_fp16")];
+            tensor<int32, [4]> var_1884_begin_0 = const()[name = string("op_1884_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_1884_end_0 = const()[name = string("op_1884_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_1884_end_mask_0 = const()[name = string("op_1884_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1884_squeeze_mask_0 = const()[name = string("op_1884_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1884_cast_fp16 = slice_by_index(begin = var_1884_begin_0, end = var_1884_end_0, end_mask = var_1884_end_mask_0, squeeze_mask = var_1884_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1884_cast_fp16")];
+            bool var_1885_transpose_x_0 = const()[name = string("op_1885_transpose_x_0"), val = bool(false)];
+            bool var_1885_transpose_y_0 = const()[name = string("op_1885_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1885_cast_fp16 = matmul(transpose_x = var_1885_transpose_x_0, transpose_y = var_1885_transpose_y_0, x = var_1884_cast_fp16, y = state_99_cast_fp16)[name = string("op_1885_cast_fp16")];
+            tensor<int32, [4]> var_1887_begin_0 = const()[name = string("op_1887_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_1887_end_0 = const()[name = string("op_1887_end_0"), val = tensor<int32, [4]>([32, 18, 64, 64])];
+            tensor<bool, [4]> var_1887_end_mask_0 = const()[name = string("op_1887_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1887_squeeze_mask_0 = const()[name = string("op_1887_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1887_cast_fp16 = slice_by_index(begin = var_1887_begin_0, end = var_1887_end_0, end_mask = var_1887_end_mask_0, squeeze_mask = var_1887_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1887_cast_fp16")];
+            bool var_1888_transpose_x_0 = const()[name = string("op_1888_transpose_x_0"), val = bool(false)];
+            bool var_1888_transpose_y_0 = const()[name = string("op_1888_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1888_cast_fp16 = matmul(transpose_x = var_1888_transpose_x_0, transpose_y = var_1888_transpose_y_0, x = var_1887_cast_fp16, y = v_new_99_cast_fp16)[name = string("op_1888_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1889_cast_fp16 = add(x = var_1885_cast_fp16, y = var_1888_cast_fp16)[name = string("op_1889_cast_fp16")];
+            tensor<int32, [4]> var_1891_begin_0 = const()[name = string("op_1891_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_1891_end_0 = const()[name = string("op_1891_end_0"), val = tensor<int32, [4]>([32, 18, 1, 1])];
+            tensor<bool, [4]> var_1891_end_mask_0 = const()[name = string("op_1891_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1891_squeeze_mask_0 = const()[name = string("op_1891_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1891_cast_fp16 = slice_by_index(begin = var_1891_begin_0, end = var_1891_end_0, end_mask = var_1891_end_mask_0, squeeze_mask = var_1891_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1891_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1892_cast_fp16 = mul(x = state_99_cast_fp16, y = var_1891_cast_fp16)[name = string("op_1892_cast_fp16")];
+            tensor<int32, [4]> var_1894_begin_0 = const()[name = string("op_1894_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_1894_end_0 = const()[name = string("op_1894_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_1894_end_mask_0 = const()[name = string("op_1894_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1894_squeeze_mask_0 = const()[name = string("op_1894_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1894_cast_fp16 = slice_by_index(begin = var_1894_begin_0, end = var_1894_end_0, end_mask = var_1894_end_mask_0, squeeze_mask = var_1894_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1894_cast_fp16")];
+            bool var_1896_transpose_x_1 = const()[name = string("op_1896_transpose_x_1"), val = bool(true)];
+            bool var_1896_transpose_y_1 = const()[name = string("op_1896_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1896_cast_fp16 = matmul(transpose_x = var_1896_transpose_x_1, transpose_y = var_1896_transpose_y_1, x = var_1894_cast_fp16, y = v_new_99_cast_fp16)[name = string("op_1896_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_101_cast_fp16 = add(x = var_1892_cast_fp16, y = var_1896_cast_fp16)[name = string("state_101_cast_fp16")];
+            tensor<int32, [4]> var_1899_begin_0 = const()[name = string("op_1899_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_1899_end_0 = const()[name = string("op_1899_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_1899_end_mask_0 = const()[name = string("op_1899_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1899_squeeze_mask_0 = const()[name = string("op_1899_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1899_cast_fp16 = slice_by_index(begin = var_1899_begin_0, end = var_1899_end_0, end_mask = var_1899_end_mask_0, squeeze_mask = var_1899_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1899_cast_fp16")];
+            tensor<int32, [4]> var_1901_begin_0 = const()[name = string("op_1901_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_1901_end_0 = const()[name = string("op_1901_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_1901_end_mask_0 = const()[name = string("op_1901_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1901_squeeze_mask_0 = const()[name = string("op_1901_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1901_cast_fp16 = slice_by_index(begin = var_1901_begin_0, end = var_1901_end_0, end_mask = var_1901_end_mask_0, squeeze_mask = var_1901_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1901_cast_fp16")];
+            bool var_1902_transpose_x_0 = const()[name = string("op_1902_transpose_x_0"), val = bool(false)];
+            bool var_1902_transpose_y_0 = const()[name = string("op_1902_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1902_cast_fp16 = matmul(transpose_x = var_1902_transpose_x_0, transpose_y = var_1902_transpose_y_0, x = var_1901_cast_fp16, y = state_101_cast_fp16)[name = string("op_1902_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_101_cast_fp16 = sub(x = var_1899_cast_fp16, y = var_1902_cast_fp16)[name = string("v_new_101_cast_fp16")];
+            tensor<int32, [4]> var_1905_begin_0 = const()[name = string("op_1905_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_1905_end_0 = const()[name = string("op_1905_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_1905_end_mask_0 = const()[name = string("op_1905_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1905_squeeze_mask_0 = const()[name = string("op_1905_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1905_cast_fp16 = slice_by_index(begin = var_1905_begin_0, end = var_1905_end_0, end_mask = var_1905_end_mask_0, squeeze_mask = var_1905_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1905_cast_fp16")];
+            bool var_1906_transpose_x_0 = const()[name = string("op_1906_transpose_x_0"), val = bool(false)];
+            bool var_1906_transpose_y_0 = const()[name = string("op_1906_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1906_cast_fp16 = matmul(transpose_x = var_1906_transpose_x_0, transpose_y = var_1906_transpose_y_0, x = var_1905_cast_fp16, y = state_101_cast_fp16)[name = string("op_1906_cast_fp16")];
+            tensor<int32, [4]> var_1908_begin_0 = const()[name = string("op_1908_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_1908_end_0 = const()[name = string("op_1908_end_0"), val = tensor<int32, [4]>([32, 19, 64, 64])];
+            tensor<bool, [4]> var_1908_end_mask_0 = const()[name = string("op_1908_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1908_squeeze_mask_0 = const()[name = string("op_1908_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1908_cast_fp16 = slice_by_index(begin = var_1908_begin_0, end = var_1908_end_0, end_mask = var_1908_end_mask_0, squeeze_mask = var_1908_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1908_cast_fp16")];
+            bool var_1909_transpose_x_0 = const()[name = string("op_1909_transpose_x_0"), val = bool(false)];
+            bool var_1909_transpose_y_0 = const()[name = string("op_1909_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1909_cast_fp16 = matmul(transpose_x = var_1909_transpose_x_0, transpose_y = var_1909_transpose_y_0, x = var_1908_cast_fp16, y = v_new_101_cast_fp16)[name = string("op_1909_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1910_cast_fp16 = add(x = var_1906_cast_fp16, y = var_1909_cast_fp16)[name = string("op_1910_cast_fp16")];
+            tensor<int32, [4]> var_1912_begin_0 = const()[name = string("op_1912_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_1912_end_0 = const()[name = string("op_1912_end_0"), val = tensor<int32, [4]>([32, 19, 1, 1])];
+            tensor<bool, [4]> var_1912_end_mask_0 = const()[name = string("op_1912_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1912_squeeze_mask_0 = const()[name = string("op_1912_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1912_cast_fp16 = slice_by_index(begin = var_1912_begin_0, end = var_1912_end_0, end_mask = var_1912_end_mask_0, squeeze_mask = var_1912_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1912_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1913_cast_fp16 = mul(x = state_101_cast_fp16, y = var_1912_cast_fp16)[name = string("op_1913_cast_fp16")];
+            tensor<int32, [4]> var_1915_begin_0 = const()[name = string("op_1915_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_1915_end_0 = const()[name = string("op_1915_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_1915_end_mask_0 = const()[name = string("op_1915_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1915_squeeze_mask_0 = const()[name = string("op_1915_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1915_cast_fp16 = slice_by_index(begin = var_1915_begin_0, end = var_1915_end_0, end_mask = var_1915_end_mask_0, squeeze_mask = var_1915_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1915_cast_fp16")];
+            bool var_1917_transpose_x_1 = const()[name = string("op_1917_transpose_x_1"), val = bool(true)];
+            bool var_1917_transpose_y_1 = const()[name = string("op_1917_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1917_cast_fp16 = matmul(transpose_x = var_1917_transpose_x_1, transpose_y = var_1917_transpose_y_1, x = var_1915_cast_fp16, y = v_new_101_cast_fp16)[name = string("op_1917_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_103_cast_fp16 = add(x = var_1913_cast_fp16, y = var_1917_cast_fp16)[name = string("state_103_cast_fp16")];
+            tensor<int32, [4]> var_1920_begin_0 = const()[name = string("op_1920_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_1920_end_0 = const()[name = string("op_1920_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_1920_end_mask_0 = const()[name = string("op_1920_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1920_squeeze_mask_0 = const()[name = string("op_1920_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1920_cast_fp16 = slice_by_index(begin = var_1920_begin_0, end = var_1920_end_0, end_mask = var_1920_end_mask_0, squeeze_mask = var_1920_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1920_cast_fp16")];
+            tensor<int32, [4]> var_1922_begin_0 = const()[name = string("op_1922_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_1922_end_0 = const()[name = string("op_1922_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_1922_end_mask_0 = const()[name = string("op_1922_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1922_squeeze_mask_0 = const()[name = string("op_1922_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1922_cast_fp16 = slice_by_index(begin = var_1922_begin_0, end = var_1922_end_0, end_mask = var_1922_end_mask_0, squeeze_mask = var_1922_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1922_cast_fp16")];
+            bool var_1923_transpose_x_0 = const()[name = string("op_1923_transpose_x_0"), val = bool(false)];
+            bool var_1923_transpose_y_0 = const()[name = string("op_1923_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1923_cast_fp16 = matmul(transpose_x = var_1923_transpose_x_0, transpose_y = var_1923_transpose_y_0, x = var_1922_cast_fp16, y = state_103_cast_fp16)[name = string("op_1923_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_103_cast_fp16 = sub(x = var_1920_cast_fp16, y = var_1923_cast_fp16)[name = string("v_new_103_cast_fp16")];
+            tensor<int32, [4]> var_1926_begin_0 = const()[name = string("op_1926_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_1926_end_0 = const()[name = string("op_1926_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_1926_end_mask_0 = const()[name = string("op_1926_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1926_squeeze_mask_0 = const()[name = string("op_1926_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1926_cast_fp16 = slice_by_index(begin = var_1926_begin_0, end = var_1926_end_0, end_mask = var_1926_end_mask_0, squeeze_mask = var_1926_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1926_cast_fp16")];
+            bool var_1927_transpose_x_0 = const()[name = string("op_1927_transpose_x_0"), val = bool(false)];
+            bool var_1927_transpose_y_0 = const()[name = string("op_1927_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1927_cast_fp16 = matmul(transpose_x = var_1927_transpose_x_0, transpose_y = var_1927_transpose_y_0, x = var_1926_cast_fp16, y = state_103_cast_fp16)[name = string("op_1927_cast_fp16")];
+            tensor<int32, [4]> var_1929_begin_0 = const()[name = string("op_1929_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_1929_end_0 = const()[name = string("op_1929_end_0"), val = tensor<int32, [4]>([32, 20, 64, 64])];
+            tensor<bool, [4]> var_1929_end_mask_0 = const()[name = string("op_1929_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1929_squeeze_mask_0 = const()[name = string("op_1929_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1929_cast_fp16 = slice_by_index(begin = var_1929_begin_0, end = var_1929_end_0, end_mask = var_1929_end_mask_0, squeeze_mask = var_1929_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1929_cast_fp16")];
+            bool var_1930_transpose_x_0 = const()[name = string("op_1930_transpose_x_0"), val = bool(false)];
+            bool var_1930_transpose_y_0 = const()[name = string("op_1930_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1930_cast_fp16 = matmul(transpose_x = var_1930_transpose_x_0, transpose_y = var_1930_transpose_y_0, x = var_1929_cast_fp16, y = v_new_103_cast_fp16)[name = string("op_1930_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1931_cast_fp16 = add(x = var_1927_cast_fp16, y = var_1930_cast_fp16)[name = string("op_1931_cast_fp16")];
+            tensor<int32, [4]> var_1933_begin_0 = const()[name = string("op_1933_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_1933_end_0 = const()[name = string("op_1933_end_0"), val = tensor<int32, [4]>([32, 20, 1, 1])];
+            tensor<bool, [4]> var_1933_end_mask_0 = const()[name = string("op_1933_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1933_squeeze_mask_0 = const()[name = string("op_1933_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1933_cast_fp16 = slice_by_index(begin = var_1933_begin_0, end = var_1933_end_0, end_mask = var_1933_end_mask_0, squeeze_mask = var_1933_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1933_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1934_cast_fp16 = mul(x = state_103_cast_fp16, y = var_1933_cast_fp16)[name = string("op_1934_cast_fp16")];
+            tensor<int32, [4]> var_1936_begin_0 = const()[name = string("op_1936_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_1936_end_0 = const()[name = string("op_1936_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_1936_end_mask_0 = const()[name = string("op_1936_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1936_squeeze_mask_0 = const()[name = string("op_1936_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1936_cast_fp16 = slice_by_index(begin = var_1936_begin_0, end = var_1936_end_0, end_mask = var_1936_end_mask_0, squeeze_mask = var_1936_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1936_cast_fp16")];
+            bool var_1938_transpose_x_1 = const()[name = string("op_1938_transpose_x_1"), val = bool(true)];
+            bool var_1938_transpose_y_1 = const()[name = string("op_1938_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1938_cast_fp16 = matmul(transpose_x = var_1938_transpose_x_1, transpose_y = var_1938_transpose_y_1, x = var_1936_cast_fp16, y = v_new_103_cast_fp16)[name = string("op_1938_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_105_cast_fp16 = add(x = var_1934_cast_fp16, y = var_1938_cast_fp16)[name = string("state_105_cast_fp16")];
+            tensor<int32, [4]> var_1941_begin_0 = const()[name = string("op_1941_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_1941_end_0 = const()[name = string("op_1941_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_1941_end_mask_0 = const()[name = string("op_1941_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1941_squeeze_mask_0 = const()[name = string("op_1941_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1941_cast_fp16 = slice_by_index(begin = var_1941_begin_0, end = var_1941_end_0, end_mask = var_1941_end_mask_0, squeeze_mask = var_1941_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1941_cast_fp16")];
+            tensor<int32, [4]> var_1943_begin_0 = const()[name = string("op_1943_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_1943_end_0 = const()[name = string("op_1943_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_1943_end_mask_0 = const()[name = string("op_1943_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1943_squeeze_mask_0 = const()[name = string("op_1943_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1943_cast_fp16 = slice_by_index(begin = var_1943_begin_0, end = var_1943_end_0, end_mask = var_1943_end_mask_0, squeeze_mask = var_1943_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1943_cast_fp16")];
+            bool var_1944_transpose_x_0 = const()[name = string("op_1944_transpose_x_0"), val = bool(false)];
+            bool var_1944_transpose_y_0 = const()[name = string("op_1944_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1944_cast_fp16 = matmul(transpose_x = var_1944_transpose_x_0, transpose_y = var_1944_transpose_y_0, x = var_1943_cast_fp16, y = state_105_cast_fp16)[name = string("op_1944_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_105_cast_fp16 = sub(x = var_1941_cast_fp16, y = var_1944_cast_fp16)[name = string("v_new_105_cast_fp16")];
+            tensor<int32, [4]> var_1947_begin_0 = const()[name = string("op_1947_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_1947_end_0 = const()[name = string("op_1947_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_1947_end_mask_0 = const()[name = string("op_1947_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1947_squeeze_mask_0 = const()[name = string("op_1947_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1947_cast_fp16 = slice_by_index(begin = var_1947_begin_0, end = var_1947_end_0, end_mask = var_1947_end_mask_0, squeeze_mask = var_1947_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1947_cast_fp16")];
+            bool var_1948_transpose_x_0 = const()[name = string("op_1948_transpose_x_0"), val = bool(false)];
+            bool var_1948_transpose_y_0 = const()[name = string("op_1948_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1948_cast_fp16 = matmul(transpose_x = var_1948_transpose_x_0, transpose_y = var_1948_transpose_y_0, x = var_1947_cast_fp16, y = state_105_cast_fp16)[name = string("op_1948_cast_fp16")];
+            tensor<int32, [4]> var_1950_begin_0 = const()[name = string("op_1950_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_1950_end_0 = const()[name = string("op_1950_end_0"), val = tensor<int32, [4]>([32, 21, 64, 64])];
+            tensor<bool, [4]> var_1950_end_mask_0 = const()[name = string("op_1950_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1950_squeeze_mask_0 = const()[name = string("op_1950_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1950_cast_fp16 = slice_by_index(begin = var_1950_begin_0, end = var_1950_end_0, end_mask = var_1950_end_mask_0, squeeze_mask = var_1950_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1950_cast_fp16")];
+            bool var_1951_transpose_x_0 = const()[name = string("op_1951_transpose_x_0"), val = bool(false)];
+            bool var_1951_transpose_y_0 = const()[name = string("op_1951_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1951_cast_fp16 = matmul(transpose_x = var_1951_transpose_x_0, transpose_y = var_1951_transpose_y_0, x = var_1950_cast_fp16, y = v_new_105_cast_fp16)[name = string("op_1951_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1952_cast_fp16 = add(x = var_1948_cast_fp16, y = var_1951_cast_fp16)[name = string("op_1952_cast_fp16")];
+            tensor<int32, [4]> var_1954_begin_0 = const()[name = string("op_1954_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_1954_end_0 = const()[name = string("op_1954_end_0"), val = tensor<int32, [4]>([32, 21, 1, 1])];
+            tensor<bool, [4]> var_1954_end_mask_0 = const()[name = string("op_1954_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1954_squeeze_mask_0 = const()[name = string("op_1954_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1954_cast_fp16 = slice_by_index(begin = var_1954_begin_0, end = var_1954_end_0, end_mask = var_1954_end_mask_0, squeeze_mask = var_1954_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1954_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1955_cast_fp16 = mul(x = state_105_cast_fp16, y = var_1954_cast_fp16)[name = string("op_1955_cast_fp16")];
+            tensor<int32, [4]> var_1957_begin_0 = const()[name = string("op_1957_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_1957_end_0 = const()[name = string("op_1957_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_1957_end_mask_0 = const()[name = string("op_1957_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1957_squeeze_mask_0 = const()[name = string("op_1957_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1957_cast_fp16 = slice_by_index(begin = var_1957_begin_0, end = var_1957_end_0, end_mask = var_1957_end_mask_0, squeeze_mask = var_1957_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1957_cast_fp16")];
+            bool var_1959_transpose_x_1 = const()[name = string("op_1959_transpose_x_1"), val = bool(true)];
+            bool var_1959_transpose_y_1 = const()[name = string("op_1959_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1959_cast_fp16 = matmul(transpose_x = var_1959_transpose_x_1, transpose_y = var_1959_transpose_y_1, x = var_1957_cast_fp16, y = v_new_105_cast_fp16)[name = string("op_1959_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_107_cast_fp16 = add(x = var_1955_cast_fp16, y = var_1959_cast_fp16)[name = string("state_107_cast_fp16")];
+            tensor<int32, [4]> var_1962_begin_0 = const()[name = string("op_1962_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_1962_end_0 = const()[name = string("op_1962_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_1962_end_mask_0 = const()[name = string("op_1962_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1962_squeeze_mask_0 = const()[name = string("op_1962_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1962_cast_fp16 = slice_by_index(begin = var_1962_begin_0, end = var_1962_end_0, end_mask = var_1962_end_mask_0, squeeze_mask = var_1962_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1962_cast_fp16")];
+            tensor<int32, [4]> var_1964_begin_0 = const()[name = string("op_1964_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_1964_end_0 = const()[name = string("op_1964_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_1964_end_mask_0 = const()[name = string("op_1964_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1964_squeeze_mask_0 = const()[name = string("op_1964_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1964_cast_fp16 = slice_by_index(begin = var_1964_begin_0, end = var_1964_end_0, end_mask = var_1964_end_mask_0, squeeze_mask = var_1964_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1964_cast_fp16")];
+            bool var_1965_transpose_x_0 = const()[name = string("op_1965_transpose_x_0"), val = bool(false)];
+            bool var_1965_transpose_y_0 = const()[name = string("op_1965_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1965_cast_fp16 = matmul(transpose_x = var_1965_transpose_x_0, transpose_y = var_1965_transpose_y_0, x = var_1964_cast_fp16, y = state_107_cast_fp16)[name = string("op_1965_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_107_cast_fp16 = sub(x = var_1962_cast_fp16, y = var_1965_cast_fp16)[name = string("v_new_107_cast_fp16")];
+            tensor<int32, [4]> var_1968_begin_0 = const()[name = string("op_1968_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_1968_end_0 = const()[name = string("op_1968_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_1968_end_mask_0 = const()[name = string("op_1968_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1968_squeeze_mask_0 = const()[name = string("op_1968_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1968_cast_fp16 = slice_by_index(begin = var_1968_begin_0, end = var_1968_end_0, end_mask = var_1968_end_mask_0, squeeze_mask = var_1968_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1968_cast_fp16")];
+            bool var_1969_transpose_x_0 = const()[name = string("op_1969_transpose_x_0"), val = bool(false)];
+            bool var_1969_transpose_y_0 = const()[name = string("op_1969_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1969_cast_fp16 = matmul(transpose_x = var_1969_transpose_x_0, transpose_y = var_1969_transpose_y_0, x = var_1968_cast_fp16, y = state_107_cast_fp16)[name = string("op_1969_cast_fp16")];
+            tensor<int32, [4]> var_1971_begin_0 = const()[name = string("op_1971_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_1971_end_0 = const()[name = string("op_1971_end_0"), val = tensor<int32, [4]>([32, 22, 64, 64])];
+            tensor<bool, [4]> var_1971_end_mask_0 = const()[name = string("op_1971_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1971_squeeze_mask_0 = const()[name = string("op_1971_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1971_cast_fp16 = slice_by_index(begin = var_1971_begin_0, end = var_1971_end_0, end_mask = var_1971_end_mask_0, squeeze_mask = var_1971_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1971_cast_fp16")];
+            bool var_1972_transpose_x_0 = const()[name = string("op_1972_transpose_x_0"), val = bool(false)];
+            bool var_1972_transpose_y_0 = const()[name = string("op_1972_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1972_cast_fp16 = matmul(transpose_x = var_1972_transpose_x_0, transpose_y = var_1972_transpose_y_0, x = var_1971_cast_fp16, y = v_new_107_cast_fp16)[name = string("op_1972_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1973_cast_fp16 = add(x = var_1969_cast_fp16, y = var_1972_cast_fp16)[name = string("op_1973_cast_fp16")];
+            tensor<int32, [4]> var_1975_begin_0 = const()[name = string("op_1975_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_1975_end_0 = const()[name = string("op_1975_end_0"), val = tensor<int32, [4]>([32, 22, 1, 1])];
+            tensor<bool, [4]> var_1975_end_mask_0 = const()[name = string("op_1975_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1975_squeeze_mask_0 = const()[name = string("op_1975_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1975_cast_fp16 = slice_by_index(begin = var_1975_begin_0, end = var_1975_end_0, end_mask = var_1975_end_mask_0, squeeze_mask = var_1975_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1975_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1976_cast_fp16 = mul(x = state_107_cast_fp16, y = var_1975_cast_fp16)[name = string("op_1976_cast_fp16")];
+            tensor<int32, [4]> var_1978_begin_0 = const()[name = string("op_1978_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_1978_end_0 = const()[name = string("op_1978_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_1978_end_mask_0 = const()[name = string("op_1978_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1978_squeeze_mask_0 = const()[name = string("op_1978_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1978_cast_fp16 = slice_by_index(begin = var_1978_begin_0, end = var_1978_end_0, end_mask = var_1978_end_mask_0, squeeze_mask = var_1978_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1978_cast_fp16")];
+            bool var_1980_transpose_x_1 = const()[name = string("op_1980_transpose_x_1"), val = bool(true)];
+            bool var_1980_transpose_y_1 = const()[name = string("op_1980_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_1980_cast_fp16 = matmul(transpose_x = var_1980_transpose_x_1, transpose_y = var_1980_transpose_y_1, x = var_1978_cast_fp16, y = v_new_107_cast_fp16)[name = string("op_1980_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_109_cast_fp16 = add(x = var_1976_cast_fp16, y = var_1980_cast_fp16)[name = string("state_109_cast_fp16")];
+            tensor<int32, [4]> var_1983_begin_0 = const()[name = string("op_1983_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_1983_end_0 = const()[name = string("op_1983_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_1983_end_mask_0 = const()[name = string("op_1983_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1983_squeeze_mask_0 = const()[name = string("op_1983_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1983_cast_fp16 = slice_by_index(begin = var_1983_begin_0, end = var_1983_end_0, end_mask = var_1983_end_mask_0, squeeze_mask = var_1983_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_1983_cast_fp16")];
+            tensor<int32, [4]> var_1985_begin_0 = const()[name = string("op_1985_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_1985_end_0 = const()[name = string("op_1985_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_1985_end_mask_0 = const()[name = string("op_1985_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1985_squeeze_mask_0 = const()[name = string("op_1985_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1985_cast_fp16 = slice_by_index(begin = var_1985_begin_0, end = var_1985_end_0, end_mask = var_1985_end_mask_0, squeeze_mask = var_1985_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_1985_cast_fp16")];
+            bool var_1986_transpose_x_0 = const()[name = string("op_1986_transpose_x_0"), val = bool(false)];
+            bool var_1986_transpose_y_0 = const()[name = string("op_1986_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1986_cast_fp16 = matmul(transpose_x = var_1986_transpose_x_0, transpose_y = var_1986_transpose_y_0, x = var_1985_cast_fp16, y = state_109_cast_fp16)[name = string("op_1986_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_109_cast_fp16 = sub(x = var_1983_cast_fp16, y = var_1986_cast_fp16)[name = string("v_new_109_cast_fp16")];
+            tensor<int32, [4]> var_1989_begin_0 = const()[name = string("op_1989_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_1989_end_0 = const()[name = string("op_1989_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_1989_end_mask_0 = const()[name = string("op_1989_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1989_squeeze_mask_0 = const()[name = string("op_1989_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1989_cast_fp16 = slice_by_index(begin = var_1989_begin_0, end = var_1989_end_0, end_mask = var_1989_end_mask_0, squeeze_mask = var_1989_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_1989_cast_fp16")];
+            bool var_1990_transpose_x_0 = const()[name = string("op_1990_transpose_x_0"), val = bool(false)];
+            bool var_1990_transpose_y_0 = const()[name = string("op_1990_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1990_cast_fp16 = matmul(transpose_x = var_1990_transpose_x_0, transpose_y = var_1990_transpose_y_0, x = var_1989_cast_fp16, y = state_109_cast_fp16)[name = string("op_1990_cast_fp16")];
+            tensor<int32, [4]> var_1992_begin_0 = const()[name = string("op_1992_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_1992_end_0 = const()[name = string("op_1992_end_0"), val = tensor<int32, [4]>([32, 23, 64, 64])];
+            tensor<bool, [4]> var_1992_end_mask_0 = const()[name = string("op_1992_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1992_squeeze_mask_0 = const()[name = string("op_1992_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_1992_cast_fp16 = slice_by_index(begin = var_1992_begin_0, end = var_1992_end_0, end_mask = var_1992_end_mask_0, squeeze_mask = var_1992_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_1992_cast_fp16")];
+            bool var_1993_transpose_x_0 = const()[name = string("op_1993_transpose_x_0"), val = bool(false)];
+            bool var_1993_transpose_y_0 = const()[name = string("op_1993_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_1993_cast_fp16 = matmul(transpose_x = var_1993_transpose_x_0, transpose_y = var_1993_transpose_y_0, x = var_1992_cast_fp16, y = v_new_109_cast_fp16)[name = string("op_1993_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_1994_cast_fp16 = add(x = var_1990_cast_fp16, y = var_1993_cast_fp16)[name = string("op_1994_cast_fp16")];
+            tensor<int32, [4]> var_1996_begin_0 = const()[name = string("op_1996_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_1996_end_0 = const()[name = string("op_1996_end_0"), val = tensor<int32, [4]>([32, 23, 1, 1])];
+            tensor<bool, [4]> var_1996_end_mask_0 = const()[name = string("op_1996_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1996_squeeze_mask_0 = const()[name = string("op_1996_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_1996_cast_fp16 = slice_by_index(begin = var_1996_begin_0, end = var_1996_end_0, end_mask = var_1996_end_mask_0, squeeze_mask = var_1996_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_1996_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_1997_cast_fp16 = mul(x = state_109_cast_fp16, y = var_1996_cast_fp16)[name = string("op_1997_cast_fp16")];
+            tensor<int32, [4]> var_1999_begin_0 = const()[name = string("op_1999_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_1999_end_0 = const()[name = string("op_1999_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_1999_end_mask_0 = const()[name = string("op_1999_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_1999_squeeze_mask_0 = const()[name = string("op_1999_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_1999_cast_fp16 = slice_by_index(begin = var_1999_begin_0, end = var_1999_end_0, end_mask = var_1999_end_mask_0, squeeze_mask = var_1999_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_1999_cast_fp16")];
+            bool var_2001_transpose_x_1 = const()[name = string("op_2001_transpose_x_1"), val = bool(true)];
+            bool var_2001_transpose_y_1 = const()[name = string("op_2001_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2001_cast_fp16 = matmul(transpose_x = var_2001_transpose_x_1, transpose_y = var_2001_transpose_y_1, x = var_1999_cast_fp16, y = v_new_109_cast_fp16)[name = string("op_2001_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_111_cast_fp16 = add(x = var_1997_cast_fp16, y = var_2001_cast_fp16)[name = string("state_111_cast_fp16")];
+            tensor<int32, [4]> var_2004_begin_0 = const()[name = string("op_2004_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_2004_end_0 = const()[name = string("op_2004_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_2004_end_mask_0 = const()[name = string("op_2004_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2004_squeeze_mask_0 = const()[name = string("op_2004_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2004_cast_fp16 = slice_by_index(begin = var_2004_begin_0, end = var_2004_end_0, end_mask = var_2004_end_mask_0, squeeze_mask = var_2004_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2004_cast_fp16")];
+            tensor<int32, [4]> var_2006_begin_0 = const()[name = string("op_2006_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_2006_end_0 = const()[name = string("op_2006_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_2006_end_mask_0 = const()[name = string("op_2006_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2006_squeeze_mask_0 = const()[name = string("op_2006_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2006_cast_fp16 = slice_by_index(begin = var_2006_begin_0, end = var_2006_end_0, end_mask = var_2006_end_mask_0, squeeze_mask = var_2006_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2006_cast_fp16")];
+            bool var_2007_transpose_x_0 = const()[name = string("op_2007_transpose_x_0"), val = bool(false)];
+            bool var_2007_transpose_y_0 = const()[name = string("op_2007_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2007_cast_fp16 = matmul(transpose_x = var_2007_transpose_x_0, transpose_y = var_2007_transpose_y_0, x = var_2006_cast_fp16, y = state_111_cast_fp16)[name = string("op_2007_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_111_cast_fp16 = sub(x = var_2004_cast_fp16, y = var_2007_cast_fp16)[name = string("v_new_111_cast_fp16")];
+            tensor<int32, [4]> var_2010_begin_0 = const()[name = string("op_2010_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_2010_end_0 = const()[name = string("op_2010_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_2010_end_mask_0 = const()[name = string("op_2010_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2010_squeeze_mask_0 = const()[name = string("op_2010_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2010_cast_fp16 = slice_by_index(begin = var_2010_begin_0, end = var_2010_end_0, end_mask = var_2010_end_mask_0, squeeze_mask = var_2010_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2010_cast_fp16")];
+            bool var_2011_transpose_x_0 = const()[name = string("op_2011_transpose_x_0"), val = bool(false)];
+            bool var_2011_transpose_y_0 = const()[name = string("op_2011_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2011_cast_fp16 = matmul(transpose_x = var_2011_transpose_x_0, transpose_y = var_2011_transpose_y_0, x = var_2010_cast_fp16, y = state_111_cast_fp16)[name = string("op_2011_cast_fp16")];
+            tensor<int32, [4]> var_2013_begin_0 = const()[name = string("op_2013_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_2013_end_0 = const()[name = string("op_2013_end_0"), val = tensor<int32, [4]>([32, 24, 64, 64])];
+            tensor<bool, [4]> var_2013_end_mask_0 = const()[name = string("op_2013_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2013_squeeze_mask_0 = const()[name = string("op_2013_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2013_cast_fp16 = slice_by_index(begin = var_2013_begin_0, end = var_2013_end_0, end_mask = var_2013_end_mask_0, squeeze_mask = var_2013_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2013_cast_fp16")];
+            bool var_2014_transpose_x_0 = const()[name = string("op_2014_transpose_x_0"), val = bool(false)];
+            bool var_2014_transpose_y_0 = const()[name = string("op_2014_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2014_cast_fp16 = matmul(transpose_x = var_2014_transpose_x_0, transpose_y = var_2014_transpose_y_0, x = var_2013_cast_fp16, y = v_new_111_cast_fp16)[name = string("op_2014_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2015_cast_fp16 = add(x = var_2011_cast_fp16, y = var_2014_cast_fp16)[name = string("op_2015_cast_fp16")];
+            tensor<int32, [4]> var_2017_begin_0 = const()[name = string("op_2017_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_2017_end_0 = const()[name = string("op_2017_end_0"), val = tensor<int32, [4]>([32, 24, 1, 1])];
+            tensor<bool, [4]> var_2017_end_mask_0 = const()[name = string("op_2017_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2017_squeeze_mask_0 = const()[name = string("op_2017_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2017_cast_fp16 = slice_by_index(begin = var_2017_begin_0, end = var_2017_end_0, end_mask = var_2017_end_mask_0, squeeze_mask = var_2017_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2017_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2018_cast_fp16 = mul(x = state_111_cast_fp16, y = var_2017_cast_fp16)[name = string("op_2018_cast_fp16")];
+            tensor<int32, [4]> var_2020_begin_0 = const()[name = string("op_2020_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_2020_end_0 = const()[name = string("op_2020_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_2020_end_mask_0 = const()[name = string("op_2020_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2020_squeeze_mask_0 = const()[name = string("op_2020_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2020_cast_fp16 = slice_by_index(begin = var_2020_begin_0, end = var_2020_end_0, end_mask = var_2020_end_mask_0, squeeze_mask = var_2020_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2020_cast_fp16")];
+            bool var_2022_transpose_x_1 = const()[name = string("op_2022_transpose_x_1"), val = bool(true)];
+            bool var_2022_transpose_y_1 = const()[name = string("op_2022_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2022_cast_fp16 = matmul(transpose_x = var_2022_transpose_x_1, transpose_y = var_2022_transpose_y_1, x = var_2020_cast_fp16, y = v_new_111_cast_fp16)[name = string("op_2022_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_113_cast_fp16 = add(x = var_2018_cast_fp16, y = var_2022_cast_fp16)[name = string("state_113_cast_fp16")];
+            tensor<int32, [4]> var_2025_begin_0 = const()[name = string("op_2025_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_2025_end_0 = const()[name = string("op_2025_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_2025_end_mask_0 = const()[name = string("op_2025_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2025_squeeze_mask_0 = const()[name = string("op_2025_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2025_cast_fp16 = slice_by_index(begin = var_2025_begin_0, end = var_2025_end_0, end_mask = var_2025_end_mask_0, squeeze_mask = var_2025_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2025_cast_fp16")];
+            tensor<int32, [4]> var_2027_begin_0 = const()[name = string("op_2027_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_2027_end_0 = const()[name = string("op_2027_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_2027_end_mask_0 = const()[name = string("op_2027_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2027_squeeze_mask_0 = const()[name = string("op_2027_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2027_cast_fp16 = slice_by_index(begin = var_2027_begin_0, end = var_2027_end_0, end_mask = var_2027_end_mask_0, squeeze_mask = var_2027_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2027_cast_fp16")];
+            bool var_2028_transpose_x_0 = const()[name = string("op_2028_transpose_x_0"), val = bool(false)];
+            bool var_2028_transpose_y_0 = const()[name = string("op_2028_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2028_cast_fp16 = matmul(transpose_x = var_2028_transpose_x_0, transpose_y = var_2028_transpose_y_0, x = var_2027_cast_fp16, y = state_113_cast_fp16)[name = string("op_2028_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_113_cast_fp16 = sub(x = var_2025_cast_fp16, y = var_2028_cast_fp16)[name = string("v_new_113_cast_fp16")];
+            tensor<int32, [4]> var_2031_begin_0 = const()[name = string("op_2031_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_2031_end_0 = const()[name = string("op_2031_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_2031_end_mask_0 = const()[name = string("op_2031_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2031_squeeze_mask_0 = const()[name = string("op_2031_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2031_cast_fp16 = slice_by_index(begin = var_2031_begin_0, end = var_2031_end_0, end_mask = var_2031_end_mask_0, squeeze_mask = var_2031_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2031_cast_fp16")];
+            bool var_2032_transpose_x_0 = const()[name = string("op_2032_transpose_x_0"), val = bool(false)];
+            bool var_2032_transpose_y_0 = const()[name = string("op_2032_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2032_cast_fp16 = matmul(transpose_x = var_2032_transpose_x_0, transpose_y = var_2032_transpose_y_0, x = var_2031_cast_fp16, y = state_113_cast_fp16)[name = string("op_2032_cast_fp16")];
+            tensor<int32, [4]> var_2034_begin_0 = const()[name = string("op_2034_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_2034_end_0 = const()[name = string("op_2034_end_0"), val = tensor<int32, [4]>([32, 25, 64, 64])];
+            tensor<bool, [4]> var_2034_end_mask_0 = const()[name = string("op_2034_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2034_squeeze_mask_0 = const()[name = string("op_2034_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2034_cast_fp16 = slice_by_index(begin = var_2034_begin_0, end = var_2034_end_0, end_mask = var_2034_end_mask_0, squeeze_mask = var_2034_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2034_cast_fp16")];
+            bool var_2035_transpose_x_0 = const()[name = string("op_2035_transpose_x_0"), val = bool(false)];
+            bool var_2035_transpose_y_0 = const()[name = string("op_2035_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2035_cast_fp16 = matmul(transpose_x = var_2035_transpose_x_0, transpose_y = var_2035_transpose_y_0, x = var_2034_cast_fp16, y = v_new_113_cast_fp16)[name = string("op_2035_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2036_cast_fp16 = add(x = var_2032_cast_fp16, y = var_2035_cast_fp16)[name = string("op_2036_cast_fp16")];
+            tensor<int32, [4]> var_2038_begin_0 = const()[name = string("op_2038_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_2038_end_0 = const()[name = string("op_2038_end_0"), val = tensor<int32, [4]>([32, 25, 1, 1])];
+            tensor<bool, [4]> var_2038_end_mask_0 = const()[name = string("op_2038_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2038_squeeze_mask_0 = const()[name = string("op_2038_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2038_cast_fp16 = slice_by_index(begin = var_2038_begin_0, end = var_2038_end_0, end_mask = var_2038_end_mask_0, squeeze_mask = var_2038_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2038_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2039_cast_fp16 = mul(x = state_113_cast_fp16, y = var_2038_cast_fp16)[name = string("op_2039_cast_fp16")];
+            tensor<int32, [4]> var_2041_begin_0 = const()[name = string("op_2041_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_2041_end_0 = const()[name = string("op_2041_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_2041_end_mask_0 = const()[name = string("op_2041_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2041_squeeze_mask_0 = const()[name = string("op_2041_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2041_cast_fp16 = slice_by_index(begin = var_2041_begin_0, end = var_2041_end_0, end_mask = var_2041_end_mask_0, squeeze_mask = var_2041_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2041_cast_fp16")];
+            bool var_2043_transpose_x_1 = const()[name = string("op_2043_transpose_x_1"), val = bool(true)];
+            bool var_2043_transpose_y_1 = const()[name = string("op_2043_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2043_cast_fp16 = matmul(transpose_x = var_2043_transpose_x_1, transpose_y = var_2043_transpose_y_1, x = var_2041_cast_fp16, y = v_new_113_cast_fp16)[name = string("op_2043_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_115_cast_fp16 = add(x = var_2039_cast_fp16, y = var_2043_cast_fp16)[name = string("state_115_cast_fp16")];
+            tensor<int32, [4]> var_2046_begin_0 = const()[name = string("op_2046_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_2046_end_0 = const()[name = string("op_2046_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_2046_end_mask_0 = const()[name = string("op_2046_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2046_squeeze_mask_0 = const()[name = string("op_2046_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2046_cast_fp16 = slice_by_index(begin = var_2046_begin_0, end = var_2046_end_0, end_mask = var_2046_end_mask_0, squeeze_mask = var_2046_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2046_cast_fp16")];
+            tensor<int32, [4]> var_2048_begin_0 = const()[name = string("op_2048_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_2048_end_0 = const()[name = string("op_2048_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_2048_end_mask_0 = const()[name = string("op_2048_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2048_squeeze_mask_0 = const()[name = string("op_2048_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2048_cast_fp16 = slice_by_index(begin = var_2048_begin_0, end = var_2048_end_0, end_mask = var_2048_end_mask_0, squeeze_mask = var_2048_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2048_cast_fp16")];
+            bool var_2049_transpose_x_0 = const()[name = string("op_2049_transpose_x_0"), val = bool(false)];
+            bool var_2049_transpose_y_0 = const()[name = string("op_2049_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2049_cast_fp16 = matmul(transpose_x = var_2049_transpose_x_0, transpose_y = var_2049_transpose_y_0, x = var_2048_cast_fp16, y = state_115_cast_fp16)[name = string("op_2049_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_115_cast_fp16 = sub(x = var_2046_cast_fp16, y = var_2049_cast_fp16)[name = string("v_new_115_cast_fp16")];
+            tensor<int32, [4]> var_2052_begin_0 = const()[name = string("op_2052_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_2052_end_0 = const()[name = string("op_2052_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_2052_end_mask_0 = const()[name = string("op_2052_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2052_squeeze_mask_0 = const()[name = string("op_2052_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2052_cast_fp16 = slice_by_index(begin = var_2052_begin_0, end = var_2052_end_0, end_mask = var_2052_end_mask_0, squeeze_mask = var_2052_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2052_cast_fp16")];
+            bool var_2053_transpose_x_0 = const()[name = string("op_2053_transpose_x_0"), val = bool(false)];
+            bool var_2053_transpose_y_0 = const()[name = string("op_2053_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2053_cast_fp16 = matmul(transpose_x = var_2053_transpose_x_0, transpose_y = var_2053_transpose_y_0, x = var_2052_cast_fp16, y = state_115_cast_fp16)[name = string("op_2053_cast_fp16")];
+            tensor<int32, [4]> var_2055_begin_0 = const()[name = string("op_2055_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_2055_end_0 = const()[name = string("op_2055_end_0"), val = tensor<int32, [4]>([32, 26, 64, 64])];
+            tensor<bool, [4]> var_2055_end_mask_0 = const()[name = string("op_2055_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2055_squeeze_mask_0 = const()[name = string("op_2055_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2055_cast_fp16 = slice_by_index(begin = var_2055_begin_0, end = var_2055_end_0, end_mask = var_2055_end_mask_0, squeeze_mask = var_2055_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2055_cast_fp16")];
+            bool var_2056_transpose_x_0 = const()[name = string("op_2056_transpose_x_0"), val = bool(false)];
+            bool var_2056_transpose_y_0 = const()[name = string("op_2056_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2056_cast_fp16 = matmul(transpose_x = var_2056_transpose_x_0, transpose_y = var_2056_transpose_y_0, x = var_2055_cast_fp16, y = v_new_115_cast_fp16)[name = string("op_2056_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2057_cast_fp16 = add(x = var_2053_cast_fp16, y = var_2056_cast_fp16)[name = string("op_2057_cast_fp16")];
+            tensor<int32, [4]> var_2059_begin_0 = const()[name = string("op_2059_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_2059_end_0 = const()[name = string("op_2059_end_0"), val = tensor<int32, [4]>([32, 26, 1, 1])];
+            tensor<bool, [4]> var_2059_end_mask_0 = const()[name = string("op_2059_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2059_squeeze_mask_0 = const()[name = string("op_2059_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2059_cast_fp16 = slice_by_index(begin = var_2059_begin_0, end = var_2059_end_0, end_mask = var_2059_end_mask_0, squeeze_mask = var_2059_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2059_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2060_cast_fp16 = mul(x = state_115_cast_fp16, y = var_2059_cast_fp16)[name = string("op_2060_cast_fp16")];
+            tensor<int32, [4]> var_2062_begin_0 = const()[name = string("op_2062_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_2062_end_0 = const()[name = string("op_2062_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_2062_end_mask_0 = const()[name = string("op_2062_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2062_squeeze_mask_0 = const()[name = string("op_2062_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2062_cast_fp16 = slice_by_index(begin = var_2062_begin_0, end = var_2062_end_0, end_mask = var_2062_end_mask_0, squeeze_mask = var_2062_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2062_cast_fp16")];
+            bool var_2064_transpose_x_1 = const()[name = string("op_2064_transpose_x_1"), val = bool(true)];
+            bool var_2064_transpose_y_1 = const()[name = string("op_2064_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2064_cast_fp16 = matmul(transpose_x = var_2064_transpose_x_1, transpose_y = var_2064_transpose_y_1, x = var_2062_cast_fp16, y = v_new_115_cast_fp16)[name = string("op_2064_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_117_cast_fp16 = add(x = var_2060_cast_fp16, y = var_2064_cast_fp16)[name = string("state_117_cast_fp16")];
+            tensor<int32, [4]> var_2067_begin_0 = const()[name = string("op_2067_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_2067_end_0 = const()[name = string("op_2067_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_2067_end_mask_0 = const()[name = string("op_2067_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2067_squeeze_mask_0 = const()[name = string("op_2067_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2067_cast_fp16 = slice_by_index(begin = var_2067_begin_0, end = var_2067_end_0, end_mask = var_2067_end_mask_0, squeeze_mask = var_2067_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2067_cast_fp16")];
+            tensor<int32, [4]> var_2069_begin_0 = const()[name = string("op_2069_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_2069_end_0 = const()[name = string("op_2069_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_2069_end_mask_0 = const()[name = string("op_2069_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2069_squeeze_mask_0 = const()[name = string("op_2069_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2069_cast_fp16 = slice_by_index(begin = var_2069_begin_0, end = var_2069_end_0, end_mask = var_2069_end_mask_0, squeeze_mask = var_2069_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2069_cast_fp16")];
+            bool var_2070_transpose_x_0 = const()[name = string("op_2070_transpose_x_0"), val = bool(false)];
+            bool var_2070_transpose_y_0 = const()[name = string("op_2070_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2070_cast_fp16 = matmul(transpose_x = var_2070_transpose_x_0, transpose_y = var_2070_transpose_y_0, x = var_2069_cast_fp16, y = state_117_cast_fp16)[name = string("op_2070_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_117_cast_fp16 = sub(x = var_2067_cast_fp16, y = var_2070_cast_fp16)[name = string("v_new_117_cast_fp16")];
+            tensor<int32, [4]> var_2073_begin_0 = const()[name = string("op_2073_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_2073_end_0 = const()[name = string("op_2073_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_2073_end_mask_0 = const()[name = string("op_2073_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2073_squeeze_mask_0 = const()[name = string("op_2073_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2073_cast_fp16 = slice_by_index(begin = var_2073_begin_0, end = var_2073_end_0, end_mask = var_2073_end_mask_0, squeeze_mask = var_2073_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2073_cast_fp16")];
+            bool var_2074_transpose_x_0 = const()[name = string("op_2074_transpose_x_0"), val = bool(false)];
+            bool var_2074_transpose_y_0 = const()[name = string("op_2074_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2074_cast_fp16 = matmul(transpose_x = var_2074_transpose_x_0, transpose_y = var_2074_transpose_y_0, x = var_2073_cast_fp16, y = state_117_cast_fp16)[name = string("op_2074_cast_fp16")];
+            tensor<int32, [4]> var_2076_begin_0 = const()[name = string("op_2076_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_2076_end_0 = const()[name = string("op_2076_end_0"), val = tensor<int32, [4]>([32, 27, 64, 64])];
+            tensor<bool, [4]> var_2076_end_mask_0 = const()[name = string("op_2076_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2076_squeeze_mask_0 = const()[name = string("op_2076_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2076_cast_fp16 = slice_by_index(begin = var_2076_begin_0, end = var_2076_end_0, end_mask = var_2076_end_mask_0, squeeze_mask = var_2076_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2076_cast_fp16")];
+            bool var_2077_transpose_x_0 = const()[name = string("op_2077_transpose_x_0"), val = bool(false)];
+            bool var_2077_transpose_y_0 = const()[name = string("op_2077_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2077_cast_fp16 = matmul(transpose_x = var_2077_transpose_x_0, transpose_y = var_2077_transpose_y_0, x = var_2076_cast_fp16, y = v_new_117_cast_fp16)[name = string("op_2077_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2078_cast_fp16 = add(x = var_2074_cast_fp16, y = var_2077_cast_fp16)[name = string("op_2078_cast_fp16")];
+            tensor<int32, [4]> var_2080_begin_0 = const()[name = string("op_2080_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_2080_end_0 = const()[name = string("op_2080_end_0"), val = tensor<int32, [4]>([32, 27, 1, 1])];
+            tensor<bool, [4]> var_2080_end_mask_0 = const()[name = string("op_2080_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2080_squeeze_mask_0 = const()[name = string("op_2080_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2080_cast_fp16 = slice_by_index(begin = var_2080_begin_0, end = var_2080_end_0, end_mask = var_2080_end_mask_0, squeeze_mask = var_2080_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2080_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2081_cast_fp16 = mul(x = state_117_cast_fp16, y = var_2080_cast_fp16)[name = string("op_2081_cast_fp16")];
+            tensor<int32, [4]> var_2083_begin_0 = const()[name = string("op_2083_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_2083_end_0 = const()[name = string("op_2083_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_2083_end_mask_0 = const()[name = string("op_2083_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2083_squeeze_mask_0 = const()[name = string("op_2083_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2083_cast_fp16 = slice_by_index(begin = var_2083_begin_0, end = var_2083_end_0, end_mask = var_2083_end_mask_0, squeeze_mask = var_2083_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2083_cast_fp16")];
+            bool var_2085_transpose_x_1 = const()[name = string("op_2085_transpose_x_1"), val = bool(true)];
+            bool var_2085_transpose_y_1 = const()[name = string("op_2085_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2085_cast_fp16 = matmul(transpose_x = var_2085_transpose_x_1, transpose_y = var_2085_transpose_y_1, x = var_2083_cast_fp16, y = v_new_117_cast_fp16)[name = string("op_2085_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_119_cast_fp16 = add(x = var_2081_cast_fp16, y = var_2085_cast_fp16)[name = string("state_119_cast_fp16")];
+            tensor<int32, [4]> var_2088_begin_0 = const()[name = string("op_2088_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_2088_end_0 = const()[name = string("op_2088_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_2088_end_mask_0 = const()[name = string("op_2088_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2088_squeeze_mask_0 = const()[name = string("op_2088_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2088_cast_fp16 = slice_by_index(begin = var_2088_begin_0, end = var_2088_end_0, end_mask = var_2088_end_mask_0, squeeze_mask = var_2088_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2088_cast_fp16")];
+            tensor<int32, [4]> var_2090_begin_0 = const()[name = string("op_2090_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_2090_end_0 = const()[name = string("op_2090_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_2090_end_mask_0 = const()[name = string("op_2090_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2090_squeeze_mask_0 = const()[name = string("op_2090_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2090_cast_fp16 = slice_by_index(begin = var_2090_begin_0, end = var_2090_end_0, end_mask = var_2090_end_mask_0, squeeze_mask = var_2090_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2090_cast_fp16")];
+            bool var_2091_transpose_x_0 = const()[name = string("op_2091_transpose_x_0"), val = bool(false)];
+            bool var_2091_transpose_y_0 = const()[name = string("op_2091_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2091_cast_fp16 = matmul(transpose_x = var_2091_transpose_x_0, transpose_y = var_2091_transpose_y_0, x = var_2090_cast_fp16, y = state_119_cast_fp16)[name = string("op_2091_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_119_cast_fp16 = sub(x = var_2088_cast_fp16, y = var_2091_cast_fp16)[name = string("v_new_119_cast_fp16")];
+            tensor<int32, [4]> var_2094_begin_0 = const()[name = string("op_2094_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_2094_end_0 = const()[name = string("op_2094_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_2094_end_mask_0 = const()[name = string("op_2094_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2094_squeeze_mask_0 = const()[name = string("op_2094_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2094_cast_fp16 = slice_by_index(begin = var_2094_begin_0, end = var_2094_end_0, end_mask = var_2094_end_mask_0, squeeze_mask = var_2094_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2094_cast_fp16")];
+            bool var_2095_transpose_x_0 = const()[name = string("op_2095_transpose_x_0"), val = bool(false)];
+            bool var_2095_transpose_y_0 = const()[name = string("op_2095_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2095_cast_fp16 = matmul(transpose_x = var_2095_transpose_x_0, transpose_y = var_2095_transpose_y_0, x = var_2094_cast_fp16, y = state_119_cast_fp16)[name = string("op_2095_cast_fp16")];
+            tensor<int32, [4]> var_2097_begin_0 = const()[name = string("op_2097_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_2097_end_0 = const()[name = string("op_2097_end_0"), val = tensor<int32, [4]>([32, 28, 64, 64])];
+            tensor<bool, [4]> var_2097_end_mask_0 = const()[name = string("op_2097_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2097_squeeze_mask_0 = const()[name = string("op_2097_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2097_cast_fp16 = slice_by_index(begin = var_2097_begin_0, end = var_2097_end_0, end_mask = var_2097_end_mask_0, squeeze_mask = var_2097_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2097_cast_fp16")];
+            bool var_2098_transpose_x_0 = const()[name = string("op_2098_transpose_x_0"), val = bool(false)];
+            bool var_2098_transpose_y_0 = const()[name = string("op_2098_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2098_cast_fp16 = matmul(transpose_x = var_2098_transpose_x_0, transpose_y = var_2098_transpose_y_0, x = var_2097_cast_fp16, y = v_new_119_cast_fp16)[name = string("op_2098_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2099_cast_fp16 = add(x = var_2095_cast_fp16, y = var_2098_cast_fp16)[name = string("op_2099_cast_fp16")];
+            tensor<int32, [4]> var_2101_begin_0 = const()[name = string("op_2101_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_2101_end_0 = const()[name = string("op_2101_end_0"), val = tensor<int32, [4]>([32, 28, 1, 1])];
+            tensor<bool, [4]> var_2101_end_mask_0 = const()[name = string("op_2101_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2101_squeeze_mask_0 = const()[name = string("op_2101_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2101_cast_fp16 = slice_by_index(begin = var_2101_begin_0, end = var_2101_end_0, end_mask = var_2101_end_mask_0, squeeze_mask = var_2101_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2101_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2102_cast_fp16 = mul(x = state_119_cast_fp16, y = var_2101_cast_fp16)[name = string("op_2102_cast_fp16")];
+            tensor<int32, [4]> var_2104_begin_0 = const()[name = string("op_2104_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_2104_end_0 = const()[name = string("op_2104_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_2104_end_mask_0 = const()[name = string("op_2104_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2104_squeeze_mask_0 = const()[name = string("op_2104_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2104_cast_fp16 = slice_by_index(begin = var_2104_begin_0, end = var_2104_end_0, end_mask = var_2104_end_mask_0, squeeze_mask = var_2104_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2104_cast_fp16")];
+            bool var_2106_transpose_x_1 = const()[name = string("op_2106_transpose_x_1"), val = bool(true)];
+            bool var_2106_transpose_y_1 = const()[name = string("op_2106_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2106_cast_fp16 = matmul(transpose_x = var_2106_transpose_x_1, transpose_y = var_2106_transpose_y_1, x = var_2104_cast_fp16, y = v_new_119_cast_fp16)[name = string("op_2106_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_121_cast_fp16 = add(x = var_2102_cast_fp16, y = var_2106_cast_fp16)[name = string("state_121_cast_fp16")];
+            tensor<int32, [4]> var_2109_begin_0 = const()[name = string("op_2109_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_2109_end_0 = const()[name = string("op_2109_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_2109_end_mask_0 = const()[name = string("op_2109_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2109_squeeze_mask_0 = const()[name = string("op_2109_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2109_cast_fp16 = slice_by_index(begin = var_2109_begin_0, end = var_2109_end_0, end_mask = var_2109_end_mask_0, squeeze_mask = var_2109_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2109_cast_fp16")];
+            tensor<int32, [4]> var_2111_begin_0 = const()[name = string("op_2111_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_2111_end_0 = const()[name = string("op_2111_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_2111_end_mask_0 = const()[name = string("op_2111_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2111_squeeze_mask_0 = const()[name = string("op_2111_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2111_cast_fp16 = slice_by_index(begin = var_2111_begin_0, end = var_2111_end_0, end_mask = var_2111_end_mask_0, squeeze_mask = var_2111_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2111_cast_fp16")];
+            bool var_2112_transpose_x_0 = const()[name = string("op_2112_transpose_x_0"), val = bool(false)];
+            bool var_2112_transpose_y_0 = const()[name = string("op_2112_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2112_cast_fp16 = matmul(transpose_x = var_2112_transpose_x_0, transpose_y = var_2112_transpose_y_0, x = var_2111_cast_fp16, y = state_121_cast_fp16)[name = string("op_2112_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_121_cast_fp16 = sub(x = var_2109_cast_fp16, y = var_2112_cast_fp16)[name = string("v_new_121_cast_fp16")];
+            tensor<int32, [4]> var_2115_begin_0 = const()[name = string("op_2115_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_2115_end_0 = const()[name = string("op_2115_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_2115_end_mask_0 = const()[name = string("op_2115_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2115_squeeze_mask_0 = const()[name = string("op_2115_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2115_cast_fp16 = slice_by_index(begin = var_2115_begin_0, end = var_2115_end_0, end_mask = var_2115_end_mask_0, squeeze_mask = var_2115_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2115_cast_fp16")];
+            bool var_2116_transpose_x_0 = const()[name = string("op_2116_transpose_x_0"), val = bool(false)];
+            bool var_2116_transpose_y_0 = const()[name = string("op_2116_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2116_cast_fp16 = matmul(transpose_x = var_2116_transpose_x_0, transpose_y = var_2116_transpose_y_0, x = var_2115_cast_fp16, y = state_121_cast_fp16)[name = string("op_2116_cast_fp16")];
+            tensor<int32, [4]> var_2118_begin_0 = const()[name = string("op_2118_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_2118_end_0 = const()[name = string("op_2118_end_0"), val = tensor<int32, [4]>([32, 29, 64, 64])];
+            tensor<bool, [4]> var_2118_end_mask_0 = const()[name = string("op_2118_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2118_squeeze_mask_0 = const()[name = string("op_2118_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2118_cast_fp16 = slice_by_index(begin = var_2118_begin_0, end = var_2118_end_0, end_mask = var_2118_end_mask_0, squeeze_mask = var_2118_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2118_cast_fp16")];
+            bool var_2119_transpose_x_0 = const()[name = string("op_2119_transpose_x_0"), val = bool(false)];
+            bool var_2119_transpose_y_0 = const()[name = string("op_2119_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2119_cast_fp16 = matmul(transpose_x = var_2119_transpose_x_0, transpose_y = var_2119_transpose_y_0, x = var_2118_cast_fp16, y = v_new_121_cast_fp16)[name = string("op_2119_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2120_cast_fp16 = add(x = var_2116_cast_fp16, y = var_2119_cast_fp16)[name = string("op_2120_cast_fp16")];
+            tensor<int32, [4]> var_2122_begin_0 = const()[name = string("op_2122_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_2122_end_0 = const()[name = string("op_2122_end_0"), val = tensor<int32, [4]>([32, 29, 1, 1])];
+            tensor<bool, [4]> var_2122_end_mask_0 = const()[name = string("op_2122_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2122_squeeze_mask_0 = const()[name = string("op_2122_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2122_cast_fp16 = slice_by_index(begin = var_2122_begin_0, end = var_2122_end_0, end_mask = var_2122_end_mask_0, squeeze_mask = var_2122_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2122_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2123_cast_fp16 = mul(x = state_121_cast_fp16, y = var_2122_cast_fp16)[name = string("op_2123_cast_fp16")];
+            tensor<int32, [4]> var_2125_begin_0 = const()[name = string("op_2125_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_2125_end_0 = const()[name = string("op_2125_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_2125_end_mask_0 = const()[name = string("op_2125_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2125_squeeze_mask_0 = const()[name = string("op_2125_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2125_cast_fp16 = slice_by_index(begin = var_2125_begin_0, end = var_2125_end_0, end_mask = var_2125_end_mask_0, squeeze_mask = var_2125_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2125_cast_fp16")];
+            bool var_2127_transpose_x_1 = const()[name = string("op_2127_transpose_x_1"), val = bool(true)];
+            bool var_2127_transpose_y_1 = const()[name = string("op_2127_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2127_cast_fp16 = matmul(transpose_x = var_2127_transpose_x_1, transpose_y = var_2127_transpose_y_1, x = var_2125_cast_fp16, y = v_new_121_cast_fp16)[name = string("op_2127_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_123_cast_fp16 = add(x = var_2123_cast_fp16, y = var_2127_cast_fp16)[name = string("state_123_cast_fp16")];
+            tensor<int32, [4]> var_2130_begin_0 = const()[name = string("op_2130_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_2130_end_0 = const()[name = string("op_2130_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_2130_end_mask_0 = const()[name = string("op_2130_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2130_squeeze_mask_0 = const()[name = string("op_2130_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2130_cast_fp16 = slice_by_index(begin = var_2130_begin_0, end = var_2130_end_0, end_mask = var_2130_end_mask_0, squeeze_mask = var_2130_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2130_cast_fp16")];
+            tensor<int32, [4]> var_2132_begin_0 = const()[name = string("op_2132_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_2132_end_0 = const()[name = string("op_2132_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_2132_end_mask_0 = const()[name = string("op_2132_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2132_squeeze_mask_0 = const()[name = string("op_2132_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2132_cast_fp16 = slice_by_index(begin = var_2132_begin_0, end = var_2132_end_0, end_mask = var_2132_end_mask_0, squeeze_mask = var_2132_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2132_cast_fp16")];
+            bool var_2133_transpose_x_0 = const()[name = string("op_2133_transpose_x_0"), val = bool(false)];
+            bool var_2133_transpose_y_0 = const()[name = string("op_2133_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2133_cast_fp16 = matmul(transpose_x = var_2133_transpose_x_0, transpose_y = var_2133_transpose_y_0, x = var_2132_cast_fp16, y = state_123_cast_fp16)[name = string("op_2133_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_123_cast_fp16 = sub(x = var_2130_cast_fp16, y = var_2133_cast_fp16)[name = string("v_new_123_cast_fp16")];
+            tensor<int32, [4]> var_2136_begin_0 = const()[name = string("op_2136_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_2136_end_0 = const()[name = string("op_2136_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_2136_end_mask_0 = const()[name = string("op_2136_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2136_squeeze_mask_0 = const()[name = string("op_2136_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2136_cast_fp16 = slice_by_index(begin = var_2136_begin_0, end = var_2136_end_0, end_mask = var_2136_end_mask_0, squeeze_mask = var_2136_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2136_cast_fp16")];
+            bool var_2137_transpose_x_0 = const()[name = string("op_2137_transpose_x_0"), val = bool(false)];
+            bool var_2137_transpose_y_0 = const()[name = string("op_2137_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2137_cast_fp16 = matmul(transpose_x = var_2137_transpose_x_0, transpose_y = var_2137_transpose_y_0, x = var_2136_cast_fp16, y = state_123_cast_fp16)[name = string("op_2137_cast_fp16")];
+            tensor<int32, [4]> var_2139_begin_0 = const()[name = string("op_2139_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_2139_end_0 = const()[name = string("op_2139_end_0"), val = tensor<int32, [4]>([32, 30, 64, 64])];
+            tensor<bool, [4]> var_2139_end_mask_0 = const()[name = string("op_2139_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2139_squeeze_mask_0 = const()[name = string("op_2139_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2139_cast_fp16 = slice_by_index(begin = var_2139_begin_0, end = var_2139_end_0, end_mask = var_2139_end_mask_0, squeeze_mask = var_2139_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2139_cast_fp16")];
+            bool var_2140_transpose_x_0 = const()[name = string("op_2140_transpose_x_0"), val = bool(false)];
+            bool var_2140_transpose_y_0 = const()[name = string("op_2140_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2140_cast_fp16 = matmul(transpose_x = var_2140_transpose_x_0, transpose_y = var_2140_transpose_y_0, x = var_2139_cast_fp16, y = v_new_123_cast_fp16)[name = string("op_2140_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2141_cast_fp16 = add(x = var_2137_cast_fp16, y = var_2140_cast_fp16)[name = string("op_2141_cast_fp16")];
+            tensor<int32, [4]> var_2143_begin_0 = const()[name = string("op_2143_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_2143_end_0 = const()[name = string("op_2143_end_0"), val = tensor<int32, [4]>([32, 30, 1, 1])];
+            tensor<bool, [4]> var_2143_end_mask_0 = const()[name = string("op_2143_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2143_squeeze_mask_0 = const()[name = string("op_2143_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2143_cast_fp16 = slice_by_index(begin = var_2143_begin_0, end = var_2143_end_0, end_mask = var_2143_end_mask_0, squeeze_mask = var_2143_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2143_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2144_cast_fp16 = mul(x = state_123_cast_fp16, y = var_2143_cast_fp16)[name = string("op_2144_cast_fp16")];
+            tensor<int32, [4]> var_2146_begin_0 = const()[name = string("op_2146_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_2146_end_0 = const()[name = string("op_2146_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_2146_end_mask_0 = const()[name = string("op_2146_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2146_squeeze_mask_0 = const()[name = string("op_2146_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2146_cast_fp16 = slice_by_index(begin = var_2146_begin_0, end = var_2146_end_0, end_mask = var_2146_end_mask_0, squeeze_mask = var_2146_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2146_cast_fp16")];
+            bool var_2148_transpose_x_1 = const()[name = string("op_2148_transpose_x_1"), val = bool(true)];
+            bool var_2148_transpose_y_1 = const()[name = string("op_2148_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2148_cast_fp16 = matmul(transpose_x = var_2148_transpose_x_1, transpose_y = var_2148_transpose_y_1, x = var_2146_cast_fp16, y = v_new_123_cast_fp16)[name = string("op_2148_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_125_cast_fp16 = add(x = var_2144_cast_fp16, y = var_2148_cast_fp16)[name = string("state_125_cast_fp16")];
+            tensor<int32, [4]> var_2151_begin_0 = const()[name = string("op_2151_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_2151_end_0 = const()[name = string("op_2151_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_2151_end_mask_0 = const()[name = string("op_2151_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2151_squeeze_mask_0 = const()[name = string("op_2151_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2151_cast_fp16 = slice_by_index(begin = var_2151_begin_0, end = var_2151_end_0, end_mask = var_2151_end_mask_0, squeeze_mask = var_2151_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2151_cast_fp16")];
+            tensor<int32, [4]> var_2153_begin_0 = const()[name = string("op_2153_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_2153_end_0 = const()[name = string("op_2153_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_2153_end_mask_0 = const()[name = string("op_2153_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2153_squeeze_mask_0 = const()[name = string("op_2153_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2153_cast_fp16 = slice_by_index(begin = var_2153_begin_0, end = var_2153_end_0, end_mask = var_2153_end_mask_0, squeeze_mask = var_2153_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2153_cast_fp16")];
+            bool var_2154_transpose_x_0 = const()[name = string("op_2154_transpose_x_0"), val = bool(false)];
+            bool var_2154_transpose_y_0 = const()[name = string("op_2154_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2154_cast_fp16 = matmul(transpose_x = var_2154_transpose_x_0, transpose_y = var_2154_transpose_y_0, x = var_2153_cast_fp16, y = state_125_cast_fp16)[name = string("op_2154_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_125_cast_fp16 = sub(x = var_2151_cast_fp16, y = var_2154_cast_fp16)[name = string("v_new_125_cast_fp16")];
+            tensor<int32, [4]> var_2157_begin_0 = const()[name = string("op_2157_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_2157_end_0 = const()[name = string("op_2157_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_2157_end_mask_0 = const()[name = string("op_2157_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2157_squeeze_mask_0 = const()[name = string("op_2157_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2157_cast_fp16 = slice_by_index(begin = var_2157_begin_0, end = var_2157_end_0, end_mask = var_2157_end_mask_0, squeeze_mask = var_2157_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2157_cast_fp16")];
+            bool var_2158_transpose_x_0 = const()[name = string("op_2158_transpose_x_0"), val = bool(false)];
+            bool var_2158_transpose_y_0 = const()[name = string("op_2158_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2158_cast_fp16 = matmul(transpose_x = var_2158_transpose_x_0, transpose_y = var_2158_transpose_y_0, x = var_2157_cast_fp16, y = state_125_cast_fp16)[name = string("op_2158_cast_fp16")];
+            tensor<int32, [4]> var_2160_begin_0 = const()[name = string("op_2160_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_2160_end_0 = const()[name = string("op_2160_end_0"), val = tensor<int32, [4]>([32, 31, 64, 64])];
+            tensor<bool, [4]> var_2160_end_mask_0 = const()[name = string("op_2160_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2160_squeeze_mask_0 = const()[name = string("op_2160_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2160_cast_fp16 = slice_by_index(begin = var_2160_begin_0, end = var_2160_end_0, end_mask = var_2160_end_mask_0, squeeze_mask = var_2160_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2160_cast_fp16")];
+            bool var_2161_transpose_x_0 = const()[name = string("op_2161_transpose_x_0"), val = bool(false)];
+            bool var_2161_transpose_y_0 = const()[name = string("op_2161_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2161_cast_fp16 = matmul(transpose_x = var_2161_transpose_x_0, transpose_y = var_2161_transpose_y_0, x = var_2160_cast_fp16, y = v_new_125_cast_fp16)[name = string("op_2161_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2162_cast_fp16 = add(x = var_2158_cast_fp16, y = var_2161_cast_fp16)[name = string("op_2162_cast_fp16")];
+            tensor<int32, [4]> var_2164_begin_0 = const()[name = string("op_2164_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_2164_end_0 = const()[name = string("op_2164_end_0"), val = tensor<int32, [4]>([32, 31, 1, 1])];
+            tensor<bool, [4]> var_2164_end_mask_0 = const()[name = string("op_2164_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2164_squeeze_mask_0 = const()[name = string("op_2164_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2164_cast_fp16 = slice_by_index(begin = var_2164_begin_0, end = var_2164_end_0, end_mask = var_2164_end_mask_0, squeeze_mask = var_2164_squeeze_mask_0, x = chunk_decay_3_cast_fp16)[name = string("op_2164_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2165_cast_fp16 = mul(x = state_125_cast_fp16, y = var_2164_cast_fp16)[name = string("op_2165_cast_fp16")];
+            tensor<int32, [4]> var_2167_begin_0 = const()[name = string("op_2167_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_2167_end_0 = const()[name = string("op_2167_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_2167_end_mask_0 = const()[name = string("op_2167_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2167_squeeze_mask_0 = const()[name = string("op_2167_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2167_cast_fp16 = slice_by_index(begin = var_2167_begin_0, end = var_2167_end_0, end_mask = var_2167_end_mask_0, squeeze_mask = var_2167_squeeze_mask_0, x = k_dec_3_cast_fp16)[name = string("op_2167_cast_fp16")];
+            bool var_2169_transpose_x_1 = const()[name = string("op_2169_transpose_x_1"), val = bool(true)];
+            bool var_2169_transpose_y_1 = const()[name = string("op_2169_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2169_cast_fp16 = matmul(transpose_x = var_2169_transpose_x_1, transpose_y = var_2169_transpose_y_1, x = var_2167_cast_fp16, y = v_new_125_cast_fp16)[name = string("op_2169_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_127_cast_fp16 = add(x = var_2165_cast_fp16, y = var_2169_cast_fp16)[name = string("state_127_cast_fp16")];
+            tensor<int32, [4]> var_2172_begin_0 = const()[name = string("op_2172_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_2172_end_0 = const()[name = string("op_2172_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_2172_end_mask_0 = const()[name = string("op_2172_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2172_squeeze_mask_0 = const()[name = string("op_2172_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2172_cast_fp16 = slice_by_index(begin = var_2172_begin_0, end = var_2172_end_0, end_mask = var_2172_end_mask_0, squeeze_mask = var_2172_squeeze_mask_0, x = new_v_3_cast_fp16)[name = string("op_2172_cast_fp16")];
+            tensor<int32, [4]> var_2174_begin_0 = const()[name = string("op_2174_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_2174_end_0 = const()[name = string("op_2174_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_2174_end_mask_0 = const()[name = string("op_2174_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2174_squeeze_mask_0 = const()[name = string("op_2174_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2174_cast_fp16 = slice_by_index(begin = var_2174_begin_0, end = var_2174_end_0, end_mask = var_2174_end_mask_0, squeeze_mask = var_2174_squeeze_mask_0, x = k_cumdecay_3_cast_fp16)[name = string("op_2174_cast_fp16")];
+            bool var_2175_transpose_x_0 = const()[name = string("op_2175_transpose_x_0"), val = bool(false)];
+            bool var_2175_transpose_y_0 = const()[name = string("op_2175_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2175_cast_fp16 = matmul(transpose_x = var_2175_transpose_x_0, transpose_y = var_2175_transpose_y_0, x = var_2174_cast_fp16, y = state_127_cast_fp16)[name = string("op_2175_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_127_cast_fp16 = sub(x = var_2172_cast_fp16, y = var_2175_cast_fp16)[name = string("v_new_127_cast_fp16")];
+            tensor<int32, [4]> var_2178_begin_0 = const()[name = string("op_2178_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_2178_end_0 = const()[name = string("op_2178_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_2178_end_mask_0 = const()[name = string("op_2178_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2178_squeeze_mask_0 = const()[name = string("op_2178_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2178_cast_fp16 = slice_by_index(begin = var_2178_begin_0, end = var_2178_end_0, end_mask = var_2178_end_mask_0, squeeze_mask = var_2178_squeeze_mask_0, x = q_dec_3_cast_fp16)[name = string("op_2178_cast_fp16")];
+            bool var_2179_transpose_x_0 = const()[name = string("op_2179_transpose_x_0"), val = bool(false)];
+            bool var_2179_transpose_y_0 = const()[name = string("op_2179_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2179_cast_fp16 = matmul(transpose_x = var_2179_transpose_x_0, transpose_y = var_2179_transpose_y_0, x = var_2178_cast_fp16, y = state_127_cast_fp16)[name = string("op_2179_cast_fp16")];
+            tensor<int32, [4]> var_2181_begin_0 = const()[name = string("op_2181_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_2181_end_0 = const()[name = string("op_2181_end_0"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<bool, [4]> var_2181_end_mask_0 = const()[name = string("op_2181_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2181_squeeze_mask_0 = const()[name = string("op_2181_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2181_cast_fp16 = slice_by_index(begin = var_2181_begin_0, end = var_2181_end_0, end_mask = var_2181_end_mask_0, squeeze_mask = var_2181_squeeze_mask_0, x = attn_intra_3_cast_fp16)[name = string("op_2181_cast_fp16")];
+            bool var_2182_transpose_x_0 = const()[name = string("op_2182_transpose_x_0"), val = bool(false)];
+            bool var_2182_transpose_y_0 = const()[name = string("op_2182_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2182_cast_fp16 = matmul(transpose_x = var_2182_transpose_x_0, transpose_y = var_2182_transpose_y_0, x = var_2181_cast_fp16, y = v_new_127_cast_fp16)[name = string("op_2182_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2183_cast_fp16 = add(x = var_2179_cast_fp16, y = var_2182_cast_fp16)[name = string("op_2183_cast_fp16")];
+            int32 var_2185_axis_0 = const()[name = string("op_2185_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 32, 64, 128]> var_2185_cast_fp16 = stack(axis = var_2185_axis_0, values = (var_1532_cast_fp16, var_1553_cast_fp16, var_1574_cast_fp16, var_1595_cast_fp16, var_1616_cast_fp16, var_1637_cast_fp16, var_1658_cast_fp16, var_1679_cast_fp16, var_1700_cast_fp16, var_1721_cast_fp16, var_1742_cast_fp16, var_1763_cast_fp16, var_1784_cast_fp16, var_1805_cast_fp16, var_1826_cast_fp16, var_1847_cast_fp16, var_1868_cast_fp16, var_1889_cast_fp16, var_1910_cast_fp16, var_1931_cast_fp16, var_1952_cast_fp16, var_1973_cast_fp16, var_1994_cast_fp16, var_2015_cast_fp16, var_2036_cast_fp16, var_2057_cast_fp16, var_2078_cast_fp16, var_2099_cast_fp16, var_2120_cast_fp16, var_2141_cast_fp16, var_2162_cast_fp16, var_2183_cast_fp16))[name = string("op_2185_cast_fp16")];
+            tensor<int32, [3]> var_2186 = const()[name = string("op_2186"), val = tensor<int32, [3]>([32, 2048, 128])];
+            tensor<fp16, [32, 2048, 128]> var_2187_cast_fp16 = reshape(shape = var_2186, x = var_2185_cast_fp16)[name = string("op_2187_cast_fp16")];
+            tensor<int32, [3]> var_2188 = const()[name = string("op_2188"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_23_begin_0 = const()[name = string("input_23_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_23_end_0 = const()[name = string("input_23_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> input_23_end_mask_0 = const()[name = string("input_23_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_23_squeeze_mask_0 = const()[name = string("input_23_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> input_23_cast_fp16 = slice_by_index(begin = input_23_begin_0, end = input_23_end_0, end_mask = input_23_end_mask_0, squeeze_mask = input_23_squeeze_mask_0, x = z_3_cast_fp16)[name = string("input_23_cast_fp16")];
+            fp16 var_1168_promoted_1_to_fp16 = const()[name = string("op_1168_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [2048, 32, 128]> x_43_cast_fp16 = transpose(perm = var_2188, x = var_2187_cast_fp16)[name = string("transpose_40")];
+            tensor<fp16, [2048, 32, 128]> var_2192_cast_fp16 = pow(x = x_43_cast_fp16, y = var_1168_promoted_1_to_fp16)[name = string("op_2192_cast_fp16")];
+            tensor<int32, [1]> var_2194_axes_0 = const()[name = string("op_2194_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2194_keep_dims_0 = const()[name = string("op_2194_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [2048, 32, 1]> var_2194_cast_fp16 = reduce_mean(axes = var_2194_axes_0, keep_dims = var_2194_keep_dims_0, x = var_2192_cast_fp16)[name = string("op_2194_cast_fp16")];
+            fp16 var_2195_to_fp16 = const()[name = string("op_2195_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [2048, 32, 1]> var_2196_cast_fp16 = add(x = var_2194_cast_fp16, y = var_2195_to_fp16)[name = string("op_2196_cast_fp16")];
+            fp32 var_2197_epsilon_0 = const()[name = string("op_2197_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [2048, 32, 1]> var_2197_cast_fp16 = rsqrt(epsilon = var_2197_epsilon_0, x = var_2196_cast_fp16)[name = string("op_2197_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> x_45_cast_fp16 = mul(x = x_43_cast_fp16, y = var_2197_cast_fp16)[name = string("x_45_cast_fp16")];
+            tensor<fp16, [128]> layers_1_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_1_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295080448)))];
+            tensor<fp16, [2048, 32, 128]> var_2199_cast_fp16 = mul(x = layers_1_linear_attn_norm_weight_to_fp16, y = x_45_cast_fp16)[name = string("op_2199_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> var_2200_cast_fp16 = silu(x = input_23_cast_fp16)[name = string("op_2200_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> core_3_cast_fp16 = mul(x = var_2199_cast_fp16, y = var_2200_cast_fp16)[name = string("core_3_cast_fp16")];
+            tensor<int32, [3]> var_2202 = const()[name = string("op_2202"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> input_25_cast_fp16 = reshape(shape = var_2202, x = core_3_cast_fp16)[name = string("input_25_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_1_linear_attn_out_proj_weight_to_fp16 = const()[name = string("layers_1_linear_attn_out_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(295080768)))];
+            tensor<fp16, [1, 2048, 2560]> linear_12_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_1_linear_attn_out_proj_weight_to_fp16, x = input_25_cast_fp16)[name = string("linear_12_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_47_cast_fp16 = add(x = x_25_cast_fp16, y = linear_12_cast_fp16)[name = string("x_47_cast_fp16")];
+            fp16 var_1168_promoted_2_to_fp16 = const()[name = string("op_1168_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_2208_cast_fp16 = pow(x = x_47_cast_fp16, y = var_1168_promoted_2_to_fp16)[name = string("op_2208_cast_fp16")];
+            tensor<int32, [1]> var_2210_axes_0 = const()[name = string("op_2210_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2210_keep_dims_0 = const()[name = string("op_2210_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_2210_cast_fp16 = reduce_mean(axes = var_2210_axes_0, keep_dims = var_2210_keep_dims_0, x = var_2208_cast_fp16)[name = string("op_2210_cast_fp16")];
+            fp16 var_2211_to_fp16 = const()[name = string("op_2211_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_2212_cast_fp16 = add(x = var_2210_cast_fp16, y = var_2211_to_fp16)[name = string("op_2212_cast_fp16")];
+            fp32 var_2213_epsilon_0 = const()[name = string("op_2213_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_2213_cast_fp16 = rsqrt(epsilon = var_2213_epsilon_0, x = var_2212_cast_fp16)[name = string("op_2213_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_49_cast_fp16 = mul(x = x_47_cast_fp16, y = var_2213_cast_fp16)[name = string("x_49_cast_fp16")];
+            tensor<fp16, [2560]> var_2216_to_fp16 = const()[name = string("op_2216_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(316052352)))];
+            tensor<fp16, [1, 2048, 2560]> input_27_cast_fp16 = mul(x = x_49_cast_fp16, y = var_2216_to_fp16)[name = string("input_27_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_1_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_1_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(316057536)))];
+            tensor<fp16, [1, 2048, 9216]> linear_13_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_1_mlp_gate_proj_weight_to_fp16, x = input_27_cast_fp16)[name = string("linear_13_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_2223_cast_fp16 = silu(x = linear_13_cast_fp16)[name = string("op_2223_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_1_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_1_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(363243520)))];
+            tensor<fp16, [1, 2048, 9216]> linear_14_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_1_mlp_up_proj_weight_to_fp16, x = input_27_cast_fp16)[name = string("linear_14_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_31_cast_fp16 = mul(x = var_2223_cast_fp16, y = linear_14_cast_fp16)[name = string("input_31_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_1_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_1_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(410429504)))];
+            tensor<fp16, [1, 2048, 2560]> linear_15_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_1_mlp_down_proj_weight_to_fp16, x = input_31_cast_fp16)[name = string("linear_15_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_51_cast_fp16 = add(x = x_47_cast_fp16, y = linear_15_cast_fp16)[name = string("x_51_cast_fp16")];
+            int32 var_2243 = const()[name = string("op_2243"), val = int32(-2)];
+            int32 var_2272 = const()[name = string("op_2272"), val = int32(-1)];
+            fp16 var_2271_promoted_to_fp16 = const()[name = string("op_2271_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_2281_cast_fp16 = pow(x = x_51_cast_fp16, y = var_2271_promoted_to_fp16)[name = string("op_2281_cast_fp16")];
+            tensor<int32, [1]> var_2283_axes_0 = const()[name = string("op_2283_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2283_keep_dims_0 = const()[name = string("op_2283_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_2283_cast_fp16 = reduce_mean(axes = var_2283_axes_0, keep_dims = var_2283_keep_dims_0, x = var_2281_cast_fp16)[name = string("op_2283_cast_fp16")];
+            fp16 var_2284_to_fp16 = const()[name = string("op_2284_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_2285_cast_fp16 = add(x = var_2283_cast_fp16, y = var_2284_to_fp16)[name = string("op_2285_cast_fp16")];
+            fp32 var_2286_epsilon_0 = const()[name = string("op_2286_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_2286_cast_fp16 = rsqrt(epsilon = var_2286_epsilon_0, x = var_2285_cast_fp16)[name = string("op_2286_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_53_cast_fp16 = mul(x = x_51_cast_fp16, y = var_2286_cast_fp16)[name = string("x_53_cast_fp16")];
+            tensor<fp16, [2560]> var_2289_to_fp16 = const()[name = string("op_2289_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(457615488)))];
+            tensor<fp16, [1, 2048, 2560]> x_55_cast_fp16 = mul(x = x_53_cast_fp16, y = var_2289_to_fp16)[name = string("x_55_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_2_linear_attn_in_proj_qkv_weight_to_fp16 = const()[name = string("layers_2_linear_attn_in_proj_qkv_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(457620672)))];
+            tensor<fp16, [1, 2048, 8192]> linear_16_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_qkv_weight_to_fp16, x = x_55_cast_fp16)[name = string("linear_16_cast_fp16")];
+            tensor<int32, [3]> input_33_perm_0 = const()[name = string("input_33_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_35_pad_0 = const()[name = string("input_35_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_35_mode_0 = const()[name = string("input_35_mode_0"), val = string("constant")];
+            fp16 const_25_to_fp16 = const()[name = string("const_25_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 2048]> input_33_cast_fp16 = transpose(perm = input_33_perm_0, x = linear_16_cast_fp16)[name = string("transpose_39")];
+            tensor<fp16, [1, 8192, 2051]> input_35_cast_fp16 = pad(constant_val = const_25_to_fp16, mode = input_35_mode_0, pad = input_35_pad_0, x = input_33_cast_fp16)[name = string("input_35_cast_fp16")];
+            string input_37_pad_type_0 = const()[name = string("input_37_pad_type_0"), val = string("valid")];
+            int32 input_37_groups_0 = const()[name = string("input_37_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_37_strides_0 = const()[name = string("input_37_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_37_pad_0 = const()[name = string("input_37_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_37_dilations_0 = const()[name = string("input_37_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_2_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_2_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(499563776)))];
+            tensor<fp16, [1, 8192, 2048]> input_37_cast_fp16 = conv(dilations = input_37_dilations_0, groups = input_37_groups_0, pad = input_37_pad_0, pad_type = input_37_pad_type_0, strides = input_37_strides_0, weight = layers_2_linear_attn_conv1d_weight_to_fp16, x = input_35_cast_fp16)[name = string("input_37_cast_fp16")];
+            tensor<fp16, [1, 8192, 2048]> var_2324_cast_fp16 = silu(x = input_37_cast_fp16)[name = string("op_2324_cast_fp16")];
+            tensor<int32, [3]> qkv_5_perm_0 = const()[name = string("qkv_5_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_2326_begin_0 = const()[name = string("op_2326_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_2326_end_0 = const()[name = string("op_2326_end_0"), val = tensor<int32, [3]>([1, 2048, 2048])];
+            tensor<bool, [3]> var_2326_end_mask_0 = const()[name = string("op_2326_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 8192]> qkv_5_cast_fp16 = transpose(perm = qkv_5_perm_0, x = var_2324_cast_fp16)[name = string("transpose_38")];
+            tensor<fp16, [1, 2048, 2048]> var_2326_cast_fp16 = slice_by_index(begin = var_2326_begin_0, end = var_2326_end_0, end_mask = var_2326_end_mask_0, x = qkv_5_cast_fp16)[name = string("op_2326_cast_fp16")];
+            tensor<int32, [4]> var_2327 = const()[name = string("op_2327"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> q_21_cast_fp16 = reshape(shape = var_2327, x = var_2326_cast_fp16)[name = string("q_21_cast_fp16")];
+            tensor<int32, [3]> var_2329_begin_0 = const()[name = string("op_2329_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_2329_end_0 = const()[name = string("op_2329_end_0"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<bool, [3]> var_2329_end_mask_0 = const()[name = string("op_2329_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 2048]> var_2329_cast_fp16 = slice_by_index(begin = var_2329_begin_0, end = var_2329_end_0, end_mask = var_2329_end_mask_0, x = qkv_5_cast_fp16)[name = string("op_2329_cast_fp16")];
+            tensor<int32, [4]> var_2330 = const()[name = string("op_2330"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> k_17_cast_fp16 = reshape(shape = var_2330, x = var_2329_cast_fp16)[name = string("k_17_cast_fp16")];
+            tensor<int32, [3]> var_2332_begin_0 = const()[name = string("op_2332_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_2332_end_0 = const()[name = string("op_2332_end_0"), val = tensor<int32, [3]>([1, 2048, 8192])];
+            tensor<bool, [3]> var_2332_end_mask_0 = const()[name = string("op_2332_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 2048, 4096]> var_2332_cast_fp16 = slice_by_index(begin = var_2332_begin_0, end = var_2332_end_0, end_mask = var_2332_end_mask_0, x = qkv_5_cast_fp16)[name = string("op_2332_cast_fp16")];
+            tensor<int32, [4]> var_2333 = const()[name = string("op_2333"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> v_9_cast_fp16 = reshape(shape = var_2333, x = var_2332_cast_fp16)[name = string("v_9_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_2_linear_attn_in_proj_z_weight_to_fp16 = const()[name = string("layers_2_linear_attn_in_proj_z_weight_to_fp16"), val = tensor<fp16, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(499629376)))];
+            tensor<fp16, [1, 2048, 4096]> linear_17_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_z_weight_to_fp16, x = x_55_cast_fp16)[name = string("linear_17_cast_fp16")];
+            tensor<int32, [4]> var_2337 = const()[name = string("op_2337"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> z_5_cast_fp16 = reshape(shape = var_2337, x = linear_17_cast_fp16)[name = string("z_5_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_2_linear_attn_in_proj_b_weight_to_fp16 = const()[name = string("layers_2_linear_attn_in_proj_b_weight_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520600960)))];
+            tensor<fp16, [32]> linear_18_bias_0_to_fp16 = const()[name = string("linear_18_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520764864)))];
+            tensor<fp16, [1, 2048, 32]> linear_18_cast_fp16 = linear(bias = linear_18_bias_0_to_fp16, weight = layers_2_linear_attn_in_proj_b_weight_to_fp16, x = x_55_cast_fp16)[name = string("linear_18_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> beta_9_cast_fp16 = sigmoid(x = linear_18_cast_fp16)[name = string("beta_9_cast_fp16")];
+            tensor<fp16, [32, 2560]> var_2346_weight_0_to_fp16 = const()[name = string("op_2346_weight_0_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520764992)))];
+            tensor<fp16, [32]> var_2346_bias_0_to_fp16 = const()[name = string("op_2346_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520928896)))];
+            tensor<fp16, [1, 2048, 32]> var_2346_cast_fp16 = linear(bias = var_2346_bias_0_to_fp16, weight = var_2346_weight_0_to_fp16, x = x_55_cast_fp16)[name = string("op_2346_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> var_2347_cast_fp16 = softplus(x = var_2346_cast_fp16)[name = string("op_2347_cast_fp16")];
+            tensor<fp16, [32]> var_2343_to_fp16 = const()[name = string("op_2343_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520929024)))];
+            tensor<fp16, [1, 2048, 32]> g_9_cast_fp16 = mul(x = var_2343_to_fp16, y = var_2347_cast_fp16)[name = string("g_9_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_2349_cast_fp16 = mul(x = q_21_cast_fp16, y = q_21_cast_fp16)[name = string("op_2349_cast_fp16")];
+            tensor<int32, [1]> var_2351_axes_0 = const()[name = string("op_2351_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2351_keep_dims_0 = const()[name = string("op_2351_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_2351_cast_fp16 = reduce_sum(axes = var_2351_axes_0, keep_dims = var_2351_keep_dims_0, x = var_2349_cast_fp16)[name = string("op_2351_cast_fp16")];
+            fp16 var_2352_to_fp16 = const()[name = string("op_2352_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_2353_cast_fp16 = add(x = var_2351_cast_fp16, y = var_2352_to_fp16)[name = string("op_2353_cast_fp16")];
+            fp32 var_2354_epsilon_0 = const()[name = string("op_2354_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_2354_cast_fp16 = rsqrt(epsilon = var_2354_epsilon_0, x = var_2353_cast_fp16)[name = string("op_2354_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> q_23_cast_fp16 = mul(x = q_21_cast_fp16, y = var_2354_cast_fp16)[name = string("q_23_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_2356_cast_fp16 = mul(x = k_17_cast_fp16, y = k_17_cast_fp16)[name = string("op_2356_cast_fp16")];
+            tensor<int32, [1]> var_2358_axes_0 = const()[name = string("op_2358_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_2358_keep_dims_0 = const()[name = string("op_2358_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_2358_cast_fp16 = reduce_sum(axes = var_2358_axes_0, keep_dims = var_2358_keep_dims_0, x = var_2356_cast_fp16)[name = string("op_2358_cast_fp16")];
+            fp16 var_2359_to_fp16 = const()[name = string("op_2359_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_2360_cast_fp16 = add(x = var_2358_cast_fp16, y = var_2359_to_fp16)[name = string("op_2360_cast_fp16")];
+            fp32 var_2361_epsilon_0 = const()[name = string("op_2361_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_2361_cast_fp16 = rsqrt(epsilon = var_2361_epsilon_0, x = var_2360_cast_fp16)[name = string("op_2361_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> k_19_cast_fp16 = mul(x = k_17_cast_fp16, y = var_2361_cast_fp16)[name = string("k_19_cast_fp16")];
+            tensor<int32, [1]> var_2366_axes_0 = const()[name = string("op_2366_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_2366_cast_fp16 = expand_dims(axes = var_2366_axes_0, x = q_23_cast_fp16)[name = string("op_2366_cast_fp16")];
+            tensor<int32, [5]> var_2368_reps_0 = const()[name = string("op_2368_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_2368_cast_fp16 = tile(reps = var_2368_reps_0, x = var_2366_cast_fp16)[name = string("op_2368_cast_fp16")];
+            tensor<int32, [4]> var_2369 = const()[name = string("op_2369"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> q_25_cast_fp16 = reshape(shape = var_2369, x = var_2368_cast_fp16)[name = string("q_25_cast_fp16")];
+            tensor<int32, [1]> var_2374_axes_0 = const()[name = string("op_2374_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_2374_cast_fp16 = expand_dims(axes = var_2374_axes_0, x = k_19_cast_fp16)[name = string("op_2374_cast_fp16")];
+            tensor<int32, [5]> var_2376_reps_0 = const()[name = string("op_2376_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_2376_cast_fp16 = tile(reps = var_2376_reps_0, x = var_2374_cast_fp16)[name = string("op_2376_cast_fp16")];
+            tensor<int32, [4]> var_2377 = const()[name = string("op_2377"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> k_21_cast_fp16 = reshape(shape = var_2377, x = var_2376_cast_fp16)[name = string("k_21_cast_fp16")];
+            fp16 var_2379_to_fp16 = const()[name = string("op_2379_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 2048, 32, 128]> q_27_cast_fp16 = mul(x = q_25_cast_fp16, y = var_2379_to_fp16)[name = string("q_27_cast_fp16")];
+            tensor<int32, [4]> var_2381_begin_0 = const()[name = string("op_2381_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2381_end_0 = const()[name = string("op_2381_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_2381_end_mask_0 = const()[name = string("op_2381_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_2381_squeeze_mask_0 = const()[name = string("op_2381_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_2381_cast_fp16 = slice_by_index(begin = var_2381_begin_0, end = var_2381_end_0, end_mask = var_2381_end_mask_0, squeeze_mask = var_2381_squeeze_mask_0, x = q_27_cast_fp16)[name = string("op_2381_cast_fp16")];
+            tensor<int32, [3]> var_2382 = const()[name = string("op_2382"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_2384 = const()[name = string("op_2384"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_2383_cast_fp16 = transpose(perm = var_2382, x = var_2381_cast_fp16)[name = string("transpose_37")];
+            tensor<fp16, [32, 32, 64, 128]> q_29_cast_fp16 = reshape(shape = var_2384, x = var_2383_cast_fp16)[name = string("q_29_cast_fp16")];
+            tensor<int32, [4]> var_2386_begin_0 = const()[name = string("op_2386_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2386_end_0 = const()[name = string("op_2386_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_2386_end_mask_0 = const()[name = string("op_2386_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_2386_squeeze_mask_0 = const()[name = string("op_2386_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_2386_cast_fp16 = slice_by_index(begin = var_2386_begin_0, end = var_2386_end_0, end_mask = var_2386_end_mask_0, squeeze_mask = var_2386_squeeze_mask_0, x = k_21_cast_fp16)[name = string("op_2386_cast_fp16")];
+            tensor<int32, [3]> var_2387 = const()[name = string("op_2387"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_2389 = const()[name = string("op_2389"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_2388_cast_fp16 = transpose(perm = var_2387, x = var_2386_cast_fp16)[name = string("transpose_36")];
+            tensor<fp16, [32, 32, 64, 128]> k_23_cast_fp16 = reshape(shape = var_2389, x = var_2388_cast_fp16)[name = string("k_23_cast_fp16")];
+            tensor<int32, [4]> var_2391_begin_0 = const()[name = string("op_2391_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2391_end_0 = const()[name = string("op_2391_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_2391_end_mask_0 = const()[name = string("op_2391_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_2391_squeeze_mask_0 = const()[name = string("op_2391_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_2391_cast_fp16 = slice_by_index(begin = var_2391_begin_0, end = var_2391_end_0, end_mask = var_2391_end_mask_0, squeeze_mask = var_2391_squeeze_mask_0, x = v_9_cast_fp16)[name = string("op_2391_cast_fp16")];
+            tensor<int32, [3]> var_2392 = const()[name = string("op_2392"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_2394 = const()[name = string("op_2394"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_2393_cast_fp16 = transpose(perm = var_2392, x = var_2391_cast_fp16)[name = string("transpose_35")];
+            tensor<fp16, [32, 32, 64, 128]> v_11_cast_fp16 = reshape(shape = var_2394, x = var_2393_cast_fp16)[name = string("v_11_cast_fp16")];
+            tensor<int32, [3]> var_2396_begin_0 = const()[name = string("op_2396_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_2396_end_0 = const()[name = string("op_2396_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_2396_end_mask_0 = const()[name = string("op_2396_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_2396_squeeze_mask_0 = const()[name = string("op_2396_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_2396_cast_fp16 = slice_by_index(begin = var_2396_begin_0, end = var_2396_end_0, end_mask = var_2396_end_mask_0, squeeze_mask = var_2396_squeeze_mask_0, x = beta_9_cast_fp16)[name = string("op_2396_cast_fp16")];
+            tensor<int32, [2]> var_2397_perm_0 = const()[name = string("op_2397_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_2398 = const()[name = string("op_2398"), val = tensor<int32, [4]>([32, 32, 64, 1])];
+            tensor<fp16, [32, 2048]> var_2397_cast_fp16 = transpose(perm = var_2397_perm_0, x = var_2396_cast_fp16)[name = string("transpose_34")];
+            tensor<fp16, [32, 32, 64, 1]> beta_11_cast_fp16 = reshape(shape = var_2398, x = var_2397_cast_fp16)[name = string("beta_11_cast_fp16")];
+            tensor<int32, [3]> var_2400_begin_0 = const()[name = string("op_2400_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_2400_end_0 = const()[name = string("op_2400_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_2400_end_mask_0 = const()[name = string("op_2400_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_2400_squeeze_mask_0 = const()[name = string("op_2400_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_2400_cast_fp16 = slice_by_index(begin = var_2400_begin_0, end = var_2400_end_0, end_mask = var_2400_end_mask_0, squeeze_mask = var_2400_squeeze_mask_0, x = g_9_cast_fp16)[name = string("op_2400_cast_fp16")];
+            tensor<int32, [2]> var_2401_perm_0 = const()[name = string("op_2401_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_2402 = const()[name = string("op_2402"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<fp16, [32, 2048]> var_2401_cast_fp16 = transpose(perm = var_2401_perm_0, x = var_2400_cast_fp16)[name = string("transpose_33")];
+            tensor<fp16, [32, 32, 64]> g_11_cast_fp16 = reshape(shape = var_2402, x = var_2401_cast_fp16)[name = string("g_11_cast_fp16")];
+            tensor<int32, [1]> g_col_5_axes_0 = const()[name = string("g_col_5_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64, 1]> g_col_5_cast_fp16 = expand_dims(axes = g_col_5_axes_0, x = g_11_cast_fp16)[name = string("g_col_5_cast_fp16")];
+            bool var_2405_transpose_x_0 = const()[name = string("op_2405_transpose_x_0"), val = bool(false)];
+            bool var_2405_transpose_y_0 = const()[name = string("op_2405_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_2405_cast_fp16 = matmul(transpose_x = var_2405_transpose_x_0, transpose_y = var_2405_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_5_cast_fp16)[name = string("op_2405_cast_fp16")];
+            tensor<int32, [1]> cum_5_axes_0 = const()[name = string("cum_5_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> cum_5_cast_fp16 = squeeze(axes = cum_5_axes_0, x = var_2405_cast_fp16)[name = string("cum_5_cast_fp16")];
+            bool var_2407_transpose_x_0 = const()[name = string("op_2407_transpose_x_0"), val = bool(false)];
+            bool var_2407_transpose_y_0 = const()[name = string("op_2407_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_2407_cast_fp16 = matmul(transpose_x = var_2407_transpose_x_0, transpose_y = var_2407_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_5_cast_fp16)[name = string("op_2407_cast_fp16")];
+            tensor<int32, [1]> to_end_5_axes_0 = const()[name = string("to_end_5_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> to_end_5_cast_fp16 = squeeze(axes = to_end_5_axes_0, x = var_2407_cast_fp16)[name = string("to_end_5_cast_fp16")];
+            bool var_2409_transpose_x_0 = const()[name = string("op_2409_transpose_x_0"), val = bool(false)];
+            bool var_2409_transpose_y_0 = const()[name = string("op_2409_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 4096, 1]> var_2409_cast_fp16 = matmul(transpose_x = var_2409_transpose_x_0, transpose_y = var_2409_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_5_cast_fp16)[name = string("op_2409_cast_fp16")];
+            tensor<int32, [4]> var_2410 = const()[name = string("op_2410"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> pair_29_cast_fp16 = reshape(shape = var_2410, x = var_2409_cast_fp16)[name = string("pair_29_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_2412_cast_fp16 = exp(x = pair_29_cast_fp16)[name = string("op_2412_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> pair_decay_5_cast_fp16 = mul(x = var_2412_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_5_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_beta_5_cast_fp16 = mul(x = k_23_cast_fp16, y = beta_11_cast_fp16)[name = string("k_beta_5_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> v_beta_5_cast_fp16 = mul(x = v_11_cast_fp16, y = beta_11_cast_fp16)[name = string("v_beta_5_cast_fp16")];
+            bool var_2417_transpose_x_1 = const()[name = string("op_2417_transpose_x_1"), val = bool(false)];
+            bool var_2417_transpose_y_1 = const()[name = string("op_2417_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_2417_cast_fp16 = matmul(transpose_x = var_2417_transpose_x_1, transpose_y = var_2417_transpose_y_1, x = k_beta_5_cast_fp16, y = k_23_cast_fp16)[name = string("op_2417_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> kkt_5_cast_fp16 = mul(x = var_2417_cast_fp16, y = pair_decay_5_cast_fp16)[name = string("kkt_5_cast_fp16")];
+            bool var_2420_transpose_x_1 = const()[name = string("op_2420_transpose_x_1"), val = bool(false)];
+            bool var_2420_transpose_y_1 = const()[name = string("op_2420_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_2420_cast_fp16 = matmul(transpose_x = var_2420_transpose_x_1, transpose_y = var_2420_transpose_y_1, x = q_29_cast_fp16, y = k_23_cast_fp16)[name = string("op_2420_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> attn_intra_5_cast_fp16 = mul(x = var_2420_cast_fp16, y = pair_decay_5_cast_fp16)[name = string("attn_intra_5_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_2422_cast_fp16 = mul(x = kkt_5_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_2422_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> t_9_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_2422_cast_fp16)[name = string("t_9_cast_fp16")];
+            tensor<int32, [3]> var_2426 = const()[name = string("op_2426"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [1024, 64, 64]> t_11_cast_fp16 = reshape(shape = var_2426, x = t_9_cast_fp16)[name = string("t_11_cast_fp16")];
+            tensor<int32, [5]> var_2431 = const()[name = string("op_2431"), val = tensor<int32, [5]>([1024, 32, 2, 32, 2])];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_2432_cast_fp16 = reshape(shape = var_2431, x = t_11_cast_fp16)[name = string("op_2432_cast_fp16")];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_2437_cast_fp16 = mul(x = var_2432_cast_fp16, y = var_230_to_fp16)[name = string("op_2437_cast_fp16")];
+            tensor<int32, [1]> blocks_25_axes_0 = const()[name = string("blocks_25_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_25_keep_dims_0 = const()[name = string("blocks_25_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> blocks_25_cast_fp16 = reduce_sum(axes = blocks_25_axes_0, keep_dims = blocks_25_keep_dims_0, x = var_2437_cast_fp16)[name = string("blocks_25_cast_fp16")];
+            tensor<int32, [4]> var_2440_begin_0 = const()[name = string("op_2440_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_2440_end_0 = const()[name = string("op_2440_end_0"), val = tensor<int32, [4]>([1024, 32, 2, 2])];
+            tensor<bool, [4]> var_2440_end_mask_0 = const()[name = string("op_2440_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 32, 1, 2]> var_2440_cast_fp16 = slice_by_index(begin = var_2440_begin_0, end = var_2440_end_0, end_mask = var_2440_end_mask_0, x = blocks_25_cast_fp16)[name = string("op_2440_cast_fp16")];
+            tensor<int32, [4]> x_57_begin_0 = const()[name = string("x_57_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_57_end_0 = const()[name = string("x_57_end_0"), val = tensor<int32, [4]>([1024, 32, 1, 1])];
+            tensor<bool, [4]> x_57_end_mask_0 = const()[name = string("x_57_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 32, 1, 1]> x_57_cast_fp16 = slice_by_index(begin = x_57_begin_0, end = x_57_end_0, end_mask = x_57_end_mask_0, x = var_2440_cast_fp16)[name = string("x_57_cast_fp16")];
+            bool var_2450_transpose_x_0 = const()[name = string("op_2450_transpose_x_0"), val = bool(false)];
+            bool var_2450_transpose_y_0 = const()[name = string("op_2450_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_2450_cast_fp16 = matmul(transpose_x = var_2450_transpose_x_0, transpose_y = var_2450_transpose_y_0, x = x_57_cast_fp16, y = var_238_to_fp16)[name = string("op_2450_cast_fp16")];
+            bool var_2451_transpose_x_0 = const()[name = string("op_2451_transpose_x_0"), val = bool(false)];
+            bool var_2451_transpose_y_0 = const()[name = string("op_2451_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_2451_cast_fp16 = matmul(transpose_x = var_2451_transpose_x_0, transpose_y = var_2451_transpose_y_0, x = var_238_to_fp16, y = var_2450_cast_fp16)[name = string("op_2451_cast_fp16")];
+            fp16 const_30_promoted_to_fp16 = const()[name = string("const_30_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 32, 1, 1]> lower_25_cast_fp16 = mul(x = var_2451_cast_fp16, y = const_30_promoted_to_fp16)[name = string("lower_25_cast_fp16")];
+            bool bottom_25_interleave_0 = const()[name = string("bottom_25_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> bottom_25_cast_fp16 = concat(axis = var_2272, interleave = bottom_25_interleave_0, values = (lower_25_cast_fp16, var_238_to_fp16))[name = string("bottom_25_cast_fp16")];
+            bool inv_35_interleave_0 = const()[name = string("inv_35_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> inv_35_cast_fp16 = concat(axis = var_2243, interleave = inv_35_interleave_0, values = (top_1_to_fp16, bottom_25_cast_fp16))[name = string("inv_35_cast_fp16")];
+            tensor<int32, [5]> var_2460 = const()[name = string("op_2460"), val = tensor<int32, [5]>([1024, 16, 4, 16, 4])];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_2461_cast_fp16 = reshape(shape = var_2460, x = t_11_cast_fp16)[name = string("op_2461_cast_fp16")];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_2466_cast_fp16 = mul(x = var_2461_cast_fp16, y = var_259_to_fp16)[name = string("op_2466_cast_fp16")];
+            tensor<int32, [1]> blocks_27_axes_0 = const()[name = string("blocks_27_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_27_keep_dims_0 = const()[name = string("blocks_27_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> blocks_27_cast_fp16 = reduce_sum(axes = blocks_27_axes_0, keep_dims = blocks_27_keep_dims_0, x = var_2466_cast_fp16)[name = string("blocks_27_cast_fp16")];
+            tensor<int32, [4]> var_2469_begin_0 = const()[name = string("op_2469_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_2469_end_0 = const()[name = string("op_2469_end_0"), val = tensor<int32, [4]>([1024, 16, 4, 4])];
+            tensor<bool, [4]> var_2469_end_mask_0 = const()[name = string("op_2469_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 16, 2, 4]> var_2469_cast_fp16 = slice_by_index(begin = var_2469_begin_0, end = var_2469_end_0, end_mask = var_2469_end_mask_0, x = blocks_27_cast_fp16)[name = string("op_2469_cast_fp16")];
+            tensor<int32, [4]> x_59_begin_0 = const()[name = string("x_59_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_59_end_0 = const()[name = string("x_59_end_0"), val = tensor<int32, [4]>([1024, 16, 2, 2])];
+            tensor<bool, [4]> x_59_end_mask_0 = const()[name = string("x_59_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 16, 2, 2]> x_59_cast_fp16 = slice_by_index(begin = x_59_begin_0, end = x_59_end_0, end_mask = x_59_end_mask_0, x = var_2469_cast_fp16)[name = string("x_59_cast_fp16")];
+            tensor<int32, [5]> var_2471 = const()[name = string("op_2471"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<fp16, [1024, 16, 2, 2, 2]> pair_33_cast_fp16 = reshape(shape = var_2471, x = inv_35_cast_fp16)[name = string("pair_33_cast_fp16")];
+            tensor<int32, [5]> var_2473_begin_0 = const()[name = string("op_2473_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2473_end_0 = const()[name = string("op_2473_end_0"), val = tensor<int32, [5]>([1024, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_2473_end_mask_0 = const()[name = string("op_2473_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2473_squeeze_mask_0 = const()[name = string("op_2473_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_2473_cast_fp16 = slice_by_index(begin = var_2473_begin_0, end = var_2473_end_0, end_mask = var_2473_end_mask_0, squeeze_mask = var_2473_squeeze_mask_0, x = pair_33_cast_fp16)[name = string("op_2473_cast_fp16")];
+            tensor<int32, [5]> var_2476_begin_0 = const()[name = string("op_2476_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2476_end_0 = const()[name = string("op_2476_end_0"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_2476_end_mask_0 = const()[name = string("op_2476_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2476_squeeze_mask_0 = const()[name = string("op_2476_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_2476_cast_fp16 = slice_by_index(begin = var_2476_begin_0, end = var_2476_end_0, end_mask = var_2476_end_mask_0, squeeze_mask = var_2476_squeeze_mask_0, x = pair_33_cast_fp16)[name = string("op_2476_cast_fp16")];
+            bool var_2479_transpose_x_0 = const()[name = string("op_2479_transpose_x_0"), val = bool(false)];
+            bool var_2479_transpose_y_0 = const()[name = string("op_2479_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_2479_cast_fp16 = matmul(transpose_x = var_2479_transpose_x_0, transpose_y = var_2479_transpose_y_0, x = x_59_cast_fp16, y = var_2473_cast_fp16)[name = string("op_2479_cast_fp16")];
+            bool var_2480_transpose_x_0 = const()[name = string("op_2480_transpose_x_0"), val = bool(false)];
+            bool var_2480_transpose_y_0 = const()[name = string("op_2480_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_2480_cast_fp16 = matmul(transpose_x = var_2480_transpose_x_0, transpose_y = var_2480_transpose_y_0, x = var_2476_cast_fp16, y = var_2479_cast_fp16)[name = string("op_2480_cast_fp16")];
+            fp16 const_31_promoted_to_fp16 = const()[name = string("const_31_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 16, 2, 2]> lower_27_cast_fp16 = mul(x = var_2480_cast_fp16, y = const_31_promoted_to_fp16)[name = string("lower_27_cast_fp16")];
+            bool top_27_interleave_0 = const()[name = string("top_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> top_27_cast_fp16 = concat(axis = var_2272, interleave = top_27_interleave_0, values = (var_2473_cast_fp16, var_276_to_fp16))[name = string("top_27_cast_fp16")];
+            bool bottom_27_interleave_0 = const()[name = string("bottom_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> bottom_27_cast_fp16 = concat(axis = var_2272, interleave = bottom_27_interleave_0, values = (lower_27_cast_fp16, var_2476_cast_fp16))[name = string("bottom_27_cast_fp16")];
+            bool inv_37_interleave_0 = const()[name = string("inv_37_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> inv_37_cast_fp16 = concat(axis = var_2243, interleave = inv_37_interleave_0, values = (top_27_cast_fp16, bottom_27_cast_fp16))[name = string("inv_37_cast_fp16")];
+            tensor<int32, [5]> var_2489 = const()[name = string("op_2489"), val = tensor<int32, [5]>([1024, 8, 8, 8, 8])];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_2490_cast_fp16 = reshape(shape = var_2489, x = t_11_cast_fp16)[name = string("op_2490_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_2494_to_fp16 = const()[name = string("op_2494_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520929152)))];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_2495_cast_fp16 = mul(x = var_2490_cast_fp16, y = var_2494_to_fp16)[name = string("op_2495_cast_fp16")];
+            tensor<int32, [1]> blocks_29_axes_0 = const()[name = string("blocks_29_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_29_keep_dims_0 = const()[name = string("blocks_29_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> blocks_29_cast_fp16 = reduce_sum(axes = blocks_29_axes_0, keep_dims = blocks_29_keep_dims_0, x = var_2495_cast_fp16)[name = string("blocks_29_cast_fp16")];
+            tensor<int32, [4]> var_2498_begin_0 = const()[name = string("op_2498_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_2498_end_0 = const()[name = string("op_2498_end_0"), val = tensor<int32, [4]>([1024, 8, 8, 8])];
+            tensor<bool, [4]> var_2498_end_mask_0 = const()[name = string("op_2498_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 8, 4, 8]> var_2498_cast_fp16 = slice_by_index(begin = var_2498_begin_0, end = var_2498_end_0, end_mask = var_2498_end_mask_0, x = blocks_29_cast_fp16)[name = string("op_2498_cast_fp16")];
+            tensor<int32, [4]> x_61_begin_0 = const()[name = string("x_61_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_61_end_0 = const()[name = string("x_61_end_0"), val = tensor<int32, [4]>([1024, 8, 4, 4])];
+            tensor<bool, [4]> x_61_end_mask_0 = const()[name = string("x_61_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 8, 4, 4]> x_61_cast_fp16 = slice_by_index(begin = x_61_begin_0, end = x_61_end_0, end_mask = x_61_end_mask_0, x = var_2498_cast_fp16)[name = string("x_61_cast_fp16")];
+            tensor<int32, [5]> var_2500 = const()[name = string("op_2500"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<fp16, [1024, 8, 2, 4, 4]> pair_35_cast_fp16 = reshape(shape = var_2500, x = inv_37_cast_fp16)[name = string("pair_35_cast_fp16")];
+            tensor<int32, [5]> var_2502_begin_0 = const()[name = string("op_2502_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2502_end_0 = const()[name = string("op_2502_end_0"), val = tensor<int32, [5]>([1024, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_2502_end_mask_0 = const()[name = string("op_2502_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2502_squeeze_mask_0 = const()[name = string("op_2502_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_2502_cast_fp16 = slice_by_index(begin = var_2502_begin_0, end = var_2502_end_0, end_mask = var_2502_end_mask_0, squeeze_mask = var_2502_squeeze_mask_0, x = pair_35_cast_fp16)[name = string("op_2502_cast_fp16")];
+            tensor<int32, [5]> var_2505_begin_0 = const()[name = string("op_2505_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2505_end_0 = const()[name = string("op_2505_end_0"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_2505_end_mask_0 = const()[name = string("op_2505_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2505_squeeze_mask_0 = const()[name = string("op_2505_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_2505_cast_fp16 = slice_by_index(begin = var_2505_begin_0, end = var_2505_end_0, end_mask = var_2505_end_mask_0, squeeze_mask = var_2505_squeeze_mask_0, x = pair_35_cast_fp16)[name = string("op_2505_cast_fp16")];
+            bool var_2508_transpose_x_0 = const()[name = string("op_2508_transpose_x_0"), val = bool(false)];
+            bool var_2508_transpose_y_0 = const()[name = string("op_2508_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_2508_cast_fp16 = matmul(transpose_x = var_2508_transpose_x_0, transpose_y = var_2508_transpose_y_0, x = x_61_cast_fp16, y = var_2502_cast_fp16)[name = string("op_2508_cast_fp16")];
+            bool var_2509_transpose_x_0 = const()[name = string("op_2509_transpose_x_0"), val = bool(false)];
+            bool var_2509_transpose_y_0 = const()[name = string("op_2509_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_2509_cast_fp16 = matmul(transpose_x = var_2509_transpose_x_0, transpose_y = var_2509_transpose_y_0, x = var_2505_cast_fp16, y = var_2508_cast_fp16)[name = string("op_2509_cast_fp16")];
+            fp16 const_32_promoted_to_fp16 = const()[name = string("const_32_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 8, 4, 4]> lower_29_cast_fp16 = mul(x = var_2509_cast_fp16, y = const_32_promoted_to_fp16)[name = string("lower_29_cast_fp16")];
+            bool top_29_interleave_0 = const()[name = string("top_29_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> top_29_cast_fp16 = concat(axis = var_2272, interleave = top_29_interleave_0, values = (var_2502_cast_fp16, var_305_to_fp16))[name = string("top_29_cast_fp16")];
+            bool bottom_29_interleave_0 = const()[name = string("bottom_29_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> bottom_29_cast_fp16 = concat(axis = var_2272, interleave = bottom_29_interleave_0, values = (lower_29_cast_fp16, var_2505_cast_fp16))[name = string("bottom_29_cast_fp16")];
+            bool inv_39_interleave_0 = const()[name = string("inv_39_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> inv_39_cast_fp16 = concat(axis = var_2243, interleave = inv_39_interleave_0, values = (top_29_cast_fp16, bottom_29_cast_fp16))[name = string("inv_39_cast_fp16")];
+            tensor<int32, [5]> var_2518 = const()[name = string("op_2518"), val = tensor<int32, [5]>([1024, 4, 16, 4, 16])];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_2519_cast_fp16 = reshape(shape = var_2518, x = t_11_cast_fp16)[name = string("op_2519_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_2523_to_fp16 = const()[name = string("op_2523_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520929344)))];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_2524_cast_fp16 = mul(x = var_2519_cast_fp16, y = var_2523_to_fp16)[name = string("op_2524_cast_fp16")];
+            tensor<int32, [1]> blocks_31_axes_0 = const()[name = string("blocks_31_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_31_keep_dims_0 = const()[name = string("blocks_31_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> blocks_31_cast_fp16 = reduce_sum(axes = blocks_31_axes_0, keep_dims = blocks_31_keep_dims_0, x = var_2524_cast_fp16)[name = string("blocks_31_cast_fp16")];
+            tensor<int32, [4]> var_2527_begin_0 = const()[name = string("op_2527_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_2527_end_0 = const()[name = string("op_2527_end_0"), val = tensor<int32, [4]>([1024, 4, 16, 16])];
+            tensor<bool, [4]> var_2527_end_mask_0 = const()[name = string("op_2527_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 4, 8, 16]> var_2527_cast_fp16 = slice_by_index(begin = var_2527_begin_0, end = var_2527_end_0, end_mask = var_2527_end_mask_0, x = blocks_31_cast_fp16)[name = string("op_2527_cast_fp16")];
+            tensor<int32, [4]> x_63_begin_0 = const()[name = string("x_63_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_63_end_0 = const()[name = string("x_63_end_0"), val = tensor<int32, [4]>([1024, 4, 8, 8])];
+            tensor<bool, [4]> x_63_end_mask_0 = const()[name = string("x_63_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 4, 8, 8]> x_63_cast_fp16 = slice_by_index(begin = x_63_begin_0, end = x_63_end_0, end_mask = x_63_end_mask_0, x = var_2527_cast_fp16)[name = string("x_63_cast_fp16")];
+            tensor<int32, [5]> var_2529 = const()[name = string("op_2529"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<fp16, [1024, 4, 2, 8, 8]> pair_37_cast_fp16 = reshape(shape = var_2529, x = inv_39_cast_fp16)[name = string("pair_37_cast_fp16")];
+            tensor<int32, [5]> var_2531_begin_0 = const()[name = string("op_2531_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2531_end_0 = const()[name = string("op_2531_end_0"), val = tensor<int32, [5]>([1024, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_2531_end_mask_0 = const()[name = string("op_2531_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2531_squeeze_mask_0 = const()[name = string("op_2531_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_2531_cast_fp16 = slice_by_index(begin = var_2531_begin_0, end = var_2531_end_0, end_mask = var_2531_end_mask_0, squeeze_mask = var_2531_squeeze_mask_0, x = pair_37_cast_fp16)[name = string("op_2531_cast_fp16")];
+            tensor<int32, [5]> var_2534_begin_0 = const()[name = string("op_2534_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2534_end_0 = const()[name = string("op_2534_end_0"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_2534_end_mask_0 = const()[name = string("op_2534_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2534_squeeze_mask_0 = const()[name = string("op_2534_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_2534_cast_fp16 = slice_by_index(begin = var_2534_begin_0, end = var_2534_end_0, end_mask = var_2534_end_mask_0, squeeze_mask = var_2534_squeeze_mask_0, x = pair_37_cast_fp16)[name = string("op_2534_cast_fp16")];
+            bool var_2537_transpose_x_0 = const()[name = string("op_2537_transpose_x_0"), val = bool(false)];
+            bool var_2537_transpose_y_0 = const()[name = string("op_2537_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_2537_cast_fp16 = matmul(transpose_x = var_2537_transpose_x_0, transpose_y = var_2537_transpose_y_0, x = x_63_cast_fp16, y = var_2531_cast_fp16)[name = string("op_2537_cast_fp16")];
+            bool var_2538_transpose_x_0 = const()[name = string("op_2538_transpose_x_0"), val = bool(false)];
+            bool var_2538_transpose_y_0 = const()[name = string("op_2538_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_2538_cast_fp16 = matmul(transpose_x = var_2538_transpose_x_0, transpose_y = var_2538_transpose_y_0, x = var_2534_cast_fp16, y = var_2537_cast_fp16)[name = string("op_2538_cast_fp16")];
+            fp16 const_33_promoted_to_fp16 = const()[name = string("const_33_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 4, 8, 8]> lower_31_cast_fp16 = mul(x = var_2538_cast_fp16, y = const_33_promoted_to_fp16)[name = string("lower_31_cast_fp16")];
+            bool top_31_interleave_0 = const()[name = string("top_31_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> top_31_cast_fp16 = concat(axis = var_2272, interleave = top_31_interleave_0, values = (var_2531_cast_fp16, var_334_to_fp16))[name = string("top_31_cast_fp16")];
+            bool bottom_31_interleave_0 = const()[name = string("bottom_31_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> bottom_31_cast_fp16 = concat(axis = var_2272, interleave = bottom_31_interleave_0, values = (lower_31_cast_fp16, var_2534_cast_fp16))[name = string("bottom_31_cast_fp16")];
+            bool inv_41_interleave_0 = const()[name = string("inv_41_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> inv_41_cast_fp16 = concat(axis = var_2243, interleave = inv_41_interleave_0, values = (top_31_cast_fp16, bottom_31_cast_fp16))[name = string("inv_41_cast_fp16")];
+            tensor<int32, [5]> var_2547 = const()[name = string("op_2547"), val = tensor<int32, [5]>([1024, 2, 32, 2, 32])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_2548_cast_fp16 = reshape(shape = var_2547, x = t_11_cast_fp16)[name = string("op_2548_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_2552_to_fp16 = const()[name = string("op_2552_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_2553_cast_fp16 = mul(x = var_2548_cast_fp16, y = var_2552_to_fp16)[name = string("op_2553_cast_fp16")];
+            tensor<int32, [1]> blocks_33_axes_0 = const()[name = string("blocks_33_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_33_keep_dims_0 = const()[name = string("blocks_33_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> blocks_33_cast_fp16 = reduce_sum(axes = blocks_33_axes_0, keep_dims = blocks_33_keep_dims_0, x = var_2553_cast_fp16)[name = string("blocks_33_cast_fp16")];
+            tensor<int32, [4]> var_2556_begin_0 = const()[name = string("op_2556_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_2556_end_0 = const()[name = string("op_2556_end_0"), val = tensor<int32, [4]>([1024, 2, 32, 32])];
+            tensor<bool, [4]> var_2556_end_mask_0 = const()[name = string("op_2556_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 2, 16, 32]> var_2556_cast_fp16 = slice_by_index(begin = var_2556_begin_0, end = var_2556_end_0, end_mask = var_2556_end_mask_0, x = blocks_33_cast_fp16)[name = string("op_2556_cast_fp16")];
+            tensor<int32, [4]> x_65_begin_0 = const()[name = string("x_65_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_65_end_0 = const()[name = string("x_65_end_0"), val = tensor<int32, [4]>([1024, 2, 16, 16])];
+            tensor<bool, [4]> x_65_end_mask_0 = const()[name = string("x_65_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 2, 16, 16]> x_65_cast_fp16 = slice_by_index(begin = x_65_begin_0, end = x_65_end_0, end_mask = x_65_end_mask_0, x = var_2556_cast_fp16)[name = string("x_65_cast_fp16")];
+            tensor<int32, [5]> var_2558 = const()[name = string("op_2558"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<fp16, [1024, 2, 2, 16, 16]> pair_39_cast_fp16 = reshape(shape = var_2558, x = inv_41_cast_fp16)[name = string("pair_39_cast_fp16")];
+            tensor<int32, [5]> var_2560_begin_0 = const()[name = string("op_2560_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2560_end_0 = const()[name = string("op_2560_end_0"), val = tensor<int32, [5]>([1024, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_2560_end_mask_0 = const()[name = string("op_2560_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2560_squeeze_mask_0 = const()[name = string("op_2560_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_2560_cast_fp16 = slice_by_index(begin = var_2560_begin_0, end = var_2560_end_0, end_mask = var_2560_end_mask_0, squeeze_mask = var_2560_squeeze_mask_0, x = pair_39_cast_fp16)[name = string("op_2560_cast_fp16")];
+            tensor<int32, [5]> var_2563_begin_0 = const()[name = string("op_2563_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2563_end_0 = const()[name = string("op_2563_end_0"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_2563_end_mask_0 = const()[name = string("op_2563_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2563_squeeze_mask_0 = const()[name = string("op_2563_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_2563_cast_fp16 = slice_by_index(begin = var_2563_begin_0, end = var_2563_end_0, end_mask = var_2563_end_mask_0, squeeze_mask = var_2563_squeeze_mask_0, x = pair_39_cast_fp16)[name = string("op_2563_cast_fp16")];
+            bool var_2566_transpose_x_0 = const()[name = string("op_2566_transpose_x_0"), val = bool(false)];
+            bool var_2566_transpose_y_0 = const()[name = string("op_2566_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_2566_cast_fp16 = matmul(transpose_x = var_2566_transpose_x_0, transpose_y = var_2566_transpose_y_0, x = x_65_cast_fp16, y = var_2560_cast_fp16)[name = string("op_2566_cast_fp16")];
+            bool var_2567_transpose_x_0 = const()[name = string("op_2567_transpose_x_0"), val = bool(false)];
+            bool var_2567_transpose_y_0 = const()[name = string("op_2567_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_2567_cast_fp16 = matmul(transpose_x = var_2567_transpose_x_0, transpose_y = var_2567_transpose_y_0, x = var_2563_cast_fp16, y = var_2566_cast_fp16)[name = string("op_2567_cast_fp16")];
+            fp16 const_34_promoted_to_fp16 = const()[name = string("const_34_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 2, 16, 16]> lower_33_cast_fp16 = mul(x = var_2567_cast_fp16, y = const_34_promoted_to_fp16)[name = string("lower_33_cast_fp16")];
+            bool top_33_interleave_0 = const()[name = string("top_33_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> top_33_cast_fp16 = concat(axis = var_2272, interleave = top_33_interleave_0, values = (var_2560_cast_fp16, var_363_to_fp16))[name = string("top_33_cast_fp16")];
+            bool bottom_33_interleave_0 = const()[name = string("bottom_33_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> bottom_33_cast_fp16 = concat(axis = var_2272, interleave = bottom_33_interleave_0, values = (lower_33_cast_fp16, var_2563_cast_fp16))[name = string("bottom_33_cast_fp16")];
+            bool inv_43_interleave_0 = const()[name = string("inv_43_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> inv_43_cast_fp16 = concat(axis = var_2243, interleave = inv_43_interleave_0, values = (top_33_cast_fp16, bottom_33_cast_fp16))[name = string("inv_43_cast_fp16")];
+            tensor<int32, [5]> var_2576 = const()[name = string("op_2576"), val = tensor<int32, [5]>([1024, 1, 64, 1, 64])];
+            tensor<fp16, [1024, 1, 64, 1, 64]> var_2577_cast_fp16 = reshape(shape = var_2576, x = t_11_cast_fp16)[name = string("op_2577_cast_fp16")];
+            tensor<int32, [1]> blocks_35_axes_0 = const()[name = string("blocks_35_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_35_keep_dims_0 = const()[name = string("blocks_35_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> blocks_35_cast_fp16 = reduce_sum(axes = blocks_35_axes_0, keep_dims = blocks_35_keep_dims_0, x = var_2577_cast_fp16)[name = string("blocks_35_cast_fp16")];
+            tensor<int32, [4]> var_2585_begin_0 = const()[name = string("op_2585_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_2585_end_0 = const()[name = string("op_2585_end_0"), val = tensor<int32, [4]>([1024, 1, 64, 64])];
+            tensor<bool, [4]> var_2585_end_mask_0 = const()[name = string("op_2585_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 1, 32, 64]> var_2585_cast_fp16 = slice_by_index(begin = var_2585_begin_0, end = var_2585_end_0, end_mask = var_2585_end_mask_0, x = blocks_35_cast_fp16)[name = string("op_2585_cast_fp16")];
+            tensor<int32, [4]> x_67_begin_0 = const()[name = string("x_67_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_67_end_0 = const()[name = string("x_67_end_0"), val = tensor<int32, [4]>([1024, 1, 32, 32])];
+            tensor<bool, [4]> x_67_end_mask_0 = const()[name = string("x_67_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 1, 32, 32]> x_67_cast_fp16 = slice_by_index(begin = x_67_begin_0, end = x_67_end_0, end_mask = x_67_end_mask_0, x = var_2585_cast_fp16)[name = string("x_67_cast_fp16")];
+            tensor<int32, [5]> var_2587 = const()[name = string("op_2587"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<fp16, [1024, 1, 2, 32, 32]> pair_41_cast_fp16 = reshape(shape = var_2587, x = inv_43_cast_fp16)[name = string("pair_41_cast_fp16")];
+            tensor<int32, [5]> var_2589_begin_0 = const()[name = string("op_2589_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_2589_end_0 = const()[name = string("op_2589_end_0"), val = tensor<int32, [5]>([1024, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_2589_end_mask_0 = const()[name = string("op_2589_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2589_squeeze_mask_0 = const()[name = string("op_2589_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_2589_cast_fp16 = slice_by_index(begin = var_2589_begin_0, end = var_2589_end_0, end_mask = var_2589_end_mask_0, squeeze_mask = var_2589_squeeze_mask_0, x = pair_41_cast_fp16)[name = string("op_2589_cast_fp16")];
+            tensor<int32, [5]> var_2592_begin_0 = const()[name = string("op_2592_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_2592_end_0 = const()[name = string("op_2592_end_0"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_2592_end_mask_0 = const()[name = string("op_2592_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_2592_squeeze_mask_0 = const()[name = string("op_2592_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_2592_cast_fp16 = slice_by_index(begin = var_2592_begin_0, end = var_2592_end_0, end_mask = var_2592_end_mask_0, squeeze_mask = var_2592_squeeze_mask_0, x = pair_41_cast_fp16)[name = string("op_2592_cast_fp16")];
+            bool var_2595_transpose_x_0 = const()[name = string("op_2595_transpose_x_0"), val = bool(false)];
+            bool var_2595_transpose_y_0 = const()[name = string("op_2595_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_2595_cast_fp16 = matmul(transpose_x = var_2595_transpose_x_0, transpose_y = var_2595_transpose_y_0, x = x_67_cast_fp16, y = var_2589_cast_fp16)[name = string("op_2595_cast_fp16")];
+            bool var_2596_transpose_x_0 = const()[name = string("op_2596_transpose_x_0"), val = bool(false)];
+            bool var_2596_transpose_y_0 = const()[name = string("op_2596_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_2596_cast_fp16 = matmul(transpose_x = var_2596_transpose_x_0, transpose_y = var_2596_transpose_y_0, x = var_2592_cast_fp16, y = var_2595_cast_fp16)[name = string("op_2596_cast_fp16")];
+            fp16 const_35_promoted_to_fp16 = const()[name = string("const_35_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 1, 32, 32]> lower_35_cast_fp16 = mul(x = var_2596_cast_fp16, y = const_35_promoted_to_fp16)[name = string("lower_35_cast_fp16")];
+            bool top_35_interleave_0 = const()[name = string("top_35_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> top_35_cast_fp16 = concat(axis = var_2272, interleave = top_35_interleave_0, values = (var_2589_cast_fp16, var_392_to_fp16))[name = string("top_35_cast_fp16")];
+            bool bottom_35_interleave_0 = const()[name = string("bottom_35_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> bottom_35_cast_fp16 = concat(axis = var_2272, interleave = bottom_35_interleave_0, values = (lower_35_cast_fp16, var_2592_cast_fp16))[name = string("bottom_35_cast_fp16")];
+            bool inv_45_interleave_0 = const()[name = string("inv_45_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> inv_45_cast_fp16 = concat(axis = var_2243, interleave = inv_45_interleave_0, values = (top_35_cast_fp16, bottom_35_cast_fp16))[name = string("inv_45_cast_fp16")];
+            tensor<int32, [4]> var_2605 = const()[name = string("op_2605"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> inv_47_cast_fp16 = reshape(shape = var_2605, x = inv_45_cast_fp16)[name = string("inv_47_cast_fp16")];
+            bool new_v_5_transpose_x_0 = const()[name = string("new_v_5_transpose_x_0"), val = bool(false)];
+            bool new_v_5_transpose_y_0 = const()[name = string("new_v_5_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> new_v_5_cast_fp16 = matmul(transpose_x = new_v_5_transpose_x_0, transpose_y = new_v_5_transpose_y_0, x = inv_47_cast_fp16, y = v_beta_5_cast_fp16)[name = string("new_v_5_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_2608_cast_fp16 = exp(x = cum_5_cast_fp16)[name = string("op_2608_cast_fp16")];
+            tensor<int32, [1]> var_2609_axes_0 = const()[name = string("op_2609_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_2609_cast_fp16 = expand_dims(axes = var_2609_axes_0, x = var_2608_cast_fp16)[name = string("op_2609_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> var_2610_cast_fp16 = mul(x = k_beta_5_cast_fp16, y = var_2609_cast_fp16)[name = string("op_2610_cast_fp16")];
+            bool k_cumdecay_5_transpose_x_0 = const()[name = string("k_cumdecay_5_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_5_transpose_y_0 = const()[name = string("k_cumdecay_5_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> k_cumdecay_5_cast_fp16 = matmul(transpose_x = k_cumdecay_5_transpose_x_0, transpose_y = k_cumdecay_5_transpose_y_0, x = inv_47_cast_fp16, y = var_2610_cast_fp16)[name = string("k_cumdecay_5_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> q_dec_5_cast_fp16 = mul(x = q_29_cast_fp16, y = var_2609_cast_fp16)[name = string("q_dec_5_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_2615_cast_fp16 = exp(x = to_end_5_cast_fp16)[name = string("op_2615_cast_fp16")];
+            tensor<int32, [1]> var_2616_axes_0 = const()[name = string("op_2616_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_2616_cast_fp16 = expand_dims(axes = var_2616_axes_0, x = var_2615_cast_fp16)[name = string("op_2616_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_dec_5_cast_fp16 = mul(x = k_23_cast_fp16, y = var_2616_cast_fp16)[name = string("k_dec_5_cast_fp16")];
+            tensor<int32, [3]> var_2618_begin_0 = const()[name = string("op_2618_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_2618_end_0 = const()[name = string("op_2618_end_0"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<bool, [3]> var_2618_end_mask_0 = const()[name = string("op_2618_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 32, 1]> var_2618_cast_fp16 = slice_by_index(begin = var_2618_begin_0, end = var_2618_end_0, end_mask = var_2618_end_mask_0, x = cum_5_cast_fp16)[name = string("op_2618_cast_fp16")];
+            tensor<fp16, [32, 32, 1]> var_2619_cast_fp16 = exp(x = var_2618_cast_fp16)[name = string("op_2619_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_5_axes_0 = const()[name = string("chunk_decay_5_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 1, 1]> chunk_decay_5_cast_fp16 = expand_dims(axes = chunk_decay_5_axes_0, x = var_2619_cast_fp16)[name = string("chunk_decay_5_cast_fp16")];
+            tensor<int32, [4]> var_2624_begin_0 = const()[name = string("op_2624_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2624_end_0 = const()[name = string("op_2624_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2624_end_mask_0 = const()[name = string("op_2624_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2624_squeeze_mask_0 = const()[name = string("op_2624_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2624_cast_fp16 = slice_by_index(begin = var_2624_begin_0, end = var_2624_end_0, end_mask = var_2624_end_mask_0, squeeze_mask = var_2624_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2624_cast_fp16")];
+            tensor<int32, [4]> var_2626_begin_0 = const()[name = string("op_2626_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2626_end_0 = const()[name = string("op_2626_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2626_end_mask_0 = const()[name = string("op_2626_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2626_squeeze_mask_0 = const()[name = string("op_2626_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2626_cast_fp16 = slice_by_index(begin = var_2626_begin_0, end = var_2626_end_0, end_mask = var_2626_end_mask_0, squeeze_mask = var_2626_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2626_cast_fp16")];
+            bool var_2627_transpose_x_0 = const()[name = string("op_2627_transpose_x_0"), val = bool(false)];
+            bool var_2627_transpose_y_0 = const()[name = string("op_2627_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2627_cast_fp16 = matmul(transpose_x = var_2627_transpose_x_0, transpose_y = var_2627_transpose_y_0, x = var_2626_cast_fp16, y = state_1_to_fp16)[name = string("op_2627_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_129_cast_fp16 = sub(x = var_2624_cast_fp16, y = var_2627_cast_fp16)[name = string("v_new_129_cast_fp16")];
+            tensor<int32, [4]> var_2630_begin_0 = const()[name = string("op_2630_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2630_end_0 = const()[name = string("op_2630_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2630_end_mask_0 = const()[name = string("op_2630_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2630_squeeze_mask_0 = const()[name = string("op_2630_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2630_cast_fp16 = slice_by_index(begin = var_2630_begin_0, end = var_2630_end_0, end_mask = var_2630_end_mask_0, squeeze_mask = var_2630_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2630_cast_fp16")];
+            bool var_2631_transpose_x_0 = const()[name = string("op_2631_transpose_x_0"), val = bool(false)];
+            bool var_2631_transpose_y_0 = const()[name = string("op_2631_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2631_cast_fp16 = matmul(transpose_x = var_2631_transpose_x_0, transpose_y = var_2631_transpose_y_0, x = var_2630_cast_fp16, y = state_1_to_fp16)[name = string("op_2631_cast_fp16")];
+            tensor<int32, [4]> var_2633_begin_0 = const()[name = string("op_2633_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2633_end_0 = const()[name = string("op_2633_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_2633_end_mask_0 = const()[name = string("op_2633_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2633_squeeze_mask_0 = const()[name = string("op_2633_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2633_cast_fp16 = slice_by_index(begin = var_2633_begin_0, end = var_2633_end_0, end_mask = var_2633_end_mask_0, squeeze_mask = var_2633_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2633_cast_fp16")];
+            bool var_2634_transpose_x_0 = const()[name = string("op_2634_transpose_x_0"), val = bool(false)];
+            bool var_2634_transpose_y_0 = const()[name = string("op_2634_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2634_cast_fp16 = matmul(transpose_x = var_2634_transpose_x_0, transpose_y = var_2634_transpose_y_0, x = var_2633_cast_fp16, y = v_new_129_cast_fp16)[name = string("op_2634_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2635_cast_fp16 = add(x = var_2631_cast_fp16, y = var_2634_cast_fp16)[name = string("op_2635_cast_fp16")];
+            tensor<int32, [4]> var_2637_begin_0 = const()[name = string("op_2637_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2637_end_0 = const()[name = string("op_2637_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_2637_end_mask_0 = const()[name = string("op_2637_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2637_squeeze_mask_0 = const()[name = string("op_2637_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2637_cast_fp16 = slice_by_index(begin = var_2637_begin_0, end = var_2637_end_0, end_mask = var_2637_end_mask_0, squeeze_mask = var_2637_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2637_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2638_cast_fp16 = mul(x = state_1_to_fp16, y = var_2637_cast_fp16)[name = string("op_2638_cast_fp16")];
+            tensor<int32, [4]> var_2640_begin_0 = const()[name = string("op_2640_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_2640_end_0 = const()[name = string("op_2640_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_2640_end_mask_0 = const()[name = string("op_2640_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2640_squeeze_mask_0 = const()[name = string("op_2640_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2640_cast_fp16 = slice_by_index(begin = var_2640_begin_0, end = var_2640_end_0, end_mask = var_2640_end_mask_0, squeeze_mask = var_2640_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2640_cast_fp16")];
+            bool var_2642_transpose_x_1 = const()[name = string("op_2642_transpose_x_1"), val = bool(true)];
+            bool var_2642_transpose_y_1 = const()[name = string("op_2642_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2642_cast_fp16 = matmul(transpose_x = var_2642_transpose_x_1, transpose_y = var_2642_transpose_y_1, x = var_2640_cast_fp16, y = v_new_129_cast_fp16)[name = string("op_2642_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_131_cast_fp16 = add(x = var_2638_cast_fp16, y = var_2642_cast_fp16)[name = string("state_131_cast_fp16")];
+            tensor<int32, [4]> var_2645_begin_0 = const()[name = string("op_2645_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2645_end_0 = const()[name = string("op_2645_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2645_end_mask_0 = const()[name = string("op_2645_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2645_squeeze_mask_0 = const()[name = string("op_2645_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2645_cast_fp16 = slice_by_index(begin = var_2645_begin_0, end = var_2645_end_0, end_mask = var_2645_end_mask_0, squeeze_mask = var_2645_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2645_cast_fp16")];
+            tensor<int32, [4]> var_2647_begin_0 = const()[name = string("op_2647_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2647_end_0 = const()[name = string("op_2647_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2647_end_mask_0 = const()[name = string("op_2647_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2647_squeeze_mask_0 = const()[name = string("op_2647_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2647_cast_fp16 = slice_by_index(begin = var_2647_begin_0, end = var_2647_end_0, end_mask = var_2647_end_mask_0, squeeze_mask = var_2647_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2647_cast_fp16")];
+            bool var_2648_transpose_x_0 = const()[name = string("op_2648_transpose_x_0"), val = bool(false)];
+            bool var_2648_transpose_y_0 = const()[name = string("op_2648_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2648_cast_fp16 = matmul(transpose_x = var_2648_transpose_x_0, transpose_y = var_2648_transpose_y_0, x = var_2647_cast_fp16, y = state_131_cast_fp16)[name = string("op_2648_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_131_cast_fp16 = sub(x = var_2645_cast_fp16, y = var_2648_cast_fp16)[name = string("v_new_131_cast_fp16")];
+            tensor<int32, [4]> var_2651_begin_0 = const()[name = string("op_2651_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2651_end_0 = const()[name = string("op_2651_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2651_end_mask_0 = const()[name = string("op_2651_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2651_squeeze_mask_0 = const()[name = string("op_2651_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2651_cast_fp16 = slice_by_index(begin = var_2651_begin_0, end = var_2651_end_0, end_mask = var_2651_end_mask_0, squeeze_mask = var_2651_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2651_cast_fp16")];
+            bool var_2652_transpose_x_0 = const()[name = string("op_2652_transpose_x_0"), val = bool(false)];
+            bool var_2652_transpose_y_0 = const()[name = string("op_2652_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2652_cast_fp16 = matmul(transpose_x = var_2652_transpose_x_0, transpose_y = var_2652_transpose_y_0, x = var_2651_cast_fp16, y = state_131_cast_fp16)[name = string("op_2652_cast_fp16")];
+            tensor<int32, [4]> var_2654_begin_0 = const()[name = string("op_2654_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2654_end_0 = const()[name = string("op_2654_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_2654_end_mask_0 = const()[name = string("op_2654_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2654_squeeze_mask_0 = const()[name = string("op_2654_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2654_cast_fp16 = slice_by_index(begin = var_2654_begin_0, end = var_2654_end_0, end_mask = var_2654_end_mask_0, squeeze_mask = var_2654_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2654_cast_fp16")];
+            bool var_2655_transpose_x_0 = const()[name = string("op_2655_transpose_x_0"), val = bool(false)];
+            bool var_2655_transpose_y_0 = const()[name = string("op_2655_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2655_cast_fp16 = matmul(transpose_x = var_2655_transpose_x_0, transpose_y = var_2655_transpose_y_0, x = var_2654_cast_fp16, y = v_new_131_cast_fp16)[name = string("op_2655_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2656_cast_fp16 = add(x = var_2652_cast_fp16, y = var_2655_cast_fp16)[name = string("op_2656_cast_fp16")];
+            tensor<int32, [4]> var_2658_begin_0 = const()[name = string("op_2658_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2658_end_0 = const()[name = string("op_2658_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_2658_end_mask_0 = const()[name = string("op_2658_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2658_squeeze_mask_0 = const()[name = string("op_2658_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2658_cast_fp16 = slice_by_index(begin = var_2658_begin_0, end = var_2658_end_0, end_mask = var_2658_end_mask_0, squeeze_mask = var_2658_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2658_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2659_cast_fp16 = mul(x = state_131_cast_fp16, y = var_2658_cast_fp16)[name = string("op_2659_cast_fp16")];
+            tensor<int32, [4]> var_2661_begin_0 = const()[name = string("op_2661_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_2661_end_0 = const()[name = string("op_2661_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_2661_end_mask_0 = const()[name = string("op_2661_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2661_squeeze_mask_0 = const()[name = string("op_2661_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2661_cast_fp16 = slice_by_index(begin = var_2661_begin_0, end = var_2661_end_0, end_mask = var_2661_end_mask_0, squeeze_mask = var_2661_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2661_cast_fp16")];
+            bool var_2663_transpose_x_1 = const()[name = string("op_2663_transpose_x_1"), val = bool(true)];
+            bool var_2663_transpose_y_1 = const()[name = string("op_2663_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2663_cast_fp16 = matmul(transpose_x = var_2663_transpose_x_1, transpose_y = var_2663_transpose_y_1, x = var_2661_cast_fp16, y = v_new_131_cast_fp16)[name = string("op_2663_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_133_cast_fp16 = add(x = var_2659_cast_fp16, y = var_2663_cast_fp16)[name = string("state_133_cast_fp16")];
+            tensor<int32, [4]> var_2666_begin_0 = const()[name = string("op_2666_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2666_end_0 = const()[name = string("op_2666_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2666_end_mask_0 = const()[name = string("op_2666_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2666_squeeze_mask_0 = const()[name = string("op_2666_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2666_cast_fp16 = slice_by_index(begin = var_2666_begin_0, end = var_2666_end_0, end_mask = var_2666_end_mask_0, squeeze_mask = var_2666_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2666_cast_fp16")];
+            tensor<int32, [4]> var_2668_begin_0 = const()[name = string("op_2668_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2668_end_0 = const()[name = string("op_2668_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2668_end_mask_0 = const()[name = string("op_2668_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2668_squeeze_mask_0 = const()[name = string("op_2668_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2668_cast_fp16 = slice_by_index(begin = var_2668_begin_0, end = var_2668_end_0, end_mask = var_2668_end_mask_0, squeeze_mask = var_2668_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2668_cast_fp16")];
+            bool var_2669_transpose_x_0 = const()[name = string("op_2669_transpose_x_0"), val = bool(false)];
+            bool var_2669_transpose_y_0 = const()[name = string("op_2669_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2669_cast_fp16 = matmul(transpose_x = var_2669_transpose_x_0, transpose_y = var_2669_transpose_y_0, x = var_2668_cast_fp16, y = state_133_cast_fp16)[name = string("op_2669_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_133_cast_fp16 = sub(x = var_2666_cast_fp16, y = var_2669_cast_fp16)[name = string("v_new_133_cast_fp16")];
+            tensor<int32, [4]> var_2672_begin_0 = const()[name = string("op_2672_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2672_end_0 = const()[name = string("op_2672_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2672_end_mask_0 = const()[name = string("op_2672_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2672_squeeze_mask_0 = const()[name = string("op_2672_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2672_cast_fp16 = slice_by_index(begin = var_2672_begin_0, end = var_2672_end_0, end_mask = var_2672_end_mask_0, squeeze_mask = var_2672_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2672_cast_fp16")];
+            bool var_2673_transpose_x_0 = const()[name = string("op_2673_transpose_x_0"), val = bool(false)];
+            bool var_2673_transpose_y_0 = const()[name = string("op_2673_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2673_cast_fp16 = matmul(transpose_x = var_2673_transpose_x_0, transpose_y = var_2673_transpose_y_0, x = var_2672_cast_fp16, y = state_133_cast_fp16)[name = string("op_2673_cast_fp16")];
+            tensor<int32, [4]> var_2675_begin_0 = const()[name = string("op_2675_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2675_end_0 = const()[name = string("op_2675_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_2675_end_mask_0 = const()[name = string("op_2675_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2675_squeeze_mask_0 = const()[name = string("op_2675_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2675_cast_fp16 = slice_by_index(begin = var_2675_begin_0, end = var_2675_end_0, end_mask = var_2675_end_mask_0, squeeze_mask = var_2675_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2675_cast_fp16")];
+            bool var_2676_transpose_x_0 = const()[name = string("op_2676_transpose_x_0"), val = bool(false)];
+            bool var_2676_transpose_y_0 = const()[name = string("op_2676_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2676_cast_fp16 = matmul(transpose_x = var_2676_transpose_x_0, transpose_y = var_2676_transpose_y_0, x = var_2675_cast_fp16, y = v_new_133_cast_fp16)[name = string("op_2676_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2677_cast_fp16 = add(x = var_2673_cast_fp16, y = var_2676_cast_fp16)[name = string("op_2677_cast_fp16")];
+            tensor<int32, [4]> var_2679_begin_0 = const()[name = string("op_2679_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2679_end_0 = const()[name = string("op_2679_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_2679_end_mask_0 = const()[name = string("op_2679_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2679_squeeze_mask_0 = const()[name = string("op_2679_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2679_cast_fp16 = slice_by_index(begin = var_2679_begin_0, end = var_2679_end_0, end_mask = var_2679_end_mask_0, squeeze_mask = var_2679_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2679_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2680_cast_fp16 = mul(x = state_133_cast_fp16, y = var_2679_cast_fp16)[name = string("op_2680_cast_fp16")];
+            tensor<int32, [4]> var_2682_begin_0 = const()[name = string("op_2682_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_2682_end_0 = const()[name = string("op_2682_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_2682_end_mask_0 = const()[name = string("op_2682_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2682_squeeze_mask_0 = const()[name = string("op_2682_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2682_cast_fp16 = slice_by_index(begin = var_2682_begin_0, end = var_2682_end_0, end_mask = var_2682_end_mask_0, squeeze_mask = var_2682_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2682_cast_fp16")];
+            bool var_2684_transpose_x_1 = const()[name = string("op_2684_transpose_x_1"), val = bool(true)];
+            bool var_2684_transpose_y_1 = const()[name = string("op_2684_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2684_cast_fp16 = matmul(transpose_x = var_2684_transpose_x_1, transpose_y = var_2684_transpose_y_1, x = var_2682_cast_fp16, y = v_new_133_cast_fp16)[name = string("op_2684_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_135_cast_fp16 = add(x = var_2680_cast_fp16, y = var_2684_cast_fp16)[name = string("state_135_cast_fp16")];
+            tensor<int32, [4]> var_2687_begin_0 = const()[name = string("op_2687_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2687_end_0 = const()[name = string("op_2687_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2687_end_mask_0 = const()[name = string("op_2687_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2687_squeeze_mask_0 = const()[name = string("op_2687_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2687_cast_fp16 = slice_by_index(begin = var_2687_begin_0, end = var_2687_end_0, end_mask = var_2687_end_mask_0, squeeze_mask = var_2687_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2687_cast_fp16")];
+            tensor<int32, [4]> var_2689_begin_0 = const()[name = string("op_2689_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2689_end_0 = const()[name = string("op_2689_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2689_end_mask_0 = const()[name = string("op_2689_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2689_squeeze_mask_0 = const()[name = string("op_2689_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2689_cast_fp16 = slice_by_index(begin = var_2689_begin_0, end = var_2689_end_0, end_mask = var_2689_end_mask_0, squeeze_mask = var_2689_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2689_cast_fp16")];
+            bool var_2690_transpose_x_0 = const()[name = string("op_2690_transpose_x_0"), val = bool(false)];
+            bool var_2690_transpose_y_0 = const()[name = string("op_2690_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2690_cast_fp16 = matmul(transpose_x = var_2690_transpose_x_0, transpose_y = var_2690_transpose_y_0, x = var_2689_cast_fp16, y = state_135_cast_fp16)[name = string("op_2690_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_135_cast_fp16 = sub(x = var_2687_cast_fp16, y = var_2690_cast_fp16)[name = string("v_new_135_cast_fp16")];
+            tensor<int32, [4]> var_2693_begin_0 = const()[name = string("op_2693_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2693_end_0 = const()[name = string("op_2693_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2693_end_mask_0 = const()[name = string("op_2693_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2693_squeeze_mask_0 = const()[name = string("op_2693_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2693_cast_fp16 = slice_by_index(begin = var_2693_begin_0, end = var_2693_end_0, end_mask = var_2693_end_mask_0, squeeze_mask = var_2693_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2693_cast_fp16")];
+            bool var_2694_transpose_x_0 = const()[name = string("op_2694_transpose_x_0"), val = bool(false)];
+            bool var_2694_transpose_y_0 = const()[name = string("op_2694_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2694_cast_fp16 = matmul(transpose_x = var_2694_transpose_x_0, transpose_y = var_2694_transpose_y_0, x = var_2693_cast_fp16, y = state_135_cast_fp16)[name = string("op_2694_cast_fp16")];
+            tensor<int32, [4]> var_2696_begin_0 = const()[name = string("op_2696_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2696_end_0 = const()[name = string("op_2696_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_2696_end_mask_0 = const()[name = string("op_2696_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2696_squeeze_mask_0 = const()[name = string("op_2696_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2696_cast_fp16 = slice_by_index(begin = var_2696_begin_0, end = var_2696_end_0, end_mask = var_2696_end_mask_0, squeeze_mask = var_2696_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2696_cast_fp16")];
+            bool var_2697_transpose_x_0 = const()[name = string("op_2697_transpose_x_0"), val = bool(false)];
+            bool var_2697_transpose_y_0 = const()[name = string("op_2697_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2697_cast_fp16 = matmul(transpose_x = var_2697_transpose_x_0, transpose_y = var_2697_transpose_y_0, x = var_2696_cast_fp16, y = v_new_135_cast_fp16)[name = string("op_2697_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2698_cast_fp16 = add(x = var_2694_cast_fp16, y = var_2697_cast_fp16)[name = string("op_2698_cast_fp16")];
+            tensor<int32, [4]> var_2700_begin_0 = const()[name = string("op_2700_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2700_end_0 = const()[name = string("op_2700_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_2700_end_mask_0 = const()[name = string("op_2700_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2700_squeeze_mask_0 = const()[name = string("op_2700_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2700_cast_fp16 = slice_by_index(begin = var_2700_begin_0, end = var_2700_end_0, end_mask = var_2700_end_mask_0, squeeze_mask = var_2700_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2700_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2701_cast_fp16 = mul(x = state_135_cast_fp16, y = var_2700_cast_fp16)[name = string("op_2701_cast_fp16")];
+            tensor<int32, [4]> var_2703_begin_0 = const()[name = string("op_2703_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_2703_end_0 = const()[name = string("op_2703_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_2703_end_mask_0 = const()[name = string("op_2703_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2703_squeeze_mask_0 = const()[name = string("op_2703_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2703_cast_fp16 = slice_by_index(begin = var_2703_begin_0, end = var_2703_end_0, end_mask = var_2703_end_mask_0, squeeze_mask = var_2703_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2703_cast_fp16")];
+            bool var_2705_transpose_x_1 = const()[name = string("op_2705_transpose_x_1"), val = bool(true)];
+            bool var_2705_transpose_y_1 = const()[name = string("op_2705_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2705_cast_fp16 = matmul(transpose_x = var_2705_transpose_x_1, transpose_y = var_2705_transpose_y_1, x = var_2703_cast_fp16, y = v_new_135_cast_fp16)[name = string("op_2705_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_137_cast_fp16 = add(x = var_2701_cast_fp16, y = var_2705_cast_fp16)[name = string("state_137_cast_fp16")];
+            tensor<int32, [4]> var_2708_begin_0 = const()[name = string("op_2708_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2708_end_0 = const()[name = string("op_2708_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2708_end_mask_0 = const()[name = string("op_2708_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2708_squeeze_mask_0 = const()[name = string("op_2708_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2708_cast_fp16 = slice_by_index(begin = var_2708_begin_0, end = var_2708_end_0, end_mask = var_2708_end_mask_0, squeeze_mask = var_2708_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2708_cast_fp16")];
+            tensor<int32, [4]> var_2710_begin_0 = const()[name = string("op_2710_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2710_end_0 = const()[name = string("op_2710_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2710_end_mask_0 = const()[name = string("op_2710_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2710_squeeze_mask_0 = const()[name = string("op_2710_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2710_cast_fp16 = slice_by_index(begin = var_2710_begin_0, end = var_2710_end_0, end_mask = var_2710_end_mask_0, squeeze_mask = var_2710_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2710_cast_fp16")];
+            bool var_2711_transpose_x_0 = const()[name = string("op_2711_transpose_x_0"), val = bool(false)];
+            bool var_2711_transpose_y_0 = const()[name = string("op_2711_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2711_cast_fp16 = matmul(transpose_x = var_2711_transpose_x_0, transpose_y = var_2711_transpose_y_0, x = var_2710_cast_fp16, y = state_137_cast_fp16)[name = string("op_2711_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_137_cast_fp16 = sub(x = var_2708_cast_fp16, y = var_2711_cast_fp16)[name = string("v_new_137_cast_fp16")];
+            tensor<int32, [4]> var_2714_begin_0 = const()[name = string("op_2714_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2714_end_0 = const()[name = string("op_2714_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2714_end_mask_0 = const()[name = string("op_2714_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2714_squeeze_mask_0 = const()[name = string("op_2714_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2714_cast_fp16 = slice_by_index(begin = var_2714_begin_0, end = var_2714_end_0, end_mask = var_2714_end_mask_0, squeeze_mask = var_2714_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2714_cast_fp16")];
+            bool var_2715_transpose_x_0 = const()[name = string("op_2715_transpose_x_0"), val = bool(false)];
+            bool var_2715_transpose_y_0 = const()[name = string("op_2715_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2715_cast_fp16 = matmul(transpose_x = var_2715_transpose_x_0, transpose_y = var_2715_transpose_y_0, x = var_2714_cast_fp16, y = state_137_cast_fp16)[name = string("op_2715_cast_fp16")];
+            tensor<int32, [4]> var_2717_begin_0 = const()[name = string("op_2717_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2717_end_0 = const()[name = string("op_2717_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_2717_end_mask_0 = const()[name = string("op_2717_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2717_squeeze_mask_0 = const()[name = string("op_2717_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2717_cast_fp16 = slice_by_index(begin = var_2717_begin_0, end = var_2717_end_0, end_mask = var_2717_end_mask_0, squeeze_mask = var_2717_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2717_cast_fp16")];
+            bool var_2718_transpose_x_0 = const()[name = string("op_2718_transpose_x_0"), val = bool(false)];
+            bool var_2718_transpose_y_0 = const()[name = string("op_2718_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2718_cast_fp16 = matmul(transpose_x = var_2718_transpose_x_0, transpose_y = var_2718_transpose_y_0, x = var_2717_cast_fp16, y = v_new_137_cast_fp16)[name = string("op_2718_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2719_cast_fp16 = add(x = var_2715_cast_fp16, y = var_2718_cast_fp16)[name = string("op_2719_cast_fp16")];
+            tensor<int32, [4]> var_2721_begin_0 = const()[name = string("op_2721_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2721_end_0 = const()[name = string("op_2721_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_2721_end_mask_0 = const()[name = string("op_2721_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2721_squeeze_mask_0 = const()[name = string("op_2721_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2721_cast_fp16 = slice_by_index(begin = var_2721_begin_0, end = var_2721_end_0, end_mask = var_2721_end_mask_0, squeeze_mask = var_2721_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2721_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2722_cast_fp16 = mul(x = state_137_cast_fp16, y = var_2721_cast_fp16)[name = string("op_2722_cast_fp16")];
+            tensor<int32, [4]> var_2724_begin_0 = const()[name = string("op_2724_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_2724_end_0 = const()[name = string("op_2724_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_2724_end_mask_0 = const()[name = string("op_2724_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2724_squeeze_mask_0 = const()[name = string("op_2724_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2724_cast_fp16 = slice_by_index(begin = var_2724_begin_0, end = var_2724_end_0, end_mask = var_2724_end_mask_0, squeeze_mask = var_2724_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2724_cast_fp16")];
+            bool var_2726_transpose_x_1 = const()[name = string("op_2726_transpose_x_1"), val = bool(true)];
+            bool var_2726_transpose_y_1 = const()[name = string("op_2726_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2726_cast_fp16 = matmul(transpose_x = var_2726_transpose_x_1, transpose_y = var_2726_transpose_y_1, x = var_2724_cast_fp16, y = v_new_137_cast_fp16)[name = string("op_2726_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_139_cast_fp16 = add(x = var_2722_cast_fp16, y = var_2726_cast_fp16)[name = string("state_139_cast_fp16")];
+            tensor<int32, [4]> var_2729_begin_0 = const()[name = string("op_2729_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2729_end_0 = const()[name = string("op_2729_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2729_end_mask_0 = const()[name = string("op_2729_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2729_squeeze_mask_0 = const()[name = string("op_2729_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2729_cast_fp16 = slice_by_index(begin = var_2729_begin_0, end = var_2729_end_0, end_mask = var_2729_end_mask_0, squeeze_mask = var_2729_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2729_cast_fp16")];
+            tensor<int32, [4]> var_2731_begin_0 = const()[name = string("op_2731_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2731_end_0 = const()[name = string("op_2731_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2731_end_mask_0 = const()[name = string("op_2731_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2731_squeeze_mask_0 = const()[name = string("op_2731_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2731_cast_fp16 = slice_by_index(begin = var_2731_begin_0, end = var_2731_end_0, end_mask = var_2731_end_mask_0, squeeze_mask = var_2731_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2731_cast_fp16")];
+            bool var_2732_transpose_x_0 = const()[name = string("op_2732_transpose_x_0"), val = bool(false)];
+            bool var_2732_transpose_y_0 = const()[name = string("op_2732_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2732_cast_fp16 = matmul(transpose_x = var_2732_transpose_x_0, transpose_y = var_2732_transpose_y_0, x = var_2731_cast_fp16, y = state_139_cast_fp16)[name = string("op_2732_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_139_cast_fp16 = sub(x = var_2729_cast_fp16, y = var_2732_cast_fp16)[name = string("v_new_139_cast_fp16")];
+            tensor<int32, [4]> var_2735_begin_0 = const()[name = string("op_2735_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2735_end_0 = const()[name = string("op_2735_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2735_end_mask_0 = const()[name = string("op_2735_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2735_squeeze_mask_0 = const()[name = string("op_2735_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2735_cast_fp16 = slice_by_index(begin = var_2735_begin_0, end = var_2735_end_0, end_mask = var_2735_end_mask_0, squeeze_mask = var_2735_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2735_cast_fp16")];
+            bool var_2736_transpose_x_0 = const()[name = string("op_2736_transpose_x_0"), val = bool(false)];
+            bool var_2736_transpose_y_0 = const()[name = string("op_2736_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2736_cast_fp16 = matmul(transpose_x = var_2736_transpose_x_0, transpose_y = var_2736_transpose_y_0, x = var_2735_cast_fp16, y = state_139_cast_fp16)[name = string("op_2736_cast_fp16")];
+            tensor<int32, [4]> var_2738_begin_0 = const()[name = string("op_2738_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2738_end_0 = const()[name = string("op_2738_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_2738_end_mask_0 = const()[name = string("op_2738_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2738_squeeze_mask_0 = const()[name = string("op_2738_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2738_cast_fp16 = slice_by_index(begin = var_2738_begin_0, end = var_2738_end_0, end_mask = var_2738_end_mask_0, squeeze_mask = var_2738_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2738_cast_fp16")];
+            bool var_2739_transpose_x_0 = const()[name = string("op_2739_transpose_x_0"), val = bool(false)];
+            bool var_2739_transpose_y_0 = const()[name = string("op_2739_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2739_cast_fp16 = matmul(transpose_x = var_2739_transpose_x_0, transpose_y = var_2739_transpose_y_0, x = var_2738_cast_fp16, y = v_new_139_cast_fp16)[name = string("op_2739_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2740_cast_fp16 = add(x = var_2736_cast_fp16, y = var_2739_cast_fp16)[name = string("op_2740_cast_fp16")];
+            tensor<int32, [4]> var_2742_begin_0 = const()[name = string("op_2742_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2742_end_0 = const()[name = string("op_2742_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_2742_end_mask_0 = const()[name = string("op_2742_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2742_squeeze_mask_0 = const()[name = string("op_2742_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2742_cast_fp16 = slice_by_index(begin = var_2742_begin_0, end = var_2742_end_0, end_mask = var_2742_end_mask_0, squeeze_mask = var_2742_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2742_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2743_cast_fp16 = mul(x = state_139_cast_fp16, y = var_2742_cast_fp16)[name = string("op_2743_cast_fp16")];
+            tensor<int32, [4]> var_2745_begin_0 = const()[name = string("op_2745_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_2745_end_0 = const()[name = string("op_2745_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_2745_end_mask_0 = const()[name = string("op_2745_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2745_squeeze_mask_0 = const()[name = string("op_2745_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2745_cast_fp16 = slice_by_index(begin = var_2745_begin_0, end = var_2745_end_0, end_mask = var_2745_end_mask_0, squeeze_mask = var_2745_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2745_cast_fp16")];
+            bool var_2747_transpose_x_1 = const()[name = string("op_2747_transpose_x_1"), val = bool(true)];
+            bool var_2747_transpose_y_1 = const()[name = string("op_2747_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2747_cast_fp16 = matmul(transpose_x = var_2747_transpose_x_1, transpose_y = var_2747_transpose_y_1, x = var_2745_cast_fp16, y = v_new_139_cast_fp16)[name = string("op_2747_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_141_cast_fp16 = add(x = var_2743_cast_fp16, y = var_2747_cast_fp16)[name = string("state_141_cast_fp16")];
+            tensor<int32, [4]> var_2750_begin_0 = const()[name = string("op_2750_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2750_end_0 = const()[name = string("op_2750_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2750_end_mask_0 = const()[name = string("op_2750_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2750_squeeze_mask_0 = const()[name = string("op_2750_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2750_cast_fp16 = slice_by_index(begin = var_2750_begin_0, end = var_2750_end_0, end_mask = var_2750_end_mask_0, squeeze_mask = var_2750_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2750_cast_fp16")];
+            tensor<int32, [4]> var_2752_begin_0 = const()[name = string("op_2752_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2752_end_0 = const()[name = string("op_2752_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2752_end_mask_0 = const()[name = string("op_2752_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2752_squeeze_mask_0 = const()[name = string("op_2752_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2752_cast_fp16 = slice_by_index(begin = var_2752_begin_0, end = var_2752_end_0, end_mask = var_2752_end_mask_0, squeeze_mask = var_2752_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2752_cast_fp16")];
+            bool var_2753_transpose_x_0 = const()[name = string("op_2753_transpose_x_0"), val = bool(false)];
+            bool var_2753_transpose_y_0 = const()[name = string("op_2753_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2753_cast_fp16 = matmul(transpose_x = var_2753_transpose_x_0, transpose_y = var_2753_transpose_y_0, x = var_2752_cast_fp16, y = state_141_cast_fp16)[name = string("op_2753_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_141_cast_fp16 = sub(x = var_2750_cast_fp16, y = var_2753_cast_fp16)[name = string("v_new_141_cast_fp16")];
+            tensor<int32, [4]> var_2756_begin_0 = const()[name = string("op_2756_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2756_end_0 = const()[name = string("op_2756_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2756_end_mask_0 = const()[name = string("op_2756_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2756_squeeze_mask_0 = const()[name = string("op_2756_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2756_cast_fp16 = slice_by_index(begin = var_2756_begin_0, end = var_2756_end_0, end_mask = var_2756_end_mask_0, squeeze_mask = var_2756_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2756_cast_fp16")];
+            bool var_2757_transpose_x_0 = const()[name = string("op_2757_transpose_x_0"), val = bool(false)];
+            bool var_2757_transpose_y_0 = const()[name = string("op_2757_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2757_cast_fp16 = matmul(transpose_x = var_2757_transpose_x_0, transpose_y = var_2757_transpose_y_0, x = var_2756_cast_fp16, y = state_141_cast_fp16)[name = string("op_2757_cast_fp16")];
+            tensor<int32, [4]> var_2759_begin_0 = const()[name = string("op_2759_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2759_end_0 = const()[name = string("op_2759_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_2759_end_mask_0 = const()[name = string("op_2759_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2759_squeeze_mask_0 = const()[name = string("op_2759_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2759_cast_fp16 = slice_by_index(begin = var_2759_begin_0, end = var_2759_end_0, end_mask = var_2759_end_mask_0, squeeze_mask = var_2759_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2759_cast_fp16")];
+            bool var_2760_transpose_x_0 = const()[name = string("op_2760_transpose_x_0"), val = bool(false)];
+            bool var_2760_transpose_y_0 = const()[name = string("op_2760_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2760_cast_fp16 = matmul(transpose_x = var_2760_transpose_x_0, transpose_y = var_2760_transpose_y_0, x = var_2759_cast_fp16, y = v_new_141_cast_fp16)[name = string("op_2760_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2761_cast_fp16 = add(x = var_2757_cast_fp16, y = var_2760_cast_fp16)[name = string("op_2761_cast_fp16")];
+            tensor<int32, [4]> var_2763_begin_0 = const()[name = string("op_2763_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2763_end_0 = const()[name = string("op_2763_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_2763_end_mask_0 = const()[name = string("op_2763_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2763_squeeze_mask_0 = const()[name = string("op_2763_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2763_cast_fp16 = slice_by_index(begin = var_2763_begin_0, end = var_2763_end_0, end_mask = var_2763_end_mask_0, squeeze_mask = var_2763_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2763_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2764_cast_fp16 = mul(x = state_141_cast_fp16, y = var_2763_cast_fp16)[name = string("op_2764_cast_fp16")];
+            tensor<int32, [4]> var_2766_begin_0 = const()[name = string("op_2766_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_2766_end_0 = const()[name = string("op_2766_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_2766_end_mask_0 = const()[name = string("op_2766_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2766_squeeze_mask_0 = const()[name = string("op_2766_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2766_cast_fp16 = slice_by_index(begin = var_2766_begin_0, end = var_2766_end_0, end_mask = var_2766_end_mask_0, squeeze_mask = var_2766_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2766_cast_fp16")];
+            bool var_2768_transpose_x_1 = const()[name = string("op_2768_transpose_x_1"), val = bool(true)];
+            bool var_2768_transpose_y_1 = const()[name = string("op_2768_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2768_cast_fp16 = matmul(transpose_x = var_2768_transpose_x_1, transpose_y = var_2768_transpose_y_1, x = var_2766_cast_fp16, y = v_new_141_cast_fp16)[name = string("op_2768_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_143_cast_fp16 = add(x = var_2764_cast_fp16, y = var_2768_cast_fp16)[name = string("state_143_cast_fp16")];
+            tensor<int32, [4]> var_2771_begin_0 = const()[name = string("op_2771_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2771_end_0 = const()[name = string("op_2771_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2771_end_mask_0 = const()[name = string("op_2771_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2771_squeeze_mask_0 = const()[name = string("op_2771_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2771_cast_fp16 = slice_by_index(begin = var_2771_begin_0, end = var_2771_end_0, end_mask = var_2771_end_mask_0, squeeze_mask = var_2771_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2771_cast_fp16")];
+            tensor<int32, [4]> var_2773_begin_0 = const()[name = string("op_2773_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2773_end_0 = const()[name = string("op_2773_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2773_end_mask_0 = const()[name = string("op_2773_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2773_squeeze_mask_0 = const()[name = string("op_2773_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2773_cast_fp16 = slice_by_index(begin = var_2773_begin_0, end = var_2773_end_0, end_mask = var_2773_end_mask_0, squeeze_mask = var_2773_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2773_cast_fp16")];
+            bool var_2774_transpose_x_0 = const()[name = string("op_2774_transpose_x_0"), val = bool(false)];
+            bool var_2774_transpose_y_0 = const()[name = string("op_2774_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2774_cast_fp16 = matmul(transpose_x = var_2774_transpose_x_0, transpose_y = var_2774_transpose_y_0, x = var_2773_cast_fp16, y = state_143_cast_fp16)[name = string("op_2774_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_143_cast_fp16 = sub(x = var_2771_cast_fp16, y = var_2774_cast_fp16)[name = string("v_new_143_cast_fp16")];
+            tensor<int32, [4]> var_2777_begin_0 = const()[name = string("op_2777_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2777_end_0 = const()[name = string("op_2777_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2777_end_mask_0 = const()[name = string("op_2777_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2777_squeeze_mask_0 = const()[name = string("op_2777_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2777_cast_fp16 = slice_by_index(begin = var_2777_begin_0, end = var_2777_end_0, end_mask = var_2777_end_mask_0, squeeze_mask = var_2777_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2777_cast_fp16")];
+            bool var_2778_transpose_x_0 = const()[name = string("op_2778_transpose_x_0"), val = bool(false)];
+            bool var_2778_transpose_y_0 = const()[name = string("op_2778_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2778_cast_fp16 = matmul(transpose_x = var_2778_transpose_x_0, transpose_y = var_2778_transpose_y_0, x = var_2777_cast_fp16, y = state_143_cast_fp16)[name = string("op_2778_cast_fp16")];
+            tensor<int32, [4]> var_2780_begin_0 = const()[name = string("op_2780_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2780_end_0 = const()[name = string("op_2780_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_2780_end_mask_0 = const()[name = string("op_2780_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2780_squeeze_mask_0 = const()[name = string("op_2780_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2780_cast_fp16 = slice_by_index(begin = var_2780_begin_0, end = var_2780_end_0, end_mask = var_2780_end_mask_0, squeeze_mask = var_2780_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2780_cast_fp16")];
+            bool var_2781_transpose_x_0 = const()[name = string("op_2781_transpose_x_0"), val = bool(false)];
+            bool var_2781_transpose_y_0 = const()[name = string("op_2781_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2781_cast_fp16 = matmul(transpose_x = var_2781_transpose_x_0, transpose_y = var_2781_transpose_y_0, x = var_2780_cast_fp16, y = v_new_143_cast_fp16)[name = string("op_2781_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2782_cast_fp16 = add(x = var_2778_cast_fp16, y = var_2781_cast_fp16)[name = string("op_2782_cast_fp16")];
+            tensor<int32, [4]> var_2784_begin_0 = const()[name = string("op_2784_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2784_end_0 = const()[name = string("op_2784_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_2784_end_mask_0 = const()[name = string("op_2784_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2784_squeeze_mask_0 = const()[name = string("op_2784_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2784_cast_fp16 = slice_by_index(begin = var_2784_begin_0, end = var_2784_end_0, end_mask = var_2784_end_mask_0, squeeze_mask = var_2784_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2784_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2785_cast_fp16 = mul(x = state_143_cast_fp16, y = var_2784_cast_fp16)[name = string("op_2785_cast_fp16")];
+            tensor<int32, [4]> var_2787_begin_0 = const()[name = string("op_2787_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_2787_end_0 = const()[name = string("op_2787_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_2787_end_mask_0 = const()[name = string("op_2787_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2787_squeeze_mask_0 = const()[name = string("op_2787_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2787_cast_fp16 = slice_by_index(begin = var_2787_begin_0, end = var_2787_end_0, end_mask = var_2787_end_mask_0, squeeze_mask = var_2787_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2787_cast_fp16")];
+            bool var_2789_transpose_x_1 = const()[name = string("op_2789_transpose_x_1"), val = bool(true)];
+            bool var_2789_transpose_y_1 = const()[name = string("op_2789_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2789_cast_fp16 = matmul(transpose_x = var_2789_transpose_x_1, transpose_y = var_2789_transpose_y_1, x = var_2787_cast_fp16, y = v_new_143_cast_fp16)[name = string("op_2789_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_145_cast_fp16 = add(x = var_2785_cast_fp16, y = var_2789_cast_fp16)[name = string("state_145_cast_fp16")];
+            tensor<int32, [4]> var_2792_begin_0 = const()[name = string("op_2792_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2792_end_0 = const()[name = string("op_2792_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2792_end_mask_0 = const()[name = string("op_2792_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2792_squeeze_mask_0 = const()[name = string("op_2792_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2792_cast_fp16 = slice_by_index(begin = var_2792_begin_0, end = var_2792_end_0, end_mask = var_2792_end_mask_0, squeeze_mask = var_2792_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2792_cast_fp16")];
+            tensor<int32, [4]> var_2794_begin_0 = const()[name = string("op_2794_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2794_end_0 = const()[name = string("op_2794_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2794_end_mask_0 = const()[name = string("op_2794_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2794_squeeze_mask_0 = const()[name = string("op_2794_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2794_cast_fp16 = slice_by_index(begin = var_2794_begin_0, end = var_2794_end_0, end_mask = var_2794_end_mask_0, squeeze_mask = var_2794_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2794_cast_fp16")];
+            bool var_2795_transpose_x_0 = const()[name = string("op_2795_transpose_x_0"), val = bool(false)];
+            bool var_2795_transpose_y_0 = const()[name = string("op_2795_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2795_cast_fp16 = matmul(transpose_x = var_2795_transpose_x_0, transpose_y = var_2795_transpose_y_0, x = var_2794_cast_fp16, y = state_145_cast_fp16)[name = string("op_2795_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_145_cast_fp16 = sub(x = var_2792_cast_fp16, y = var_2795_cast_fp16)[name = string("v_new_145_cast_fp16")];
+            tensor<int32, [4]> var_2798_begin_0 = const()[name = string("op_2798_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2798_end_0 = const()[name = string("op_2798_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2798_end_mask_0 = const()[name = string("op_2798_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2798_squeeze_mask_0 = const()[name = string("op_2798_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2798_cast_fp16 = slice_by_index(begin = var_2798_begin_0, end = var_2798_end_0, end_mask = var_2798_end_mask_0, squeeze_mask = var_2798_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2798_cast_fp16")];
+            bool var_2799_transpose_x_0 = const()[name = string("op_2799_transpose_x_0"), val = bool(false)];
+            bool var_2799_transpose_y_0 = const()[name = string("op_2799_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2799_cast_fp16 = matmul(transpose_x = var_2799_transpose_x_0, transpose_y = var_2799_transpose_y_0, x = var_2798_cast_fp16, y = state_145_cast_fp16)[name = string("op_2799_cast_fp16")];
+            tensor<int32, [4]> var_2801_begin_0 = const()[name = string("op_2801_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2801_end_0 = const()[name = string("op_2801_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_2801_end_mask_0 = const()[name = string("op_2801_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2801_squeeze_mask_0 = const()[name = string("op_2801_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2801_cast_fp16 = slice_by_index(begin = var_2801_begin_0, end = var_2801_end_0, end_mask = var_2801_end_mask_0, squeeze_mask = var_2801_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2801_cast_fp16")];
+            bool var_2802_transpose_x_0 = const()[name = string("op_2802_transpose_x_0"), val = bool(false)];
+            bool var_2802_transpose_y_0 = const()[name = string("op_2802_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2802_cast_fp16 = matmul(transpose_x = var_2802_transpose_x_0, transpose_y = var_2802_transpose_y_0, x = var_2801_cast_fp16, y = v_new_145_cast_fp16)[name = string("op_2802_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2803_cast_fp16 = add(x = var_2799_cast_fp16, y = var_2802_cast_fp16)[name = string("op_2803_cast_fp16")];
+            tensor<int32, [4]> var_2805_begin_0 = const()[name = string("op_2805_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2805_end_0 = const()[name = string("op_2805_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_2805_end_mask_0 = const()[name = string("op_2805_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2805_squeeze_mask_0 = const()[name = string("op_2805_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2805_cast_fp16 = slice_by_index(begin = var_2805_begin_0, end = var_2805_end_0, end_mask = var_2805_end_mask_0, squeeze_mask = var_2805_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2805_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2806_cast_fp16 = mul(x = state_145_cast_fp16, y = var_2805_cast_fp16)[name = string("op_2806_cast_fp16")];
+            tensor<int32, [4]> var_2808_begin_0 = const()[name = string("op_2808_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_2808_end_0 = const()[name = string("op_2808_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_2808_end_mask_0 = const()[name = string("op_2808_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2808_squeeze_mask_0 = const()[name = string("op_2808_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2808_cast_fp16 = slice_by_index(begin = var_2808_begin_0, end = var_2808_end_0, end_mask = var_2808_end_mask_0, squeeze_mask = var_2808_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2808_cast_fp16")];
+            bool var_2810_transpose_x_1 = const()[name = string("op_2810_transpose_x_1"), val = bool(true)];
+            bool var_2810_transpose_y_1 = const()[name = string("op_2810_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2810_cast_fp16 = matmul(transpose_x = var_2810_transpose_x_1, transpose_y = var_2810_transpose_y_1, x = var_2808_cast_fp16, y = v_new_145_cast_fp16)[name = string("op_2810_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_147_cast_fp16 = add(x = var_2806_cast_fp16, y = var_2810_cast_fp16)[name = string("state_147_cast_fp16")];
+            tensor<int32, [4]> var_2813_begin_0 = const()[name = string("op_2813_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2813_end_0 = const()[name = string("op_2813_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2813_end_mask_0 = const()[name = string("op_2813_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2813_squeeze_mask_0 = const()[name = string("op_2813_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2813_cast_fp16 = slice_by_index(begin = var_2813_begin_0, end = var_2813_end_0, end_mask = var_2813_end_mask_0, squeeze_mask = var_2813_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2813_cast_fp16")];
+            tensor<int32, [4]> var_2815_begin_0 = const()[name = string("op_2815_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2815_end_0 = const()[name = string("op_2815_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2815_end_mask_0 = const()[name = string("op_2815_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2815_squeeze_mask_0 = const()[name = string("op_2815_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2815_cast_fp16 = slice_by_index(begin = var_2815_begin_0, end = var_2815_end_0, end_mask = var_2815_end_mask_0, squeeze_mask = var_2815_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2815_cast_fp16")];
+            bool var_2816_transpose_x_0 = const()[name = string("op_2816_transpose_x_0"), val = bool(false)];
+            bool var_2816_transpose_y_0 = const()[name = string("op_2816_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2816_cast_fp16 = matmul(transpose_x = var_2816_transpose_x_0, transpose_y = var_2816_transpose_y_0, x = var_2815_cast_fp16, y = state_147_cast_fp16)[name = string("op_2816_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_147_cast_fp16 = sub(x = var_2813_cast_fp16, y = var_2816_cast_fp16)[name = string("v_new_147_cast_fp16")];
+            tensor<int32, [4]> var_2819_begin_0 = const()[name = string("op_2819_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2819_end_0 = const()[name = string("op_2819_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2819_end_mask_0 = const()[name = string("op_2819_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2819_squeeze_mask_0 = const()[name = string("op_2819_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2819_cast_fp16 = slice_by_index(begin = var_2819_begin_0, end = var_2819_end_0, end_mask = var_2819_end_mask_0, squeeze_mask = var_2819_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2819_cast_fp16")];
+            bool var_2820_transpose_x_0 = const()[name = string("op_2820_transpose_x_0"), val = bool(false)];
+            bool var_2820_transpose_y_0 = const()[name = string("op_2820_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2820_cast_fp16 = matmul(transpose_x = var_2820_transpose_x_0, transpose_y = var_2820_transpose_y_0, x = var_2819_cast_fp16, y = state_147_cast_fp16)[name = string("op_2820_cast_fp16")];
+            tensor<int32, [4]> var_2822_begin_0 = const()[name = string("op_2822_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2822_end_0 = const()[name = string("op_2822_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_2822_end_mask_0 = const()[name = string("op_2822_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2822_squeeze_mask_0 = const()[name = string("op_2822_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2822_cast_fp16 = slice_by_index(begin = var_2822_begin_0, end = var_2822_end_0, end_mask = var_2822_end_mask_0, squeeze_mask = var_2822_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2822_cast_fp16")];
+            bool var_2823_transpose_x_0 = const()[name = string("op_2823_transpose_x_0"), val = bool(false)];
+            bool var_2823_transpose_y_0 = const()[name = string("op_2823_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2823_cast_fp16 = matmul(transpose_x = var_2823_transpose_x_0, transpose_y = var_2823_transpose_y_0, x = var_2822_cast_fp16, y = v_new_147_cast_fp16)[name = string("op_2823_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2824_cast_fp16 = add(x = var_2820_cast_fp16, y = var_2823_cast_fp16)[name = string("op_2824_cast_fp16")];
+            tensor<int32, [4]> var_2826_begin_0 = const()[name = string("op_2826_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2826_end_0 = const()[name = string("op_2826_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_2826_end_mask_0 = const()[name = string("op_2826_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2826_squeeze_mask_0 = const()[name = string("op_2826_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2826_cast_fp16 = slice_by_index(begin = var_2826_begin_0, end = var_2826_end_0, end_mask = var_2826_end_mask_0, squeeze_mask = var_2826_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2826_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2827_cast_fp16 = mul(x = state_147_cast_fp16, y = var_2826_cast_fp16)[name = string("op_2827_cast_fp16")];
+            tensor<int32, [4]> var_2829_begin_0 = const()[name = string("op_2829_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_2829_end_0 = const()[name = string("op_2829_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_2829_end_mask_0 = const()[name = string("op_2829_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2829_squeeze_mask_0 = const()[name = string("op_2829_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2829_cast_fp16 = slice_by_index(begin = var_2829_begin_0, end = var_2829_end_0, end_mask = var_2829_end_mask_0, squeeze_mask = var_2829_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2829_cast_fp16")];
+            bool var_2831_transpose_x_1 = const()[name = string("op_2831_transpose_x_1"), val = bool(true)];
+            bool var_2831_transpose_y_1 = const()[name = string("op_2831_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2831_cast_fp16 = matmul(transpose_x = var_2831_transpose_x_1, transpose_y = var_2831_transpose_y_1, x = var_2829_cast_fp16, y = v_new_147_cast_fp16)[name = string("op_2831_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_149_cast_fp16 = add(x = var_2827_cast_fp16, y = var_2831_cast_fp16)[name = string("state_149_cast_fp16")];
+            tensor<int32, [4]> var_2834_begin_0 = const()[name = string("op_2834_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2834_end_0 = const()[name = string("op_2834_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2834_end_mask_0 = const()[name = string("op_2834_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2834_squeeze_mask_0 = const()[name = string("op_2834_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2834_cast_fp16 = slice_by_index(begin = var_2834_begin_0, end = var_2834_end_0, end_mask = var_2834_end_mask_0, squeeze_mask = var_2834_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2834_cast_fp16")];
+            tensor<int32, [4]> var_2836_begin_0 = const()[name = string("op_2836_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2836_end_0 = const()[name = string("op_2836_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2836_end_mask_0 = const()[name = string("op_2836_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2836_squeeze_mask_0 = const()[name = string("op_2836_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2836_cast_fp16 = slice_by_index(begin = var_2836_begin_0, end = var_2836_end_0, end_mask = var_2836_end_mask_0, squeeze_mask = var_2836_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2836_cast_fp16")];
+            bool var_2837_transpose_x_0 = const()[name = string("op_2837_transpose_x_0"), val = bool(false)];
+            bool var_2837_transpose_y_0 = const()[name = string("op_2837_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2837_cast_fp16 = matmul(transpose_x = var_2837_transpose_x_0, transpose_y = var_2837_transpose_y_0, x = var_2836_cast_fp16, y = state_149_cast_fp16)[name = string("op_2837_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_149_cast_fp16 = sub(x = var_2834_cast_fp16, y = var_2837_cast_fp16)[name = string("v_new_149_cast_fp16")];
+            tensor<int32, [4]> var_2840_begin_0 = const()[name = string("op_2840_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2840_end_0 = const()[name = string("op_2840_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2840_end_mask_0 = const()[name = string("op_2840_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2840_squeeze_mask_0 = const()[name = string("op_2840_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2840_cast_fp16 = slice_by_index(begin = var_2840_begin_0, end = var_2840_end_0, end_mask = var_2840_end_mask_0, squeeze_mask = var_2840_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2840_cast_fp16")];
+            bool var_2841_transpose_x_0 = const()[name = string("op_2841_transpose_x_0"), val = bool(false)];
+            bool var_2841_transpose_y_0 = const()[name = string("op_2841_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2841_cast_fp16 = matmul(transpose_x = var_2841_transpose_x_0, transpose_y = var_2841_transpose_y_0, x = var_2840_cast_fp16, y = state_149_cast_fp16)[name = string("op_2841_cast_fp16")];
+            tensor<int32, [4]> var_2843_begin_0 = const()[name = string("op_2843_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2843_end_0 = const()[name = string("op_2843_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_2843_end_mask_0 = const()[name = string("op_2843_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2843_squeeze_mask_0 = const()[name = string("op_2843_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2843_cast_fp16 = slice_by_index(begin = var_2843_begin_0, end = var_2843_end_0, end_mask = var_2843_end_mask_0, squeeze_mask = var_2843_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2843_cast_fp16")];
+            bool var_2844_transpose_x_0 = const()[name = string("op_2844_transpose_x_0"), val = bool(false)];
+            bool var_2844_transpose_y_0 = const()[name = string("op_2844_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2844_cast_fp16 = matmul(transpose_x = var_2844_transpose_x_0, transpose_y = var_2844_transpose_y_0, x = var_2843_cast_fp16, y = v_new_149_cast_fp16)[name = string("op_2844_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2845_cast_fp16 = add(x = var_2841_cast_fp16, y = var_2844_cast_fp16)[name = string("op_2845_cast_fp16")];
+            tensor<int32, [4]> var_2847_begin_0 = const()[name = string("op_2847_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2847_end_0 = const()[name = string("op_2847_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_2847_end_mask_0 = const()[name = string("op_2847_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2847_squeeze_mask_0 = const()[name = string("op_2847_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2847_cast_fp16 = slice_by_index(begin = var_2847_begin_0, end = var_2847_end_0, end_mask = var_2847_end_mask_0, squeeze_mask = var_2847_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2847_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2848_cast_fp16 = mul(x = state_149_cast_fp16, y = var_2847_cast_fp16)[name = string("op_2848_cast_fp16")];
+            tensor<int32, [4]> var_2850_begin_0 = const()[name = string("op_2850_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_2850_end_0 = const()[name = string("op_2850_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_2850_end_mask_0 = const()[name = string("op_2850_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2850_squeeze_mask_0 = const()[name = string("op_2850_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2850_cast_fp16 = slice_by_index(begin = var_2850_begin_0, end = var_2850_end_0, end_mask = var_2850_end_mask_0, squeeze_mask = var_2850_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2850_cast_fp16")];
+            bool var_2852_transpose_x_1 = const()[name = string("op_2852_transpose_x_1"), val = bool(true)];
+            bool var_2852_transpose_y_1 = const()[name = string("op_2852_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2852_cast_fp16 = matmul(transpose_x = var_2852_transpose_x_1, transpose_y = var_2852_transpose_y_1, x = var_2850_cast_fp16, y = v_new_149_cast_fp16)[name = string("op_2852_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_151_cast_fp16 = add(x = var_2848_cast_fp16, y = var_2852_cast_fp16)[name = string("state_151_cast_fp16")];
+            tensor<int32, [4]> var_2855_begin_0 = const()[name = string("op_2855_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2855_end_0 = const()[name = string("op_2855_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2855_end_mask_0 = const()[name = string("op_2855_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2855_squeeze_mask_0 = const()[name = string("op_2855_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2855_cast_fp16 = slice_by_index(begin = var_2855_begin_0, end = var_2855_end_0, end_mask = var_2855_end_mask_0, squeeze_mask = var_2855_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2855_cast_fp16")];
+            tensor<int32, [4]> var_2857_begin_0 = const()[name = string("op_2857_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2857_end_0 = const()[name = string("op_2857_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2857_end_mask_0 = const()[name = string("op_2857_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2857_squeeze_mask_0 = const()[name = string("op_2857_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2857_cast_fp16 = slice_by_index(begin = var_2857_begin_0, end = var_2857_end_0, end_mask = var_2857_end_mask_0, squeeze_mask = var_2857_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2857_cast_fp16")];
+            bool var_2858_transpose_x_0 = const()[name = string("op_2858_transpose_x_0"), val = bool(false)];
+            bool var_2858_transpose_y_0 = const()[name = string("op_2858_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2858_cast_fp16 = matmul(transpose_x = var_2858_transpose_x_0, transpose_y = var_2858_transpose_y_0, x = var_2857_cast_fp16, y = state_151_cast_fp16)[name = string("op_2858_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_151_cast_fp16 = sub(x = var_2855_cast_fp16, y = var_2858_cast_fp16)[name = string("v_new_151_cast_fp16")];
+            tensor<int32, [4]> var_2861_begin_0 = const()[name = string("op_2861_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2861_end_0 = const()[name = string("op_2861_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2861_end_mask_0 = const()[name = string("op_2861_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2861_squeeze_mask_0 = const()[name = string("op_2861_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2861_cast_fp16 = slice_by_index(begin = var_2861_begin_0, end = var_2861_end_0, end_mask = var_2861_end_mask_0, squeeze_mask = var_2861_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2861_cast_fp16")];
+            bool var_2862_transpose_x_0 = const()[name = string("op_2862_transpose_x_0"), val = bool(false)];
+            bool var_2862_transpose_y_0 = const()[name = string("op_2862_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2862_cast_fp16 = matmul(transpose_x = var_2862_transpose_x_0, transpose_y = var_2862_transpose_y_0, x = var_2861_cast_fp16, y = state_151_cast_fp16)[name = string("op_2862_cast_fp16")];
+            tensor<int32, [4]> var_2864_begin_0 = const()[name = string("op_2864_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2864_end_0 = const()[name = string("op_2864_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_2864_end_mask_0 = const()[name = string("op_2864_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2864_squeeze_mask_0 = const()[name = string("op_2864_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2864_cast_fp16 = slice_by_index(begin = var_2864_begin_0, end = var_2864_end_0, end_mask = var_2864_end_mask_0, squeeze_mask = var_2864_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2864_cast_fp16")];
+            bool var_2865_transpose_x_0 = const()[name = string("op_2865_transpose_x_0"), val = bool(false)];
+            bool var_2865_transpose_y_0 = const()[name = string("op_2865_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2865_cast_fp16 = matmul(transpose_x = var_2865_transpose_x_0, transpose_y = var_2865_transpose_y_0, x = var_2864_cast_fp16, y = v_new_151_cast_fp16)[name = string("op_2865_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2866_cast_fp16 = add(x = var_2862_cast_fp16, y = var_2865_cast_fp16)[name = string("op_2866_cast_fp16")];
+            tensor<int32, [4]> var_2868_begin_0 = const()[name = string("op_2868_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2868_end_0 = const()[name = string("op_2868_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_2868_end_mask_0 = const()[name = string("op_2868_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2868_squeeze_mask_0 = const()[name = string("op_2868_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2868_cast_fp16 = slice_by_index(begin = var_2868_begin_0, end = var_2868_end_0, end_mask = var_2868_end_mask_0, squeeze_mask = var_2868_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2868_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2869_cast_fp16 = mul(x = state_151_cast_fp16, y = var_2868_cast_fp16)[name = string("op_2869_cast_fp16")];
+            tensor<int32, [4]> var_2871_begin_0 = const()[name = string("op_2871_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_2871_end_0 = const()[name = string("op_2871_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_2871_end_mask_0 = const()[name = string("op_2871_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2871_squeeze_mask_0 = const()[name = string("op_2871_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2871_cast_fp16 = slice_by_index(begin = var_2871_begin_0, end = var_2871_end_0, end_mask = var_2871_end_mask_0, squeeze_mask = var_2871_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2871_cast_fp16")];
+            bool var_2873_transpose_x_1 = const()[name = string("op_2873_transpose_x_1"), val = bool(true)];
+            bool var_2873_transpose_y_1 = const()[name = string("op_2873_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2873_cast_fp16 = matmul(transpose_x = var_2873_transpose_x_1, transpose_y = var_2873_transpose_y_1, x = var_2871_cast_fp16, y = v_new_151_cast_fp16)[name = string("op_2873_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_153_cast_fp16 = add(x = var_2869_cast_fp16, y = var_2873_cast_fp16)[name = string("state_153_cast_fp16")];
+            tensor<int32, [4]> var_2876_begin_0 = const()[name = string("op_2876_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2876_end_0 = const()[name = string("op_2876_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2876_end_mask_0 = const()[name = string("op_2876_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2876_squeeze_mask_0 = const()[name = string("op_2876_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2876_cast_fp16 = slice_by_index(begin = var_2876_begin_0, end = var_2876_end_0, end_mask = var_2876_end_mask_0, squeeze_mask = var_2876_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2876_cast_fp16")];
+            tensor<int32, [4]> var_2878_begin_0 = const()[name = string("op_2878_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2878_end_0 = const()[name = string("op_2878_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2878_end_mask_0 = const()[name = string("op_2878_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2878_squeeze_mask_0 = const()[name = string("op_2878_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2878_cast_fp16 = slice_by_index(begin = var_2878_begin_0, end = var_2878_end_0, end_mask = var_2878_end_mask_0, squeeze_mask = var_2878_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2878_cast_fp16")];
+            bool var_2879_transpose_x_0 = const()[name = string("op_2879_transpose_x_0"), val = bool(false)];
+            bool var_2879_transpose_y_0 = const()[name = string("op_2879_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2879_cast_fp16 = matmul(transpose_x = var_2879_transpose_x_0, transpose_y = var_2879_transpose_y_0, x = var_2878_cast_fp16, y = state_153_cast_fp16)[name = string("op_2879_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_153_cast_fp16 = sub(x = var_2876_cast_fp16, y = var_2879_cast_fp16)[name = string("v_new_153_cast_fp16")];
+            tensor<int32, [4]> var_2882_begin_0 = const()[name = string("op_2882_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2882_end_0 = const()[name = string("op_2882_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2882_end_mask_0 = const()[name = string("op_2882_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2882_squeeze_mask_0 = const()[name = string("op_2882_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2882_cast_fp16 = slice_by_index(begin = var_2882_begin_0, end = var_2882_end_0, end_mask = var_2882_end_mask_0, squeeze_mask = var_2882_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2882_cast_fp16")];
+            bool var_2883_transpose_x_0 = const()[name = string("op_2883_transpose_x_0"), val = bool(false)];
+            bool var_2883_transpose_y_0 = const()[name = string("op_2883_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2883_cast_fp16 = matmul(transpose_x = var_2883_transpose_x_0, transpose_y = var_2883_transpose_y_0, x = var_2882_cast_fp16, y = state_153_cast_fp16)[name = string("op_2883_cast_fp16")];
+            tensor<int32, [4]> var_2885_begin_0 = const()[name = string("op_2885_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2885_end_0 = const()[name = string("op_2885_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_2885_end_mask_0 = const()[name = string("op_2885_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2885_squeeze_mask_0 = const()[name = string("op_2885_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2885_cast_fp16 = slice_by_index(begin = var_2885_begin_0, end = var_2885_end_0, end_mask = var_2885_end_mask_0, squeeze_mask = var_2885_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2885_cast_fp16")];
+            bool var_2886_transpose_x_0 = const()[name = string("op_2886_transpose_x_0"), val = bool(false)];
+            bool var_2886_transpose_y_0 = const()[name = string("op_2886_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2886_cast_fp16 = matmul(transpose_x = var_2886_transpose_x_0, transpose_y = var_2886_transpose_y_0, x = var_2885_cast_fp16, y = v_new_153_cast_fp16)[name = string("op_2886_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2887_cast_fp16 = add(x = var_2883_cast_fp16, y = var_2886_cast_fp16)[name = string("op_2887_cast_fp16")];
+            tensor<int32, [4]> var_2889_begin_0 = const()[name = string("op_2889_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2889_end_0 = const()[name = string("op_2889_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_2889_end_mask_0 = const()[name = string("op_2889_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2889_squeeze_mask_0 = const()[name = string("op_2889_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2889_cast_fp16 = slice_by_index(begin = var_2889_begin_0, end = var_2889_end_0, end_mask = var_2889_end_mask_0, squeeze_mask = var_2889_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2889_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2890_cast_fp16 = mul(x = state_153_cast_fp16, y = var_2889_cast_fp16)[name = string("op_2890_cast_fp16")];
+            tensor<int32, [4]> var_2892_begin_0 = const()[name = string("op_2892_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_2892_end_0 = const()[name = string("op_2892_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_2892_end_mask_0 = const()[name = string("op_2892_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2892_squeeze_mask_0 = const()[name = string("op_2892_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2892_cast_fp16 = slice_by_index(begin = var_2892_begin_0, end = var_2892_end_0, end_mask = var_2892_end_mask_0, squeeze_mask = var_2892_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2892_cast_fp16")];
+            bool var_2894_transpose_x_1 = const()[name = string("op_2894_transpose_x_1"), val = bool(true)];
+            bool var_2894_transpose_y_1 = const()[name = string("op_2894_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2894_cast_fp16 = matmul(transpose_x = var_2894_transpose_x_1, transpose_y = var_2894_transpose_y_1, x = var_2892_cast_fp16, y = v_new_153_cast_fp16)[name = string("op_2894_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_155_cast_fp16 = add(x = var_2890_cast_fp16, y = var_2894_cast_fp16)[name = string("state_155_cast_fp16")];
+            tensor<int32, [4]> var_2897_begin_0 = const()[name = string("op_2897_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2897_end_0 = const()[name = string("op_2897_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2897_end_mask_0 = const()[name = string("op_2897_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2897_squeeze_mask_0 = const()[name = string("op_2897_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2897_cast_fp16 = slice_by_index(begin = var_2897_begin_0, end = var_2897_end_0, end_mask = var_2897_end_mask_0, squeeze_mask = var_2897_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2897_cast_fp16")];
+            tensor<int32, [4]> var_2899_begin_0 = const()[name = string("op_2899_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2899_end_0 = const()[name = string("op_2899_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2899_end_mask_0 = const()[name = string("op_2899_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2899_squeeze_mask_0 = const()[name = string("op_2899_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2899_cast_fp16 = slice_by_index(begin = var_2899_begin_0, end = var_2899_end_0, end_mask = var_2899_end_mask_0, squeeze_mask = var_2899_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2899_cast_fp16")];
+            bool var_2900_transpose_x_0 = const()[name = string("op_2900_transpose_x_0"), val = bool(false)];
+            bool var_2900_transpose_y_0 = const()[name = string("op_2900_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2900_cast_fp16 = matmul(transpose_x = var_2900_transpose_x_0, transpose_y = var_2900_transpose_y_0, x = var_2899_cast_fp16, y = state_155_cast_fp16)[name = string("op_2900_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_155_cast_fp16 = sub(x = var_2897_cast_fp16, y = var_2900_cast_fp16)[name = string("v_new_155_cast_fp16")];
+            tensor<int32, [4]> var_2903_begin_0 = const()[name = string("op_2903_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2903_end_0 = const()[name = string("op_2903_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2903_end_mask_0 = const()[name = string("op_2903_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2903_squeeze_mask_0 = const()[name = string("op_2903_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2903_cast_fp16 = slice_by_index(begin = var_2903_begin_0, end = var_2903_end_0, end_mask = var_2903_end_mask_0, squeeze_mask = var_2903_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2903_cast_fp16")];
+            bool var_2904_transpose_x_0 = const()[name = string("op_2904_transpose_x_0"), val = bool(false)];
+            bool var_2904_transpose_y_0 = const()[name = string("op_2904_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2904_cast_fp16 = matmul(transpose_x = var_2904_transpose_x_0, transpose_y = var_2904_transpose_y_0, x = var_2903_cast_fp16, y = state_155_cast_fp16)[name = string("op_2904_cast_fp16")];
+            tensor<int32, [4]> var_2906_begin_0 = const()[name = string("op_2906_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2906_end_0 = const()[name = string("op_2906_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_2906_end_mask_0 = const()[name = string("op_2906_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2906_squeeze_mask_0 = const()[name = string("op_2906_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2906_cast_fp16 = slice_by_index(begin = var_2906_begin_0, end = var_2906_end_0, end_mask = var_2906_end_mask_0, squeeze_mask = var_2906_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2906_cast_fp16")];
+            bool var_2907_transpose_x_0 = const()[name = string("op_2907_transpose_x_0"), val = bool(false)];
+            bool var_2907_transpose_y_0 = const()[name = string("op_2907_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2907_cast_fp16 = matmul(transpose_x = var_2907_transpose_x_0, transpose_y = var_2907_transpose_y_0, x = var_2906_cast_fp16, y = v_new_155_cast_fp16)[name = string("op_2907_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2908_cast_fp16 = add(x = var_2904_cast_fp16, y = var_2907_cast_fp16)[name = string("op_2908_cast_fp16")];
+            tensor<int32, [4]> var_2910_begin_0 = const()[name = string("op_2910_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2910_end_0 = const()[name = string("op_2910_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_2910_end_mask_0 = const()[name = string("op_2910_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2910_squeeze_mask_0 = const()[name = string("op_2910_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2910_cast_fp16 = slice_by_index(begin = var_2910_begin_0, end = var_2910_end_0, end_mask = var_2910_end_mask_0, squeeze_mask = var_2910_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2910_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2911_cast_fp16 = mul(x = state_155_cast_fp16, y = var_2910_cast_fp16)[name = string("op_2911_cast_fp16")];
+            tensor<int32, [4]> var_2913_begin_0 = const()[name = string("op_2913_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_2913_end_0 = const()[name = string("op_2913_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_2913_end_mask_0 = const()[name = string("op_2913_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2913_squeeze_mask_0 = const()[name = string("op_2913_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2913_cast_fp16 = slice_by_index(begin = var_2913_begin_0, end = var_2913_end_0, end_mask = var_2913_end_mask_0, squeeze_mask = var_2913_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2913_cast_fp16")];
+            bool var_2915_transpose_x_1 = const()[name = string("op_2915_transpose_x_1"), val = bool(true)];
+            bool var_2915_transpose_y_1 = const()[name = string("op_2915_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2915_cast_fp16 = matmul(transpose_x = var_2915_transpose_x_1, transpose_y = var_2915_transpose_y_1, x = var_2913_cast_fp16, y = v_new_155_cast_fp16)[name = string("op_2915_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_157_cast_fp16 = add(x = var_2911_cast_fp16, y = var_2915_cast_fp16)[name = string("state_157_cast_fp16")];
+            tensor<int32, [4]> var_2918_begin_0 = const()[name = string("op_2918_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2918_end_0 = const()[name = string("op_2918_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2918_end_mask_0 = const()[name = string("op_2918_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2918_squeeze_mask_0 = const()[name = string("op_2918_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2918_cast_fp16 = slice_by_index(begin = var_2918_begin_0, end = var_2918_end_0, end_mask = var_2918_end_mask_0, squeeze_mask = var_2918_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2918_cast_fp16")];
+            tensor<int32, [4]> var_2920_begin_0 = const()[name = string("op_2920_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2920_end_0 = const()[name = string("op_2920_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2920_end_mask_0 = const()[name = string("op_2920_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2920_squeeze_mask_0 = const()[name = string("op_2920_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2920_cast_fp16 = slice_by_index(begin = var_2920_begin_0, end = var_2920_end_0, end_mask = var_2920_end_mask_0, squeeze_mask = var_2920_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2920_cast_fp16")];
+            bool var_2921_transpose_x_0 = const()[name = string("op_2921_transpose_x_0"), val = bool(false)];
+            bool var_2921_transpose_y_0 = const()[name = string("op_2921_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2921_cast_fp16 = matmul(transpose_x = var_2921_transpose_x_0, transpose_y = var_2921_transpose_y_0, x = var_2920_cast_fp16, y = state_157_cast_fp16)[name = string("op_2921_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_157_cast_fp16 = sub(x = var_2918_cast_fp16, y = var_2921_cast_fp16)[name = string("v_new_157_cast_fp16")];
+            tensor<int32, [4]> var_2924_begin_0 = const()[name = string("op_2924_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2924_end_0 = const()[name = string("op_2924_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2924_end_mask_0 = const()[name = string("op_2924_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2924_squeeze_mask_0 = const()[name = string("op_2924_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2924_cast_fp16 = slice_by_index(begin = var_2924_begin_0, end = var_2924_end_0, end_mask = var_2924_end_mask_0, squeeze_mask = var_2924_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2924_cast_fp16")];
+            bool var_2925_transpose_x_0 = const()[name = string("op_2925_transpose_x_0"), val = bool(false)];
+            bool var_2925_transpose_y_0 = const()[name = string("op_2925_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2925_cast_fp16 = matmul(transpose_x = var_2925_transpose_x_0, transpose_y = var_2925_transpose_y_0, x = var_2924_cast_fp16, y = state_157_cast_fp16)[name = string("op_2925_cast_fp16")];
+            tensor<int32, [4]> var_2927_begin_0 = const()[name = string("op_2927_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2927_end_0 = const()[name = string("op_2927_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_2927_end_mask_0 = const()[name = string("op_2927_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2927_squeeze_mask_0 = const()[name = string("op_2927_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2927_cast_fp16 = slice_by_index(begin = var_2927_begin_0, end = var_2927_end_0, end_mask = var_2927_end_mask_0, squeeze_mask = var_2927_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2927_cast_fp16")];
+            bool var_2928_transpose_x_0 = const()[name = string("op_2928_transpose_x_0"), val = bool(false)];
+            bool var_2928_transpose_y_0 = const()[name = string("op_2928_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2928_cast_fp16 = matmul(transpose_x = var_2928_transpose_x_0, transpose_y = var_2928_transpose_y_0, x = var_2927_cast_fp16, y = v_new_157_cast_fp16)[name = string("op_2928_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2929_cast_fp16 = add(x = var_2925_cast_fp16, y = var_2928_cast_fp16)[name = string("op_2929_cast_fp16")];
+            tensor<int32, [4]> var_2931_begin_0 = const()[name = string("op_2931_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2931_end_0 = const()[name = string("op_2931_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_2931_end_mask_0 = const()[name = string("op_2931_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2931_squeeze_mask_0 = const()[name = string("op_2931_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2931_cast_fp16 = slice_by_index(begin = var_2931_begin_0, end = var_2931_end_0, end_mask = var_2931_end_mask_0, squeeze_mask = var_2931_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2931_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2932_cast_fp16 = mul(x = state_157_cast_fp16, y = var_2931_cast_fp16)[name = string("op_2932_cast_fp16")];
+            tensor<int32, [4]> var_2934_begin_0 = const()[name = string("op_2934_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_2934_end_0 = const()[name = string("op_2934_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_2934_end_mask_0 = const()[name = string("op_2934_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2934_squeeze_mask_0 = const()[name = string("op_2934_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2934_cast_fp16 = slice_by_index(begin = var_2934_begin_0, end = var_2934_end_0, end_mask = var_2934_end_mask_0, squeeze_mask = var_2934_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2934_cast_fp16")];
+            bool var_2936_transpose_x_1 = const()[name = string("op_2936_transpose_x_1"), val = bool(true)];
+            bool var_2936_transpose_y_1 = const()[name = string("op_2936_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2936_cast_fp16 = matmul(transpose_x = var_2936_transpose_x_1, transpose_y = var_2936_transpose_y_1, x = var_2934_cast_fp16, y = v_new_157_cast_fp16)[name = string("op_2936_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_159_cast_fp16 = add(x = var_2932_cast_fp16, y = var_2936_cast_fp16)[name = string("state_159_cast_fp16")];
+            tensor<int32, [4]> var_2939_begin_0 = const()[name = string("op_2939_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2939_end_0 = const()[name = string("op_2939_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2939_end_mask_0 = const()[name = string("op_2939_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2939_squeeze_mask_0 = const()[name = string("op_2939_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2939_cast_fp16 = slice_by_index(begin = var_2939_begin_0, end = var_2939_end_0, end_mask = var_2939_end_mask_0, squeeze_mask = var_2939_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2939_cast_fp16")];
+            tensor<int32, [4]> var_2941_begin_0 = const()[name = string("op_2941_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2941_end_0 = const()[name = string("op_2941_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2941_end_mask_0 = const()[name = string("op_2941_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2941_squeeze_mask_0 = const()[name = string("op_2941_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2941_cast_fp16 = slice_by_index(begin = var_2941_begin_0, end = var_2941_end_0, end_mask = var_2941_end_mask_0, squeeze_mask = var_2941_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2941_cast_fp16")];
+            bool var_2942_transpose_x_0 = const()[name = string("op_2942_transpose_x_0"), val = bool(false)];
+            bool var_2942_transpose_y_0 = const()[name = string("op_2942_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2942_cast_fp16 = matmul(transpose_x = var_2942_transpose_x_0, transpose_y = var_2942_transpose_y_0, x = var_2941_cast_fp16, y = state_159_cast_fp16)[name = string("op_2942_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_159_cast_fp16 = sub(x = var_2939_cast_fp16, y = var_2942_cast_fp16)[name = string("v_new_159_cast_fp16")];
+            tensor<int32, [4]> var_2945_begin_0 = const()[name = string("op_2945_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2945_end_0 = const()[name = string("op_2945_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2945_end_mask_0 = const()[name = string("op_2945_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2945_squeeze_mask_0 = const()[name = string("op_2945_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2945_cast_fp16 = slice_by_index(begin = var_2945_begin_0, end = var_2945_end_0, end_mask = var_2945_end_mask_0, squeeze_mask = var_2945_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2945_cast_fp16")];
+            bool var_2946_transpose_x_0 = const()[name = string("op_2946_transpose_x_0"), val = bool(false)];
+            bool var_2946_transpose_y_0 = const()[name = string("op_2946_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2946_cast_fp16 = matmul(transpose_x = var_2946_transpose_x_0, transpose_y = var_2946_transpose_y_0, x = var_2945_cast_fp16, y = state_159_cast_fp16)[name = string("op_2946_cast_fp16")];
+            tensor<int32, [4]> var_2948_begin_0 = const()[name = string("op_2948_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2948_end_0 = const()[name = string("op_2948_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_2948_end_mask_0 = const()[name = string("op_2948_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2948_squeeze_mask_0 = const()[name = string("op_2948_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2948_cast_fp16 = slice_by_index(begin = var_2948_begin_0, end = var_2948_end_0, end_mask = var_2948_end_mask_0, squeeze_mask = var_2948_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2948_cast_fp16")];
+            bool var_2949_transpose_x_0 = const()[name = string("op_2949_transpose_x_0"), val = bool(false)];
+            bool var_2949_transpose_y_0 = const()[name = string("op_2949_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2949_cast_fp16 = matmul(transpose_x = var_2949_transpose_x_0, transpose_y = var_2949_transpose_y_0, x = var_2948_cast_fp16, y = v_new_159_cast_fp16)[name = string("op_2949_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2950_cast_fp16 = add(x = var_2946_cast_fp16, y = var_2949_cast_fp16)[name = string("op_2950_cast_fp16")];
+            tensor<int32, [4]> var_2952_begin_0 = const()[name = string("op_2952_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2952_end_0 = const()[name = string("op_2952_end_0"), val = tensor<int32, [4]>([32, 16, 1, 1])];
+            tensor<bool, [4]> var_2952_end_mask_0 = const()[name = string("op_2952_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2952_squeeze_mask_0 = const()[name = string("op_2952_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2952_cast_fp16 = slice_by_index(begin = var_2952_begin_0, end = var_2952_end_0, end_mask = var_2952_end_mask_0, squeeze_mask = var_2952_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2952_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2953_cast_fp16 = mul(x = state_159_cast_fp16, y = var_2952_cast_fp16)[name = string("op_2953_cast_fp16")];
+            tensor<int32, [4]> var_2955_begin_0 = const()[name = string("op_2955_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_2955_end_0 = const()[name = string("op_2955_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_2955_end_mask_0 = const()[name = string("op_2955_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2955_squeeze_mask_0 = const()[name = string("op_2955_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2955_cast_fp16 = slice_by_index(begin = var_2955_begin_0, end = var_2955_end_0, end_mask = var_2955_end_mask_0, squeeze_mask = var_2955_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2955_cast_fp16")];
+            bool var_2957_transpose_x_1 = const()[name = string("op_2957_transpose_x_1"), val = bool(true)];
+            bool var_2957_transpose_y_1 = const()[name = string("op_2957_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2957_cast_fp16 = matmul(transpose_x = var_2957_transpose_x_1, transpose_y = var_2957_transpose_y_1, x = var_2955_cast_fp16, y = v_new_159_cast_fp16)[name = string("op_2957_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_161_cast_fp16 = add(x = var_2953_cast_fp16, y = var_2957_cast_fp16)[name = string("state_161_cast_fp16")];
+            tensor<int32, [4]> var_2960_begin_0 = const()[name = string("op_2960_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_2960_end_0 = const()[name = string("op_2960_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_2960_end_mask_0 = const()[name = string("op_2960_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2960_squeeze_mask_0 = const()[name = string("op_2960_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2960_cast_fp16 = slice_by_index(begin = var_2960_begin_0, end = var_2960_end_0, end_mask = var_2960_end_mask_0, squeeze_mask = var_2960_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2960_cast_fp16")];
+            tensor<int32, [4]> var_2962_begin_0 = const()[name = string("op_2962_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_2962_end_0 = const()[name = string("op_2962_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_2962_end_mask_0 = const()[name = string("op_2962_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2962_squeeze_mask_0 = const()[name = string("op_2962_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2962_cast_fp16 = slice_by_index(begin = var_2962_begin_0, end = var_2962_end_0, end_mask = var_2962_end_mask_0, squeeze_mask = var_2962_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2962_cast_fp16")];
+            bool var_2963_transpose_x_0 = const()[name = string("op_2963_transpose_x_0"), val = bool(false)];
+            bool var_2963_transpose_y_0 = const()[name = string("op_2963_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2963_cast_fp16 = matmul(transpose_x = var_2963_transpose_x_0, transpose_y = var_2963_transpose_y_0, x = var_2962_cast_fp16, y = state_161_cast_fp16)[name = string("op_2963_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_161_cast_fp16 = sub(x = var_2960_cast_fp16, y = var_2963_cast_fp16)[name = string("v_new_161_cast_fp16")];
+            tensor<int32, [4]> var_2966_begin_0 = const()[name = string("op_2966_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_2966_end_0 = const()[name = string("op_2966_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_2966_end_mask_0 = const()[name = string("op_2966_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2966_squeeze_mask_0 = const()[name = string("op_2966_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2966_cast_fp16 = slice_by_index(begin = var_2966_begin_0, end = var_2966_end_0, end_mask = var_2966_end_mask_0, squeeze_mask = var_2966_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2966_cast_fp16")];
+            bool var_2967_transpose_x_0 = const()[name = string("op_2967_transpose_x_0"), val = bool(false)];
+            bool var_2967_transpose_y_0 = const()[name = string("op_2967_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2967_cast_fp16 = matmul(transpose_x = var_2967_transpose_x_0, transpose_y = var_2967_transpose_y_0, x = var_2966_cast_fp16, y = state_161_cast_fp16)[name = string("op_2967_cast_fp16")];
+            tensor<int32, [4]> var_2969_begin_0 = const()[name = string("op_2969_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_2969_end_0 = const()[name = string("op_2969_end_0"), val = tensor<int32, [4]>([32, 17, 64, 64])];
+            tensor<bool, [4]> var_2969_end_mask_0 = const()[name = string("op_2969_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2969_squeeze_mask_0 = const()[name = string("op_2969_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2969_cast_fp16 = slice_by_index(begin = var_2969_begin_0, end = var_2969_end_0, end_mask = var_2969_end_mask_0, squeeze_mask = var_2969_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2969_cast_fp16")];
+            bool var_2970_transpose_x_0 = const()[name = string("op_2970_transpose_x_0"), val = bool(false)];
+            bool var_2970_transpose_y_0 = const()[name = string("op_2970_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2970_cast_fp16 = matmul(transpose_x = var_2970_transpose_x_0, transpose_y = var_2970_transpose_y_0, x = var_2969_cast_fp16, y = v_new_161_cast_fp16)[name = string("op_2970_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2971_cast_fp16 = add(x = var_2967_cast_fp16, y = var_2970_cast_fp16)[name = string("op_2971_cast_fp16")];
+            tensor<int32, [4]> var_2973_begin_0 = const()[name = string("op_2973_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_2973_end_0 = const()[name = string("op_2973_end_0"), val = tensor<int32, [4]>([32, 17, 1, 1])];
+            tensor<bool, [4]> var_2973_end_mask_0 = const()[name = string("op_2973_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2973_squeeze_mask_0 = const()[name = string("op_2973_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2973_cast_fp16 = slice_by_index(begin = var_2973_begin_0, end = var_2973_end_0, end_mask = var_2973_end_mask_0, squeeze_mask = var_2973_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2973_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2974_cast_fp16 = mul(x = state_161_cast_fp16, y = var_2973_cast_fp16)[name = string("op_2974_cast_fp16")];
+            tensor<int32, [4]> var_2976_begin_0 = const()[name = string("op_2976_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_2976_end_0 = const()[name = string("op_2976_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_2976_end_mask_0 = const()[name = string("op_2976_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2976_squeeze_mask_0 = const()[name = string("op_2976_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2976_cast_fp16 = slice_by_index(begin = var_2976_begin_0, end = var_2976_end_0, end_mask = var_2976_end_mask_0, squeeze_mask = var_2976_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2976_cast_fp16")];
+            bool var_2978_transpose_x_1 = const()[name = string("op_2978_transpose_x_1"), val = bool(true)];
+            bool var_2978_transpose_y_1 = const()[name = string("op_2978_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2978_cast_fp16 = matmul(transpose_x = var_2978_transpose_x_1, transpose_y = var_2978_transpose_y_1, x = var_2976_cast_fp16, y = v_new_161_cast_fp16)[name = string("op_2978_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_163_cast_fp16 = add(x = var_2974_cast_fp16, y = var_2978_cast_fp16)[name = string("state_163_cast_fp16")];
+            tensor<int32, [4]> var_2981_begin_0 = const()[name = string("op_2981_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_2981_end_0 = const()[name = string("op_2981_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_2981_end_mask_0 = const()[name = string("op_2981_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2981_squeeze_mask_0 = const()[name = string("op_2981_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2981_cast_fp16 = slice_by_index(begin = var_2981_begin_0, end = var_2981_end_0, end_mask = var_2981_end_mask_0, squeeze_mask = var_2981_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_2981_cast_fp16")];
+            tensor<int32, [4]> var_2983_begin_0 = const()[name = string("op_2983_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_2983_end_0 = const()[name = string("op_2983_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_2983_end_mask_0 = const()[name = string("op_2983_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2983_squeeze_mask_0 = const()[name = string("op_2983_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2983_cast_fp16 = slice_by_index(begin = var_2983_begin_0, end = var_2983_end_0, end_mask = var_2983_end_mask_0, squeeze_mask = var_2983_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_2983_cast_fp16")];
+            bool var_2984_transpose_x_0 = const()[name = string("op_2984_transpose_x_0"), val = bool(false)];
+            bool var_2984_transpose_y_0 = const()[name = string("op_2984_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2984_cast_fp16 = matmul(transpose_x = var_2984_transpose_x_0, transpose_y = var_2984_transpose_y_0, x = var_2983_cast_fp16, y = state_163_cast_fp16)[name = string("op_2984_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_163_cast_fp16 = sub(x = var_2981_cast_fp16, y = var_2984_cast_fp16)[name = string("v_new_163_cast_fp16")];
+            tensor<int32, [4]> var_2987_begin_0 = const()[name = string("op_2987_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_2987_end_0 = const()[name = string("op_2987_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_2987_end_mask_0 = const()[name = string("op_2987_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2987_squeeze_mask_0 = const()[name = string("op_2987_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2987_cast_fp16 = slice_by_index(begin = var_2987_begin_0, end = var_2987_end_0, end_mask = var_2987_end_mask_0, squeeze_mask = var_2987_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_2987_cast_fp16")];
+            bool var_2988_transpose_x_0 = const()[name = string("op_2988_transpose_x_0"), val = bool(false)];
+            bool var_2988_transpose_y_0 = const()[name = string("op_2988_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2988_cast_fp16 = matmul(transpose_x = var_2988_transpose_x_0, transpose_y = var_2988_transpose_y_0, x = var_2987_cast_fp16, y = state_163_cast_fp16)[name = string("op_2988_cast_fp16")];
+            tensor<int32, [4]> var_2990_begin_0 = const()[name = string("op_2990_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_2990_end_0 = const()[name = string("op_2990_end_0"), val = tensor<int32, [4]>([32, 18, 64, 64])];
+            tensor<bool, [4]> var_2990_end_mask_0 = const()[name = string("op_2990_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2990_squeeze_mask_0 = const()[name = string("op_2990_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_2990_cast_fp16 = slice_by_index(begin = var_2990_begin_0, end = var_2990_end_0, end_mask = var_2990_end_mask_0, squeeze_mask = var_2990_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_2990_cast_fp16")];
+            bool var_2991_transpose_x_0 = const()[name = string("op_2991_transpose_x_0"), val = bool(false)];
+            bool var_2991_transpose_y_0 = const()[name = string("op_2991_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_2991_cast_fp16 = matmul(transpose_x = var_2991_transpose_x_0, transpose_y = var_2991_transpose_y_0, x = var_2990_cast_fp16, y = v_new_163_cast_fp16)[name = string("op_2991_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_2992_cast_fp16 = add(x = var_2988_cast_fp16, y = var_2991_cast_fp16)[name = string("op_2992_cast_fp16")];
+            tensor<int32, [4]> var_2994_begin_0 = const()[name = string("op_2994_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_2994_end_0 = const()[name = string("op_2994_end_0"), val = tensor<int32, [4]>([32, 18, 1, 1])];
+            tensor<bool, [4]> var_2994_end_mask_0 = const()[name = string("op_2994_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2994_squeeze_mask_0 = const()[name = string("op_2994_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_2994_cast_fp16 = slice_by_index(begin = var_2994_begin_0, end = var_2994_end_0, end_mask = var_2994_end_mask_0, squeeze_mask = var_2994_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_2994_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_2995_cast_fp16 = mul(x = state_163_cast_fp16, y = var_2994_cast_fp16)[name = string("op_2995_cast_fp16")];
+            tensor<int32, [4]> var_2997_begin_0 = const()[name = string("op_2997_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_2997_end_0 = const()[name = string("op_2997_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_2997_end_mask_0 = const()[name = string("op_2997_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_2997_squeeze_mask_0 = const()[name = string("op_2997_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_2997_cast_fp16 = slice_by_index(begin = var_2997_begin_0, end = var_2997_end_0, end_mask = var_2997_end_mask_0, squeeze_mask = var_2997_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_2997_cast_fp16")];
+            bool var_2999_transpose_x_1 = const()[name = string("op_2999_transpose_x_1"), val = bool(true)];
+            bool var_2999_transpose_y_1 = const()[name = string("op_2999_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_2999_cast_fp16 = matmul(transpose_x = var_2999_transpose_x_1, transpose_y = var_2999_transpose_y_1, x = var_2997_cast_fp16, y = v_new_163_cast_fp16)[name = string("op_2999_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_165_cast_fp16 = add(x = var_2995_cast_fp16, y = var_2999_cast_fp16)[name = string("state_165_cast_fp16")];
+            tensor<int32, [4]> var_3002_begin_0 = const()[name = string("op_3002_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_3002_end_0 = const()[name = string("op_3002_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_3002_end_mask_0 = const()[name = string("op_3002_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3002_squeeze_mask_0 = const()[name = string("op_3002_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3002_cast_fp16 = slice_by_index(begin = var_3002_begin_0, end = var_3002_end_0, end_mask = var_3002_end_mask_0, squeeze_mask = var_3002_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3002_cast_fp16")];
+            tensor<int32, [4]> var_3004_begin_0 = const()[name = string("op_3004_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_3004_end_0 = const()[name = string("op_3004_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_3004_end_mask_0 = const()[name = string("op_3004_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3004_squeeze_mask_0 = const()[name = string("op_3004_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3004_cast_fp16 = slice_by_index(begin = var_3004_begin_0, end = var_3004_end_0, end_mask = var_3004_end_mask_0, squeeze_mask = var_3004_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3004_cast_fp16")];
+            bool var_3005_transpose_x_0 = const()[name = string("op_3005_transpose_x_0"), val = bool(false)];
+            bool var_3005_transpose_y_0 = const()[name = string("op_3005_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3005_cast_fp16 = matmul(transpose_x = var_3005_transpose_x_0, transpose_y = var_3005_transpose_y_0, x = var_3004_cast_fp16, y = state_165_cast_fp16)[name = string("op_3005_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_165_cast_fp16 = sub(x = var_3002_cast_fp16, y = var_3005_cast_fp16)[name = string("v_new_165_cast_fp16")];
+            tensor<int32, [4]> var_3008_begin_0 = const()[name = string("op_3008_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_3008_end_0 = const()[name = string("op_3008_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_3008_end_mask_0 = const()[name = string("op_3008_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3008_squeeze_mask_0 = const()[name = string("op_3008_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3008_cast_fp16 = slice_by_index(begin = var_3008_begin_0, end = var_3008_end_0, end_mask = var_3008_end_mask_0, squeeze_mask = var_3008_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3008_cast_fp16")];
+            bool var_3009_transpose_x_0 = const()[name = string("op_3009_transpose_x_0"), val = bool(false)];
+            bool var_3009_transpose_y_0 = const()[name = string("op_3009_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3009_cast_fp16 = matmul(transpose_x = var_3009_transpose_x_0, transpose_y = var_3009_transpose_y_0, x = var_3008_cast_fp16, y = state_165_cast_fp16)[name = string("op_3009_cast_fp16")];
+            tensor<int32, [4]> var_3011_begin_0 = const()[name = string("op_3011_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_3011_end_0 = const()[name = string("op_3011_end_0"), val = tensor<int32, [4]>([32, 19, 64, 64])];
+            tensor<bool, [4]> var_3011_end_mask_0 = const()[name = string("op_3011_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3011_squeeze_mask_0 = const()[name = string("op_3011_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3011_cast_fp16 = slice_by_index(begin = var_3011_begin_0, end = var_3011_end_0, end_mask = var_3011_end_mask_0, squeeze_mask = var_3011_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3011_cast_fp16")];
+            bool var_3012_transpose_x_0 = const()[name = string("op_3012_transpose_x_0"), val = bool(false)];
+            bool var_3012_transpose_y_0 = const()[name = string("op_3012_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3012_cast_fp16 = matmul(transpose_x = var_3012_transpose_x_0, transpose_y = var_3012_transpose_y_0, x = var_3011_cast_fp16, y = v_new_165_cast_fp16)[name = string("op_3012_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3013_cast_fp16 = add(x = var_3009_cast_fp16, y = var_3012_cast_fp16)[name = string("op_3013_cast_fp16")];
+            tensor<int32, [4]> var_3015_begin_0 = const()[name = string("op_3015_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_3015_end_0 = const()[name = string("op_3015_end_0"), val = tensor<int32, [4]>([32, 19, 1, 1])];
+            tensor<bool, [4]> var_3015_end_mask_0 = const()[name = string("op_3015_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3015_squeeze_mask_0 = const()[name = string("op_3015_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3015_cast_fp16 = slice_by_index(begin = var_3015_begin_0, end = var_3015_end_0, end_mask = var_3015_end_mask_0, squeeze_mask = var_3015_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3015_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3016_cast_fp16 = mul(x = state_165_cast_fp16, y = var_3015_cast_fp16)[name = string("op_3016_cast_fp16")];
+            tensor<int32, [4]> var_3018_begin_0 = const()[name = string("op_3018_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_3018_end_0 = const()[name = string("op_3018_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_3018_end_mask_0 = const()[name = string("op_3018_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3018_squeeze_mask_0 = const()[name = string("op_3018_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3018_cast_fp16 = slice_by_index(begin = var_3018_begin_0, end = var_3018_end_0, end_mask = var_3018_end_mask_0, squeeze_mask = var_3018_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3018_cast_fp16")];
+            bool var_3020_transpose_x_1 = const()[name = string("op_3020_transpose_x_1"), val = bool(true)];
+            bool var_3020_transpose_y_1 = const()[name = string("op_3020_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3020_cast_fp16 = matmul(transpose_x = var_3020_transpose_x_1, transpose_y = var_3020_transpose_y_1, x = var_3018_cast_fp16, y = v_new_165_cast_fp16)[name = string("op_3020_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_167_cast_fp16 = add(x = var_3016_cast_fp16, y = var_3020_cast_fp16)[name = string("state_167_cast_fp16")];
+            tensor<int32, [4]> var_3023_begin_0 = const()[name = string("op_3023_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_3023_end_0 = const()[name = string("op_3023_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_3023_end_mask_0 = const()[name = string("op_3023_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3023_squeeze_mask_0 = const()[name = string("op_3023_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3023_cast_fp16 = slice_by_index(begin = var_3023_begin_0, end = var_3023_end_0, end_mask = var_3023_end_mask_0, squeeze_mask = var_3023_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3023_cast_fp16")];
+            tensor<int32, [4]> var_3025_begin_0 = const()[name = string("op_3025_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_3025_end_0 = const()[name = string("op_3025_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_3025_end_mask_0 = const()[name = string("op_3025_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3025_squeeze_mask_0 = const()[name = string("op_3025_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3025_cast_fp16 = slice_by_index(begin = var_3025_begin_0, end = var_3025_end_0, end_mask = var_3025_end_mask_0, squeeze_mask = var_3025_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3025_cast_fp16")];
+            bool var_3026_transpose_x_0 = const()[name = string("op_3026_transpose_x_0"), val = bool(false)];
+            bool var_3026_transpose_y_0 = const()[name = string("op_3026_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3026_cast_fp16 = matmul(transpose_x = var_3026_transpose_x_0, transpose_y = var_3026_transpose_y_0, x = var_3025_cast_fp16, y = state_167_cast_fp16)[name = string("op_3026_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_167_cast_fp16 = sub(x = var_3023_cast_fp16, y = var_3026_cast_fp16)[name = string("v_new_167_cast_fp16")];
+            tensor<int32, [4]> var_3029_begin_0 = const()[name = string("op_3029_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_3029_end_0 = const()[name = string("op_3029_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_3029_end_mask_0 = const()[name = string("op_3029_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3029_squeeze_mask_0 = const()[name = string("op_3029_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3029_cast_fp16 = slice_by_index(begin = var_3029_begin_0, end = var_3029_end_0, end_mask = var_3029_end_mask_0, squeeze_mask = var_3029_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3029_cast_fp16")];
+            bool var_3030_transpose_x_0 = const()[name = string("op_3030_transpose_x_0"), val = bool(false)];
+            bool var_3030_transpose_y_0 = const()[name = string("op_3030_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3030_cast_fp16 = matmul(transpose_x = var_3030_transpose_x_0, transpose_y = var_3030_transpose_y_0, x = var_3029_cast_fp16, y = state_167_cast_fp16)[name = string("op_3030_cast_fp16")];
+            tensor<int32, [4]> var_3032_begin_0 = const()[name = string("op_3032_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_3032_end_0 = const()[name = string("op_3032_end_0"), val = tensor<int32, [4]>([32, 20, 64, 64])];
+            tensor<bool, [4]> var_3032_end_mask_0 = const()[name = string("op_3032_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3032_squeeze_mask_0 = const()[name = string("op_3032_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3032_cast_fp16 = slice_by_index(begin = var_3032_begin_0, end = var_3032_end_0, end_mask = var_3032_end_mask_0, squeeze_mask = var_3032_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3032_cast_fp16")];
+            bool var_3033_transpose_x_0 = const()[name = string("op_3033_transpose_x_0"), val = bool(false)];
+            bool var_3033_transpose_y_0 = const()[name = string("op_3033_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3033_cast_fp16 = matmul(transpose_x = var_3033_transpose_x_0, transpose_y = var_3033_transpose_y_0, x = var_3032_cast_fp16, y = v_new_167_cast_fp16)[name = string("op_3033_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3034_cast_fp16 = add(x = var_3030_cast_fp16, y = var_3033_cast_fp16)[name = string("op_3034_cast_fp16")];
+            tensor<int32, [4]> var_3036_begin_0 = const()[name = string("op_3036_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_3036_end_0 = const()[name = string("op_3036_end_0"), val = tensor<int32, [4]>([32, 20, 1, 1])];
+            tensor<bool, [4]> var_3036_end_mask_0 = const()[name = string("op_3036_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3036_squeeze_mask_0 = const()[name = string("op_3036_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3036_cast_fp16 = slice_by_index(begin = var_3036_begin_0, end = var_3036_end_0, end_mask = var_3036_end_mask_0, squeeze_mask = var_3036_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3036_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3037_cast_fp16 = mul(x = state_167_cast_fp16, y = var_3036_cast_fp16)[name = string("op_3037_cast_fp16")];
+            tensor<int32, [4]> var_3039_begin_0 = const()[name = string("op_3039_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_3039_end_0 = const()[name = string("op_3039_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_3039_end_mask_0 = const()[name = string("op_3039_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3039_squeeze_mask_0 = const()[name = string("op_3039_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3039_cast_fp16 = slice_by_index(begin = var_3039_begin_0, end = var_3039_end_0, end_mask = var_3039_end_mask_0, squeeze_mask = var_3039_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3039_cast_fp16")];
+            bool var_3041_transpose_x_1 = const()[name = string("op_3041_transpose_x_1"), val = bool(true)];
+            bool var_3041_transpose_y_1 = const()[name = string("op_3041_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3041_cast_fp16 = matmul(transpose_x = var_3041_transpose_x_1, transpose_y = var_3041_transpose_y_1, x = var_3039_cast_fp16, y = v_new_167_cast_fp16)[name = string("op_3041_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_169_cast_fp16 = add(x = var_3037_cast_fp16, y = var_3041_cast_fp16)[name = string("state_169_cast_fp16")];
+            tensor<int32, [4]> var_3044_begin_0 = const()[name = string("op_3044_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_3044_end_0 = const()[name = string("op_3044_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_3044_end_mask_0 = const()[name = string("op_3044_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3044_squeeze_mask_0 = const()[name = string("op_3044_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3044_cast_fp16 = slice_by_index(begin = var_3044_begin_0, end = var_3044_end_0, end_mask = var_3044_end_mask_0, squeeze_mask = var_3044_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3044_cast_fp16")];
+            tensor<int32, [4]> var_3046_begin_0 = const()[name = string("op_3046_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_3046_end_0 = const()[name = string("op_3046_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_3046_end_mask_0 = const()[name = string("op_3046_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3046_squeeze_mask_0 = const()[name = string("op_3046_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3046_cast_fp16 = slice_by_index(begin = var_3046_begin_0, end = var_3046_end_0, end_mask = var_3046_end_mask_0, squeeze_mask = var_3046_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3046_cast_fp16")];
+            bool var_3047_transpose_x_0 = const()[name = string("op_3047_transpose_x_0"), val = bool(false)];
+            bool var_3047_transpose_y_0 = const()[name = string("op_3047_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3047_cast_fp16 = matmul(transpose_x = var_3047_transpose_x_0, transpose_y = var_3047_transpose_y_0, x = var_3046_cast_fp16, y = state_169_cast_fp16)[name = string("op_3047_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_169_cast_fp16 = sub(x = var_3044_cast_fp16, y = var_3047_cast_fp16)[name = string("v_new_169_cast_fp16")];
+            tensor<int32, [4]> var_3050_begin_0 = const()[name = string("op_3050_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_3050_end_0 = const()[name = string("op_3050_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_3050_end_mask_0 = const()[name = string("op_3050_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3050_squeeze_mask_0 = const()[name = string("op_3050_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3050_cast_fp16 = slice_by_index(begin = var_3050_begin_0, end = var_3050_end_0, end_mask = var_3050_end_mask_0, squeeze_mask = var_3050_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3050_cast_fp16")];
+            bool var_3051_transpose_x_0 = const()[name = string("op_3051_transpose_x_0"), val = bool(false)];
+            bool var_3051_transpose_y_0 = const()[name = string("op_3051_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3051_cast_fp16 = matmul(transpose_x = var_3051_transpose_x_0, transpose_y = var_3051_transpose_y_0, x = var_3050_cast_fp16, y = state_169_cast_fp16)[name = string("op_3051_cast_fp16")];
+            tensor<int32, [4]> var_3053_begin_0 = const()[name = string("op_3053_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_3053_end_0 = const()[name = string("op_3053_end_0"), val = tensor<int32, [4]>([32, 21, 64, 64])];
+            tensor<bool, [4]> var_3053_end_mask_0 = const()[name = string("op_3053_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3053_squeeze_mask_0 = const()[name = string("op_3053_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3053_cast_fp16 = slice_by_index(begin = var_3053_begin_0, end = var_3053_end_0, end_mask = var_3053_end_mask_0, squeeze_mask = var_3053_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3053_cast_fp16")];
+            bool var_3054_transpose_x_0 = const()[name = string("op_3054_transpose_x_0"), val = bool(false)];
+            bool var_3054_transpose_y_0 = const()[name = string("op_3054_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3054_cast_fp16 = matmul(transpose_x = var_3054_transpose_x_0, transpose_y = var_3054_transpose_y_0, x = var_3053_cast_fp16, y = v_new_169_cast_fp16)[name = string("op_3054_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3055_cast_fp16 = add(x = var_3051_cast_fp16, y = var_3054_cast_fp16)[name = string("op_3055_cast_fp16")];
+            tensor<int32, [4]> var_3057_begin_0 = const()[name = string("op_3057_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_3057_end_0 = const()[name = string("op_3057_end_0"), val = tensor<int32, [4]>([32, 21, 1, 1])];
+            tensor<bool, [4]> var_3057_end_mask_0 = const()[name = string("op_3057_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3057_squeeze_mask_0 = const()[name = string("op_3057_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3057_cast_fp16 = slice_by_index(begin = var_3057_begin_0, end = var_3057_end_0, end_mask = var_3057_end_mask_0, squeeze_mask = var_3057_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3057_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3058_cast_fp16 = mul(x = state_169_cast_fp16, y = var_3057_cast_fp16)[name = string("op_3058_cast_fp16")];
+            tensor<int32, [4]> var_3060_begin_0 = const()[name = string("op_3060_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_3060_end_0 = const()[name = string("op_3060_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_3060_end_mask_0 = const()[name = string("op_3060_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3060_squeeze_mask_0 = const()[name = string("op_3060_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3060_cast_fp16 = slice_by_index(begin = var_3060_begin_0, end = var_3060_end_0, end_mask = var_3060_end_mask_0, squeeze_mask = var_3060_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3060_cast_fp16")];
+            bool var_3062_transpose_x_1 = const()[name = string("op_3062_transpose_x_1"), val = bool(true)];
+            bool var_3062_transpose_y_1 = const()[name = string("op_3062_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3062_cast_fp16 = matmul(transpose_x = var_3062_transpose_x_1, transpose_y = var_3062_transpose_y_1, x = var_3060_cast_fp16, y = v_new_169_cast_fp16)[name = string("op_3062_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_171_cast_fp16 = add(x = var_3058_cast_fp16, y = var_3062_cast_fp16)[name = string("state_171_cast_fp16")];
+            tensor<int32, [4]> var_3065_begin_0 = const()[name = string("op_3065_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_3065_end_0 = const()[name = string("op_3065_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_3065_end_mask_0 = const()[name = string("op_3065_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3065_squeeze_mask_0 = const()[name = string("op_3065_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3065_cast_fp16 = slice_by_index(begin = var_3065_begin_0, end = var_3065_end_0, end_mask = var_3065_end_mask_0, squeeze_mask = var_3065_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3065_cast_fp16")];
+            tensor<int32, [4]> var_3067_begin_0 = const()[name = string("op_3067_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_3067_end_0 = const()[name = string("op_3067_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_3067_end_mask_0 = const()[name = string("op_3067_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3067_squeeze_mask_0 = const()[name = string("op_3067_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3067_cast_fp16 = slice_by_index(begin = var_3067_begin_0, end = var_3067_end_0, end_mask = var_3067_end_mask_0, squeeze_mask = var_3067_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3067_cast_fp16")];
+            bool var_3068_transpose_x_0 = const()[name = string("op_3068_transpose_x_0"), val = bool(false)];
+            bool var_3068_transpose_y_0 = const()[name = string("op_3068_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3068_cast_fp16 = matmul(transpose_x = var_3068_transpose_x_0, transpose_y = var_3068_transpose_y_0, x = var_3067_cast_fp16, y = state_171_cast_fp16)[name = string("op_3068_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_171_cast_fp16 = sub(x = var_3065_cast_fp16, y = var_3068_cast_fp16)[name = string("v_new_171_cast_fp16")];
+            tensor<int32, [4]> var_3071_begin_0 = const()[name = string("op_3071_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_3071_end_0 = const()[name = string("op_3071_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_3071_end_mask_0 = const()[name = string("op_3071_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3071_squeeze_mask_0 = const()[name = string("op_3071_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3071_cast_fp16 = slice_by_index(begin = var_3071_begin_0, end = var_3071_end_0, end_mask = var_3071_end_mask_0, squeeze_mask = var_3071_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3071_cast_fp16")];
+            bool var_3072_transpose_x_0 = const()[name = string("op_3072_transpose_x_0"), val = bool(false)];
+            bool var_3072_transpose_y_0 = const()[name = string("op_3072_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3072_cast_fp16 = matmul(transpose_x = var_3072_transpose_x_0, transpose_y = var_3072_transpose_y_0, x = var_3071_cast_fp16, y = state_171_cast_fp16)[name = string("op_3072_cast_fp16")];
+            tensor<int32, [4]> var_3074_begin_0 = const()[name = string("op_3074_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_3074_end_0 = const()[name = string("op_3074_end_0"), val = tensor<int32, [4]>([32, 22, 64, 64])];
+            tensor<bool, [4]> var_3074_end_mask_0 = const()[name = string("op_3074_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3074_squeeze_mask_0 = const()[name = string("op_3074_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3074_cast_fp16 = slice_by_index(begin = var_3074_begin_0, end = var_3074_end_0, end_mask = var_3074_end_mask_0, squeeze_mask = var_3074_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3074_cast_fp16")];
+            bool var_3075_transpose_x_0 = const()[name = string("op_3075_transpose_x_0"), val = bool(false)];
+            bool var_3075_transpose_y_0 = const()[name = string("op_3075_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3075_cast_fp16 = matmul(transpose_x = var_3075_transpose_x_0, transpose_y = var_3075_transpose_y_0, x = var_3074_cast_fp16, y = v_new_171_cast_fp16)[name = string("op_3075_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3076_cast_fp16 = add(x = var_3072_cast_fp16, y = var_3075_cast_fp16)[name = string("op_3076_cast_fp16")];
+            tensor<int32, [4]> var_3078_begin_0 = const()[name = string("op_3078_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_3078_end_0 = const()[name = string("op_3078_end_0"), val = tensor<int32, [4]>([32, 22, 1, 1])];
+            tensor<bool, [4]> var_3078_end_mask_0 = const()[name = string("op_3078_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3078_squeeze_mask_0 = const()[name = string("op_3078_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3078_cast_fp16 = slice_by_index(begin = var_3078_begin_0, end = var_3078_end_0, end_mask = var_3078_end_mask_0, squeeze_mask = var_3078_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3078_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3079_cast_fp16 = mul(x = state_171_cast_fp16, y = var_3078_cast_fp16)[name = string("op_3079_cast_fp16")];
+            tensor<int32, [4]> var_3081_begin_0 = const()[name = string("op_3081_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_3081_end_0 = const()[name = string("op_3081_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_3081_end_mask_0 = const()[name = string("op_3081_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3081_squeeze_mask_0 = const()[name = string("op_3081_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3081_cast_fp16 = slice_by_index(begin = var_3081_begin_0, end = var_3081_end_0, end_mask = var_3081_end_mask_0, squeeze_mask = var_3081_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3081_cast_fp16")];
+            bool var_3083_transpose_x_1 = const()[name = string("op_3083_transpose_x_1"), val = bool(true)];
+            bool var_3083_transpose_y_1 = const()[name = string("op_3083_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3083_cast_fp16 = matmul(transpose_x = var_3083_transpose_x_1, transpose_y = var_3083_transpose_y_1, x = var_3081_cast_fp16, y = v_new_171_cast_fp16)[name = string("op_3083_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_173_cast_fp16 = add(x = var_3079_cast_fp16, y = var_3083_cast_fp16)[name = string("state_173_cast_fp16")];
+            tensor<int32, [4]> var_3086_begin_0 = const()[name = string("op_3086_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_3086_end_0 = const()[name = string("op_3086_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_3086_end_mask_0 = const()[name = string("op_3086_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3086_squeeze_mask_0 = const()[name = string("op_3086_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3086_cast_fp16 = slice_by_index(begin = var_3086_begin_0, end = var_3086_end_0, end_mask = var_3086_end_mask_0, squeeze_mask = var_3086_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3086_cast_fp16")];
+            tensor<int32, [4]> var_3088_begin_0 = const()[name = string("op_3088_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_3088_end_0 = const()[name = string("op_3088_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_3088_end_mask_0 = const()[name = string("op_3088_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3088_squeeze_mask_0 = const()[name = string("op_3088_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3088_cast_fp16 = slice_by_index(begin = var_3088_begin_0, end = var_3088_end_0, end_mask = var_3088_end_mask_0, squeeze_mask = var_3088_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3088_cast_fp16")];
+            bool var_3089_transpose_x_0 = const()[name = string("op_3089_transpose_x_0"), val = bool(false)];
+            bool var_3089_transpose_y_0 = const()[name = string("op_3089_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3089_cast_fp16 = matmul(transpose_x = var_3089_transpose_x_0, transpose_y = var_3089_transpose_y_0, x = var_3088_cast_fp16, y = state_173_cast_fp16)[name = string("op_3089_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_173_cast_fp16 = sub(x = var_3086_cast_fp16, y = var_3089_cast_fp16)[name = string("v_new_173_cast_fp16")];
+            tensor<int32, [4]> var_3092_begin_0 = const()[name = string("op_3092_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_3092_end_0 = const()[name = string("op_3092_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_3092_end_mask_0 = const()[name = string("op_3092_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3092_squeeze_mask_0 = const()[name = string("op_3092_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3092_cast_fp16 = slice_by_index(begin = var_3092_begin_0, end = var_3092_end_0, end_mask = var_3092_end_mask_0, squeeze_mask = var_3092_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3092_cast_fp16")];
+            bool var_3093_transpose_x_0 = const()[name = string("op_3093_transpose_x_0"), val = bool(false)];
+            bool var_3093_transpose_y_0 = const()[name = string("op_3093_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3093_cast_fp16 = matmul(transpose_x = var_3093_transpose_x_0, transpose_y = var_3093_transpose_y_0, x = var_3092_cast_fp16, y = state_173_cast_fp16)[name = string("op_3093_cast_fp16")];
+            tensor<int32, [4]> var_3095_begin_0 = const()[name = string("op_3095_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_3095_end_0 = const()[name = string("op_3095_end_0"), val = tensor<int32, [4]>([32, 23, 64, 64])];
+            tensor<bool, [4]> var_3095_end_mask_0 = const()[name = string("op_3095_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3095_squeeze_mask_0 = const()[name = string("op_3095_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3095_cast_fp16 = slice_by_index(begin = var_3095_begin_0, end = var_3095_end_0, end_mask = var_3095_end_mask_0, squeeze_mask = var_3095_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3095_cast_fp16")];
+            bool var_3096_transpose_x_0 = const()[name = string("op_3096_transpose_x_0"), val = bool(false)];
+            bool var_3096_transpose_y_0 = const()[name = string("op_3096_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3096_cast_fp16 = matmul(transpose_x = var_3096_transpose_x_0, transpose_y = var_3096_transpose_y_0, x = var_3095_cast_fp16, y = v_new_173_cast_fp16)[name = string("op_3096_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3097_cast_fp16 = add(x = var_3093_cast_fp16, y = var_3096_cast_fp16)[name = string("op_3097_cast_fp16")];
+            tensor<int32, [4]> var_3099_begin_0 = const()[name = string("op_3099_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_3099_end_0 = const()[name = string("op_3099_end_0"), val = tensor<int32, [4]>([32, 23, 1, 1])];
+            tensor<bool, [4]> var_3099_end_mask_0 = const()[name = string("op_3099_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3099_squeeze_mask_0 = const()[name = string("op_3099_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3099_cast_fp16 = slice_by_index(begin = var_3099_begin_0, end = var_3099_end_0, end_mask = var_3099_end_mask_0, squeeze_mask = var_3099_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3099_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3100_cast_fp16 = mul(x = state_173_cast_fp16, y = var_3099_cast_fp16)[name = string("op_3100_cast_fp16")];
+            tensor<int32, [4]> var_3102_begin_0 = const()[name = string("op_3102_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_3102_end_0 = const()[name = string("op_3102_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_3102_end_mask_0 = const()[name = string("op_3102_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3102_squeeze_mask_0 = const()[name = string("op_3102_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3102_cast_fp16 = slice_by_index(begin = var_3102_begin_0, end = var_3102_end_0, end_mask = var_3102_end_mask_0, squeeze_mask = var_3102_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3102_cast_fp16")];
+            bool var_3104_transpose_x_1 = const()[name = string("op_3104_transpose_x_1"), val = bool(true)];
+            bool var_3104_transpose_y_1 = const()[name = string("op_3104_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3104_cast_fp16 = matmul(transpose_x = var_3104_transpose_x_1, transpose_y = var_3104_transpose_y_1, x = var_3102_cast_fp16, y = v_new_173_cast_fp16)[name = string("op_3104_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_175_cast_fp16 = add(x = var_3100_cast_fp16, y = var_3104_cast_fp16)[name = string("state_175_cast_fp16")];
+            tensor<int32, [4]> var_3107_begin_0 = const()[name = string("op_3107_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_3107_end_0 = const()[name = string("op_3107_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_3107_end_mask_0 = const()[name = string("op_3107_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3107_squeeze_mask_0 = const()[name = string("op_3107_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3107_cast_fp16 = slice_by_index(begin = var_3107_begin_0, end = var_3107_end_0, end_mask = var_3107_end_mask_0, squeeze_mask = var_3107_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3107_cast_fp16")];
+            tensor<int32, [4]> var_3109_begin_0 = const()[name = string("op_3109_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_3109_end_0 = const()[name = string("op_3109_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_3109_end_mask_0 = const()[name = string("op_3109_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3109_squeeze_mask_0 = const()[name = string("op_3109_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3109_cast_fp16 = slice_by_index(begin = var_3109_begin_0, end = var_3109_end_0, end_mask = var_3109_end_mask_0, squeeze_mask = var_3109_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3109_cast_fp16")];
+            bool var_3110_transpose_x_0 = const()[name = string("op_3110_transpose_x_0"), val = bool(false)];
+            bool var_3110_transpose_y_0 = const()[name = string("op_3110_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3110_cast_fp16 = matmul(transpose_x = var_3110_transpose_x_0, transpose_y = var_3110_transpose_y_0, x = var_3109_cast_fp16, y = state_175_cast_fp16)[name = string("op_3110_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_175_cast_fp16 = sub(x = var_3107_cast_fp16, y = var_3110_cast_fp16)[name = string("v_new_175_cast_fp16")];
+            tensor<int32, [4]> var_3113_begin_0 = const()[name = string("op_3113_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_3113_end_0 = const()[name = string("op_3113_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_3113_end_mask_0 = const()[name = string("op_3113_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3113_squeeze_mask_0 = const()[name = string("op_3113_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3113_cast_fp16 = slice_by_index(begin = var_3113_begin_0, end = var_3113_end_0, end_mask = var_3113_end_mask_0, squeeze_mask = var_3113_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3113_cast_fp16")];
+            bool var_3114_transpose_x_0 = const()[name = string("op_3114_transpose_x_0"), val = bool(false)];
+            bool var_3114_transpose_y_0 = const()[name = string("op_3114_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3114_cast_fp16 = matmul(transpose_x = var_3114_transpose_x_0, transpose_y = var_3114_transpose_y_0, x = var_3113_cast_fp16, y = state_175_cast_fp16)[name = string("op_3114_cast_fp16")];
+            tensor<int32, [4]> var_3116_begin_0 = const()[name = string("op_3116_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_3116_end_0 = const()[name = string("op_3116_end_0"), val = tensor<int32, [4]>([32, 24, 64, 64])];
+            tensor<bool, [4]> var_3116_end_mask_0 = const()[name = string("op_3116_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3116_squeeze_mask_0 = const()[name = string("op_3116_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3116_cast_fp16 = slice_by_index(begin = var_3116_begin_0, end = var_3116_end_0, end_mask = var_3116_end_mask_0, squeeze_mask = var_3116_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3116_cast_fp16")];
+            bool var_3117_transpose_x_0 = const()[name = string("op_3117_transpose_x_0"), val = bool(false)];
+            bool var_3117_transpose_y_0 = const()[name = string("op_3117_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3117_cast_fp16 = matmul(transpose_x = var_3117_transpose_x_0, transpose_y = var_3117_transpose_y_0, x = var_3116_cast_fp16, y = v_new_175_cast_fp16)[name = string("op_3117_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3118_cast_fp16 = add(x = var_3114_cast_fp16, y = var_3117_cast_fp16)[name = string("op_3118_cast_fp16")];
+            tensor<int32, [4]> var_3120_begin_0 = const()[name = string("op_3120_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_3120_end_0 = const()[name = string("op_3120_end_0"), val = tensor<int32, [4]>([32, 24, 1, 1])];
+            tensor<bool, [4]> var_3120_end_mask_0 = const()[name = string("op_3120_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3120_squeeze_mask_0 = const()[name = string("op_3120_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3120_cast_fp16 = slice_by_index(begin = var_3120_begin_0, end = var_3120_end_0, end_mask = var_3120_end_mask_0, squeeze_mask = var_3120_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3120_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3121_cast_fp16 = mul(x = state_175_cast_fp16, y = var_3120_cast_fp16)[name = string("op_3121_cast_fp16")];
+            tensor<int32, [4]> var_3123_begin_0 = const()[name = string("op_3123_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_3123_end_0 = const()[name = string("op_3123_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_3123_end_mask_0 = const()[name = string("op_3123_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3123_squeeze_mask_0 = const()[name = string("op_3123_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3123_cast_fp16 = slice_by_index(begin = var_3123_begin_0, end = var_3123_end_0, end_mask = var_3123_end_mask_0, squeeze_mask = var_3123_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3123_cast_fp16")];
+            bool var_3125_transpose_x_1 = const()[name = string("op_3125_transpose_x_1"), val = bool(true)];
+            bool var_3125_transpose_y_1 = const()[name = string("op_3125_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3125_cast_fp16 = matmul(transpose_x = var_3125_transpose_x_1, transpose_y = var_3125_transpose_y_1, x = var_3123_cast_fp16, y = v_new_175_cast_fp16)[name = string("op_3125_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_177_cast_fp16 = add(x = var_3121_cast_fp16, y = var_3125_cast_fp16)[name = string("state_177_cast_fp16")];
+            tensor<int32, [4]> var_3128_begin_0 = const()[name = string("op_3128_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_3128_end_0 = const()[name = string("op_3128_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_3128_end_mask_0 = const()[name = string("op_3128_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3128_squeeze_mask_0 = const()[name = string("op_3128_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3128_cast_fp16 = slice_by_index(begin = var_3128_begin_0, end = var_3128_end_0, end_mask = var_3128_end_mask_0, squeeze_mask = var_3128_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3128_cast_fp16")];
+            tensor<int32, [4]> var_3130_begin_0 = const()[name = string("op_3130_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_3130_end_0 = const()[name = string("op_3130_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_3130_end_mask_0 = const()[name = string("op_3130_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3130_squeeze_mask_0 = const()[name = string("op_3130_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3130_cast_fp16 = slice_by_index(begin = var_3130_begin_0, end = var_3130_end_0, end_mask = var_3130_end_mask_0, squeeze_mask = var_3130_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3130_cast_fp16")];
+            bool var_3131_transpose_x_0 = const()[name = string("op_3131_transpose_x_0"), val = bool(false)];
+            bool var_3131_transpose_y_0 = const()[name = string("op_3131_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3131_cast_fp16 = matmul(transpose_x = var_3131_transpose_x_0, transpose_y = var_3131_transpose_y_0, x = var_3130_cast_fp16, y = state_177_cast_fp16)[name = string("op_3131_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_177_cast_fp16 = sub(x = var_3128_cast_fp16, y = var_3131_cast_fp16)[name = string("v_new_177_cast_fp16")];
+            tensor<int32, [4]> var_3134_begin_0 = const()[name = string("op_3134_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_3134_end_0 = const()[name = string("op_3134_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_3134_end_mask_0 = const()[name = string("op_3134_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3134_squeeze_mask_0 = const()[name = string("op_3134_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3134_cast_fp16 = slice_by_index(begin = var_3134_begin_0, end = var_3134_end_0, end_mask = var_3134_end_mask_0, squeeze_mask = var_3134_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3134_cast_fp16")];
+            bool var_3135_transpose_x_0 = const()[name = string("op_3135_transpose_x_0"), val = bool(false)];
+            bool var_3135_transpose_y_0 = const()[name = string("op_3135_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3135_cast_fp16 = matmul(transpose_x = var_3135_transpose_x_0, transpose_y = var_3135_transpose_y_0, x = var_3134_cast_fp16, y = state_177_cast_fp16)[name = string("op_3135_cast_fp16")];
+            tensor<int32, [4]> var_3137_begin_0 = const()[name = string("op_3137_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_3137_end_0 = const()[name = string("op_3137_end_0"), val = tensor<int32, [4]>([32, 25, 64, 64])];
+            tensor<bool, [4]> var_3137_end_mask_0 = const()[name = string("op_3137_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3137_squeeze_mask_0 = const()[name = string("op_3137_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3137_cast_fp16 = slice_by_index(begin = var_3137_begin_0, end = var_3137_end_0, end_mask = var_3137_end_mask_0, squeeze_mask = var_3137_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3137_cast_fp16")];
+            bool var_3138_transpose_x_0 = const()[name = string("op_3138_transpose_x_0"), val = bool(false)];
+            bool var_3138_transpose_y_0 = const()[name = string("op_3138_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3138_cast_fp16 = matmul(transpose_x = var_3138_transpose_x_0, transpose_y = var_3138_transpose_y_0, x = var_3137_cast_fp16, y = v_new_177_cast_fp16)[name = string("op_3138_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3139_cast_fp16 = add(x = var_3135_cast_fp16, y = var_3138_cast_fp16)[name = string("op_3139_cast_fp16")];
+            tensor<int32, [4]> var_3141_begin_0 = const()[name = string("op_3141_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_3141_end_0 = const()[name = string("op_3141_end_0"), val = tensor<int32, [4]>([32, 25, 1, 1])];
+            tensor<bool, [4]> var_3141_end_mask_0 = const()[name = string("op_3141_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3141_squeeze_mask_0 = const()[name = string("op_3141_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3141_cast_fp16 = slice_by_index(begin = var_3141_begin_0, end = var_3141_end_0, end_mask = var_3141_end_mask_0, squeeze_mask = var_3141_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3141_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3142_cast_fp16 = mul(x = state_177_cast_fp16, y = var_3141_cast_fp16)[name = string("op_3142_cast_fp16")];
+            tensor<int32, [4]> var_3144_begin_0 = const()[name = string("op_3144_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_3144_end_0 = const()[name = string("op_3144_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_3144_end_mask_0 = const()[name = string("op_3144_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3144_squeeze_mask_0 = const()[name = string("op_3144_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3144_cast_fp16 = slice_by_index(begin = var_3144_begin_0, end = var_3144_end_0, end_mask = var_3144_end_mask_0, squeeze_mask = var_3144_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3144_cast_fp16")];
+            bool var_3146_transpose_x_1 = const()[name = string("op_3146_transpose_x_1"), val = bool(true)];
+            bool var_3146_transpose_y_1 = const()[name = string("op_3146_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3146_cast_fp16 = matmul(transpose_x = var_3146_transpose_x_1, transpose_y = var_3146_transpose_y_1, x = var_3144_cast_fp16, y = v_new_177_cast_fp16)[name = string("op_3146_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_179_cast_fp16 = add(x = var_3142_cast_fp16, y = var_3146_cast_fp16)[name = string("state_179_cast_fp16")];
+            tensor<int32, [4]> var_3149_begin_0 = const()[name = string("op_3149_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_3149_end_0 = const()[name = string("op_3149_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_3149_end_mask_0 = const()[name = string("op_3149_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3149_squeeze_mask_0 = const()[name = string("op_3149_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3149_cast_fp16 = slice_by_index(begin = var_3149_begin_0, end = var_3149_end_0, end_mask = var_3149_end_mask_0, squeeze_mask = var_3149_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3149_cast_fp16")];
+            tensor<int32, [4]> var_3151_begin_0 = const()[name = string("op_3151_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_3151_end_0 = const()[name = string("op_3151_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_3151_end_mask_0 = const()[name = string("op_3151_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3151_squeeze_mask_0 = const()[name = string("op_3151_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3151_cast_fp16 = slice_by_index(begin = var_3151_begin_0, end = var_3151_end_0, end_mask = var_3151_end_mask_0, squeeze_mask = var_3151_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3151_cast_fp16")];
+            bool var_3152_transpose_x_0 = const()[name = string("op_3152_transpose_x_0"), val = bool(false)];
+            bool var_3152_transpose_y_0 = const()[name = string("op_3152_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3152_cast_fp16 = matmul(transpose_x = var_3152_transpose_x_0, transpose_y = var_3152_transpose_y_0, x = var_3151_cast_fp16, y = state_179_cast_fp16)[name = string("op_3152_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_179_cast_fp16 = sub(x = var_3149_cast_fp16, y = var_3152_cast_fp16)[name = string("v_new_179_cast_fp16")];
+            tensor<int32, [4]> var_3155_begin_0 = const()[name = string("op_3155_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_3155_end_0 = const()[name = string("op_3155_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_3155_end_mask_0 = const()[name = string("op_3155_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3155_squeeze_mask_0 = const()[name = string("op_3155_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3155_cast_fp16 = slice_by_index(begin = var_3155_begin_0, end = var_3155_end_0, end_mask = var_3155_end_mask_0, squeeze_mask = var_3155_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3155_cast_fp16")];
+            bool var_3156_transpose_x_0 = const()[name = string("op_3156_transpose_x_0"), val = bool(false)];
+            bool var_3156_transpose_y_0 = const()[name = string("op_3156_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3156_cast_fp16 = matmul(transpose_x = var_3156_transpose_x_0, transpose_y = var_3156_transpose_y_0, x = var_3155_cast_fp16, y = state_179_cast_fp16)[name = string("op_3156_cast_fp16")];
+            tensor<int32, [4]> var_3158_begin_0 = const()[name = string("op_3158_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_3158_end_0 = const()[name = string("op_3158_end_0"), val = tensor<int32, [4]>([32, 26, 64, 64])];
+            tensor<bool, [4]> var_3158_end_mask_0 = const()[name = string("op_3158_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3158_squeeze_mask_0 = const()[name = string("op_3158_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3158_cast_fp16 = slice_by_index(begin = var_3158_begin_0, end = var_3158_end_0, end_mask = var_3158_end_mask_0, squeeze_mask = var_3158_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3158_cast_fp16")];
+            bool var_3159_transpose_x_0 = const()[name = string("op_3159_transpose_x_0"), val = bool(false)];
+            bool var_3159_transpose_y_0 = const()[name = string("op_3159_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3159_cast_fp16 = matmul(transpose_x = var_3159_transpose_x_0, transpose_y = var_3159_transpose_y_0, x = var_3158_cast_fp16, y = v_new_179_cast_fp16)[name = string("op_3159_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3160_cast_fp16 = add(x = var_3156_cast_fp16, y = var_3159_cast_fp16)[name = string("op_3160_cast_fp16")];
+            tensor<int32, [4]> var_3162_begin_0 = const()[name = string("op_3162_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_3162_end_0 = const()[name = string("op_3162_end_0"), val = tensor<int32, [4]>([32, 26, 1, 1])];
+            tensor<bool, [4]> var_3162_end_mask_0 = const()[name = string("op_3162_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3162_squeeze_mask_0 = const()[name = string("op_3162_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3162_cast_fp16 = slice_by_index(begin = var_3162_begin_0, end = var_3162_end_0, end_mask = var_3162_end_mask_0, squeeze_mask = var_3162_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3162_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3163_cast_fp16 = mul(x = state_179_cast_fp16, y = var_3162_cast_fp16)[name = string("op_3163_cast_fp16")];
+            tensor<int32, [4]> var_3165_begin_0 = const()[name = string("op_3165_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_3165_end_0 = const()[name = string("op_3165_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_3165_end_mask_0 = const()[name = string("op_3165_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3165_squeeze_mask_0 = const()[name = string("op_3165_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3165_cast_fp16 = slice_by_index(begin = var_3165_begin_0, end = var_3165_end_0, end_mask = var_3165_end_mask_0, squeeze_mask = var_3165_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3165_cast_fp16")];
+            bool var_3167_transpose_x_1 = const()[name = string("op_3167_transpose_x_1"), val = bool(true)];
+            bool var_3167_transpose_y_1 = const()[name = string("op_3167_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3167_cast_fp16 = matmul(transpose_x = var_3167_transpose_x_1, transpose_y = var_3167_transpose_y_1, x = var_3165_cast_fp16, y = v_new_179_cast_fp16)[name = string("op_3167_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_181_cast_fp16 = add(x = var_3163_cast_fp16, y = var_3167_cast_fp16)[name = string("state_181_cast_fp16")];
+            tensor<int32, [4]> var_3170_begin_0 = const()[name = string("op_3170_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_3170_end_0 = const()[name = string("op_3170_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_3170_end_mask_0 = const()[name = string("op_3170_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3170_squeeze_mask_0 = const()[name = string("op_3170_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3170_cast_fp16 = slice_by_index(begin = var_3170_begin_0, end = var_3170_end_0, end_mask = var_3170_end_mask_0, squeeze_mask = var_3170_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3170_cast_fp16")];
+            tensor<int32, [4]> var_3172_begin_0 = const()[name = string("op_3172_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_3172_end_0 = const()[name = string("op_3172_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_3172_end_mask_0 = const()[name = string("op_3172_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3172_squeeze_mask_0 = const()[name = string("op_3172_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3172_cast_fp16 = slice_by_index(begin = var_3172_begin_0, end = var_3172_end_0, end_mask = var_3172_end_mask_0, squeeze_mask = var_3172_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3172_cast_fp16")];
+            bool var_3173_transpose_x_0 = const()[name = string("op_3173_transpose_x_0"), val = bool(false)];
+            bool var_3173_transpose_y_0 = const()[name = string("op_3173_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3173_cast_fp16 = matmul(transpose_x = var_3173_transpose_x_0, transpose_y = var_3173_transpose_y_0, x = var_3172_cast_fp16, y = state_181_cast_fp16)[name = string("op_3173_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_181_cast_fp16 = sub(x = var_3170_cast_fp16, y = var_3173_cast_fp16)[name = string("v_new_181_cast_fp16")];
+            tensor<int32, [4]> var_3176_begin_0 = const()[name = string("op_3176_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_3176_end_0 = const()[name = string("op_3176_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_3176_end_mask_0 = const()[name = string("op_3176_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3176_squeeze_mask_0 = const()[name = string("op_3176_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3176_cast_fp16 = slice_by_index(begin = var_3176_begin_0, end = var_3176_end_0, end_mask = var_3176_end_mask_0, squeeze_mask = var_3176_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3176_cast_fp16")];
+            bool var_3177_transpose_x_0 = const()[name = string("op_3177_transpose_x_0"), val = bool(false)];
+            bool var_3177_transpose_y_0 = const()[name = string("op_3177_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3177_cast_fp16 = matmul(transpose_x = var_3177_transpose_x_0, transpose_y = var_3177_transpose_y_0, x = var_3176_cast_fp16, y = state_181_cast_fp16)[name = string("op_3177_cast_fp16")];
+            tensor<int32, [4]> var_3179_begin_0 = const()[name = string("op_3179_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_3179_end_0 = const()[name = string("op_3179_end_0"), val = tensor<int32, [4]>([32, 27, 64, 64])];
+            tensor<bool, [4]> var_3179_end_mask_0 = const()[name = string("op_3179_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3179_squeeze_mask_0 = const()[name = string("op_3179_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3179_cast_fp16 = slice_by_index(begin = var_3179_begin_0, end = var_3179_end_0, end_mask = var_3179_end_mask_0, squeeze_mask = var_3179_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3179_cast_fp16")];
+            bool var_3180_transpose_x_0 = const()[name = string("op_3180_transpose_x_0"), val = bool(false)];
+            bool var_3180_transpose_y_0 = const()[name = string("op_3180_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3180_cast_fp16 = matmul(transpose_x = var_3180_transpose_x_0, transpose_y = var_3180_transpose_y_0, x = var_3179_cast_fp16, y = v_new_181_cast_fp16)[name = string("op_3180_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3181_cast_fp16 = add(x = var_3177_cast_fp16, y = var_3180_cast_fp16)[name = string("op_3181_cast_fp16")];
+            tensor<int32, [4]> var_3183_begin_0 = const()[name = string("op_3183_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_3183_end_0 = const()[name = string("op_3183_end_0"), val = tensor<int32, [4]>([32, 27, 1, 1])];
+            tensor<bool, [4]> var_3183_end_mask_0 = const()[name = string("op_3183_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3183_squeeze_mask_0 = const()[name = string("op_3183_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3183_cast_fp16 = slice_by_index(begin = var_3183_begin_0, end = var_3183_end_0, end_mask = var_3183_end_mask_0, squeeze_mask = var_3183_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3183_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3184_cast_fp16 = mul(x = state_181_cast_fp16, y = var_3183_cast_fp16)[name = string("op_3184_cast_fp16")];
+            tensor<int32, [4]> var_3186_begin_0 = const()[name = string("op_3186_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_3186_end_0 = const()[name = string("op_3186_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_3186_end_mask_0 = const()[name = string("op_3186_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3186_squeeze_mask_0 = const()[name = string("op_3186_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3186_cast_fp16 = slice_by_index(begin = var_3186_begin_0, end = var_3186_end_0, end_mask = var_3186_end_mask_0, squeeze_mask = var_3186_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3186_cast_fp16")];
+            bool var_3188_transpose_x_1 = const()[name = string("op_3188_transpose_x_1"), val = bool(true)];
+            bool var_3188_transpose_y_1 = const()[name = string("op_3188_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3188_cast_fp16 = matmul(transpose_x = var_3188_transpose_x_1, transpose_y = var_3188_transpose_y_1, x = var_3186_cast_fp16, y = v_new_181_cast_fp16)[name = string("op_3188_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_183_cast_fp16 = add(x = var_3184_cast_fp16, y = var_3188_cast_fp16)[name = string("state_183_cast_fp16")];
+            tensor<int32, [4]> var_3191_begin_0 = const()[name = string("op_3191_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_3191_end_0 = const()[name = string("op_3191_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_3191_end_mask_0 = const()[name = string("op_3191_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3191_squeeze_mask_0 = const()[name = string("op_3191_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3191_cast_fp16 = slice_by_index(begin = var_3191_begin_0, end = var_3191_end_0, end_mask = var_3191_end_mask_0, squeeze_mask = var_3191_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3191_cast_fp16")];
+            tensor<int32, [4]> var_3193_begin_0 = const()[name = string("op_3193_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_3193_end_0 = const()[name = string("op_3193_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_3193_end_mask_0 = const()[name = string("op_3193_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3193_squeeze_mask_0 = const()[name = string("op_3193_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3193_cast_fp16 = slice_by_index(begin = var_3193_begin_0, end = var_3193_end_0, end_mask = var_3193_end_mask_0, squeeze_mask = var_3193_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3193_cast_fp16")];
+            bool var_3194_transpose_x_0 = const()[name = string("op_3194_transpose_x_0"), val = bool(false)];
+            bool var_3194_transpose_y_0 = const()[name = string("op_3194_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3194_cast_fp16 = matmul(transpose_x = var_3194_transpose_x_0, transpose_y = var_3194_transpose_y_0, x = var_3193_cast_fp16, y = state_183_cast_fp16)[name = string("op_3194_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_183_cast_fp16 = sub(x = var_3191_cast_fp16, y = var_3194_cast_fp16)[name = string("v_new_183_cast_fp16")];
+            tensor<int32, [4]> var_3197_begin_0 = const()[name = string("op_3197_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_3197_end_0 = const()[name = string("op_3197_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_3197_end_mask_0 = const()[name = string("op_3197_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3197_squeeze_mask_0 = const()[name = string("op_3197_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3197_cast_fp16 = slice_by_index(begin = var_3197_begin_0, end = var_3197_end_0, end_mask = var_3197_end_mask_0, squeeze_mask = var_3197_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3197_cast_fp16")];
+            bool var_3198_transpose_x_0 = const()[name = string("op_3198_transpose_x_0"), val = bool(false)];
+            bool var_3198_transpose_y_0 = const()[name = string("op_3198_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3198_cast_fp16 = matmul(transpose_x = var_3198_transpose_x_0, transpose_y = var_3198_transpose_y_0, x = var_3197_cast_fp16, y = state_183_cast_fp16)[name = string("op_3198_cast_fp16")];
+            tensor<int32, [4]> var_3200_begin_0 = const()[name = string("op_3200_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_3200_end_0 = const()[name = string("op_3200_end_0"), val = tensor<int32, [4]>([32, 28, 64, 64])];
+            tensor<bool, [4]> var_3200_end_mask_0 = const()[name = string("op_3200_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3200_squeeze_mask_0 = const()[name = string("op_3200_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3200_cast_fp16 = slice_by_index(begin = var_3200_begin_0, end = var_3200_end_0, end_mask = var_3200_end_mask_0, squeeze_mask = var_3200_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3200_cast_fp16")];
+            bool var_3201_transpose_x_0 = const()[name = string("op_3201_transpose_x_0"), val = bool(false)];
+            bool var_3201_transpose_y_0 = const()[name = string("op_3201_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3201_cast_fp16 = matmul(transpose_x = var_3201_transpose_x_0, transpose_y = var_3201_transpose_y_0, x = var_3200_cast_fp16, y = v_new_183_cast_fp16)[name = string("op_3201_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3202_cast_fp16 = add(x = var_3198_cast_fp16, y = var_3201_cast_fp16)[name = string("op_3202_cast_fp16")];
+            tensor<int32, [4]> var_3204_begin_0 = const()[name = string("op_3204_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_3204_end_0 = const()[name = string("op_3204_end_0"), val = tensor<int32, [4]>([32, 28, 1, 1])];
+            tensor<bool, [4]> var_3204_end_mask_0 = const()[name = string("op_3204_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3204_squeeze_mask_0 = const()[name = string("op_3204_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3204_cast_fp16 = slice_by_index(begin = var_3204_begin_0, end = var_3204_end_0, end_mask = var_3204_end_mask_0, squeeze_mask = var_3204_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3204_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3205_cast_fp16 = mul(x = state_183_cast_fp16, y = var_3204_cast_fp16)[name = string("op_3205_cast_fp16")];
+            tensor<int32, [4]> var_3207_begin_0 = const()[name = string("op_3207_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_3207_end_0 = const()[name = string("op_3207_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_3207_end_mask_0 = const()[name = string("op_3207_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3207_squeeze_mask_0 = const()[name = string("op_3207_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3207_cast_fp16 = slice_by_index(begin = var_3207_begin_0, end = var_3207_end_0, end_mask = var_3207_end_mask_0, squeeze_mask = var_3207_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3207_cast_fp16")];
+            bool var_3209_transpose_x_1 = const()[name = string("op_3209_transpose_x_1"), val = bool(true)];
+            bool var_3209_transpose_y_1 = const()[name = string("op_3209_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3209_cast_fp16 = matmul(transpose_x = var_3209_transpose_x_1, transpose_y = var_3209_transpose_y_1, x = var_3207_cast_fp16, y = v_new_183_cast_fp16)[name = string("op_3209_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_185_cast_fp16 = add(x = var_3205_cast_fp16, y = var_3209_cast_fp16)[name = string("state_185_cast_fp16")];
+            tensor<int32, [4]> var_3212_begin_0 = const()[name = string("op_3212_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_3212_end_0 = const()[name = string("op_3212_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_3212_end_mask_0 = const()[name = string("op_3212_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3212_squeeze_mask_0 = const()[name = string("op_3212_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3212_cast_fp16 = slice_by_index(begin = var_3212_begin_0, end = var_3212_end_0, end_mask = var_3212_end_mask_0, squeeze_mask = var_3212_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3212_cast_fp16")];
+            tensor<int32, [4]> var_3214_begin_0 = const()[name = string("op_3214_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_3214_end_0 = const()[name = string("op_3214_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_3214_end_mask_0 = const()[name = string("op_3214_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3214_squeeze_mask_0 = const()[name = string("op_3214_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3214_cast_fp16 = slice_by_index(begin = var_3214_begin_0, end = var_3214_end_0, end_mask = var_3214_end_mask_0, squeeze_mask = var_3214_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3214_cast_fp16")];
+            bool var_3215_transpose_x_0 = const()[name = string("op_3215_transpose_x_0"), val = bool(false)];
+            bool var_3215_transpose_y_0 = const()[name = string("op_3215_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3215_cast_fp16 = matmul(transpose_x = var_3215_transpose_x_0, transpose_y = var_3215_transpose_y_0, x = var_3214_cast_fp16, y = state_185_cast_fp16)[name = string("op_3215_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_185_cast_fp16 = sub(x = var_3212_cast_fp16, y = var_3215_cast_fp16)[name = string("v_new_185_cast_fp16")];
+            tensor<int32, [4]> var_3218_begin_0 = const()[name = string("op_3218_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_3218_end_0 = const()[name = string("op_3218_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_3218_end_mask_0 = const()[name = string("op_3218_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3218_squeeze_mask_0 = const()[name = string("op_3218_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3218_cast_fp16 = slice_by_index(begin = var_3218_begin_0, end = var_3218_end_0, end_mask = var_3218_end_mask_0, squeeze_mask = var_3218_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3218_cast_fp16")];
+            bool var_3219_transpose_x_0 = const()[name = string("op_3219_transpose_x_0"), val = bool(false)];
+            bool var_3219_transpose_y_0 = const()[name = string("op_3219_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3219_cast_fp16 = matmul(transpose_x = var_3219_transpose_x_0, transpose_y = var_3219_transpose_y_0, x = var_3218_cast_fp16, y = state_185_cast_fp16)[name = string("op_3219_cast_fp16")];
+            tensor<int32, [4]> var_3221_begin_0 = const()[name = string("op_3221_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_3221_end_0 = const()[name = string("op_3221_end_0"), val = tensor<int32, [4]>([32, 29, 64, 64])];
+            tensor<bool, [4]> var_3221_end_mask_0 = const()[name = string("op_3221_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3221_squeeze_mask_0 = const()[name = string("op_3221_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3221_cast_fp16 = slice_by_index(begin = var_3221_begin_0, end = var_3221_end_0, end_mask = var_3221_end_mask_0, squeeze_mask = var_3221_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3221_cast_fp16")];
+            bool var_3222_transpose_x_0 = const()[name = string("op_3222_transpose_x_0"), val = bool(false)];
+            bool var_3222_transpose_y_0 = const()[name = string("op_3222_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3222_cast_fp16 = matmul(transpose_x = var_3222_transpose_x_0, transpose_y = var_3222_transpose_y_0, x = var_3221_cast_fp16, y = v_new_185_cast_fp16)[name = string("op_3222_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3223_cast_fp16 = add(x = var_3219_cast_fp16, y = var_3222_cast_fp16)[name = string("op_3223_cast_fp16")];
+            tensor<int32, [4]> var_3225_begin_0 = const()[name = string("op_3225_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_3225_end_0 = const()[name = string("op_3225_end_0"), val = tensor<int32, [4]>([32, 29, 1, 1])];
+            tensor<bool, [4]> var_3225_end_mask_0 = const()[name = string("op_3225_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3225_squeeze_mask_0 = const()[name = string("op_3225_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3225_cast_fp16 = slice_by_index(begin = var_3225_begin_0, end = var_3225_end_0, end_mask = var_3225_end_mask_0, squeeze_mask = var_3225_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3225_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3226_cast_fp16 = mul(x = state_185_cast_fp16, y = var_3225_cast_fp16)[name = string("op_3226_cast_fp16")];
+            tensor<int32, [4]> var_3228_begin_0 = const()[name = string("op_3228_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_3228_end_0 = const()[name = string("op_3228_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_3228_end_mask_0 = const()[name = string("op_3228_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3228_squeeze_mask_0 = const()[name = string("op_3228_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3228_cast_fp16 = slice_by_index(begin = var_3228_begin_0, end = var_3228_end_0, end_mask = var_3228_end_mask_0, squeeze_mask = var_3228_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3228_cast_fp16")];
+            bool var_3230_transpose_x_1 = const()[name = string("op_3230_transpose_x_1"), val = bool(true)];
+            bool var_3230_transpose_y_1 = const()[name = string("op_3230_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3230_cast_fp16 = matmul(transpose_x = var_3230_transpose_x_1, transpose_y = var_3230_transpose_y_1, x = var_3228_cast_fp16, y = v_new_185_cast_fp16)[name = string("op_3230_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_187_cast_fp16 = add(x = var_3226_cast_fp16, y = var_3230_cast_fp16)[name = string("state_187_cast_fp16")];
+            tensor<int32, [4]> var_3233_begin_0 = const()[name = string("op_3233_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_3233_end_0 = const()[name = string("op_3233_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_3233_end_mask_0 = const()[name = string("op_3233_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3233_squeeze_mask_0 = const()[name = string("op_3233_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3233_cast_fp16 = slice_by_index(begin = var_3233_begin_0, end = var_3233_end_0, end_mask = var_3233_end_mask_0, squeeze_mask = var_3233_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3233_cast_fp16")];
+            tensor<int32, [4]> var_3235_begin_0 = const()[name = string("op_3235_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_3235_end_0 = const()[name = string("op_3235_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_3235_end_mask_0 = const()[name = string("op_3235_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3235_squeeze_mask_0 = const()[name = string("op_3235_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3235_cast_fp16 = slice_by_index(begin = var_3235_begin_0, end = var_3235_end_0, end_mask = var_3235_end_mask_0, squeeze_mask = var_3235_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3235_cast_fp16")];
+            bool var_3236_transpose_x_0 = const()[name = string("op_3236_transpose_x_0"), val = bool(false)];
+            bool var_3236_transpose_y_0 = const()[name = string("op_3236_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3236_cast_fp16 = matmul(transpose_x = var_3236_transpose_x_0, transpose_y = var_3236_transpose_y_0, x = var_3235_cast_fp16, y = state_187_cast_fp16)[name = string("op_3236_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_187_cast_fp16 = sub(x = var_3233_cast_fp16, y = var_3236_cast_fp16)[name = string("v_new_187_cast_fp16")];
+            tensor<int32, [4]> var_3239_begin_0 = const()[name = string("op_3239_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_3239_end_0 = const()[name = string("op_3239_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_3239_end_mask_0 = const()[name = string("op_3239_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3239_squeeze_mask_0 = const()[name = string("op_3239_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3239_cast_fp16 = slice_by_index(begin = var_3239_begin_0, end = var_3239_end_0, end_mask = var_3239_end_mask_0, squeeze_mask = var_3239_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3239_cast_fp16")];
+            bool var_3240_transpose_x_0 = const()[name = string("op_3240_transpose_x_0"), val = bool(false)];
+            bool var_3240_transpose_y_0 = const()[name = string("op_3240_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3240_cast_fp16 = matmul(transpose_x = var_3240_transpose_x_0, transpose_y = var_3240_transpose_y_0, x = var_3239_cast_fp16, y = state_187_cast_fp16)[name = string("op_3240_cast_fp16")];
+            tensor<int32, [4]> var_3242_begin_0 = const()[name = string("op_3242_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_3242_end_0 = const()[name = string("op_3242_end_0"), val = tensor<int32, [4]>([32, 30, 64, 64])];
+            tensor<bool, [4]> var_3242_end_mask_0 = const()[name = string("op_3242_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3242_squeeze_mask_0 = const()[name = string("op_3242_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3242_cast_fp16 = slice_by_index(begin = var_3242_begin_0, end = var_3242_end_0, end_mask = var_3242_end_mask_0, squeeze_mask = var_3242_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3242_cast_fp16")];
+            bool var_3243_transpose_x_0 = const()[name = string("op_3243_transpose_x_0"), val = bool(false)];
+            bool var_3243_transpose_y_0 = const()[name = string("op_3243_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3243_cast_fp16 = matmul(transpose_x = var_3243_transpose_x_0, transpose_y = var_3243_transpose_y_0, x = var_3242_cast_fp16, y = v_new_187_cast_fp16)[name = string("op_3243_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3244_cast_fp16 = add(x = var_3240_cast_fp16, y = var_3243_cast_fp16)[name = string("op_3244_cast_fp16")];
+            tensor<int32, [4]> var_3246_begin_0 = const()[name = string("op_3246_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_3246_end_0 = const()[name = string("op_3246_end_0"), val = tensor<int32, [4]>([32, 30, 1, 1])];
+            tensor<bool, [4]> var_3246_end_mask_0 = const()[name = string("op_3246_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3246_squeeze_mask_0 = const()[name = string("op_3246_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3246_cast_fp16 = slice_by_index(begin = var_3246_begin_0, end = var_3246_end_0, end_mask = var_3246_end_mask_0, squeeze_mask = var_3246_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3246_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3247_cast_fp16 = mul(x = state_187_cast_fp16, y = var_3246_cast_fp16)[name = string("op_3247_cast_fp16")];
+            tensor<int32, [4]> var_3249_begin_0 = const()[name = string("op_3249_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_3249_end_0 = const()[name = string("op_3249_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_3249_end_mask_0 = const()[name = string("op_3249_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3249_squeeze_mask_0 = const()[name = string("op_3249_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3249_cast_fp16 = slice_by_index(begin = var_3249_begin_0, end = var_3249_end_0, end_mask = var_3249_end_mask_0, squeeze_mask = var_3249_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3249_cast_fp16")];
+            bool var_3251_transpose_x_1 = const()[name = string("op_3251_transpose_x_1"), val = bool(true)];
+            bool var_3251_transpose_y_1 = const()[name = string("op_3251_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3251_cast_fp16 = matmul(transpose_x = var_3251_transpose_x_1, transpose_y = var_3251_transpose_y_1, x = var_3249_cast_fp16, y = v_new_187_cast_fp16)[name = string("op_3251_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_189_cast_fp16 = add(x = var_3247_cast_fp16, y = var_3251_cast_fp16)[name = string("state_189_cast_fp16")];
+            tensor<int32, [4]> var_3254_begin_0 = const()[name = string("op_3254_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_3254_end_0 = const()[name = string("op_3254_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_3254_end_mask_0 = const()[name = string("op_3254_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3254_squeeze_mask_0 = const()[name = string("op_3254_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3254_cast_fp16 = slice_by_index(begin = var_3254_begin_0, end = var_3254_end_0, end_mask = var_3254_end_mask_0, squeeze_mask = var_3254_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3254_cast_fp16")];
+            tensor<int32, [4]> var_3256_begin_0 = const()[name = string("op_3256_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_3256_end_0 = const()[name = string("op_3256_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_3256_end_mask_0 = const()[name = string("op_3256_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3256_squeeze_mask_0 = const()[name = string("op_3256_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3256_cast_fp16 = slice_by_index(begin = var_3256_begin_0, end = var_3256_end_0, end_mask = var_3256_end_mask_0, squeeze_mask = var_3256_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3256_cast_fp16")];
+            bool var_3257_transpose_x_0 = const()[name = string("op_3257_transpose_x_0"), val = bool(false)];
+            bool var_3257_transpose_y_0 = const()[name = string("op_3257_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3257_cast_fp16 = matmul(transpose_x = var_3257_transpose_x_0, transpose_y = var_3257_transpose_y_0, x = var_3256_cast_fp16, y = state_189_cast_fp16)[name = string("op_3257_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_189_cast_fp16 = sub(x = var_3254_cast_fp16, y = var_3257_cast_fp16)[name = string("v_new_189_cast_fp16")];
+            tensor<int32, [4]> var_3260_begin_0 = const()[name = string("op_3260_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_3260_end_0 = const()[name = string("op_3260_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_3260_end_mask_0 = const()[name = string("op_3260_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3260_squeeze_mask_0 = const()[name = string("op_3260_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3260_cast_fp16 = slice_by_index(begin = var_3260_begin_0, end = var_3260_end_0, end_mask = var_3260_end_mask_0, squeeze_mask = var_3260_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3260_cast_fp16")];
+            bool var_3261_transpose_x_0 = const()[name = string("op_3261_transpose_x_0"), val = bool(false)];
+            bool var_3261_transpose_y_0 = const()[name = string("op_3261_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3261_cast_fp16 = matmul(transpose_x = var_3261_transpose_x_0, transpose_y = var_3261_transpose_y_0, x = var_3260_cast_fp16, y = state_189_cast_fp16)[name = string("op_3261_cast_fp16")];
+            tensor<int32, [4]> var_3263_begin_0 = const()[name = string("op_3263_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_3263_end_0 = const()[name = string("op_3263_end_0"), val = tensor<int32, [4]>([32, 31, 64, 64])];
+            tensor<bool, [4]> var_3263_end_mask_0 = const()[name = string("op_3263_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3263_squeeze_mask_0 = const()[name = string("op_3263_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3263_cast_fp16 = slice_by_index(begin = var_3263_begin_0, end = var_3263_end_0, end_mask = var_3263_end_mask_0, squeeze_mask = var_3263_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3263_cast_fp16")];
+            bool var_3264_transpose_x_0 = const()[name = string("op_3264_transpose_x_0"), val = bool(false)];
+            bool var_3264_transpose_y_0 = const()[name = string("op_3264_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3264_cast_fp16 = matmul(transpose_x = var_3264_transpose_x_0, transpose_y = var_3264_transpose_y_0, x = var_3263_cast_fp16, y = v_new_189_cast_fp16)[name = string("op_3264_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3265_cast_fp16 = add(x = var_3261_cast_fp16, y = var_3264_cast_fp16)[name = string("op_3265_cast_fp16")];
+            tensor<int32, [4]> var_3267_begin_0 = const()[name = string("op_3267_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_3267_end_0 = const()[name = string("op_3267_end_0"), val = tensor<int32, [4]>([32, 31, 1, 1])];
+            tensor<bool, [4]> var_3267_end_mask_0 = const()[name = string("op_3267_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3267_squeeze_mask_0 = const()[name = string("op_3267_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3267_cast_fp16 = slice_by_index(begin = var_3267_begin_0, end = var_3267_end_0, end_mask = var_3267_end_mask_0, squeeze_mask = var_3267_squeeze_mask_0, x = chunk_decay_5_cast_fp16)[name = string("op_3267_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3268_cast_fp16 = mul(x = state_189_cast_fp16, y = var_3267_cast_fp16)[name = string("op_3268_cast_fp16")];
+            tensor<int32, [4]> var_3270_begin_0 = const()[name = string("op_3270_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_3270_end_0 = const()[name = string("op_3270_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_3270_end_mask_0 = const()[name = string("op_3270_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3270_squeeze_mask_0 = const()[name = string("op_3270_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3270_cast_fp16 = slice_by_index(begin = var_3270_begin_0, end = var_3270_end_0, end_mask = var_3270_end_mask_0, squeeze_mask = var_3270_squeeze_mask_0, x = k_dec_5_cast_fp16)[name = string("op_3270_cast_fp16")];
+            bool var_3272_transpose_x_1 = const()[name = string("op_3272_transpose_x_1"), val = bool(true)];
+            bool var_3272_transpose_y_1 = const()[name = string("op_3272_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3272_cast_fp16 = matmul(transpose_x = var_3272_transpose_x_1, transpose_y = var_3272_transpose_y_1, x = var_3270_cast_fp16, y = v_new_189_cast_fp16)[name = string("op_3272_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_191_cast_fp16 = add(x = var_3268_cast_fp16, y = var_3272_cast_fp16)[name = string("state_191_cast_fp16")];
+            tensor<int32, [4]> var_3275_begin_0 = const()[name = string("op_3275_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_3275_end_0 = const()[name = string("op_3275_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_3275_end_mask_0 = const()[name = string("op_3275_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3275_squeeze_mask_0 = const()[name = string("op_3275_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3275_cast_fp16 = slice_by_index(begin = var_3275_begin_0, end = var_3275_end_0, end_mask = var_3275_end_mask_0, squeeze_mask = var_3275_squeeze_mask_0, x = new_v_5_cast_fp16)[name = string("op_3275_cast_fp16")];
+            tensor<int32, [4]> var_3277_begin_0 = const()[name = string("op_3277_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_3277_end_0 = const()[name = string("op_3277_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_3277_end_mask_0 = const()[name = string("op_3277_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3277_squeeze_mask_0 = const()[name = string("op_3277_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3277_cast_fp16 = slice_by_index(begin = var_3277_begin_0, end = var_3277_end_0, end_mask = var_3277_end_mask_0, squeeze_mask = var_3277_squeeze_mask_0, x = k_cumdecay_5_cast_fp16)[name = string("op_3277_cast_fp16")];
+            bool var_3278_transpose_x_0 = const()[name = string("op_3278_transpose_x_0"), val = bool(false)];
+            bool var_3278_transpose_y_0 = const()[name = string("op_3278_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3278_cast_fp16 = matmul(transpose_x = var_3278_transpose_x_0, transpose_y = var_3278_transpose_y_0, x = var_3277_cast_fp16, y = state_191_cast_fp16)[name = string("op_3278_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_191_cast_fp16 = sub(x = var_3275_cast_fp16, y = var_3278_cast_fp16)[name = string("v_new_191_cast_fp16")];
+            tensor<int32, [4]> var_3281_begin_0 = const()[name = string("op_3281_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_3281_end_0 = const()[name = string("op_3281_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_3281_end_mask_0 = const()[name = string("op_3281_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3281_squeeze_mask_0 = const()[name = string("op_3281_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3281_cast_fp16 = slice_by_index(begin = var_3281_begin_0, end = var_3281_end_0, end_mask = var_3281_end_mask_0, squeeze_mask = var_3281_squeeze_mask_0, x = q_dec_5_cast_fp16)[name = string("op_3281_cast_fp16")];
+            bool var_3282_transpose_x_0 = const()[name = string("op_3282_transpose_x_0"), val = bool(false)];
+            bool var_3282_transpose_y_0 = const()[name = string("op_3282_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3282_cast_fp16 = matmul(transpose_x = var_3282_transpose_x_0, transpose_y = var_3282_transpose_y_0, x = var_3281_cast_fp16, y = state_191_cast_fp16)[name = string("op_3282_cast_fp16")];
+            tensor<int32, [4]> var_3284_begin_0 = const()[name = string("op_3284_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_3284_end_0 = const()[name = string("op_3284_end_0"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<bool, [4]> var_3284_end_mask_0 = const()[name = string("op_3284_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3284_squeeze_mask_0 = const()[name = string("op_3284_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3284_cast_fp16 = slice_by_index(begin = var_3284_begin_0, end = var_3284_end_0, end_mask = var_3284_end_mask_0, squeeze_mask = var_3284_squeeze_mask_0, x = attn_intra_5_cast_fp16)[name = string("op_3284_cast_fp16")];
+            bool var_3285_transpose_x_0 = const()[name = string("op_3285_transpose_x_0"), val = bool(false)];
+            bool var_3285_transpose_y_0 = const()[name = string("op_3285_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3285_cast_fp16 = matmul(transpose_x = var_3285_transpose_x_0, transpose_y = var_3285_transpose_y_0, x = var_3284_cast_fp16, y = v_new_191_cast_fp16)[name = string("op_3285_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3286_cast_fp16 = add(x = var_3282_cast_fp16, y = var_3285_cast_fp16)[name = string("op_3286_cast_fp16")];
+            int32 var_3288_axis_0 = const()[name = string("op_3288_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 32, 64, 128]> var_3288_cast_fp16 = stack(axis = var_3288_axis_0, values = (var_2635_cast_fp16, var_2656_cast_fp16, var_2677_cast_fp16, var_2698_cast_fp16, var_2719_cast_fp16, var_2740_cast_fp16, var_2761_cast_fp16, var_2782_cast_fp16, var_2803_cast_fp16, var_2824_cast_fp16, var_2845_cast_fp16, var_2866_cast_fp16, var_2887_cast_fp16, var_2908_cast_fp16, var_2929_cast_fp16, var_2950_cast_fp16, var_2971_cast_fp16, var_2992_cast_fp16, var_3013_cast_fp16, var_3034_cast_fp16, var_3055_cast_fp16, var_3076_cast_fp16, var_3097_cast_fp16, var_3118_cast_fp16, var_3139_cast_fp16, var_3160_cast_fp16, var_3181_cast_fp16, var_3202_cast_fp16, var_3223_cast_fp16, var_3244_cast_fp16, var_3265_cast_fp16, var_3286_cast_fp16))[name = string("op_3288_cast_fp16")];
+            tensor<int32, [3]> var_3289 = const()[name = string("op_3289"), val = tensor<int32, [3]>([32, 2048, 128])];
+            tensor<fp16, [32, 2048, 128]> var_3290_cast_fp16 = reshape(shape = var_3289, x = var_3288_cast_fp16)[name = string("op_3290_cast_fp16")];
+            tensor<int32, [3]> var_3291 = const()[name = string("op_3291"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_39_begin_0 = const()[name = string("input_39_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_39_end_0 = const()[name = string("input_39_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> input_39_end_mask_0 = const()[name = string("input_39_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_39_squeeze_mask_0 = const()[name = string("input_39_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> input_39_cast_fp16 = slice_by_index(begin = input_39_begin_0, end = input_39_end_0, end_mask = input_39_end_mask_0, squeeze_mask = input_39_squeeze_mask_0, x = z_5_cast_fp16)[name = string("input_39_cast_fp16")];
+            fp16 var_2271_promoted_1_to_fp16 = const()[name = string("op_2271_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [2048, 32, 128]> x_69_cast_fp16 = transpose(perm = var_3291, x = var_3290_cast_fp16)[name = string("transpose_32")];
+            tensor<fp16, [2048, 32, 128]> var_3295_cast_fp16 = pow(x = x_69_cast_fp16, y = var_2271_promoted_1_to_fp16)[name = string("op_3295_cast_fp16")];
+            tensor<int32, [1]> var_3297_axes_0 = const()[name = string("op_3297_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3297_keep_dims_0 = const()[name = string("op_3297_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [2048, 32, 1]> var_3297_cast_fp16 = reduce_mean(axes = var_3297_axes_0, keep_dims = var_3297_keep_dims_0, x = var_3295_cast_fp16)[name = string("op_3297_cast_fp16")];
+            fp16 var_3298_to_fp16 = const()[name = string("op_3298_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [2048, 32, 1]> var_3299_cast_fp16 = add(x = var_3297_cast_fp16, y = var_3298_to_fp16)[name = string("op_3299_cast_fp16")];
+            fp32 var_3300_epsilon_0 = const()[name = string("op_3300_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [2048, 32, 1]> var_3300_cast_fp16 = rsqrt(epsilon = var_3300_epsilon_0, x = var_3299_cast_fp16)[name = string("op_3300_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> x_71_cast_fp16 = mul(x = x_69_cast_fp16, y = var_3300_cast_fp16)[name = string("x_71_cast_fp16")];
+            tensor<fp16, [128]> layers_2_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_2_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520929472)))];
+            tensor<fp16, [2048, 32, 128]> var_3302_cast_fp16 = mul(x = layers_2_linear_attn_norm_weight_to_fp16, y = x_71_cast_fp16)[name = string("op_3302_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> var_3303_cast_fp16 = silu(x = input_39_cast_fp16)[name = string("op_3303_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> core_5_cast_fp16 = mul(x = var_3302_cast_fp16, y = var_3303_cast_fp16)[name = string("core_5_cast_fp16")];
+            tensor<int32, [3]> var_3305 = const()[name = string("op_3305"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> input_41_cast_fp16 = reshape(shape = var_3305, x = core_5_cast_fp16)[name = string("input_41_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_2_linear_attn_out_proj_weight_to_fp16 = const()[name = string("layers_2_linear_attn_out_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(520929792)))];
+            tensor<fp16, [1, 2048, 2560]> linear_20_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_2_linear_attn_out_proj_weight_to_fp16, x = input_41_cast_fp16)[name = string("linear_20_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_73_cast_fp16 = add(x = x_51_cast_fp16, y = linear_20_cast_fp16)[name = string("x_73_cast_fp16")];
+            fp16 var_2271_promoted_2_to_fp16 = const()[name = string("op_2271_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_3311_cast_fp16 = pow(x = x_73_cast_fp16, y = var_2271_promoted_2_to_fp16)[name = string("op_3311_cast_fp16")];
+            tensor<int32, [1]> var_3313_axes_0 = const()[name = string("op_3313_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3313_keep_dims_0 = const()[name = string("op_3313_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_3313_cast_fp16 = reduce_mean(axes = var_3313_axes_0, keep_dims = var_3313_keep_dims_0, x = var_3311_cast_fp16)[name = string("op_3313_cast_fp16")];
+            fp16 var_3314_to_fp16 = const()[name = string("op_3314_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_3315_cast_fp16 = add(x = var_3313_cast_fp16, y = var_3314_to_fp16)[name = string("op_3315_cast_fp16")];
+            fp32 var_3316_epsilon_0 = const()[name = string("op_3316_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_3316_cast_fp16 = rsqrt(epsilon = var_3316_epsilon_0, x = var_3315_cast_fp16)[name = string("op_3316_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_75_cast_fp16 = mul(x = x_73_cast_fp16, y = var_3316_cast_fp16)[name = string("x_75_cast_fp16")];
+            tensor<fp16, [2560]> var_3319_to_fp16 = const()[name = string("op_3319_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(541901376)))];
+            tensor<fp16, [1, 2048, 2560]> input_43_cast_fp16 = mul(x = x_75_cast_fp16, y = var_3319_to_fp16)[name = string("input_43_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_2_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_2_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(541906560)))];
+            tensor<fp16, [1, 2048, 9216]> linear_21_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_2_mlp_gate_proj_weight_to_fp16, x = input_43_cast_fp16)[name = string("linear_21_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_3326_cast_fp16 = silu(x = linear_21_cast_fp16)[name = string("op_3326_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_2_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_2_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(589092544)))];
+            tensor<fp16, [1, 2048, 9216]> linear_22_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_2_mlp_up_proj_weight_to_fp16, x = input_43_cast_fp16)[name = string("linear_22_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_47_cast_fp16 = mul(x = var_3326_cast_fp16, y = linear_22_cast_fp16)[name = string("input_47_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_2_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_2_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(636278528)))];
+            tensor<fp16, [1, 2048, 2560]> linear_23_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_2_mlp_down_proj_weight_to_fp16, x = input_47_cast_fp16)[name = string("linear_23_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_77_cast_fp16 = add(x = x_73_cast_fp16, y = linear_23_cast_fp16)[name = string("x_77_cast_fp16")];
+            int32 var_3346 = const()[name = string("op_3346"), val = int32(-1)];
+            fp16 var_3345_promoted_to_fp16 = const()[name = string("op_3345_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_3355_cast_fp16 = pow(x = x_77_cast_fp16, y = var_3345_promoted_to_fp16)[name = string("op_3355_cast_fp16")];
+            tensor<int32, [1]> var_3357_axes_0 = const()[name = string("op_3357_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3357_keep_dims_0 = const()[name = string("op_3357_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_3357_cast_fp16 = reduce_mean(axes = var_3357_axes_0, keep_dims = var_3357_keep_dims_0, x = var_3355_cast_fp16)[name = string("op_3357_cast_fp16")];
+            fp16 var_3358_to_fp16 = const()[name = string("op_3358_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_3359_cast_fp16 = add(x = var_3357_cast_fp16, y = var_3358_to_fp16)[name = string("op_3359_cast_fp16")];
+            fp32 var_3360_epsilon_0 = const()[name = string("op_3360_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_3360_cast_fp16 = rsqrt(epsilon = var_3360_epsilon_0, x = var_3359_cast_fp16)[name = string("op_3360_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_79_cast_fp16 = mul(x = x_77_cast_fp16, y = var_3360_cast_fp16)[name = string("x_79_cast_fp16")];
+            tensor<fp16, [2560]> var_3363_to_fp16 = const()[name = string("op_3363_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(683464512)))];
+            tensor<fp16, [1, 2048, 2560]> x_81_cast_fp16 = mul(x = x_79_cast_fp16, y = var_3363_to_fp16)[name = string("x_81_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_3_self_attn_q_proj_weight_to_fp16 = const()[name = string("layers_3_self_attn_q_proj_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(683469696)))];
+            tensor<fp16, [1, 2048, 8192]> linear_24_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_3_self_attn_q_proj_weight_to_fp16, x = x_81_cast_fp16)[name = string("linear_24_cast_fp16")];
+            tensor<int32, [4]> var_3374 = const()[name = string("op_3374"), val = tensor<int32, [4]>([1, 2048, 16, 512])];
+            tensor<fp16, [1, 2048, 16, 512]> qg_1_cast_fp16 = reshape(shape = var_3374, x = linear_24_cast_fp16)[name = string("qg_1_cast_fp16")];
+            tensor<int32, [4]> x_83_begin_0 = const()[name = string("x_83_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_83_end_0 = const()[name = string("x_83_end_0"), val = tensor<int32, [4]>([1, 2048, 16, 256])];
+            tensor<bool, [4]> x_83_end_mask_0 = const()[name = string("x_83_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 2048, 16, 256]> x_83_cast_fp16 = slice_by_index(begin = x_83_begin_0, end = x_83_end_0, end_mask = x_83_end_mask_0, x = qg_1_cast_fp16)[name = string("x_83_cast_fp16")];
+            tensor<int32, [4]> gate_1_begin_0 = const()[name = string("gate_1_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 256])];
+            tensor<int32, [4]> gate_1_end_0 = const()[name = string("gate_1_end_0"), val = tensor<int32, [4]>([1, 2048, 16, 512])];
+            tensor<bool, [4]> gate_1_end_mask_0 = const()[name = string("gate_1_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 2048, 16, 256]> gate_1_cast_fp16 = slice_by_index(begin = gate_1_begin_0, end = gate_1_end_0, end_mask = gate_1_end_mask_0, x = qg_1_cast_fp16)[name = string("gate_1_cast_fp16")];
+            tensor<int32, [3]> var_3378 = const()[name = string("op_3378"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> gate_3_cast_fp16 = reshape(shape = var_3378, x = gate_1_cast_fp16)[name = string("gate_3_cast_fp16")];
+            fp16 var_3345_promoted_1_to_fp16 = const()[name = string("op_3345_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 16, 256]> var_3381_cast_fp16 = pow(x = x_83_cast_fp16, y = var_3345_promoted_1_to_fp16)[name = string("op_3381_cast_fp16")];
+            tensor<int32, [1]> var_3383_axes_0 = const()[name = string("op_3383_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3383_keep_dims_0 = const()[name = string("op_3383_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3383_cast_fp16 = reduce_mean(axes = var_3383_axes_0, keep_dims = var_3383_keep_dims_0, x = var_3381_cast_fp16)[name = string("op_3383_cast_fp16")];
+            fp16 var_3384_to_fp16 = const()[name = string("op_3384_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3385_cast_fp16 = add(x = var_3383_cast_fp16, y = var_3384_to_fp16)[name = string("op_3385_cast_fp16")];
+            fp32 var_3386_epsilon_0 = const()[name = string("op_3386_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3386_cast_fp16 = rsqrt(epsilon = var_3386_epsilon_0, x = var_3385_cast_fp16)[name = string("op_3386_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 256]> x_85_cast_fp16 = mul(x = x_83_cast_fp16, y = var_3386_cast_fp16)[name = string("x_85_cast_fp16")];
+            tensor<fp16, [256]> var_3389_to_fp16 = const()[name = string("op_3389_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(725412800)))];
+            tensor<fp16, [1, 2048, 16, 256]> var_3390_cast_fp16 = mul(x = x_85_cast_fp16, y = var_3389_to_fp16)[name = string("op_3390_cast_fp16")];
+            tensor<int32, [4]> q_31_perm_0 = const()[name = string("q_31_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1024, 2560]> layers_3_self_attn_k_proj_weight_to_fp16 = const()[name = string("layers_3_self_attn_k_proj_weight_to_fp16"), val = tensor<fp16, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(725413376)))];
+            tensor<fp16, [1024]> linear_25_bias_0_to_fp16 = const()[name = string("linear_25_bias_0_to_fp16"), val = tensor<fp16, [1024]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(730656320)))];
+            tensor<fp16, [1, 2048, 1024]> linear_25_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_3_self_attn_k_proj_weight_to_fp16, x = x_81_cast_fp16)[name = string("linear_25_cast_fp16")];
+            tensor<int32, [4]> var_3394 = const()[name = string("op_3394"), val = tensor<int32, [4]>([1, 2048, 4, 256])];
+            tensor<fp16, [1, 2048, 4, 256]> x_87_cast_fp16 = reshape(shape = var_3394, x = linear_25_cast_fp16)[name = string("x_87_cast_fp16")];
+            fp16 var_3345_promoted_2_to_fp16 = const()[name = string("op_3345_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 4, 256]> var_3397_cast_fp16 = pow(x = x_87_cast_fp16, y = var_3345_promoted_2_to_fp16)[name = string("op_3397_cast_fp16")];
+            tensor<int32, [1]> var_3399_axes_0 = const()[name = string("op_3399_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3399_keep_dims_0 = const()[name = string("op_3399_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 4, 1]> var_3399_cast_fp16 = reduce_mean(axes = var_3399_axes_0, keep_dims = var_3399_keep_dims_0, x = var_3397_cast_fp16)[name = string("op_3399_cast_fp16")];
+            fp16 var_3400_to_fp16 = const()[name = string("op_3400_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 4, 1]> var_3401_cast_fp16 = add(x = var_3399_cast_fp16, y = var_3400_to_fp16)[name = string("op_3401_cast_fp16")];
+            fp32 var_3402_epsilon_0 = const()[name = string("op_3402_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 4, 1]> var_3402_cast_fp16 = rsqrt(epsilon = var_3402_epsilon_0, x = var_3401_cast_fp16)[name = string("op_3402_cast_fp16")];
+            tensor<fp16, [1, 2048, 4, 256]> x_89_cast_fp16 = mul(x = x_87_cast_fp16, y = var_3402_cast_fp16)[name = string("x_89_cast_fp16")];
+            tensor<fp16, [256]> var_3405_to_fp16 = const()[name = string("op_3405_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(730658432)))];
+            tensor<fp16, [1, 2048, 4, 256]> var_3406_cast_fp16 = mul(x = x_89_cast_fp16, y = var_3405_to_fp16)[name = string("op_3406_cast_fp16")];
+            tensor<int32, [4]> k_25_perm_0 = const()[name = string("k_25_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1024, 2560]> layers_3_self_attn_v_proj_weight_to_fp16 = const()[name = string("layers_3_self_attn_v_proj_weight_to_fp16"), val = tensor<fp16, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(730659008)))];
+            tensor<fp16, [1, 2048, 1024]> linear_26_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_3_self_attn_v_proj_weight_to_fp16, x = x_81_cast_fp16)[name = string("linear_26_cast_fp16")];
+            tensor<int32, [4]> var_3410 = const()[name = string("op_3410"), val = tensor<int32, [4]>([1, 2048, 4, 256])];
+            tensor<fp16, [1, 2048, 4, 256]> var_3411_cast_fp16 = reshape(shape = var_3410, x = linear_26_cast_fp16)[name = string("op_3411_cast_fp16")];
+            tensor<int32, [4]> v_13_perm_0 = const()[name = string("v_13_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [1]> var_3413_axes_0 = const()[name = string("op_3413_axes_0"), val = tensor<int32, [1]>([0])];
+            tensor<fp16, [1, 2048, 64]> var_3413_cast_fp16 = expand_dims(axes = var_3413_axes_0, x = cos)[name = string("op_3413_cast_fp16")];
+            tensor<int32, [1]> c_1_axes_0 = const()[name = string("c_1_axes_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [1, 1, 2048, 64]> c_1_cast_fp16 = expand_dims(axes = c_1_axes_0, x = var_3413_cast_fp16)[name = string("c_1_cast_fp16")];
+            tensor<int32, [1]> var_3415_axes_0 = const()[name = string("op_3415_axes_0"), val = tensor<int32, [1]>([0])];
+            tensor<fp16, [1, 2048, 64]> var_3415_cast_fp16 = expand_dims(axes = var_3415_axes_0, x = sin)[name = string("op_3415_cast_fp16")];
+            tensor<int32, [1]> s_1_axes_0 = const()[name = string("s_1_axes_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [1, 1, 2048, 64]> s_1_cast_fp16 = expand_dims(axes = s_1_axes_0, x = var_3415_cast_fp16)[name = string("s_1_cast_fp16")];
+            tensor<int32, [4]> var_3417_begin_0 = const()[name = string("op_3417_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3417_end_0 = const()[name = string("op_3417_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 64])];
+            tensor<bool, [4]> var_3417_end_mask_0 = const()[name = string("op_3417_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 2048, 256]> q_31_cast_fp16 = transpose(perm = q_31_perm_0, x = var_3390_cast_fp16)[name = string("transpose_31")];
+            tensor<fp16, [1, 16, 2048, 64]> var_3417_cast_fp16 = slice_by_index(begin = var_3417_begin_0, end = var_3417_end_0, end_mask = var_3417_end_mask_0, x = q_31_cast_fp16)[name = string("op_3417_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 64]> var_3418_cast_fp16 = mul(x = var_3417_cast_fp16, y = c_1_cast_fp16)[name = string("op_3418_cast_fp16")];
+            tensor<int32, [4]> var_3425_begin_0 = const()[name = string("op_3425_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_3425_end_0 = const()[name = string("op_3425_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 64])];
+            tensor<bool, [4]> var_3425_end_mask_0 = const()[name = string("op_3425_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 2048, 32]> var_3425_cast_fp16 = slice_by_index(begin = var_3425_begin_0, end = var_3425_end_0, end_mask = var_3425_end_mask_0, x = var_3417_cast_fp16)[name = string("op_3425_cast_fp16")];
+            fp16 const_38_promoted_to_fp16 = const()[name = string("const_38_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 16, 2048, 32]> var_3426_cast_fp16 = mul(x = var_3425_cast_fp16, y = const_38_promoted_to_fp16)[name = string("op_3426_cast_fp16")];
+            tensor<int32, [4]> var_3427_begin_0 = const()[name = string("op_3427_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3427_end_0 = const()[name = string("op_3427_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 32])];
+            tensor<bool, [4]> var_3427_end_mask_0 = const()[name = string("op_3427_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 2048, 32]> var_3427_cast_fp16 = slice_by_index(begin = var_3427_begin_0, end = var_3427_end_0, end_mask = var_3427_end_mask_0, x = var_3417_cast_fp16)[name = string("op_3427_cast_fp16")];
+            bool var_3429_interleave_0 = const()[name = string("op_3429_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 2048, 64]> var_3429_cast_fp16 = concat(axis = var_3346, interleave = var_3429_interleave_0, values = (var_3426_cast_fp16, var_3427_cast_fp16))[name = string("op_3429_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 64]> var_3430_cast_fp16 = mul(x = var_3429_cast_fp16, y = s_1_cast_fp16)[name = string("op_3430_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 64]> var_3431_cast_fp16 = add(x = var_3418_cast_fp16, y = var_3430_cast_fp16)[name = string("op_3431_cast_fp16")];
+            tensor<int32, [4]> var_3432_begin_0 = const()[name = string("op_3432_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_3432_end_0 = const()[name = string("op_3432_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 256])];
+            tensor<bool, [4]> var_3432_end_mask_0 = const()[name = string("op_3432_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 2048, 192]> var_3432_cast_fp16 = slice_by_index(begin = var_3432_begin_0, end = var_3432_end_0, end_mask = var_3432_end_mask_0, x = q_31_cast_fp16)[name = string("op_3432_cast_fp16")];
+            bool q_33_interleave_0 = const()[name = string("q_33_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 2048, 256]> q_33_cast_fp16 = concat(axis = var_3346, interleave = q_33_interleave_0, values = (var_3431_cast_fp16, var_3432_cast_fp16))[name = string("q_33_cast_fp16")];
+            tensor<int32, [4]> var_3435_begin_0 = const()[name = string("op_3435_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3435_end_0 = const()[name = string("op_3435_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 64])];
+            tensor<bool, [4]> var_3435_end_mask_0 = const()[name = string("op_3435_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 2048, 256]> k_25_cast_fp16 = transpose(perm = k_25_perm_0, x = var_3406_cast_fp16)[name = string("transpose_30")];
+            tensor<fp16, [1, 4, 2048, 64]> var_3435_cast_fp16 = slice_by_index(begin = var_3435_begin_0, end = var_3435_end_0, end_mask = var_3435_end_mask_0, x = k_25_cast_fp16)[name = string("op_3435_cast_fp16")];
+            tensor<fp16, [1, 4, 2048, 64]> var_3436_cast_fp16 = mul(x = var_3435_cast_fp16, y = c_1_cast_fp16)[name = string("op_3436_cast_fp16")];
+            tensor<int32, [4]> var_3443_begin_0 = const()[name = string("op_3443_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_3443_end_0 = const()[name = string("op_3443_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 64])];
+            tensor<bool, [4]> var_3443_end_mask_0 = const()[name = string("op_3443_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 2048, 32]> var_3443_cast_fp16 = slice_by_index(begin = var_3443_begin_0, end = var_3443_end_0, end_mask = var_3443_end_mask_0, x = var_3435_cast_fp16)[name = string("op_3443_cast_fp16")];
+            fp16 const_40_promoted_to_fp16 = const()[name = string("const_40_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 4, 2048, 32]> var_3444_cast_fp16 = mul(x = var_3443_cast_fp16, y = const_40_promoted_to_fp16)[name = string("op_3444_cast_fp16")];
+            tensor<int32, [4]> var_3445_begin_0 = const()[name = string("op_3445_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3445_end_0 = const()[name = string("op_3445_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 32])];
+            tensor<bool, [4]> var_3445_end_mask_0 = const()[name = string("op_3445_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 2048, 32]> var_3445_cast_fp16 = slice_by_index(begin = var_3445_begin_0, end = var_3445_end_0, end_mask = var_3445_end_mask_0, x = var_3435_cast_fp16)[name = string("op_3445_cast_fp16")];
+            bool var_3447_interleave_0 = const()[name = string("op_3447_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 2048, 64]> var_3447_cast_fp16 = concat(axis = var_3346, interleave = var_3447_interleave_0, values = (var_3444_cast_fp16, var_3445_cast_fp16))[name = string("op_3447_cast_fp16")];
+            tensor<fp16, [1, 4, 2048, 64]> var_3448_cast_fp16 = mul(x = var_3447_cast_fp16, y = s_1_cast_fp16)[name = string("op_3448_cast_fp16")];
+            tensor<fp16, [1, 4, 2048, 64]> var_3449_cast_fp16 = add(x = var_3436_cast_fp16, y = var_3448_cast_fp16)[name = string("op_3449_cast_fp16")];
+            tensor<int32, [4]> var_3450_begin_0 = const()[name = string("op_3450_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_3450_end_0 = const()[name = string("op_3450_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 256])];
+            tensor<bool, [4]> var_3450_end_mask_0 = const()[name = string("op_3450_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 2048, 192]> var_3450_cast_fp16 = slice_by_index(begin = var_3450_begin_0, end = var_3450_end_0, end_mask = var_3450_end_mask_0, x = k_25_cast_fp16)[name = string("op_3450_cast_fp16")];
+            bool k_27_interleave_0 = const()[name = string("k_27_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 2048, 256]> k_27_cast_fp16 = concat(axis = var_3346, interleave = k_27_interleave_0, values = (var_3449_cast_fp16, var_3450_cast_fp16))[name = string("k_27_cast_fp16")];
+            tensor<int32, [1]> var_3455_axes_0 = const()[name = string("op_3455_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 1, 2048, 256]> var_3455_cast_fp16 = expand_dims(axes = var_3455_axes_0, x = k_27_cast_fp16)[name = string("op_3455_cast_fp16")];
+            tensor<int32, [5]> var_3457_reps_0 = const()[name = string("op_3457_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 2048, 256]> var_3457_cast_fp16 = tile(reps = var_3457_reps_0, x = var_3455_cast_fp16)[name = string("op_3457_cast_fp16")];
+            tensor<int32, [4]> var_3458 = const()[name = string("op_3458"), val = tensor<int32, [4]>([1, 16, 2048, 256])];
+            tensor<fp16, [1, 16, 2048, 256]> k_29_cast_fp16 = reshape(shape = var_3458, x = var_3457_cast_fp16)[name = string("k_29_cast_fp16")];
+            tensor<int32, [1]> var_3462_axes_0 = const()[name = string("op_3462_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 2048, 256]> v_13_cast_fp16 = transpose(perm = v_13_perm_0, x = var_3411_cast_fp16)[name = string("transpose_29")];
+            tensor<fp16, [1, 4, 1, 2048, 256]> var_3462_cast_fp16 = expand_dims(axes = var_3462_axes_0, x = v_13_cast_fp16)[name = string("op_3462_cast_fp16")];
+            tensor<int32, [5]> var_3464_reps_0 = const()[name = string("op_3464_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 2048, 256]> var_3464_cast_fp16 = tile(reps = var_3464_reps_0, x = var_3462_cast_fp16)[name = string("op_3464_cast_fp16")];
+            tensor<int32, [4]> var_3465 = const()[name = string("op_3465"), val = tensor<int32, [4]>([1, 16, 2048, 256])];
+            tensor<fp16, [1, 16, 2048, 256]> v_15_cast_fp16 = reshape(shape = var_3465, x = var_3464_cast_fp16)[name = string("v_15_cast_fp16")];
+            bool var_3468_transpose_x_1 = const()[name = string("op_3468_transpose_x_1"), val = bool(false)];
+            bool var_3468_transpose_y_1 = const()[name = string("op_3468_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [1, 16, 2048, 2048]> var_3468_cast_fp16 = matmul(transpose_x = var_3468_transpose_x_1, transpose_y = var_3468_transpose_y_1, x = q_33_cast_fp16, y = k_29_cast_fp16)[name = string("op_3468_cast_fp16")];
+            fp16 var_3469_to_fp16 = const()[name = string("op_3469_to_fp16"), val = fp16(0x1p-4)];
+            tensor<fp16, [1, 16, 2048, 2048]> var_3470_cast_fp16 = mul(x = var_3468_cast_fp16, y = var_3469_to_fp16)[name = string("op_3470_cast_fp16")];
+            tensor<fp16, [1, 1, 2048, 2048]> mask_to_fp16 = const()[name = string("mask_to_fp16"), val = tensor<fp16, [1, 1, 2048, 2048]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(735901952)))];
+            tensor<fp16, [1, 16, 2048, 2048]> scores_1_cast_fp16 = add(x = var_3470_cast_fp16, y = mask_to_fp16)[name = string("scores_1_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 2048]> var_3472_cast_fp16 = softmax(axis = var_3346, x = scores_1_cast_fp16)[name = string("op_3472_cast_fp16")];
+            bool out_1_transpose_x_0 = const()[name = string("out_1_transpose_x_0"), val = bool(false)];
+            bool out_1_transpose_y_0 = const()[name = string("out_1_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 2048, 256]> out_1_cast_fp16 = matmul(transpose_x = out_1_transpose_x_0, transpose_y = out_1_transpose_y_0, x = var_3472_cast_fp16, y = v_15_cast_fp16)[name = string("out_1_cast_fp16")];
+            tensor<int32, [4]> var_3474_perm_0 = const()[name = string("op_3474_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [3]> var_3475 = const()[name = string("op_3475"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 16, 256]> var_3474_cast_fp16 = transpose(perm = var_3474_perm_0, x = out_1_cast_fp16)[name = string("transpose_28")];
+            tensor<fp16, [1, 2048, 4096]> out_3_cast_fp16 = reshape(shape = var_3475, x = var_3474_cast_fp16)[name = string("out_3_cast_fp16")];
+            tensor<fp16, [1, 2048, 4096]> var_3477_cast_fp16 = sigmoid(x = gate_3_cast_fp16)[name = string("op_3477_cast_fp16")];
+            tensor<fp16, [1, 2048, 4096]> input_49_cast_fp16 = mul(x = out_3_cast_fp16, y = var_3477_cast_fp16)[name = string("input_49_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_3_self_attn_o_proj_weight_to_fp16 = const()[name = string("layers_3_self_attn_o_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(744290624)))];
+            tensor<fp16, [1, 2048, 2560]> linear_27_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_3_self_attn_o_proj_weight_to_fp16, x = input_49_cast_fp16)[name = string("linear_27_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_95_cast_fp16 = add(x = x_77_cast_fp16, y = linear_27_cast_fp16)[name = string("x_95_cast_fp16")];
+            fp16 var_3345_promoted_3_to_fp16 = const()[name = string("op_3345_promoted_3_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_3483_cast_fp16 = pow(x = x_95_cast_fp16, y = var_3345_promoted_3_to_fp16)[name = string("op_3483_cast_fp16")];
+            tensor<int32, [1]> var_3485_axes_0 = const()[name = string("op_3485_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3485_keep_dims_0 = const()[name = string("op_3485_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_3485_cast_fp16 = reduce_mean(axes = var_3485_axes_0, keep_dims = var_3485_keep_dims_0, x = var_3483_cast_fp16)[name = string("op_3485_cast_fp16")];
+            fp16 var_3486_to_fp16 = const()[name = string("op_3486_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_3487_cast_fp16 = add(x = var_3485_cast_fp16, y = var_3486_to_fp16)[name = string("op_3487_cast_fp16")];
+            fp32 var_3488_epsilon_0 = const()[name = string("op_3488_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_3488_cast_fp16 = rsqrt(epsilon = var_3488_epsilon_0, x = var_3487_cast_fp16)[name = string("op_3488_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_97_cast_fp16 = mul(x = x_95_cast_fp16, y = var_3488_cast_fp16)[name = string("x_97_cast_fp16")];
+            tensor<fp16, [2560]> var_3491_to_fp16 = const()[name = string("op_3491_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(765262208)))];
+            tensor<fp16, [1, 2048, 2560]> input_51_cast_fp16 = mul(x = x_97_cast_fp16, y = var_3491_to_fp16)[name = string("input_51_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_3_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_3_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(765267392)))];
+            tensor<fp16, [1, 2048, 9216]> linear_28_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_3_mlp_gate_proj_weight_to_fp16, x = input_51_cast_fp16)[name = string("linear_28_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_3498_cast_fp16 = silu(x = linear_28_cast_fp16)[name = string("op_3498_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_3_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_3_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(812453376)))];
+            tensor<fp16, [1, 2048, 9216]> linear_29_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_3_mlp_up_proj_weight_to_fp16, x = input_51_cast_fp16)[name = string("linear_29_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_55_cast_fp16 = mul(x = var_3498_cast_fp16, y = linear_29_cast_fp16)[name = string("input_55_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_3_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_3_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(859639360)))];
+            tensor<fp16, [1, 2048, 2560]> linear_30_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_3_mlp_down_proj_weight_to_fp16, x = input_55_cast_fp16)[name = string("linear_30_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_99_cast_fp16 = add(x = x_95_cast_fp16, y = linear_30_cast_fp16)[name = string("x_99_cast_fp16")];
+            int32 var_3518 = const()[name = string("op_3518"), val = int32(-2)];
+            int32 var_3547 = const()[name = string("op_3547"), val = int32(-1)];
+            fp16 var_3546_promoted_to_fp16 = const()[name = string("op_3546_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_3556_cast_fp16 = pow(x = x_99_cast_fp16, y = var_3546_promoted_to_fp16)[name = string("op_3556_cast_fp16")];
+            tensor<int32, [1]> var_3558_axes_0 = const()[name = string("op_3558_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3558_keep_dims_0 = const()[name = string("op_3558_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_3558_cast_fp16 = reduce_mean(axes = var_3558_axes_0, keep_dims = var_3558_keep_dims_0, x = var_3556_cast_fp16)[name = string("op_3558_cast_fp16")];
+            fp16 var_3559_to_fp16 = const()[name = string("op_3559_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_3560_cast_fp16 = add(x = var_3558_cast_fp16, y = var_3559_to_fp16)[name = string("op_3560_cast_fp16")];
+            fp32 var_3561_epsilon_0 = const()[name = string("op_3561_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_3561_cast_fp16 = rsqrt(epsilon = var_3561_epsilon_0, x = var_3560_cast_fp16)[name = string("op_3561_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_101_cast_fp16 = mul(x = x_99_cast_fp16, y = var_3561_cast_fp16)[name = string("x_101_cast_fp16")];
+            tensor<fp16, [2560]> var_3564_to_fp16 = const()[name = string("op_3564_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(906825344)))];
+            tensor<fp16, [1, 2048, 2560]> x_103_cast_fp16 = mul(x = x_101_cast_fp16, y = var_3564_to_fp16)[name = string("x_103_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_4_linear_attn_in_proj_qkv_weight_to_fp16 = const()[name = string("layers_4_linear_attn_in_proj_qkv_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(906830528)))];
+            tensor<fp16, [1, 2048, 8192]> linear_31_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_qkv_weight_to_fp16, x = x_103_cast_fp16)[name = string("linear_31_cast_fp16")];
+            tensor<int32, [3]> input_57_perm_0 = const()[name = string("input_57_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_59_pad_0 = const()[name = string("input_59_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_59_mode_0 = const()[name = string("input_59_mode_0"), val = string("constant")];
+            fp16 const_42_to_fp16 = const()[name = string("const_42_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 2048]> input_57_cast_fp16 = transpose(perm = input_57_perm_0, x = linear_31_cast_fp16)[name = string("transpose_27")];
+            tensor<fp16, [1, 8192, 2051]> input_59_cast_fp16 = pad(constant_val = const_42_to_fp16, mode = input_59_mode_0, pad = input_59_pad_0, x = input_57_cast_fp16)[name = string("input_59_cast_fp16")];
+            string input_61_pad_type_0 = const()[name = string("input_61_pad_type_0"), val = string("valid")];
+            int32 input_61_groups_0 = const()[name = string("input_61_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_61_strides_0 = const()[name = string("input_61_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_61_pad_0 = const()[name = string("input_61_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_61_dilations_0 = const()[name = string("input_61_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_4_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_4_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(948773632)))];
+            tensor<fp16, [1, 8192, 2048]> input_61_cast_fp16 = conv(dilations = input_61_dilations_0, groups = input_61_groups_0, pad = input_61_pad_0, pad_type = input_61_pad_type_0, strides = input_61_strides_0, weight = layers_4_linear_attn_conv1d_weight_to_fp16, x = input_59_cast_fp16)[name = string("input_61_cast_fp16")];
+            tensor<fp16, [1, 8192, 2048]> var_3599_cast_fp16 = silu(x = input_61_cast_fp16)[name = string("op_3599_cast_fp16")];
+            tensor<int32, [3]> qkv_7_perm_0 = const()[name = string("qkv_7_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_3601_begin_0 = const()[name = string("op_3601_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_3601_end_0 = const()[name = string("op_3601_end_0"), val = tensor<int32, [3]>([1, 2048, 2048])];
+            tensor<bool, [3]> var_3601_end_mask_0 = const()[name = string("op_3601_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 8192]> qkv_7_cast_fp16 = transpose(perm = qkv_7_perm_0, x = var_3599_cast_fp16)[name = string("transpose_26")];
+            tensor<fp16, [1, 2048, 2048]> var_3601_cast_fp16 = slice_by_index(begin = var_3601_begin_0, end = var_3601_end_0, end_mask = var_3601_end_mask_0, x = qkv_7_cast_fp16)[name = string("op_3601_cast_fp16")];
+            tensor<int32, [4]> var_3602 = const()[name = string("op_3602"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> q_35_cast_fp16 = reshape(shape = var_3602, x = var_3601_cast_fp16)[name = string("q_35_cast_fp16")];
+            tensor<int32, [3]> var_3604_begin_0 = const()[name = string("op_3604_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_3604_end_0 = const()[name = string("op_3604_end_0"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<bool, [3]> var_3604_end_mask_0 = const()[name = string("op_3604_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 2048]> var_3604_cast_fp16 = slice_by_index(begin = var_3604_begin_0, end = var_3604_end_0, end_mask = var_3604_end_mask_0, x = qkv_7_cast_fp16)[name = string("op_3604_cast_fp16")];
+            tensor<int32, [4]> var_3605 = const()[name = string("op_3605"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> k_31_cast_fp16 = reshape(shape = var_3605, x = var_3604_cast_fp16)[name = string("k_31_cast_fp16")];
+            tensor<int32, [3]> var_3607_begin_0 = const()[name = string("op_3607_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_3607_end_0 = const()[name = string("op_3607_end_0"), val = tensor<int32, [3]>([1, 2048, 8192])];
+            tensor<bool, [3]> var_3607_end_mask_0 = const()[name = string("op_3607_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 2048, 4096]> var_3607_cast_fp16 = slice_by_index(begin = var_3607_begin_0, end = var_3607_end_0, end_mask = var_3607_end_mask_0, x = qkv_7_cast_fp16)[name = string("op_3607_cast_fp16")];
+            tensor<int32, [4]> var_3608 = const()[name = string("op_3608"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> v_17_cast_fp16 = reshape(shape = var_3608, x = var_3607_cast_fp16)[name = string("v_17_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_4_linear_attn_in_proj_z_weight_to_fp16 = const()[name = string("layers_4_linear_attn_in_proj_z_weight_to_fp16"), val = tensor<fp16, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(948839232)))];
+            tensor<fp16, [1, 2048, 4096]> linear_32_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_z_weight_to_fp16, x = x_103_cast_fp16)[name = string("linear_32_cast_fp16")];
+            tensor<int32, [4]> var_3612 = const()[name = string("op_3612"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> z_7_cast_fp16 = reshape(shape = var_3612, x = linear_32_cast_fp16)[name = string("z_7_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_4_linear_attn_in_proj_b_weight_to_fp16 = const()[name = string("layers_4_linear_attn_in_proj_b_weight_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(969810816)))];
+            tensor<fp16, [32]> linear_33_bias_0_to_fp16 = const()[name = string("linear_33_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(969974720)))];
+            tensor<fp16, [1, 2048, 32]> linear_33_cast_fp16 = linear(bias = linear_33_bias_0_to_fp16, weight = layers_4_linear_attn_in_proj_b_weight_to_fp16, x = x_103_cast_fp16)[name = string("linear_33_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> beta_13_cast_fp16 = sigmoid(x = linear_33_cast_fp16)[name = string("beta_13_cast_fp16")];
+            tensor<fp16, [32, 2560]> var_3621_weight_0_to_fp16 = const()[name = string("op_3621_weight_0_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(969974848)))];
+            tensor<fp16, [32]> var_3621_bias_0_to_fp16 = const()[name = string("op_3621_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(970138752)))];
+            tensor<fp16, [1, 2048, 32]> var_3621_cast_fp16 = linear(bias = var_3621_bias_0_to_fp16, weight = var_3621_weight_0_to_fp16, x = x_103_cast_fp16)[name = string("op_3621_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> var_3622_cast_fp16 = softplus(x = var_3621_cast_fp16)[name = string("op_3622_cast_fp16")];
+            tensor<fp16, [32]> var_3618_to_fp16 = const()[name = string("op_3618_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(970138880)))];
+            tensor<fp16, [1, 2048, 32]> g_13_cast_fp16 = mul(x = var_3618_to_fp16, y = var_3622_cast_fp16)[name = string("g_13_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_3624_cast_fp16 = mul(x = q_35_cast_fp16, y = q_35_cast_fp16)[name = string("op_3624_cast_fp16")];
+            tensor<int32, [1]> var_3626_axes_0 = const()[name = string("op_3626_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3626_keep_dims_0 = const()[name = string("op_3626_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3626_cast_fp16 = reduce_sum(axes = var_3626_axes_0, keep_dims = var_3626_keep_dims_0, x = var_3624_cast_fp16)[name = string("op_3626_cast_fp16")];
+            fp16 var_3627_to_fp16 = const()[name = string("op_3627_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3628_cast_fp16 = add(x = var_3626_cast_fp16, y = var_3627_to_fp16)[name = string("op_3628_cast_fp16")];
+            fp32 var_3629_epsilon_0 = const()[name = string("op_3629_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3629_cast_fp16 = rsqrt(epsilon = var_3629_epsilon_0, x = var_3628_cast_fp16)[name = string("op_3629_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> q_37_cast_fp16 = mul(x = q_35_cast_fp16, y = var_3629_cast_fp16)[name = string("q_37_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_3631_cast_fp16 = mul(x = k_31_cast_fp16, y = k_31_cast_fp16)[name = string("op_3631_cast_fp16")];
+            tensor<int32, [1]> var_3633_axes_0 = const()[name = string("op_3633_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_3633_keep_dims_0 = const()[name = string("op_3633_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3633_cast_fp16 = reduce_sum(axes = var_3633_axes_0, keep_dims = var_3633_keep_dims_0, x = var_3631_cast_fp16)[name = string("op_3633_cast_fp16")];
+            fp16 var_3634_to_fp16 = const()[name = string("op_3634_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3635_cast_fp16 = add(x = var_3633_cast_fp16, y = var_3634_to_fp16)[name = string("op_3635_cast_fp16")];
+            fp32 var_3636_epsilon_0 = const()[name = string("op_3636_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_3636_cast_fp16 = rsqrt(epsilon = var_3636_epsilon_0, x = var_3635_cast_fp16)[name = string("op_3636_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> k_33_cast_fp16 = mul(x = k_31_cast_fp16, y = var_3636_cast_fp16)[name = string("k_33_cast_fp16")];
+            tensor<int32, [1]> var_3641_axes_0 = const()[name = string("op_3641_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_3641_cast_fp16 = expand_dims(axes = var_3641_axes_0, x = q_37_cast_fp16)[name = string("op_3641_cast_fp16")];
+            tensor<int32, [5]> var_3643_reps_0 = const()[name = string("op_3643_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_3643_cast_fp16 = tile(reps = var_3643_reps_0, x = var_3641_cast_fp16)[name = string("op_3643_cast_fp16")];
+            tensor<int32, [4]> var_3644 = const()[name = string("op_3644"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> q_39_cast_fp16 = reshape(shape = var_3644, x = var_3643_cast_fp16)[name = string("q_39_cast_fp16")];
+            tensor<int32, [1]> var_3649_axes_0 = const()[name = string("op_3649_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_3649_cast_fp16 = expand_dims(axes = var_3649_axes_0, x = k_33_cast_fp16)[name = string("op_3649_cast_fp16")];
+            tensor<int32, [5]> var_3651_reps_0 = const()[name = string("op_3651_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_3651_cast_fp16 = tile(reps = var_3651_reps_0, x = var_3649_cast_fp16)[name = string("op_3651_cast_fp16")];
+            tensor<int32, [4]> var_3652 = const()[name = string("op_3652"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> k_35_cast_fp16 = reshape(shape = var_3652, x = var_3651_cast_fp16)[name = string("k_35_cast_fp16")];
+            fp16 var_3654_to_fp16 = const()[name = string("op_3654_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 2048, 32, 128]> q_41_cast_fp16 = mul(x = q_39_cast_fp16, y = var_3654_to_fp16)[name = string("q_41_cast_fp16")];
+            tensor<int32, [4]> var_3656_begin_0 = const()[name = string("op_3656_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3656_end_0 = const()[name = string("op_3656_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_3656_end_mask_0 = const()[name = string("op_3656_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_3656_squeeze_mask_0 = const()[name = string("op_3656_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_3656_cast_fp16 = slice_by_index(begin = var_3656_begin_0, end = var_3656_end_0, end_mask = var_3656_end_mask_0, squeeze_mask = var_3656_squeeze_mask_0, x = q_41_cast_fp16)[name = string("op_3656_cast_fp16")];
+            tensor<int32, [3]> var_3657 = const()[name = string("op_3657"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_3659 = const()[name = string("op_3659"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_3658_cast_fp16 = transpose(perm = var_3657, x = var_3656_cast_fp16)[name = string("transpose_25")];
+            tensor<fp16, [32, 32, 64, 128]> q_43_cast_fp16 = reshape(shape = var_3659, x = var_3658_cast_fp16)[name = string("q_43_cast_fp16")];
+            tensor<int32, [4]> var_3661_begin_0 = const()[name = string("op_3661_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3661_end_0 = const()[name = string("op_3661_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_3661_end_mask_0 = const()[name = string("op_3661_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_3661_squeeze_mask_0 = const()[name = string("op_3661_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_3661_cast_fp16 = slice_by_index(begin = var_3661_begin_0, end = var_3661_end_0, end_mask = var_3661_end_mask_0, squeeze_mask = var_3661_squeeze_mask_0, x = k_35_cast_fp16)[name = string("op_3661_cast_fp16")];
+            tensor<int32, [3]> var_3662 = const()[name = string("op_3662"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_3664 = const()[name = string("op_3664"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_3663_cast_fp16 = transpose(perm = var_3662, x = var_3661_cast_fp16)[name = string("transpose_24")];
+            tensor<fp16, [32, 32, 64, 128]> k_37_cast_fp16 = reshape(shape = var_3664, x = var_3663_cast_fp16)[name = string("k_37_cast_fp16")];
+            tensor<int32, [4]> var_3666_begin_0 = const()[name = string("op_3666_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3666_end_0 = const()[name = string("op_3666_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_3666_end_mask_0 = const()[name = string("op_3666_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_3666_squeeze_mask_0 = const()[name = string("op_3666_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_3666_cast_fp16 = slice_by_index(begin = var_3666_begin_0, end = var_3666_end_0, end_mask = var_3666_end_mask_0, squeeze_mask = var_3666_squeeze_mask_0, x = v_17_cast_fp16)[name = string("op_3666_cast_fp16")];
+            tensor<int32, [3]> var_3667 = const()[name = string("op_3667"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_3669 = const()[name = string("op_3669"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_3668_cast_fp16 = transpose(perm = var_3667, x = var_3666_cast_fp16)[name = string("transpose_23")];
+            tensor<fp16, [32, 32, 64, 128]> v_19_cast_fp16 = reshape(shape = var_3669, x = var_3668_cast_fp16)[name = string("v_19_cast_fp16")];
+            tensor<int32, [3]> var_3671_begin_0 = const()[name = string("op_3671_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_3671_end_0 = const()[name = string("op_3671_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_3671_end_mask_0 = const()[name = string("op_3671_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_3671_squeeze_mask_0 = const()[name = string("op_3671_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_3671_cast_fp16 = slice_by_index(begin = var_3671_begin_0, end = var_3671_end_0, end_mask = var_3671_end_mask_0, squeeze_mask = var_3671_squeeze_mask_0, x = beta_13_cast_fp16)[name = string("op_3671_cast_fp16")];
+            tensor<int32, [2]> var_3672_perm_0 = const()[name = string("op_3672_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_3673 = const()[name = string("op_3673"), val = tensor<int32, [4]>([32, 32, 64, 1])];
+            tensor<fp16, [32, 2048]> var_3672_cast_fp16 = transpose(perm = var_3672_perm_0, x = var_3671_cast_fp16)[name = string("transpose_22")];
+            tensor<fp16, [32, 32, 64, 1]> beta_15_cast_fp16 = reshape(shape = var_3673, x = var_3672_cast_fp16)[name = string("beta_15_cast_fp16")];
+            tensor<int32, [3]> var_3675_begin_0 = const()[name = string("op_3675_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_3675_end_0 = const()[name = string("op_3675_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_3675_end_mask_0 = const()[name = string("op_3675_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_3675_squeeze_mask_0 = const()[name = string("op_3675_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_3675_cast_fp16 = slice_by_index(begin = var_3675_begin_0, end = var_3675_end_0, end_mask = var_3675_end_mask_0, squeeze_mask = var_3675_squeeze_mask_0, x = g_13_cast_fp16)[name = string("op_3675_cast_fp16")];
+            tensor<int32, [2]> var_3676_perm_0 = const()[name = string("op_3676_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_3677 = const()[name = string("op_3677"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<fp16, [32, 2048]> var_3676_cast_fp16 = transpose(perm = var_3676_perm_0, x = var_3675_cast_fp16)[name = string("transpose_21")];
+            tensor<fp16, [32, 32, 64]> g_15_cast_fp16 = reshape(shape = var_3677, x = var_3676_cast_fp16)[name = string("g_15_cast_fp16")];
+            tensor<int32, [1]> g_col_7_axes_0 = const()[name = string("g_col_7_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64, 1]> g_col_7_cast_fp16 = expand_dims(axes = g_col_7_axes_0, x = g_15_cast_fp16)[name = string("g_col_7_cast_fp16")];
+            bool var_3680_transpose_x_0 = const()[name = string("op_3680_transpose_x_0"), val = bool(false)];
+            bool var_3680_transpose_y_0 = const()[name = string("op_3680_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_3680_cast_fp16 = matmul(transpose_x = var_3680_transpose_x_0, transpose_y = var_3680_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_7_cast_fp16)[name = string("op_3680_cast_fp16")];
+            tensor<int32, [1]> cum_7_axes_0 = const()[name = string("cum_7_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> cum_7_cast_fp16 = squeeze(axes = cum_7_axes_0, x = var_3680_cast_fp16)[name = string("cum_7_cast_fp16")];
+            bool var_3682_transpose_x_0 = const()[name = string("op_3682_transpose_x_0"), val = bool(false)];
+            bool var_3682_transpose_y_0 = const()[name = string("op_3682_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_3682_cast_fp16 = matmul(transpose_x = var_3682_transpose_x_0, transpose_y = var_3682_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_7_cast_fp16)[name = string("op_3682_cast_fp16")];
+            tensor<int32, [1]> to_end_7_axes_0 = const()[name = string("to_end_7_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> to_end_7_cast_fp16 = squeeze(axes = to_end_7_axes_0, x = var_3682_cast_fp16)[name = string("to_end_7_cast_fp16")];
+            bool var_3684_transpose_x_0 = const()[name = string("op_3684_transpose_x_0"), val = bool(false)];
+            bool var_3684_transpose_y_0 = const()[name = string("op_3684_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 4096, 1]> var_3684_cast_fp16 = matmul(transpose_x = var_3684_transpose_x_0, transpose_y = var_3684_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_7_cast_fp16)[name = string("op_3684_cast_fp16")];
+            tensor<int32, [4]> var_3685 = const()[name = string("op_3685"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> pair_43_cast_fp16 = reshape(shape = var_3685, x = var_3684_cast_fp16)[name = string("pair_43_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_3687_cast_fp16 = exp(x = pair_43_cast_fp16)[name = string("op_3687_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> pair_decay_7_cast_fp16 = mul(x = var_3687_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_7_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_beta_7_cast_fp16 = mul(x = k_37_cast_fp16, y = beta_15_cast_fp16)[name = string("k_beta_7_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> v_beta_7_cast_fp16 = mul(x = v_19_cast_fp16, y = beta_15_cast_fp16)[name = string("v_beta_7_cast_fp16")];
+            bool var_3692_transpose_x_1 = const()[name = string("op_3692_transpose_x_1"), val = bool(false)];
+            bool var_3692_transpose_y_1 = const()[name = string("op_3692_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_3692_cast_fp16 = matmul(transpose_x = var_3692_transpose_x_1, transpose_y = var_3692_transpose_y_1, x = k_beta_7_cast_fp16, y = k_37_cast_fp16)[name = string("op_3692_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> kkt_7_cast_fp16 = mul(x = var_3692_cast_fp16, y = pair_decay_7_cast_fp16)[name = string("kkt_7_cast_fp16")];
+            bool var_3695_transpose_x_1 = const()[name = string("op_3695_transpose_x_1"), val = bool(false)];
+            bool var_3695_transpose_y_1 = const()[name = string("op_3695_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_3695_cast_fp16 = matmul(transpose_x = var_3695_transpose_x_1, transpose_y = var_3695_transpose_y_1, x = q_43_cast_fp16, y = k_37_cast_fp16)[name = string("op_3695_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> attn_intra_7_cast_fp16 = mul(x = var_3695_cast_fp16, y = pair_decay_7_cast_fp16)[name = string("attn_intra_7_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_3697_cast_fp16 = mul(x = kkt_7_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_3697_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> t_13_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_3697_cast_fp16)[name = string("t_13_cast_fp16")];
+            tensor<int32, [3]> var_3701 = const()[name = string("op_3701"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [1024, 64, 64]> t_15_cast_fp16 = reshape(shape = var_3701, x = t_13_cast_fp16)[name = string("t_15_cast_fp16")];
+            tensor<int32, [5]> var_3706 = const()[name = string("op_3706"), val = tensor<int32, [5]>([1024, 32, 2, 32, 2])];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_3707_cast_fp16 = reshape(shape = var_3706, x = t_15_cast_fp16)[name = string("op_3707_cast_fp16")];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_3712_cast_fp16 = mul(x = var_3707_cast_fp16, y = var_230_to_fp16)[name = string("op_3712_cast_fp16")];
+            tensor<int32, [1]> blocks_37_axes_0 = const()[name = string("blocks_37_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_37_keep_dims_0 = const()[name = string("blocks_37_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> blocks_37_cast_fp16 = reduce_sum(axes = blocks_37_axes_0, keep_dims = blocks_37_keep_dims_0, x = var_3712_cast_fp16)[name = string("blocks_37_cast_fp16")];
+            tensor<int32, [4]> var_3715_begin_0 = const()[name = string("op_3715_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_3715_end_0 = const()[name = string("op_3715_end_0"), val = tensor<int32, [4]>([1024, 32, 2, 2])];
+            tensor<bool, [4]> var_3715_end_mask_0 = const()[name = string("op_3715_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 32, 1, 2]> var_3715_cast_fp16 = slice_by_index(begin = var_3715_begin_0, end = var_3715_end_0, end_mask = var_3715_end_mask_0, x = blocks_37_cast_fp16)[name = string("op_3715_cast_fp16")];
+            tensor<int32, [4]> x_105_begin_0 = const()[name = string("x_105_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_105_end_0 = const()[name = string("x_105_end_0"), val = tensor<int32, [4]>([1024, 32, 1, 1])];
+            tensor<bool, [4]> x_105_end_mask_0 = const()[name = string("x_105_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 32, 1, 1]> x_105_cast_fp16 = slice_by_index(begin = x_105_begin_0, end = x_105_end_0, end_mask = x_105_end_mask_0, x = var_3715_cast_fp16)[name = string("x_105_cast_fp16")];
+            bool var_3725_transpose_x_0 = const()[name = string("op_3725_transpose_x_0"), val = bool(false)];
+            bool var_3725_transpose_y_0 = const()[name = string("op_3725_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_3725_cast_fp16 = matmul(transpose_x = var_3725_transpose_x_0, transpose_y = var_3725_transpose_y_0, x = x_105_cast_fp16, y = var_238_to_fp16)[name = string("op_3725_cast_fp16")];
+            bool var_3726_transpose_x_0 = const()[name = string("op_3726_transpose_x_0"), val = bool(false)];
+            bool var_3726_transpose_y_0 = const()[name = string("op_3726_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_3726_cast_fp16 = matmul(transpose_x = var_3726_transpose_x_0, transpose_y = var_3726_transpose_y_0, x = var_238_to_fp16, y = var_3725_cast_fp16)[name = string("op_3726_cast_fp16")];
+            fp16 const_47_promoted_to_fp16 = const()[name = string("const_47_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 32, 1, 1]> lower_37_cast_fp16 = mul(x = var_3726_cast_fp16, y = const_47_promoted_to_fp16)[name = string("lower_37_cast_fp16")];
+            bool bottom_37_interleave_0 = const()[name = string("bottom_37_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> bottom_37_cast_fp16 = concat(axis = var_3547, interleave = bottom_37_interleave_0, values = (lower_37_cast_fp16, var_238_to_fp16))[name = string("bottom_37_cast_fp16")];
+            bool inv_51_interleave_0 = const()[name = string("inv_51_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> inv_51_cast_fp16 = concat(axis = var_3518, interleave = inv_51_interleave_0, values = (top_1_to_fp16, bottom_37_cast_fp16))[name = string("inv_51_cast_fp16")];
+            tensor<int32, [5]> var_3735 = const()[name = string("op_3735"), val = tensor<int32, [5]>([1024, 16, 4, 16, 4])];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_3736_cast_fp16 = reshape(shape = var_3735, x = t_15_cast_fp16)[name = string("op_3736_cast_fp16")];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_3741_cast_fp16 = mul(x = var_3736_cast_fp16, y = var_259_to_fp16)[name = string("op_3741_cast_fp16")];
+            tensor<int32, [1]> blocks_39_axes_0 = const()[name = string("blocks_39_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_39_keep_dims_0 = const()[name = string("blocks_39_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> blocks_39_cast_fp16 = reduce_sum(axes = blocks_39_axes_0, keep_dims = blocks_39_keep_dims_0, x = var_3741_cast_fp16)[name = string("blocks_39_cast_fp16")];
+            tensor<int32, [4]> var_3744_begin_0 = const()[name = string("op_3744_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_3744_end_0 = const()[name = string("op_3744_end_0"), val = tensor<int32, [4]>([1024, 16, 4, 4])];
+            tensor<bool, [4]> var_3744_end_mask_0 = const()[name = string("op_3744_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 16, 2, 4]> var_3744_cast_fp16 = slice_by_index(begin = var_3744_begin_0, end = var_3744_end_0, end_mask = var_3744_end_mask_0, x = blocks_39_cast_fp16)[name = string("op_3744_cast_fp16")];
+            tensor<int32, [4]> x_107_begin_0 = const()[name = string("x_107_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_107_end_0 = const()[name = string("x_107_end_0"), val = tensor<int32, [4]>([1024, 16, 2, 2])];
+            tensor<bool, [4]> x_107_end_mask_0 = const()[name = string("x_107_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 16, 2, 2]> x_107_cast_fp16 = slice_by_index(begin = x_107_begin_0, end = x_107_end_0, end_mask = x_107_end_mask_0, x = var_3744_cast_fp16)[name = string("x_107_cast_fp16")];
+            tensor<int32, [5]> var_3746 = const()[name = string("op_3746"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<fp16, [1024, 16, 2, 2, 2]> pair_47_cast_fp16 = reshape(shape = var_3746, x = inv_51_cast_fp16)[name = string("pair_47_cast_fp16")];
+            tensor<int32, [5]> var_3748_begin_0 = const()[name = string("op_3748_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3748_end_0 = const()[name = string("op_3748_end_0"), val = tensor<int32, [5]>([1024, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_3748_end_mask_0 = const()[name = string("op_3748_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3748_squeeze_mask_0 = const()[name = string("op_3748_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_3748_cast_fp16 = slice_by_index(begin = var_3748_begin_0, end = var_3748_end_0, end_mask = var_3748_end_mask_0, squeeze_mask = var_3748_squeeze_mask_0, x = pair_47_cast_fp16)[name = string("op_3748_cast_fp16")];
+            tensor<int32, [5]> var_3751_begin_0 = const()[name = string("op_3751_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3751_end_0 = const()[name = string("op_3751_end_0"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_3751_end_mask_0 = const()[name = string("op_3751_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3751_squeeze_mask_0 = const()[name = string("op_3751_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_3751_cast_fp16 = slice_by_index(begin = var_3751_begin_0, end = var_3751_end_0, end_mask = var_3751_end_mask_0, squeeze_mask = var_3751_squeeze_mask_0, x = pair_47_cast_fp16)[name = string("op_3751_cast_fp16")];
+            bool var_3754_transpose_x_0 = const()[name = string("op_3754_transpose_x_0"), val = bool(false)];
+            bool var_3754_transpose_y_0 = const()[name = string("op_3754_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_3754_cast_fp16 = matmul(transpose_x = var_3754_transpose_x_0, transpose_y = var_3754_transpose_y_0, x = x_107_cast_fp16, y = var_3748_cast_fp16)[name = string("op_3754_cast_fp16")];
+            bool var_3755_transpose_x_0 = const()[name = string("op_3755_transpose_x_0"), val = bool(false)];
+            bool var_3755_transpose_y_0 = const()[name = string("op_3755_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_3755_cast_fp16 = matmul(transpose_x = var_3755_transpose_x_0, transpose_y = var_3755_transpose_y_0, x = var_3751_cast_fp16, y = var_3754_cast_fp16)[name = string("op_3755_cast_fp16")];
+            fp16 const_48_promoted_to_fp16 = const()[name = string("const_48_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 16, 2, 2]> lower_39_cast_fp16 = mul(x = var_3755_cast_fp16, y = const_48_promoted_to_fp16)[name = string("lower_39_cast_fp16")];
+            bool top_39_interleave_0 = const()[name = string("top_39_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> top_39_cast_fp16 = concat(axis = var_3547, interleave = top_39_interleave_0, values = (var_3748_cast_fp16, var_276_to_fp16))[name = string("top_39_cast_fp16")];
+            bool bottom_39_interleave_0 = const()[name = string("bottom_39_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> bottom_39_cast_fp16 = concat(axis = var_3547, interleave = bottom_39_interleave_0, values = (lower_39_cast_fp16, var_3751_cast_fp16))[name = string("bottom_39_cast_fp16")];
+            bool inv_53_interleave_0 = const()[name = string("inv_53_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> inv_53_cast_fp16 = concat(axis = var_3518, interleave = inv_53_interleave_0, values = (top_39_cast_fp16, bottom_39_cast_fp16))[name = string("inv_53_cast_fp16")];
+            tensor<int32, [5]> var_3764 = const()[name = string("op_3764"), val = tensor<int32, [5]>([1024, 8, 8, 8, 8])];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_3765_cast_fp16 = reshape(shape = var_3764, x = t_15_cast_fp16)[name = string("op_3765_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_3769_to_fp16 = const()[name = string("op_3769_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(970139008)))];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_3770_cast_fp16 = mul(x = var_3765_cast_fp16, y = var_3769_to_fp16)[name = string("op_3770_cast_fp16")];
+            tensor<int32, [1]> blocks_41_axes_0 = const()[name = string("blocks_41_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_41_keep_dims_0 = const()[name = string("blocks_41_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> blocks_41_cast_fp16 = reduce_sum(axes = blocks_41_axes_0, keep_dims = blocks_41_keep_dims_0, x = var_3770_cast_fp16)[name = string("blocks_41_cast_fp16")];
+            tensor<int32, [4]> var_3773_begin_0 = const()[name = string("op_3773_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_3773_end_0 = const()[name = string("op_3773_end_0"), val = tensor<int32, [4]>([1024, 8, 8, 8])];
+            tensor<bool, [4]> var_3773_end_mask_0 = const()[name = string("op_3773_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 8, 4, 8]> var_3773_cast_fp16 = slice_by_index(begin = var_3773_begin_0, end = var_3773_end_0, end_mask = var_3773_end_mask_0, x = blocks_41_cast_fp16)[name = string("op_3773_cast_fp16")];
+            tensor<int32, [4]> x_109_begin_0 = const()[name = string("x_109_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_109_end_0 = const()[name = string("x_109_end_0"), val = tensor<int32, [4]>([1024, 8, 4, 4])];
+            tensor<bool, [4]> x_109_end_mask_0 = const()[name = string("x_109_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 8, 4, 4]> x_109_cast_fp16 = slice_by_index(begin = x_109_begin_0, end = x_109_end_0, end_mask = x_109_end_mask_0, x = var_3773_cast_fp16)[name = string("x_109_cast_fp16")];
+            tensor<int32, [5]> var_3775 = const()[name = string("op_3775"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<fp16, [1024, 8, 2, 4, 4]> pair_49_cast_fp16 = reshape(shape = var_3775, x = inv_53_cast_fp16)[name = string("pair_49_cast_fp16")];
+            tensor<int32, [5]> var_3777_begin_0 = const()[name = string("op_3777_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3777_end_0 = const()[name = string("op_3777_end_0"), val = tensor<int32, [5]>([1024, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_3777_end_mask_0 = const()[name = string("op_3777_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3777_squeeze_mask_0 = const()[name = string("op_3777_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_3777_cast_fp16 = slice_by_index(begin = var_3777_begin_0, end = var_3777_end_0, end_mask = var_3777_end_mask_0, squeeze_mask = var_3777_squeeze_mask_0, x = pair_49_cast_fp16)[name = string("op_3777_cast_fp16")];
+            tensor<int32, [5]> var_3780_begin_0 = const()[name = string("op_3780_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3780_end_0 = const()[name = string("op_3780_end_0"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_3780_end_mask_0 = const()[name = string("op_3780_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3780_squeeze_mask_0 = const()[name = string("op_3780_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_3780_cast_fp16 = slice_by_index(begin = var_3780_begin_0, end = var_3780_end_0, end_mask = var_3780_end_mask_0, squeeze_mask = var_3780_squeeze_mask_0, x = pair_49_cast_fp16)[name = string("op_3780_cast_fp16")];
+            bool var_3783_transpose_x_0 = const()[name = string("op_3783_transpose_x_0"), val = bool(false)];
+            bool var_3783_transpose_y_0 = const()[name = string("op_3783_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_3783_cast_fp16 = matmul(transpose_x = var_3783_transpose_x_0, transpose_y = var_3783_transpose_y_0, x = x_109_cast_fp16, y = var_3777_cast_fp16)[name = string("op_3783_cast_fp16")];
+            bool var_3784_transpose_x_0 = const()[name = string("op_3784_transpose_x_0"), val = bool(false)];
+            bool var_3784_transpose_y_0 = const()[name = string("op_3784_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_3784_cast_fp16 = matmul(transpose_x = var_3784_transpose_x_0, transpose_y = var_3784_transpose_y_0, x = var_3780_cast_fp16, y = var_3783_cast_fp16)[name = string("op_3784_cast_fp16")];
+            fp16 const_49_promoted_to_fp16 = const()[name = string("const_49_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 8, 4, 4]> lower_41_cast_fp16 = mul(x = var_3784_cast_fp16, y = const_49_promoted_to_fp16)[name = string("lower_41_cast_fp16")];
+            bool top_41_interleave_0 = const()[name = string("top_41_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> top_41_cast_fp16 = concat(axis = var_3547, interleave = top_41_interleave_0, values = (var_3777_cast_fp16, var_305_to_fp16))[name = string("top_41_cast_fp16")];
+            bool bottom_41_interleave_0 = const()[name = string("bottom_41_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> bottom_41_cast_fp16 = concat(axis = var_3547, interleave = bottom_41_interleave_0, values = (lower_41_cast_fp16, var_3780_cast_fp16))[name = string("bottom_41_cast_fp16")];
+            bool inv_55_interleave_0 = const()[name = string("inv_55_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> inv_55_cast_fp16 = concat(axis = var_3518, interleave = inv_55_interleave_0, values = (top_41_cast_fp16, bottom_41_cast_fp16))[name = string("inv_55_cast_fp16")];
+            tensor<int32, [5]> var_3793 = const()[name = string("op_3793"), val = tensor<int32, [5]>([1024, 4, 16, 4, 16])];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_3794_cast_fp16 = reshape(shape = var_3793, x = t_15_cast_fp16)[name = string("op_3794_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_3798_to_fp16 = const()[name = string("op_3798_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(970139200)))];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_3799_cast_fp16 = mul(x = var_3794_cast_fp16, y = var_3798_to_fp16)[name = string("op_3799_cast_fp16")];
+            tensor<int32, [1]> blocks_43_axes_0 = const()[name = string("blocks_43_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_43_keep_dims_0 = const()[name = string("blocks_43_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> blocks_43_cast_fp16 = reduce_sum(axes = blocks_43_axes_0, keep_dims = blocks_43_keep_dims_0, x = var_3799_cast_fp16)[name = string("blocks_43_cast_fp16")];
+            tensor<int32, [4]> var_3802_begin_0 = const()[name = string("op_3802_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_3802_end_0 = const()[name = string("op_3802_end_0"), val = tensor<int32, [4]>([1024, 4, 16, 16])];
+            tensor<bool, [4]> var_3802_end_mask_0 = const()[name = string("op_3802_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 4, 8, 16]> var_3802_cast_fp16 = slice_by_index(begin = var_3802_begin_0, end = var_3802_end_0, end_mask = var_3802_end_mask_0, x = blocks_43_cast_fp16)[name = string("op_3802_cast_fp16")];
+            tensor<int32, [4]> x_111_begin_0 = const()[name = string("x_111_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_111_end_0 = const()[name = string("x_111_end_0"), val = tensor<int32, [4]>([1024, 4, 8, 8])];
+            tensor<bool, [4]> x_111_end_mask_0 = const()[name = string("x_111_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 4, 8, 8]> x_111_cast_fp16 = slice_by_index(begin = x_111_begin_0, end = x_111_end_0, end_mask = x_111_end_mask_0, x = var_3802_cast_fp16)[name = string("x_111_cast_fp16")];
+            tensor<int32, [5]> var_3804 = const()[name = string("op_3804"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<fp16, [1024, 4, 2, 8, 8]> pair_51_cast_fp16 = reshape(shape = var_3804, x = inv_55_cast_fp16)[name = string("pair_51_cast_fp16")];
+            tensor<int32, [5]> var_3806_begin_0 = const()[name = string("op_3806_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3806_end_0 = const()[name = string("op_3806_end_0"), val = tensor<int32, [5]>([1024, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_3806_end_mask_0 = const()[name = string("op_3806_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3806_squeeze_mask_0 = const()[name = string("op_3806_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_3806_cast_fp16 = slice_by_index(begin = var_3806_begin_0, end = var_3806_end_0, end_mask = var_3806_end_mask_0, squeeze_mask = var_3806_squeeze_mask_0, x = pair_51_cast_fp16)[name = string("op_3806_cast_fp16")];
+            tensor<int32, [5]> var_3809_begin_0 = const()[name = string("op_3809_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3809_end_0 = const()[name = string("op_3809_end_0"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_3809_end_mask_0 = const()[name = string("op_3809_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3809_squeeze_mask_0 = const()[name = string("op_3809_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_3809_cast_fp16 = slice_by_index(begin = var_3809_begin_0, end = var_3809_end_0, end_mask = var_3809_end_mask_0, squeeze_mask = var_3809_squeeze_mask_0, x = pair_51_cast_fp16)[name = string("op_3809_cast_fp16")];
+            bool var_3812_transpose_x_0 = const()[name = string("op_3812_transpose_x_0"), val = bool(false)];
+            bool var_3812_transpose_y_0 = const()[name = string("op_3812_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_3812_cast_fp16 = matmul(transpose_x = var_3812_transpose_x_0, transpose_y = var_3812_transpose_y_0, x = x_111_cast_fp16, y = var_3806_cast_fp16)[name = string("op_3812_cast_fp16")];
+            bool var_3813_transpose_x_0 = const()[name = string("op_3813_transpose_x_0"), val = bool(false)];
+            bool var_3813_transpose_y_0 = const()[name = string("op_3813_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_3813_cast_fp16 = matmul(transpose_x = var_3813_transpose_x_0, transpose_y = var_3813_transpose_y_0, x = var_3809_cast_fp16, y = var_3812_cast_fp16)[name = string("op_3813_cast_fp16")];
+            fp16 const_50_promoted_to_fp16 = const()[name = string("const_50_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 4, 8, 8]> lower_43_cast_fp16 = mul(x = var_3813_cast_fp16, y = const_50_promoted_to_fp16)[name = string("lower_43_cast_fp16")];
+            bool top_43_interleave_0 = const()[name = string("top_43_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> top_43_cast_fp16 = concat(axis = var_3547, interleave = top_43_interleave_0, values = (var_3806_cast_fp16, var_334_to_fp16))[name = string("top_43_cast_fp16")];
+            bool bottom_43_interleave_0 = const()[name = string("bottom_43_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> bottom_43_cast_fp16 = concat(axis = var_3547, interleave = bottom_43_interleave_0, values = (lower_43_cast_fp16, var_3809_cast_fp16))[name = string("bottom_43_cast_fp16")];
+            bool inv_57_interleave_0 = const()[name = string("inv_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> inv_57_cast_fp16 = concat(axis = var_3518, interleave = inv_57_interleave_0, values = (top_43_cast_fp16, bottom_43_cast_fp16))[name = string("inv_57_cast_fp16")];
+            tensor<int32, [5]> var_3822 = const()[name = string("op_3822"), val = tensor<int32, [5]>([1024, 2, 32, 2, 32])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_3823_cast_fp16 = reshape(shape = var_3822, x = t_15_cast_fp16)[name = string("op_3823_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_3827_to_fp16 = const()[name = string("op_3827_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_3828_cast_fp16 = mul(x = var_3823_cast_fp16, y = var_3827_to_fp16)[name = string("op_3828_cast_fp16")];
+            tensor<int32, [1]> blocks_45_axes_0 = const()[name = string("blocks_45_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_45_keep_dims_0 = const()[name = string("blocks_45_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> blocks_45_cast_fp16 = reduce_sum(axes = blocks_45_axes_0, keep_dims = blocks_45_keep_dims_0, x = var_3828_cast_fp16)[name = string("blocks_45_cast_fp16")];
+            tensor<int32, [4]> var_3831_begin_0 = const()[name = string("op_3831_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_3831_end_0 = const()[name = string("op_3831_end_0"), val = tensor<int32, [4]>([1024, 2, 32, 32])];
+            tensor<bool, [4]> var_3831_end_mask_0 = const()[name = string("op_3831_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 2, 16, 32]> var_3831_cast_fp16 = slice_by_index(begin = var_3831_begin_0, end = var_3831_end_0, end_mask = var_3831_end_mask_0, x = blocks_45_cast_fp16)[name = string("op_3831_cast_fp16")];
+            tensor<int32, [4]> x_113_begin_0 = const()[name = string("x_113_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_113_end_0 = const()[name = string("x_113_end_0"), val = tensor<int32, [4]>([1024, 2, 16, 16])];
+            tensor<bool, [4]> x_113_end_mask_0 = const()[name = string("x_113_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 2, 16, 16]> x_113_cast_fp16 = slice_by_index(begin = x_113_begin_0, end = x_113_end_0, end_mask = x_113_end_mask_0, x = var_3831_cast_fp16)[name = string("x_113_cast_fp16")];
+            tensor<int32, [5]> var_3833 = const()[name = string("op_3833"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<fp16, [1024, 2, 2, 16, 16]> pair_53_cast_fp16 = reshape(shape = var_3833, x = inv_57_cast_fp16)[name = string("pair_53_cast_fp16")];
+            tensor<int32, [5]> var_3835_begin_0 = const()[name = string("op_3835_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3835_end_0 = const()[name = string("op_3835_end_0"), val = tensor<int32, [5]>([1024, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_3835_end_mask_0 = const()[name = string("op_3835_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3835_squeeze_mask_0 = const()[name = string("op_3835_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_3835_cast_fp16 = slice_by_index(begin = var_3835_begin_0, end = var_3835_end_0, end_mask = var_3835_end_mask_0, squeeze_mask = var_3835_squeeze_mask_0, x = pair_53_cast_fp16)[name = string("op_3835_cast_fp16")];
+            tensor<int32, [5]> var_3838_begin_0 = const()[name = string("op_3838_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3838_end_0 = const()[name = string("op_3838_end_0"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_3838_end_mask_0 = const()[name = string("op_3838_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3838_squeeze_mask_0 = const()[name = string("op_3838_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_3838_cast_fp16 = slice_by_index(begin = var_3838_begin_0, end = var_3838_end_0, end_mask = var_3838_end_mask_0, squeeze_mask = var_3838_squeeze_mask_0, x = pair_53_cast_fp16)[name = string("op_3838_cast_fp16")];
+            bool var_3841_transpose_x_0 = const()[name = string("op_3841_transpose_x_0"), val = bool(false)];
+            bool var_3841_transpose_y_0 = const()[name = string("op_3841_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_3841_cast_fp16 = matmul(transpose_x = var_3841_transpose_x_0, transpose_y = var_3841_transpose_y_0, x = x_113_cast_fp16, y = var_3835_cast_fp16)[name = string("op_3841_cast_fp16")];
+            bool var_3842_transpose_x_0 = const()[name = string("op_3842_transpose_x_0"), val = bool(false)];
+            bool var_3842_transpose_y_0 = const()[name = string("op_3842_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_3842_cast_fp16 = matmul(transpose_x = var_3842_transpose_x_0, transpose_y = var_3842_transpose_y_0, x = var_3838_cast_fp16, y = var_3841_cast_fp16)[name = string("op_3842_cast_fp16")];
+            fp16 const_51_promoted_to_fp16 = const()[name = string("const_51_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 2, 16, 16]> lower_45_cast_fp16 = mul(x = var_3842_cast_fp16, y = const_51_promoted_to_fp16)[name = string("lower_45_cast_fp16")];
+            bool top_45_interleave_0 = const()[name = string("top_45_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> top_45_cast_fp16 = concat(axis = var_3547, interleave = top_45_interleave_0, values = (var_3835_cast_fp16, var_363_to_fp16))[name = string("top_45_cast_fp16")];
+            bool bottom_45_interleave_0 = const()[name = string("bottom_45_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> bottom_45_cast_fp16 = concat(axis = var_3547, interleave = bottom_45_interleave_0, values = (lower_45_cast_fp16, var_3838_cast_fp16))[name = string("bottom_45_cast_fp16")];
+            bool inv_59_interleave_0 = const()[name = string("inv_59_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> inv_59_cast_fp16 = concat(axis = var_3518, interleave = inv_59_interleave_0, values = (top_45_cast_fp16, bottom_45_cast_fp16))[name = string("inv_59_cast_fp16")];
+            tensor<int32, [5]> var_3851 = const()[name = string("op_3851"), val = tensor<int32, [5]>([1024, 1, 64, 1, 64])];
+            tensor<fp16, [1024, 1, 64, 1, 64]> var_3852_cast_fp16 = reshape(shape = var_3851, x = t_15_cast_fp16)[name = string("op_3852_cast_fp16")];
+            tensor<int32, [1]> blocks_47_axes_0 = const()[name = string("blocks_47_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_47_keep_dims_0 = const()[name = string("blocks_47_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> blocks_47_cast_fp16 = reduce_sum(axes = blocks_47_axes_0, keep_dims = blocks_47_keep_dims_0, x = var_3852_cast_fp16)[name = string("blocks_47_cast_fp16")];
+            tensor<int32, [4]> var_3860_begin_0 = const()[name = string("op_3860_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_3860_end_0 = const()[name = string("op_3860_end_0"), val = tensor<int32, [4]>([1024, 1, 64, 64])];
+            tensor<bool, [4]> var_3860_end_mask_0 = const()[name = string("op_3860_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 1, 32, 64]> var_3860_cast_fp16 = slice_by_index(begin = var_3860_begin_0, end = var_3860_end_0, end_mask = var_3860_end_mask_0, x = blocks_47_cast_fp16)[name = string("op_3860_cast_fp16")];
+            tensor<int32, [4]> x_115_begin_0 = const()[name = string("x_115_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_115_end_0 = const()[name = string("x_115_end_0"), val = tensor<int32, [4]>([1024, 1, 32, 32])];
+            tensor<bool, [4]> x_115_end_mask_0 = const()[name = string("x_115_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 1, 32, 32]> x_115_cast_fp16 = slice_by_index(begin = x_115_begin_0, end = x_115_end_0, end_mask = x_115_end_mask_0, x = var_3860_cast_fp16)[name = string("x_115_cast_fp16")];
+            tensor<int32, [5]> var_3862 = const()[name = string("op_3862"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<fp16, [1024, 1, 2, 32, 32]> pair_55_cast_fp16 = reshape(shape = var_3862, x = inv_59_cast_fp16)[name = string("pair_55_cast_fp16")];
+            tensor<int32, [5]> var_3864_begin_0 = const()[name = string("op_3864_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_3864_end_0 = const()[name = string("op_3864_end_0"), val = tensor<int32, [5]>([1024, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_3864_end_mask_0 = const()[name = string("op_3864_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3864_squeeze_mask_0 = const()[name = string("op_3864_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_3864_cast_fp16 = slice_by_index(begin = var_3864_begin_0, end = var_3864_end_0, end_mask = var_3864_end_mask_0, squeeze_mask = var_3864_squeeze_mask_0, x = pair_55_cast_fp16)[name = string("op_3864_cast_fp16")];
+            tensor<int32, [5]> var_3867_begin_0 = const()[name = string("op_3867_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_3867_end_0 = const()[name = string("op_3867_end_0"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_3867_end_mask_0 = const()[name = string("op_3867_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_3867_squeeze_mask_0 = const()[name = string("op_3867_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_3867_cast_fp16 = slice_by_index(begin = var_3867_begin_0, end = var_3867_end_0, end_mask = var_3867_end_mask_0, squeeze_mask = var_3867_squeeze_mask_0, x = pair_55_cast_fp16)[name = string("op_3867_cast_fp16")];
+            bool var_3870_transpose_x_0 = const()[name = string("op_3870_transpose_x_0"), val = bool(false)];
+            bool var_3870_transpose_y_0 = const()[name = string("op_3870_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_3870_cast_fp16 = matmul(transpose_x = var_3870_transpose_x_0, transpose_y = var_3870_transpose_y_0, x = x_115_cast_fp16, y = var_3864_cast_fp16)[name = string("op_3870_cast_fp16")];
+            bool var_3871_transpose_x_0 = const()[name = string("op_3871_transpose_x_0"), val = bool(false)];
+            bool var_3871_transpose_y_0 = const()[name = string("op_3871_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_3871_cast_fp16 = matmul(transpose_x = var_3871_transpose_x_0, transpose_y = var_3871_transpose_y_0, x = var_3867_cast_fp16, y = var_3870_cast_fp16)[name = string("op_3871_cast_fp16")];
+            fp16 const_52_promoted_to_fp16 = const()[name = string("const_52_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 1, 32, 32]> lower_47_cast_fp16 = mul(x = var_3871_cast_fp16, y = const_52_promoted_to_fp16)[name = string("lower_47_cast_fp16")];
+            bool top_47_interleave_0 = const()[name = string("top_47_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> top_47_cast_fp16 = concat(axis = var_3547, interleave = top_47_interleave_0, values = (var_3864_cast_fp16, var_392_to_fp16))[name = string("top_47_cast_fp16")];
+            bool bottom_47_interleave_0 = const()[name = string("bottom_47_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> bottom_47_cast_fp16 = concat(axis = var_3547, interleave = bottom_47_interleave_0, values = (lower_47_cast_fp16, var_3867_cast_fp16))[name = string("bottom_47_cast_fp16")];
+            bool inv_61_interleave_0 = const()[name = string("inv_61_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> inv_61_cast_fp16 = concat(axis = var_3518, interleave = inv_61_interleave_0, values = (top_47_cast_fp16, bottom_47_cast_fp16))[name = string("inv_61_cast_fp16")];
+            tensor<int32, [4]> var_3880 = const()[name = string("op_3880"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> inv_63_cast_fp16 = reshape(shape = var_3880, x = inv_61_cast_fp16)[name = string("inv_63_cast_fp16")];
+            bool new_v_7_transpose_x_0 = const()[name = string("new_v_7_transpose_x_0"), val = bool(false)];
+            bool new_v_7_transpose_y_0 = const()[name = string("new_v_7_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> new_v_7_cast_fp16 = matmul(transpose_x = new_v_7_transpose_x_0, transpose_y = new_v_7_transpose_y_0, x = inv_63_cast_fp16, y = v_beta_7_cast_fp16)[name = string("new_v_7_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_3883_cast_fp16 = exp(x = cum_7_cast_fp16)[name = string("op_3883_cast_fp16")];
+            tensor<int32, [1]> var_3884_axes_0 = const()[name = string("op_3884_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_3884_cast_fp16 = expand_dims(axes = var_3884_axes_0, x = var_3883_cast_fp16)[name = string("op_3884_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> var_3885_cast_fp16 = mul(x = k_beta_7_cast_fp16, y = var_3884_cast_fp16)[name = string("op_3885_cast_fp16")];
+            bool k_cumdecay_7_transpose_x_0 = const()[name = string("k_cumdecay_7_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_7_transpose_y_0 = const()[name = string("k_cumdecay_7_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> k_cumdecay_7_cast_fp16 = matmul(transpose_x = k_cumdecay_7_transpose_x_0, transpose_y = k_cumdecay_7_transpose_y_0, x = inv_63_cast_fp16, y = var_3885_cast_fp16)[name = string("k_cumdecay_7_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> q_dec_7_cast_fp16 = mul(x = q_43_cast_fp16, y = var_3884_cast_fp16)[name = string("q_dec_7_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_3890_cast_fp16 = exp(x = to_end_7_cast_fp16)[name = string("op_3890_cast_fp16")];
+            tensor<int32, [1]> var_3891_axes_0 = const()[name = string("op_3891_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_3891_cast_fp16 = expand_dims(axes = var_3891_axes_0, x = var_3890_cast_fp16)[name = string("op_3891_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_dec_7_cast_fp16 = mul(x = k_37_cast_fp16, y = var_3891_cast_fp16)[name = string("k_dec_7_cast_fp16")];
+            tensor<int32, [3]> var_3893_begin_0 = const()[name = string("op_3893_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_3893_end_0 = const()[name = string("op_3893_end_0"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<bool, [3]> var_3893_end_mask_0 = const()[name = string("op_3893_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 32, 1]> var_3893_cast_fp16 = slice_by_index(begin = var_3893_begin_0, end = var_3893_end_0, end_mask = var_3893_end_mask_0, x = cum_7_cast_fp16)[name = string("op_3893_cast_fp16")];
+            tensor<fp16, [32, 32, 1]> var_3894_cast_fp16 = exp(x = var_3893_cast_fp16)[name = string("op_3894_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_7_axes_0 = const()[name = string("chunk_decay_7_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 1, 1]> chunk_decay_7_cast_fp16 = expand_dims(axes = chunk_decay_7_axes_0, x = var_3894_cast_fp16)[name = string("chunk_decay_7_cast_fp16")];
+            tensor<int32, [4]> var_3899_begin_0 = const()[name = string("op_3899_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3899_end_0 = const()[name = string("op_3899_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3899_end_mask_0 = const()[name = string("op_3899_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3899_squeeze_mask_0 = const()[name = string("op_3899_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3899_cast_fp16 = slice_by_index(begin = var_3899_begin_0, end = var_3899_end_0, end_mask = var_3899_end_mask_0, squeeze_mask = var_3899_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3899_cast_fp16")];
+            tensor<int32, [4]> var_3901_begin_0 = const()[name = string("op_3901_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3901_end_0 = const()[name = string("op_3901_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3901_end_mask_0 = const()[name = string("op_3901_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3901_squeeze_mask_0 = const()[name = string("op_3901_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3901_cast_fp16 = slice_by_index(begin = var_3901_begin_0, end = var_3901_end_0, end_mask = var_3901_end_mask_0, squeeze_mask = var_3901_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3901_cast_fp16")];
+            bool var_3902_transpose_x_0 = const()[name = string("op_3902_transpose_x_0"), val = bool(false)];
+            bool var_3902_transpose_y_0 = const()[name = string("op_3902_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3902_cast_fp16 = matmul(transpose_x = var_3902_transpose_x_0, transpose_y = var_3902_transpose_y_0, x = var_3901_cast_fp16, y = state_1_to_fp16)[name = string("op_3902_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_193_cast_fp16 = sub(x = var_3899_cast_fp16, y = var_3902_cast_fp16)[name = string("v_new_193_cast_fp16")];
+            tensor<int32, [4]> var_3905_begin_0 = const()[name = string("op_3905_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3905_end_0 = const()[name = string("op_3905_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3905_end_mask_0 = const()[name = string("op_3905_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3905_squeeze_mask_0 = const()[name = string("op_3905_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3905_cast_fp16 = slice_by_index(begin = var_3905_begin_0, end = var_3905_end_0, end_mask = var_3905_end_mask_0, squeeze_mask = var_3905_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3905_cast_fp16")];
+            bool var_3906_transpose_x_0 = const()[name = string("op_3906_transpose_x_0"), val = bool(false)];
+            bool var_3906_transpose_y_0 = const()[name = string("op_3906_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3906_cast_fp16 = matmul(transpose_x = var_3906_transpose_x_0, transpose_y = var_3906_transpose_y_0, x = var_3905_cast_fp16, y = state_1_to_fp16)[name = string("op_3906_cast_fp16")];
+            tensor<int32, [4]> var_3908_begin_0 = const()[name = string("op_3908_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3908_end_0 = const()[name = string("op_3908_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_3908_end_mask_0 = const()[name = string("op_3908_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3908_squeeze_mask_0 = const()[name = string("op_3908_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3908_cast_fp16 = slice_by_index(begin = var_3908_begin_0, end = var_3908_end_0, end_mask = var_3908_end_mask_0, squeeze_mask = var_3908_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3908_cast_fp16")];
+            bool var_3909_transpose_x_0 = const()[name = string("op_3909_transpose_x_0"), val = bool(false)];
+            bool var_3909_transpose_y_0 = const()[name = string("op_3909_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3909_cast_fp16 = matmul(transpose_x = var_3909_transpose_x_0, transpose_y = var_3909_transpose_y_0, x = var_3908_cast_fp16, y = v_new_193_cast_fp16)[name = string("op_3909_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3910_cast_fp16 = add(x = var_3906_cast_fp16, y = var_3909_cast_fp16)[name = string("op_3910_cast_fp16")];
+            tensor<int32, [4]> var_3912_begin_0 = const()[name = string("op_3912_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3912_end_0 = const()[name = string("op_3912_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_3912_end_mask_0 = const()[name = string("op_3912_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3912_squeeze_mask_0 = const()[name = string("op_3912_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3912_cast_fp16 = slice_by_index(begin = var_3912_begin_0, end = var_3912_end_0, end_mask = var_3912_end_mask_0, squeeze_mask = var_3912_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3912_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3913_cast_fp16 = mul(x = state_1_to_fp16, y = var_3912_cast_fp16)[name = string("op_3913_cast_fp16")];
+            tensor<int32, [4]> var_3915_begin_0 = const()[name = string("op_3915_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_3915_end_0 = const()[name = string("op_3915_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_3915_end_mask_0 = const()[name = string("op_3915_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3915_squeeze_mask_0 = const()[name = string("op_3915_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3915_cast_fp16 = slice_by_index(begin = var_3915_begin_0, end = var_3915_end_0, end_mask = var_3915_end_mask_0, squeeze_mask = var_3915_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3915_cast_fp16")];
+            bool var_3917_transpose_x_1 = const()[name = string("op_3917_transpose_x_1"), val = bool(true)];
+            bool var_3917_transpose_y_1 = const()[name = string("op_3917_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3917_cast_fp16 = matmul(transpose_x = var_3917_transpose_x_1, transpose_y = var_3917_transpose_y_1, x = var_3915_cast_fp16, y = v_new_193_cast_fp16)[name = string("op_3917_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_195_cast_fp16 = add(x = var_3913_cast_fp16, y = var_3917_cast_fp16)[name = string("state_195_cast_fp16")];
+            tensor<int32, [4]> var_3920_begin_0 = const()[name = string("op_3920_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3920_end_0 = const()[name = string("op_3920_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3920_end_mask_0 = const()[name = string("op_3920_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3920_squeeze_mask_0 = const()[name = string("op_3920_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3920_cast_fp16 = slice_by_index(begin = var_3920_begin_0, end = var_3920_end_0, end_mask = var_3920_end_mask_0, squeeze_mask = var_3920_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3920_cast_fp16")];
+            tensor<int32, [4]> var_3922_begin_0 = const()[name = string("op_3922_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3922_end_0 = const()[name = string("op_3922_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3922_end_mask_0 = const()[name = string("op_3922_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3922_squeeze_mask_0 = const()[name = string("op_3922_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3922_cast_fp16 = slice_by_index(begin = var_3922_begin_0, end = var_3922_end_0, end_mask = var_3922_end_mask_0, squeeze_mask = var_3922_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3922_cast_fp16")];
+            bool var_3923_transpose_x_0 = const()[name = string("op_3923_transpose_x_0"), val = bool(false)];
+            bool var_3923_transpose_y_0 = const()[name = string("op_3923_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3923_cast_fp16 = matmul(transpose_x = var_3923_transpose_x_0, transpose_y = var_3923_transpose_y_0, x = var_3922_cast_fp16, y = state_195_cast_fp16)[name = string("op_3923_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_195_cast_fp16 = sub(x = var_3920_cast_fp16, y = var_3923_cast_fp16)[name = string("v_new_195_cast_fp16")];
+            tensor<int32, [4]> var_3926_begin_0 = const()[name = string("op_3926_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3926_end_0 = const()[name = string("op_3926_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3926_end_mask_0 = const()[name = string("op_3926_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3926_squeeze_mask_0 = const()[name = string("op_3926_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3926_cast_fp16 = slice_by_index(begin = var_3926_begin_0, end = var_3926_end_0, end_mask = var_3926_end_mask_0, squeeze_mask = var_3926_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3926_cast_fp16")];
+            bool var_3927_transpose_x_0 = const()[name = string("op_3927_transpose_x_0"), val = bool(false)];
+            bool var_3927_transpose_y_0 = const()[name = string("op_3927_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3927_cast_fp16 = matmul(transpose_x = var_3927_transpose_x_0, transpose_y = var_3927_transpose_y_0, x = var_3926_cast_fp16, y = state_195_cast_fp16)[name = string("op_3927_cast_fp16")];
+            tensor<int32, [4]> var_3929_begin_0 = const()[name = string("op_3929_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3929_end_0 = const()[name = string("op_3929_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_3929_end_mask_0 = const()[name = string("op_3929_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3929_squeeze_mask_0 = const()[name = string("op_3929_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3929_cast_fp16 = slice_by_index(begin = var_3929_begin_0, end = var_3929_end_0, end_mask = var_3929_end_mask_0, squeeze_mask = var_3929_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3929_cast_fp16")];
+            bool var_3930_transpose_x_0 = const()[name = string("op_3930_transpose_x_0"), val = bool(false)];
+            bool var_3930_transpose_y_0 = const()[name = string("op_3930_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3930_cast_fp16 = matmul(transpose_x = var_3930_transpose_x_0, transpose_y = var_3930_transpose_y_0, x = var_3929_cast_fp16, y = v_new_195_cast_fp16)[name = string("op_3930_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3931_cast_fp16 = add(x = var_3927_cast_fp16, y = var_3930_cast_fp16)[name = string("op_3931_cast_fp16")];
+            tensor<int32, [4]> var_3933_begin_0 = const()[name = string("op_3933_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3933_end_0 = const()[name = string("op_3933_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_3933_end_mask_0 = const()[name = string("op_3933_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3933_squeeze_mask_0 = const()[name = string("op_3933_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3933_cast_fp16 = slice_by_index(begin = var_3933_begin_0, end = var_3933_end_0, end_mask = var_3933_end_mask_0, squeeze_mask = var_3933_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3933_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3934_cast_fp16 = mul(x = state_195_cast_fp16, y = var_3933_cast_fp16)[name = string("op_3934_cast_fp16")];
+            tensor<int32, [4]> var_3936_begin_0 = const()[name = string("op_3936_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_3936_end_0 = const()[name = string("op_3936_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_3936_end_mask_0 = const()[name = string("op_3936_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3936_squeeze_mask_0 = const()[name = string("op_3936_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3936_cast_fp16 = slice_by_index(begin = var_3936_begin_0, end = var_3936_end_0, end_mask = var_3936_end_mask_0, squeeze_mask = var_3936_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3936_cast_fp16")];
+            bool var_3938_transpose_x_1 = const()[name = string("op_3938_transpose_x_1"), val = bool(true)];
+            bool var_3938_transpose_y_1 = const()[name = string("op_3938_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3938_cast_fp16 = matmul(transpose_x = var_3938_transpose_x_1, transpose_y = var_3938_transpose_y_1, x = var_3936_cast_fp16, y = v_new_195_cast_fp16)[name = string("op_3938_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_197_cast_fp16 = add(x = var_3934_cast_fp16, y = var_3938_cast_fp16)[name = string("state_197_cast_fp16")];
+            tensor<int32, [4]> var_3941_begin_0 = const()[name = string("op_3941_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3941_end_0 = const()[name = string("op_3941_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3941_end_mask_0 = const()[name = string("op_3941_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3941_squeeze_mask_0 = const()[name = string("op_3941_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3941_cast_fp16 = slice_by_index(begin = var_3941_begin_0, end = var_3941_end_0, end_mask = var_3941_end_mask_0, squeeze_mask = var_3941_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3941_cast_fp16")];
+            tensor<int32, [4]> var_3943_begin_0 = const()[name = string("op_3943_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3943_end_0 = const()[name = string("op_3943_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3943_end_mask_0 = const()[name = string("op_3943_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3943_squeeze_mask_0 = const()[name = string("op_3943_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3943_cast_fp16 = slice_by_index(begin = var_3943_begin_0, end = var_3943_end_0, end_mask = var_3943_end_mask_0, squeeze_mask = var_3943_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3943_cast_fp16")];
+            bool var_3944_transpose_x_0 = const()[name = string("op_3944_transpose_x_0"), val = bool(false)];
+            bool var_3944_transpose_y_0 = const()[name = string("op_3944_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3944_cast_fp16 = matmul(transpose_x = var_3944_transpose_x_0, transpose_y = var_3944_transpose_y_0, x = var_3943_cast_fp16, y = state_197_cast_fp16)[name = string("op_3944_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_197_cast_fp16 = sub(x = var_3941_cast_fp16, y = var_3944_cast_fp16)[name = string("v_new_197_cast_fp16")];
+            tensor<int32, [4]> var_3947_begin_0 = const()[name = string("op_3947_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3947_end_0 = const()[name = string("op_3947_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3947_end_mask_0 = const()[name = string("op_3947_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3947_squeeze_mask_0 = const()[name = string("op_3947_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3947_cast_fp16 = slice_by_index(begin = var_3947_begin_0, end = var_3947_end_0, end_mask = var_3947_end_mask_0, squeeze_mask = var_3947_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3947_cast_fp16")];
+            bool var_3948_transpose_x_0 = const()[name = string("op_3948_transpose_x_0"), val = bool(false)];
+            bool var_3948_transpose_y_0 = const()[name = string("op_3948_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3948_cast_fp16 = matmul(transpose_x = var_3948_transpose_x_0, transpose_y = var_3948_transpose_y_0, x = var_3947_cast_fp16, y = state_197_cast_fp16)[name = string("op_3948_cast_fp16")];
+            tensor<int32, [4]> var_3950_begin_0 = const()[name = string("op_3950_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3950_end_0 = const()[name = string("op_3950_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_3950_end_mask_0 = const()[name = string("op_3950_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3950_squeeze_mask_0 = const()[name = string("op_3950_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3950_cast_fp16 = slice_by_index(begin = var_3950_begin_0, end = var_3950_end_0, end_mask = var_3950_end_mask_0, squeeze_mask = var_3950_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3950_cast_fp16")];
+            bool var_3951_transpose_x_0 = const()[name = string("op_3951_transpose_x_0"), val = bool(false)];
+            bool var_3951_transpose_y_0 = const()[name = string("op_3951_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3951_cast_fp16 = matmul(transpose_x = var_3951_transpose_x_0, transpose_y = var_3951_transpose_y_0, x = var_3950_cast_fp16, y = v_new_197_cast_fp16)[name = string("op_3951_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3952_cast_fp16 = add(x = var_3948_cast_fp16, y = var_3951_cast_fp16)[name = string("op_3952_cast_fp16")];
+            tensor<int32, [4]> var_3954_begin_0 = const()[name = string("op_3954_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3954_end_0 = const()[name = string("op_3954_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_3954_end_mask_0 = const()[name = string("op_3954_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3954_squeeze_mask_0 = const()[name = string("op_3954_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3954_cast_fp16 = slice_by_index(begin = var_3954_begin_0, end = var_3954_end_0, end_mask = var_3954_end_mask_0, squeeze_mask = var_3954_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3954_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3955_cast_fp16 = mul(x = state_197_cast_fp16, y = var_3954_cast_fp16)[name = string("op_3955_cast_fp16")];
+            tensor<int32, [4]> var_3957_begin_0 = const()[name = string("op_3957_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_3957_end_0 = const()[name = string("op_3957_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_3957_end_mask_0 = const()[name = string("op_3957_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3957_squeeze_mask_0 = const()[name = string("op_3957_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3957_cast_fp16 = slice_by_index(begin = var_3957_begin_0, end = var_3957_end_0, end_mask = var_3957_end_mask_0, squeeze_mask = var_3957_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3957_cast_fp16")];
+            bool var_3959_transpose_x_1 = const()[name = string("op_3959_transpose_x_1"), val = bool(true)];
+            bool var_3959_transpose_y_1 = const()[name = string("op_3959_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3959_cast_fp16 = matmul(transpose_x = var_3959_transpose_x_1, transpose_y = var_3959_transpose_y_1, x = var_3957_cast_fp16, y = v_new_197_cast_fp16)[name = string("op_3959_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_199_cast_fp16 = add(x = var_3955_cast_fp16, y = var_3959_cast_fp16)[name = string("state_199_cast_fp16")];
+            tensor<int32, [4]> var_3962_begin_0 = const()[name = string("op_3962_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3962_end_0 = const()[name = string("op_3962_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3962_end_mask_0 = const()[name = string("op_3962_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3962_squeeze_mask_0 = const()[name = string("op_3962_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3962_cast_fp16 = slice_by_index(begin = var_3962_begin_0, end = var_3962_end_0, end_mask = var_3962_end_mask_0, squeeze_mask = var_3962_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3962_cast_fp16")];
+            tensor<int32, [4]> var_3964_begin_0 = const()[name = string("op_3964_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3964_end_0 = const()[name = string("op_3964_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3964_end_mask_0 = const()[name = string("op_3964_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3964_squeeze_mask_0 = const()[name = string("op_3964_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3964_cast_fp16 = slice_by_index(begin = var_3964_begin_0, end = var_3964_end_0, end_mask = var_3964_end_mask_0, squeeze_mask = var_3964_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3964_cast_fp16")];
+            bool var_3965_transpose_x_0 = const()[name = string("op_3965_transpose_x_0"), val = bool(false)];
+            bool var_3965_transpose_y_0 = const()[name = string("op_3965_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3965_cast_fp16 = matmul(transpose_x = var_3965_transpose_x_0, transpose_y = var_3965_transpose_y_0, x = var_3964_cast_fp16, y = state_199_cast_fp16)[name = string("op_3965_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_199_cast_fp16 = sub(x = var_3962_cast_fp16, y = var_3965_cast_fp16)[name = string("v_new_199_cast_fp16")];
+            tensor<int32, [4]> var_3968_begin_0 = const()[name = string("op_3968_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3968_end_0 = const()[name = string("op_3968_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3968_end_mask_0 = const()[name = string("op_3968_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3968_squeeze_mask_0 = const()[name = string("op_3968_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3968_cast_fp16 = slice_by_index(begin = var_3968_begin_0, end = var_3968_end_0, end_mask = var_3968_end_mask_0, squeeze_mask = var_3968_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3968_cast_fp16")];
+            bool var_3969_transpose_x_0 = const()[name = string("op_3969_transpose_x_0"), val = bool(false)];
+            bool var_3969_transpose_y_0 = const()[name = string("op_3969_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3969_cast_fp16 = matmul(transpose_x = var_3969_transpose_x_0, transpose_y = var_3969_transpose_y_0, x = var_3968_cast_fp16, y = state_199_cast_fp16)[name = string("op_3969_cast_fp16")];
+            tensor<int32, [4]> var_3971_begin_0 = const()[name = string("op_3971_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3971_end_0 = const()[name = string("op_3971_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_3971_end_mask_0 = const()[name = string("op_3971_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3971_squeeze_mask_0 = const()[name = string("op_3971_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3971_cast_fp16 = slice_by_index(begin = var_3971_begin_0, end = var_3971_end_0, end_mask = var_3971_end_mask_0, squeeze_mask = var_3971_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3971_cast_fp16")];
+            bool var_3972_transpose_x_0 = const()[name = string("op_3972_transpose_x_0"), val = bool(false)];
+            bool var_3972_transpose_y_0 = const()[name = string("op_3972_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3972_cast_fp16 = matmul(transpose_x = var_3972_transpose_x_0, transpose_y = var_3972_transpose_y_0, x = var_3971_cast_fp16, y = v_new_199_cast_fp16)[name = string("op_3972_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3973_cast_fp16 = add(x = var_3969_cast_fp16, y = var_3972_cast_fp16)[name = string("op_3973_cast_fp16")];
+            tensor<int32, [4]> var_3975_begin_0 = const()[name = string("op_3975_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3975_end_0 = const()[name = string("op_3975_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_3975_end_mask_0 = const()[name = string("op_3975_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3975_squeeze_mask_0 = const()[name = string("op_3975_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3975_cast_fp16 = slice_by_index(begin = var_3975_begin_0, end = var_3975_end_0, end_mask = var_3975_end_mask_0, squeeze_mask = var_3975_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3975_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3976_cast_fp16 = mul(x = state_199_cast_fp16, y = var_3975_cast_fp16)[name = string("op_3976_cast_fp16")];
+            tensor<int32, [4]> var_3978_begin_0 = const()[name = string("op_3978_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_3978_end_0 = const()[name = string("op_3978_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_3978_end_mask_0 = const()[name = string("op_3978_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3978_squeeze_mask_0 = const()[name = string("op_3978_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3978_cast_fp16 = slice_by_index(begin = var_3978_begin_0, end = var_3978_end_0, end_mask = var_3978_end_mask_0, squeeze_mask = var_3978_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3978_cast_fp16")];
+            bool var_3980_transpose_x_1 = const()[name = string("op_3980_transpose_x_1"), val = bool(true)];
+            bool var_3980_transpose_y_1 = const()[name = string("op_3980_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_3980_cast_fp16 = matmul(transpose_x = var_3980_transpose_x_1, transpose_y = var_3980_transpose_y_1, x = var_3978_cast_fp16, y = v_new_199_cast_fp16)[name = string("op_3980_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_201_cast_fp16 = add(x = var_3976_cast_fp16, y = var_3980_cast_fp16)[name = string("state_201_cast_fp16")];
+            tensor<int32, [4]> var_3983_begin_0 = const()[name = string("op_3983_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3983_end_0 = const()[name = string("op_3983_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3983_end_mask_0 = const()[name = string("op_3983_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3983_squeeze_mask_0 = const()[name = string("op_3983_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3983_cast_fp16 = slice_by_index(begin = var_3983_begin_0, end = var_3983_end_0, end_mask = var_3983_end_mask_0, squeeze_mask = var_3983_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_3983_cast_fp16")];
+            tensor<int32, [4]> var_3985_begin_0 = const()[name = string("op_3985_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3985_end_0 = const()[name = string("op_3985_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3985_end_mask_0 = const()[name = string("op_3985_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3985_squeeze_mask_0 = const()[name = string("op_3985_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3985_cast_fp16 = slice_by_index(begin = var_3985_begin_0, end = var_3985_end_0, end_mask = var_3985_end_mask_0, squeeze_mask = var_3985_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_3985_cast_fp16")];
+            bool var_3986_transpose_x_0 = const()[name = string("op_3986_transpose_x_0"), val = bool(false)];
+            bool var_3986_transpose_y_0 = const()[name = string("op_3986_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3986_cast_fp16 = matmul(transpose_x = var_3986_transpose_x_0, transpose_y = var_3986_transpose_y_0, x = var_3985_cast_fp16, y = state_201_cast_fp16)[name = string("op_3986_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_201_cast_fp16 = sub(x = var_3983_cast_fp16, y = var_3986_cast_fp16)[name = string("v_new_201_cast_fp16")];
+            tensor<int32, [4]> var_3989_begin_0 = const()[name = string("op_3989_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3989_end_0 = const()[name = string("op_3989_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3989_end_mask_0 = const()[name = string("op_3989_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3989_squeeze_mask_0 = const()[name = string("op_3989_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3989_cast_fp16 = slice_by_index(begin = var_3989_begin_0, end = var_3989_end_0, end_mask = var_3989_end_mask_0, squeeze_mask = var_3989_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_3989_cast_fp16")];
+            bool var_3990_transpose_x_0 = const()[name = string("op_3990_transpose_x_0"), val = bool(false)];
+            bool var_3990_transpose_y_0 = const()[name = string("op_3990_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3990_cast_fp16 = matmul(transpose_x = var_3990_transpose_x_0, transpose_y = var_3990_transpose_y_0, x = var_3989_cast_fp16, y = state_201_cast_fp16)[name = string("op_3990_cast_fp16")];
+            tensor<int32, [4]> var_3992_begin_0 = const()[name = string("op_3992_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3992_end_0 = const()[name = string("op_3992_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_3992_end_mask_0 = const()[name = string("op_3992_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3992_squeeze_mask_0 = const()[name = string("op_3992_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_3992_cast_fp16 = slice_by_index(begin = var_3992_begin_0, end = var_3992_end_0, end_mask = var_3992_end_mask_0, squeeze_mask = var_3992_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_3992_cast_fp16")];
+            bool var_3993_transpose_x_0 = const()[name = string("op_3993_transpose_x_0"), val = bool(false)];
+            bool var_3993_transpose_y_0 = const()[name = string("op_3993_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_3993_cast_fp16 = matmul(transpose_x = var_3993_transpose_x_0, transpose_y = var_3993_transpose_y_0, x = var_3992_cast_fp16, y = v_new_201_cast_fp16)[name = string("op_3993_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_3994_cast_fp16 = add(x = var_3990_cast_fp16, y = var_3993_cast_fp16)[name = string("op_3994_cast_fp16")];
+            tensor<int32, [4]> var_3996_begin_0 = const()[name = string("op_3996_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3996_end_0 = const()[name = string("op_3996_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_3996_end_mask_0 = const()[name = string("op_3996_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3996_squeeze_mask_0 = const()[name = string("op_3996_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_3996_cast_fp16 = slice_by_index(begin = var_3996_begin_0, end = var_3996_end_0, end_mask = var_3996_end_mask_0, squeeze_mask = var_3996_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_3996_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_3997_cast_fp16 = mul(x = state_201_cast_fp16, y = var_3996_cast_fp16)[name = string("op_3997_cast_fp16")];
+            tensor<int32, [4]> var_3999_begin_0 = const()[name = string("op_3999_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_3999_end_0 = const()[name = string("op_3999_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_3999_end_mask_0 = const()[name = string("op_3999_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_3999_squeeze_mask_0 = const()[name = string("op_3999_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_3999_cast_fp16 = slice_by_index(begin = var_3999_begin_0, end = var_3999_end_0, end_mask = var_3999_end_mask_0, squeeze_mask = var_3999_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_3999_cast_fp16")];
+            bool var_4001_transpose_x_1 = const()[name = string("op_4001_transpose_x_1"), val = bool(true)];
+            bool var_4001_transpose_y_1 = const()[name = string("op_4001_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4001_cast_fp16 = matmul(transpose_x = var_4001_transpose_x_1, transpose_y = var_4001_transpose_y_1, x = var_3999_cast_fp16, y = v_new_201_cast_fp16)[name = string("op_4001_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_203_cast_fp16 = add(x = var_3997_cast_fp16, y = var_4001_cast_fp16)[name = string("state_203_cast_fp16")];
+            tensor<int32, [4]> var_4004_begin_0 = const()[name = string("op_4004_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4004_end_0 = const()[name = string("op_4004_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4004_end_mask_0 = const()[name = string("op_4004_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4004_squeeze_mask_0 = const()[name = string("op_4004_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4004_cast_fp16 = slice_by_index(begin = var_4004_begin_0, end = var_4004_end_0, end_mask = var_4004_end_mask_0, squeeze_mask = var_4004_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4004_cast_fp16")];
+            tensor<int32, [4]> var_4006_begin_0 = const()[name = string("op_4006_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4006_end_0 = const()[name = string("op_4006_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4006_end_mask_0 = const()[name = string("op_4006_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4006_squeeze_mask_0 = const()[name = string("op_4006_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4006_cast_fp16 = slice_by_index(begin = var_4006_begin_0, end = var_4006_end_0, end_mask = var_4006_end_mask_0, squeeze_mask = var_4006_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4006_cast_fp16")];
+            bool var_4007_transpose_x_0 = const()[name = string("op_4007_transpose_x_0"), val = bool(false)];
+            bool var_4007_transpose_y_0 = const()[name = string("op_4007_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4007_cast_fp16 = matmul(transpose_x = var_4007_transpose_x_0, transpose_y = var_4007_transpose_y_0, x = var_4006_cast_fp16, y = state_203_cast_fp16)[name = string("op_4007_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_203_cast_fp16 = sub(x = var_4004_cast_fp16, y = var_4007_cast_fp16)[name = string("v_new_203_cast_fp16")];
+            tensor<int32, [4]> var_4010_begin_0 = const()[name = string("op_4010_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4010_end_0 = const()[name = string("op_4010_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4010_end_mask_0 = const()[name = string("op_4010_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4010_squeeze_mask_0 = const()[name = string("op_4010_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4010_cast_fp16 = slice_by_index(begin = var_4010_begin_0, end = var_4010_end_0, end_mask = var_4010_end_mask_0, squeeze_mask = var_4010_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4010_cast_fp16")];
+            bool var_4011_transpose_x_0 = const()[name = string("op_4011_transpose_x_0"), val = bool(false)];
+            bool var_4011_transpose_y_0 = const()[name = string("op_4011_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4011_cast_fp16 = matmul(transpose_x = var_4011_transpose_x_0, transpose_y = var_4011_transpose_y_0, x = var_4010_cast_fp16, y = state_203_cast_fp16)[name = string("op_4011_cast_fp16")];
+            tensor<int32, [4]> var_4013_begin_0 = const()[name = string("op_4013_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4013_end_0 = const()[name = string("op_4013_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_4013_end_mask_0 = const()[name = string("op_4013_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4013_squeeze_mask_0 = const()[name = string("op_4013_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4013_cast_fp16 = slice_by_index(begin = var_4013_begin_0, end = var_4013_end_0, end_mask = var_4013_end_mask_0, squeeze_mask = var_4013_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4013_cast_fp16")];
+            bool var_4014_transpose_x_0 = const()[name = string("op_4014_transpose_x_0"), val = bool(false)];
+            bool var_4014_transpose_y_0 = const()[name = string("op_4014_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4014_cast_fp16 = matmul(transpose_x = var_4014_transpose_x_0, transpose_y = var_4014_transpose_y_0, x = var_4013_cast_fp16, y = v_new_203_cast_fp16)[name = string("op_4014_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4015_cast_fp16 = add(x = var_4011_cast_fp16, y = var_4014_cast_fp16)[name = string("op_4015_cast_fp16")];
+            tensor<int32, [4]> var_4017_begin_0 = const()[name = string("op_4017_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4017_end_0 = const()[name = string("op_4017_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_4017_end_mask_0 = const()[name = string("op_4017_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4017_squeeze_mask_0 = const()[name = string("op_4017_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4017_cast_fp16 = slice_by_index(begin = var_4017_begin_0, end = var_4017_end_0, end_mask = var_4017_end_mask_0, squeeze_mask = var_4017_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4017_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4018_cast_fp16 = mul(x = state_203_cast_fp16, y = var_4017_cast_fp16)[name = string("op_4018_cast_fp16")];
+            tensor<int32, [4]> var_4020_begin_0 = const()[name = string("op_4020_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_4020_end_0 = const()[name = string("op_4020_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_4020_end_mask_0 = const()[name = string("op_4020_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4020_squeeze_mask_0 = const()[name = string("op_4020_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4020_cast_fp16 = slice_by_index(begin = var_4020_begin_0, end = var_4020_end_0, end_mask = var_4020_end_mask_0, squeeze_mask = var_4020_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4020_cast_fp16")];
+            bool var_4022_transpose_x_1 = const()[name = string("op_4022_transpose_x_1"), val = bool(true)];
+            bool var_4022_transpose_y_1 = const()[name = string("op_4022_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4022_cast_fp16 = matmul(transpose_x = var_4022_transpose_x_1, transpose_y = var_4022_transpose_y_1, x = var_4020_cast_fp16, y = v_new_203_cast_fp16)[name = string("op_4022_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_205_cast_fp16 = add(x = var_4018_cast_fp16, y = var_4022_cast_fp16)[name = string("state_205_cast_fp16")];
+            tensor<int32, [4]> var_4025_begin_0 = const()[name = string("op_4025_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4025_end_0 = const()[name = string("op_4025_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4025_end_mask_0 = const()[name = string("op_4025_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4025_squeeze_mask_0 = const()[name = string("op_4025_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4025_cast_fp16 = slice_by_index(begin = var_4025_begin_0, end = var_4025_end_0, end_mask = var_4025_end_mask_0, squeeze_mask = var_4025_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4025_cast_fp16")];
+            tensor<int32, [4]> var_4027_begin_0 = const()[name = string("op_4027_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4027_end_0 = const()[name = string("op_4027_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4027_end_mask_0 = const()[name = string("op_4027_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4027_squeeze_mask_0 = const()[name = string("op_4027_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4027_cast_fp16 = slice_by_index(begin = var_4027_begin_0, end = var_4027_end_0, end_mask = var_4027_end_mask_0, squeeze_mask = var_4027_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4027_cast_fp16")];
+            bool var_4028_transpose_x_0 = const()[name = string("op_4028_transpose_x_0"), val = bool(false)];
+            bool var_4028_transpose_y_0 = const()[name = string("op_4028_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4028_cast_fp16 = matmul(transpose_x = var_4028_transpose_x_0, transpose_y = var_4028_transpose_y_0, x = var_4027_cast_fp16, y = state_205_cast_fp16)[name = string("op_4028_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_205_cast_fp16 = sub(x = var_4025_cast_fp16, y = var_4028_cast_fp16)[name = string("v_new_205_cast_fp16")];
+            tensor<int32, [4]> var_4031_begin_0 = const()[name = string("op_4031_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4031_end_0 = const()[name = string("op_4031_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4031_end_mask_0 = const()[name = string("op_4031_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4031_squeeze_mask_0 = const()[name = string("op_4031_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4031_cast_fp16 = slice_by_index(begin = var_4031_begin_0, end = var_4031_end_0, end_mask = var_4031_end_mask_0, squeeze_mask = var_4031_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4031_cast_fp16")];
+            bool var_4032_transpose_x_0 = const()[name = string("op_4032_transpose_x_0"), val = bool(false)];
+            bool var_4032_transpose_y_0 = const()[name = string("op_4032_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4032_cast_fp16 = matmul(transpose_x = var_4032_transpose_x_0, transpose_y = var_4032_transpose_y_0, x = var_4031_cast_fp16, y = state_205_cast_fp16)[name = string("op_4032_cast_fp16")];
+            tensor<int32, [4]> var_4034_begin_0 = const()[name = string("op_4034_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4034_end_0 = const()[name = string("op_4034_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_4034_end_mask_0 = const()[name = string("op_4034_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4034_squeeze_mask_0 = const()[name = string("op_4034_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4034_cast_fp16 = slice_by_index(begin = var_4034_begin_0, end = var_4034_end_0, end_mask = var_4034_end_mask_0, squeeze_mask = var_4034_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4034_cast_fp16")];
+            bool var_4035_transpose_x_0 = const()[name = string("op_4035_transpose_x_0"), val = bool(false)];
+            bool var_4035_transpose_y_0 = const()[name = string("op_4035_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4035_cast_fp16 = matmul(transpose_x = var_4035_transpose_x_0, transpose_y = var_4035_transpose_y_0, x = var_4034_cast_fp16, y = v_new_205_cast_fp16)[name = string("op_4035_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4036_cast_fp16 = add(x = var_4032_cast_fp16, y = var_4035_cast_fp16)[name = string("op_4036_cast_fp16")];
+            tensor<int32, [4]> var_4038_begin_0 = const()[name = string("op_4038_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4038_end_0 = const()[name = string("op_4038_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_4038_end_mask_0 = const()[name = string("op_4038_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4038_squeeze_mask_0 = const()[name = string("op_4038_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4038_cast_fp16 = slice_by_index(begin = var_4038_begin_0, end = var_4038_end_0, end_mask = var_4038_end_mask_0, squeeze_mask = var_4038_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4038_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4039_cast_fp16 = mul(x = state_205_cast_fp16, y = var_4038_cast_fp16)[name = string("op_4039_cast_fp16")];
+            tensor<int32, [4]> var_4041_begin_0 = const()[name = string("op_4041_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_4041_end_0 = const()[name = string("op_4041_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_4041_end_mask_0 = const()[name = string("op_4041_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4041_squeeze_mask_0 = const()[name = string("op_4041_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4041_cast_fp16 = slice_by_index(begin = var_4041_begin_0, end = var_4041_end_0, end_mask = var_4041_end_mask_0, squeeze_mask = var_4041_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4041_cast_fp16")];
+            bool var_4043_transpose_x_1 = const()[name = string("op_4043_transpose_x_1"), val = bool(true)];
+            bool var_4043_transpose_y_1 = const()[name = string("op_4043_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4043_cast_fp16 = matmul(transpose_x = var_4043_transpose_x_1, transpose_y = var_4043_transpose_y_1, x = var_4041_cast_fp16, y = v_new_205_cast_fp16)[name = string("op_4043_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_207_cast_fp16 = add(x = var_4039_cast_fp16, y = var_4043_cast_fp16)[name = string("state_207_cast_fp16")];
+            tensor<int32, [4]> var_4046_begin_0 = const()[name = string("op_4046_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4046_end_0 = const()[name = string("op_4046_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4046_end_mask_0 = const()[name = string("op_4046_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4046_squeeze_mask_0 = const()[name = string("op_4046_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4046_cast_fp16 = slice_by_index(begin = var_4046_begin_0, end = var_4046_end_0, end_mask = var_4046_end_mask_0, squeeze_mask = var_4046_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4046_cast_fp16")];
+            tensor<int32, [4]> var_4048_begin_0 = const()[name = string("op_4048_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4048_end_0 = const()[name = string("op_4048_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4048_end_mask_0 = const()[name = string("op_4048_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4048_squeeze_mask_0 = const()[name = string("op_4048_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4048_cast_fp16 = slice_by_index(begin = var_4048_begin_0, end = var_4048_end_0, end_mask = var_4048_end_mask_0, squeeze_mask = var_4048_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4048_cast_fp16")];
+            bool var_4049_transpose_x_0 = const()[name = string("op_4049_transpose_x_0"), val = bool(false)];
+            bool var_4049_transpose_y_0 = const()[name = string("op_4049_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4049_cast_fp16 = matmul(transpose_x = var_4049_transpose_x_0, transpose_y = var_4049_transpose_y_0, x = var_4048_cast_fp16, y = state_207_cast_fp16)[name = string("op_4049_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_207_cast_fp16 = sub(x = var_4046_cast_fp16, y = var_4049_cast_fp16)[name = string("v_new_207_cast_fp16")];
+            tensor<int32, [4]> var_4052_begin_0 = const()[name = string("op_4052_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4052_end_0 = const()[name = string("op_4052_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4052_end_mask_0 = const()[name = string("op_4052_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4052_squeeze_mask_0 = const()[name = string("op_4052_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4052_cast_fp16 = slice_by_index(begin = var_4052_begin_0, end = var_4052_end_0, end_mask = var_4052_end_mask_0, squeeze_mask = var_4052_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4052_cast_fp16")];
+            bool var_4053_transpose_x_0 = const()[name = string("op_4053_transpose_x_0"), val = bool(false)];
+            bool var_4053_transpose_y_0 = const()[name = string("op_4053_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4053_cast_fp16 = matmul(transpose_x = var_4053_transpose_x_0, transpose_y = var_4053_transpose_y_0, x = var_4052_cast_fp16, y = state_207_cast_fp16)[name = string("op_4053_cast_fp16")];
+            tensor<int32, [4]> var_4055_begin_0 = const()[name = string("op_4055_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4055_end_0 = const()[name = string("op_4055_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_4055_end_mask_0 = const()[name = string("op_4055_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4055_squeeze_mask_0 = const()[name = string("op_4055_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4055_cast_fp16 = slice_by_index(begin = var_4055_begin_0, end = var_4055_end_0, end_mask = var_4055_end_mask_0, squeeze_mask = var_4055_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4055_cast_fp16")];
+            bool var_4056_transpose_x_0 = const()[name = string("op_4056_transpose_x_0"), val = bool(false)];
+            bool var_4056_transpose_y_0 = const()[name = string("op_4056_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4056_cast_fp16 = matmul(transpose_x = var_4056_transpose_x_0, transpose_y = var_4056_transpose_y_0, x = var_4055_cast_fp16, y = v_new_207_cast_fp16)[name = string("op_4056_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4057_cast_fp16 = add(x = var_4053_cast_fp16, y = var_4056_cast_fp16)[name = string("op_4057_cast_fp16")];
+            tensor<int32, [4]> var_4059_begin_0 = const()[name = string("op_4059_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4059_end_0 = const()[name = string("op_4059_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_4059_end_mask_0 = const()[name = string("op_4059_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4059_squeeze_mask_0 = const()[name = string("op_4059_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4059_cast_fp16 = slice_by_index(begin = var_4059_begin_0, end = var_4059_end_0, end_mask = var_4059_end_mask_0, squeeze_mask = var_4059_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4059_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4060_cast_fp16 = mul(x = state_207_cast_fp16, y = var_4059_cast_fp16)[name = string("op_4060_cast_fp16")];
+            tensor<int32, [4]> var_4062_begin_0 = const()[name = string("op_4062_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_4062_end_0 = const()[name = string("op_4062_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_4062_end_mask_0 = const()[name = string("op_4062_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4062_squeeze_mask_0 = const()[name = string("op_4062_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4062_cast_fp16 = slice_by_index(begin = var_4062_begin_0, end = var_4062_end_0, end_mask = var_4062_end_mask_0, squeeze_mask = var_4062_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4062_cast_fp16")];
+            bool var_4064_transpose_x_1 = const()[name = string("op_4064_transpose_x_1"), val = bool(true)];
+            bool var_4064_transpose_y_1 = const()[name = string("op_4064_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4064_cast_fp16 = matmul(transpose_x = var_4064_transpose_x_1, transpose_y = var_4064_transpose_y_1, x = var_4062_cast_fp16, y = v_new_207_cast_fp16)[name = string("op_4064_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_209_cast_fp16 = add(x = var_4060_cast_fp16, y = var_4064_cast_fp16)[name = string("state_209_cast_fp16")];
+            tensor<int32, [4]> var_4067_begin_0 = const()[name = string("op_4067_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4067_end_0 = const()[name = string("op_4067_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4067_end_mask_0 = const()[name = string("op_4067_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4067_squeeze_mask_0 = const()[name = string("op_4067_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4067_cast_fp16 = slice_by_index(begin = var_4067_begin_0, end = var_4067_end_0, end_mask = var_4067_end_mask_0, squeeze_mask = var_4067_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4067_cast_fp16")];
+            tensor<int32, [4]> var_4069_begin_0 = const()[name = string("op_4069_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4069_end_0 = const()[name = string("op_4069_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4069_end_mask_0 = const()[name = string("op_4069_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4069_squeeze_mask_0 = const()[name = string("op_4069_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4069_cast_fp16 = slice_by_index(begin = var_4069_begin_0, end = var_4069_end_0, end_mask = var_4069_end_mask_0, squeeze_mask = var_4069_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4069_cast_fp16")];
+            bool var_4070_transpose_x_0 = const()[name = string("op_4070_transpose_x_0"), val = bool(false)];
+            bool var_4070_transpose_y_0 = const()[name = string("op_4070_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4070_cast_fp16 = matmul(transpose_x = var_4070_transpose_x_0, transpose_y = var_4070_transpose_y_0, x = var_4069_cast_fp16, y = state_209_cast_fp16)[name = string("op_4070_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_209_cast_fp16 = sub(x = var_4067_cast_fp16, y = var_4070_cast_fp16)[name = string("v_new_209_cast_fp16")];
+            tensor<int32, [4]> var_4073_begin_0 = const()[name = string("op_4073_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4073_end_0 = const()[name = string("op_4073_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4073_end_mask_0 = const()[name = string("op_4073_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4073_squeeze_mask_0 = const()[name = string("op_4073_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4073_cast_fp16 = slice_by_index(begin = var_4073_begin_0, end = var_4073_end_0, end_mask = var_4073_end_mask_0, squeeze_mask = var_4073_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4073_cast_fp16")];
+            bool var_4074_transpose_x_0 = const()[name = string("op_4074_transpose_x_0"), val = bool(false)];
+            bool var_4074_transpose_y_0 = const()[name = string("op_4074_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4074_cast_fp16 = matmul(transpose_x = var_4074_transpose_x_0, transpose_y = var_4074_transpose_y_0, x = var_4073_cast_fp16, y = state_209_cast_fp16)[name = string("op_4074_cast_fp16")];
+            tensor<int32, [4]> var_4076_begin_0 = const()[name = string("op_4076_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4076_end_0 = const()[name = string("op_4076_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_4076_end_mask_0 = const()[name = string("op_4076_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4076_squeeze_mask_0 = const()[name = string("op_4076_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4076_cast_fp16 = slice_by_index(begin = var_4076_begin_0, end = var_4076_end_0, end_mask = var_4076_end_mask_0, squeeze_mask = var_4076_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4076_cast_fp16")];
+            bool var_4077_transpose_x_0 = const()[name = string("op_4077_transpose_x_0"), val = bool(false)];
+            bool var_4077_transpose_y_0 = const()[name = string("op_4077_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4077_cast_fp16 = matmul(transpose_x = var_4077_transpose_x_0, transpose_y = var_4077_transpose_y_0, x = var_4076_cast_fp16, y = v_new_209_cast_fp16)[name = string("op_4077_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4078_cast_fp16 = add(x = var_4074_cast_fp16, y = var_4077_cast_fp16)[name = string("op_4078_cast_fp16")];
+            tensor<int32, [4]> var_4080_begin_0 = const()[name = string("op_4080_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4080_end_0 = const()[name = string("op_4080_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_4080_end_mask_0 = const()[name = string("op_4080_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4080_squeeze_mask_0 = const()[name = string("op_4080_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4080_cast_fp16 = slice_by_index(begin = var_4080_begin_0, end = var_4080_end_0, end_mask = var_4080_end_mask_0, squeeze_mask = var_4080_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4080_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4081_cast_fp16 = mul(x = state_209_cast_fp16, y = var_4080_cast_fp16)[name = string("op_4081_cast_fp16")];
+            tensor<int32, [4]> var_4083_begin_0 = const()[name = string("op_4083_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_4083_end_0 = const()[name = string("op_4083_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_4083_end_mask_0 = const()[name = string("op_4083_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4083_squeeze_mask_0 = const()[name = string("op_4083_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4083_cast_fp16 = slice_by_index(begin = var_4083_begin_0, end = var_4083_end_0, end_mask = var_4083_end_mask_0, squeeze_mask = var_4083_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4083_cast_fp16")];
+            bool var_4085_transpose_x_1 = const()[name = string("op_4085_transpose_x_1"), val = bool(true)];
+            bool var_4085_transpose_y_1 = const()[name = string("op_4085_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4085_cast_fp16 = matmul(transpose_x = var_4085_transpose_x_1, transpose_y = var_4085_transpose_y_1, x = var_4083_cast_fp16, y = v_new_209_cast_fp16)[name = string("op_4085_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_211_cast_fp16 = add(x = var_4081_cast_fp16, y = var_4085_cast_fp16)[name = string("state_211_cast_fp16")];
+            tensor<int32, [4]> var_4088_begin_0 = const()[name = string("op_4088_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4088_end_0 = const()[name = string("op_4088_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4088_end_mask_0 = const()[name = string("op_4088_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4088_squeeze_mask_0 = const()[name = string("op_4088_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4088_cast_fp16 = slice_by_index(begin = var_4088_begin_0, end = var_4088_end_0, end_mask = var_4088_end_mask_0, squeeze_mask = var_4088_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4088_cast_fp16")];
+            tensor<int32, [4]> var_4090_begin_0 = const()[name = string("op_4090_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4090_end_0 = const()[name = string("op_4090_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4090_end_mask_0 = const()[name = string("op_4090_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4090_squeeze_mask_0 = const()[name = string("op_4090_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4090_cast_fp16 = slice_by_index(begin = var_4090_begin_0, end = var_4090_end_0, end_mask = var_4090_end_mask_0, squeeze_mask = var_4090_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4090_cast_fp16")];
+            bool var_4091_transpose_x_0 = const()[name = string("op_4091_transpose_x_0"), val = bool(false)];
+            bool var_4091_transpose_y_0 = const()[name = string("op_4091_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4091_cast_fp16 = matmul(transpose_x = var_4091_transpose_x_0, transpose_y = var_4091_transpose_y_0, x = var_4090_cast_fp16, y = state_211_cast_fp16)[name = string("op_4091_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_211_cast_fp16 = sub(x = var_4088_cast_fp16, y = var_4091_cast_fp16)[name = string("v_new_211_cast_fp16")];
+            tensor<int32, [4]> var_4094_begin_0 = const()[name = string("op_4094_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4094_end_0 = const()[name = string("op_4094_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4094_end_mask_0 = const()[name = string("op_4094_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4094_squeeze_mask_0 = const()[name = string("op_4094_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4094_cast_fp16 = slice_by_index(begin = var_4094_begin_0, end = var_4094_end_0, end_mask = var_4094_end_mask_0, squeeze_mask = var_4094_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4094_cast_fp16")];
+            bool var_4095_transpose_x_0 = const()[name = string("op_4095_transpose_x_0"), val = bool(false)];
+            bool var_4095_transpose_y_0 = const()[name = string("op_4095_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4095_cast_fp16 = matmul(transpose_x = var_4095_transpose_x_0, transpose_y = var_4095_transpose_y_0, x = var_4094_cast_fp16, y = state_211_cast_fp16)[name = string("op_4095_cast_fp16")];
+            tensor<int32, [4]> var_4097_begin_0 = const()[name = string("op_4097_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4097_end_0 = const()[name = string("op_4097_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_4097_end_mask_0 = const()[name = string("op_4097_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4097_squeeze_mask_0 = const()[name = string("op_4097_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4097_cast_fp16 = slice_by_index(begin = var_4097_begin_0, end = var_4097_end_0, end_mask = var_4097_end_mask_0, squeeze_mask = var_4097_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4097_cast_fp16")];
+            bool var_4098_transpose_x_0 = const()[name = string("op_4098_transpose_x_0"), val = bool(false)];
+            bool var_4098_transpose_y_0 = const()[name = string("op_4098_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4098_cast_fp16 = matmul(transpose_x = var_4098_transpose_x_0, transpose_y = var_4098_transpose_y_0, x = var_4097_cast_fp16, y = v_new_211_cast_fp16)[name = string("op_4098_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4099_cast_fp16 = add(x = var_4095_cast_fp16, y = var_4098_cast_fp16)[name = string("op_4099_cast_fp16")];
+            tensor<int32, [4]> var_4101_begin_0 = const()[name = string("op_4101_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4101_end_0 = const()[name = string("op_4101_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_4101_end_mask_0 = const()[name = string("op_4101_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4101_squeeze_mask_0 = const()[name = string("op_4101_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4101_cast_fp16 = slice_by_index(begin = var_4101_begin_0, end = var_4101_end_0, end_mask = var_4101_end_mask_0, squeeze_mask = var_4101_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4101_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4102_cast_fp16 = mul(x = state_211_cast_fp16, y = var_4101_cast_fp16)[name = string("op_4102_cast_fp16")];
+            tensor<int32, [4]> var_4104_begin_0 = const()[name = string("op_4104_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_4104_end_0 = const()[name = string("op_4104_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_4104_end_mask_0 = const()[name = string("op_4104_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4104_squeeze_mask_0 = const()[name = string("op_4104_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4104_cast_fp16 = slice_by_index(begin = var_4104_begin_0, end = var_4104_end_0, end_mask = var_4104_end_mask_0, squeeze_mask = var_4104_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4104_cast_fp16")];
+            bool var_4106_transpose_x_1 = const()[name = string("op_4106_transpose_x_1"), val = bool(true)];
+            bool var_4106_transpose_y_1 = const()[name = string("op_4106_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4106_cast_fp16 = matmul(transpose_x = var_4106_transpose_x_1, transpose_y = var_4106_transpose_y_1, x = var_4104_cast_fp16, y = v_new_211_cast_fp16)[name = string("op_4106_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_213_cast_fp16 = add(x = var_4102_cast_fp16, y = var_4106_cast_fp16)[name = string("state_213_cast_fp16")];
+            tensor<int32, [4]> var_4109_begin_0 = const()[name = string("op_4109_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4109_end_0 = const()[name = string("op_4109_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4109_end_mask_0 = const()[name = string("op_4109_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4109_squeeze_mask_0 = const()[name = string("op_4109_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4109_cast_fp16 = slice_by_index(begin = var_4109_begin_0, end = var_4109_end_0, end_mask = var_4109_end_mask_0, squeeze_mask = var_4109_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4109_cast_fp16")];
+            tensor<int32, [4]> var_4111_begin_0 = const()[name = string("op_4111_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4111_end_0 = const()[name = string("op_4111_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4111_end_mask_0 = const()[name = string("op_4111_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4111_squeeze_mask_0 = const()[name = string("op_4111_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4111_cast_fp16 = slice_by_index(begin = var_4111_begin_0, end = var_4111_end_0, end_mask = var_4111_end_mask_0, squeeze_mask = var_4111_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4111_cast_fp16")];
+            bool var_4112_transpose_x_0 = const()[name = string("op_4112_transpose_x_0"), val = bool(false)];
+            bool var_4112_transpose_y_0 = const()[name = string("op_4112_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4112_cast_fp16 = matmul(transpose_x = var_4112_transpose_x_0, transpose_y = var_4112_transpose_y_0, x = var_4111_cast_fp16, y = state_213_cast_fp16)[name = string("op_4112_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_213_cast_fp16 = sub(x = var_4109_cast_fp16, y = var_4112_cast_fp16)[name = string("v_new_213_cast_fp16")];
+            tensor<int32, [4]> var_4115_begin_0 = const()[name = string("op_4115_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4115_end_0 = const()[name = string("op_4115_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4115_end_mask_0 = const()[name = string("op_4115_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4115_squeeze_mask_0 = const()[name = string("op_4115_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4115_cast_fp16 = slice_by_index(begin = var_4115_begin_0, end = var_4115_end_0, end_mask = var_4115_end_mask_0, squeeze_mask = var_4115_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4115_cast_fp16")];
+            bool var_4116_transpose_x_0 = const()[name = string("op_4116_transpose_x_0"), val = bool(false)];
+            bool var_4116_transpose_y_0 = const()[name = string("op_4116_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4116_cast_fp16 = matmul(transpose_x = var_4116_transpose_x_0, transpose_y = var_4116_transpose_y_0, x = var_4115_cast_fp16, y = state_213_cast_fp16)[name = string("op_4116_cast_fp16")];
+            tensor<int32, [4]> var_4118_begin_0 = const()[name = string("op_4118_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4118_end_0 = const()[name = string("op_4118_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_4118_end_mask_0 = const()[name = string("op_4118_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4118_squeeze_mask_0 = const()[name = string("op_4118_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4118_cast_fp16 = slice_by_index(begin = var_4118_begin_0, end = var_4118_end_0, end_mask = var_4118_end_mask_0, squeeze_mask = var_4118_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4118_cast_fp16")];
+            bool var_4119_transpose_x_0 = const()[name = string("op_4119_transpose_x_0"), val = bool(false)];
+            bool var_4119_transpose_y_0 = const()[name = string("op_4119_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4119_cast_fp16 = matmul(transpose_x = var_4119_transpose_x_0, transpose_y = var_4119_transpose_y_0, x = var_4118_cast_fp16, y = v_new_213_cast_fp16)[name = string("op_4119_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4120_cast_fp16 = add(x = var_4116_cast_fp16, y = var_4119_cast_fp16)[name = string("op_4120_cast_fp16")];
+            tensor<int32, [4]> var_4122_begin_0 = const()[name = string("op_4122_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4122_end_0 = const()[name = string("op_4122_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_4122_end_mask_0 = const()[name = string("op_4122_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4122_squeeze_mask_0 = const()[name = string("op_4122_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4122_cast_fp16 = slice_by_index(begin = var_4122_begin_0, end = var_4122_end_0, end_mask = var_4122_end_mask_0, squeeze_mask = var_4122_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4122_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4123_cast_fp16 = mul(x = state_213_cast_fp16, y = var_4122_cast_fp16)[name = string("op_4123_cast_fp16")];
+            tensor<int32, [4]> var_4125_begin_0 = const()[name = string("op_4125_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_4125_end_0 = const()[name = string("op_4125_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_4125_end_mask_0 = const()[name = string("op_4125_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4125_squeeze_mask_0 = const()[name = string("op_4125_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4125_cast_fp16 = slice_by_index(begin = var_4125_begin_0, end = var_4125_end_0, end_mask = var_4125_end_mask_0, squeeze_mask = var_4125_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4125_cast_fp16")];
+            bool var_4127_transpose_x_1 = const()[name = string("op_4127_transpose_x_1"), val = bool(true)];
+            bool var_4127_transpose_y_1 = const()[name = string("op_4127_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4127_cast_fp16 = matmul(transpose_x = var_4127_transpose_x_1, transpose_y = var_4127_transpose_y_1, x = var_4125_cast_fp16, y = v_new_213_cast_fp16)[name = string("op_4127_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_215_cast_fp16 = add(x = var_4123_cast_fp16, y = var_4127_cast_fp16)[name = string("state_215_cast_fp16")];
+            tensor<int32, [4]> var_4130_begin_0 = const()[name = string("op_4130_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4130_end_0 = const()[name = string("op_4130_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4130_end_mask_0 = const()[name = string("op_4130_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4130_squeeze_mask_0 = const()[name = string("op_4130_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4130_cast_fp16 = slice_by_index(begin = var_4130_begin_0, end = var_4130_end_0, end_mask = var_4130_end_mask_0, squeeze_mask = var_4130_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4130_cast_fp16")];
+            tensor<int32, [4]> var_4132_begin_0 = const()[name = string("op_4132_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4132_end_0 = const()[name = string("op_4132_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4132_end_mask_0 = const()[name = string("op_4132_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4132_squeeze_mask_0 = const()[name = string("op_4132_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4132_cast_fp16 = slice_by_index(begin = var_4132_begin_0, end = var_4132_end_0, end_mask = var_4132_end_mask_0, squeeze_mask = var_4132_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4132_cast_fp16")];
+            bool var_4133_transpose_x_0 = const()[name = string("op_4133_transpose_x_0"), val = bool(false)];
+            bool var_4133_transpose_y_0 = const()[name = string("op_4133_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4133_cast_fp16 = matmul(transpose_x = var_4133_transpose_x_0, transpose_y = var_4133_transpose_y_0, x = var_4132_cast_fp16, y = state_215_cast_fp16)[name = string("op_4133_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_215_cast_fp16 = sub(x = var_4130_cast_fp16, y = var_4133_cast_fp16)[name = string("v_new_215_cast_fp16")];
+            tensor<int32, [4]> var_4136_begin_0 = const()[name = string("op_4136_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4136_end_0 = const()[name = string("op_4136_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4136_end_mask_0 = const()[name = string("op_4136_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4136_squeeze_mask_0 = const()[name = string("op_4136_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4136_cast_fp16 = slice_by_index(begin = var_4136_begin_0, end = var_4136_end_0, end_mask = var_4136_end_mask_0, squeeze_mask = var_4136_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4136_cast_fp16")];
+            bool var_4137_transpose_x_0 = const()[name = string("op_4137_transpose_x_0"), val = bool(false)];
+            bool var_4137_transpose_y_0 = const()[name = string("op_4137_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4137_cast_fp16 = matmul(transpose_x = var_4137_transpose_x_0, transpose_y = var_4137_transpose_y_0, x = var_4136_cast_fp16, y = state_215_cast_fp16)[name = string("op_4137_cast_fp16")];
+            tensor<int32, [4]> var_4139_begin_0 = const()[name = string("op_4139_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4139_end_0 = const()[name = string("op_4139_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_4139_end_mask_0 = const()[name = string("op_4139_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4139_squeeze_mask_0 = const()[name = string("op_4139_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4139_cast_fp16 = slice_by_index(begin = var_4139_begin_0, end = var_4139_end_0, end_mask = var_4139_end_mask_0, squeeze_mask = var_4139_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4139_cast_fp16")];
+            bool var_4140_transpose_x_0 = const()[name = string("op_4140_transpose_x_0"), val = bool(false)];
+            bool var_4140_transpose_y_0 = const()[name = string("op_4140_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4140_cast_fp16 = matmul(transpose_x = var_4140_transpose_x_0, transpose_y = var_4140_transpose_y_0, x = var_4139_cast_fp16, y = v_new_215_cast_fp16)[name = string("op_4140_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4141_cast_fp16 = add(x = var_4137_cast_fp16, y = var_4140_cast_fp16)[name = string("op_4141_cast_fp16")];
+            tensor<int32, [4]> var_4143_begin_0 = const()[name = string("op_4143_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4143_end_0 = const()[name = string("op_4143_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_4143_end_mask_0 = const()[name = string("op_4143_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4143_squeeze_mask_0 = const()[name = string("op_4143_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4143_cast_fp16 = slice_by_index(begin = var_4143_begin_0, end = var_4143_end_0, end_mask = var_4143_end_mask_0, squeeze_mask = var_4143_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4143_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4144_cast_fp16 = mul(x = state_215_cast_fp16, y = var_4143_cast_fp16)[name = string("op_4144_cast_fp16")];
+            tensor<int32, [4]> var_4146_begin_0 = const()[name = string("op_4146_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_4146_end_0 = const()[name = string("op_4146_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_4146_end_mask_0 = const()[name = string("op_4146_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4146_squeeze_mask_0 = const()[name = string("op_4146_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4146_cast_fp16 = slice_by_index(begin = var_4146_begin_0, end = var_4146_end_0, end_mask = var_4146_end_mask_0, squeeze_mask = var_4146_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4146_cast_fp16")];
+            bool var_4148_transpose_x_1 = const()[name = string("op_4148_transpose_x_1"), val = bool(true)];
+            bool var_4148_transpose_y_1 = const()[name = string("op_4148_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4148_cast_fp16 = matmul(transpose_x = var_4148_transpose_x_1, transpose_y = var_4148_transpose_y_1, x = var_4146_cast_fp16, y = v_new_215_cast_fp16)[name = string("op_4148_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_217_cast_fp16 = add(x = var_4144_cast_fp16, y = var_4148_cast_fp16)[name = string("state_217_cast_fp16")];
+            tensor<int32, [4]> var_4151_begin_0 = const()[name = string("op_4151_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4151_end_0 = const()[name = string("op_4151_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4151_end_mask_0 = const()[name = string("op_4151_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4151_squeeze_mask_0 = const()[name = string("op_4151_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4151_cast_fp16 = slice_by_index(begin = var_4151_begin_0, end = var_4151_end_0, end_mask = var_4151_end_mask_0, squeeze_mask = var_4151_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4151_cast_fp16")];
+            tensor<int32, [4]> var_4153_begin_0 = const()[name = string("op_4153_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4153_end_0 = const()[name = string("op_4153_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4153_end_mask_0 = const()[name = string("op_4153_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4153_squeeze_mask_0 = const()[name = string("op_4153_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4153_cast_fp16 = slice_by_index(begin = var_4153_begin_0, end = var_4153_end_0, end_mask = var_4153_end_mask_0, squeeze_mask = var_4153_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4153_cast_fp16")];
+            bool var_4154_transpose_x_0 = const()[name = string("op_4154_transpose_x_0"), val = bool(false)];
+            bool var_4154_transpose_y_0 = const()[name = string("op_4154_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4154_cast_fp16 = matmul(transpose_x = var_4154_transpose_x_0, transpose_y = var_4154_transpose_y_0, x = var_4153_cast_fp16, y = state_217_cast_fp16)[name = string("op_4154_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_217_cast_fp16 = sub(x = var_4151_cast_fp16, y = var_4154_cast_fp16)[name = string("v_new_217_cast_fp16")];
+            tensor<int32, [4]> var_4157_begin_0 = const()[name = string("op_4157_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4157_end_0 = const()[name = string("op_4157_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4157_end_mask_0 = const()[name = string("op_4157_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4157_squeeze_mask_0 = const()[name = string("op_4157_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4157_cast_fp16 = slice_by_index(begin = var_4157_begin_0, end = var_4157_end_0, end_mask = var_4157_end_mask_0, squeeze_mask = var_4157_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4157_cast_fp16")];
+            bool var_4158_transpose_x_0 = const()[name = string("op_4158_transpose_x_0"), val = bool(false)];
+            bool var_4158_transpose_y_0 = const()[name = string("op_4158_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4158_cast_fp16 = matmul(transpose_x = var_4158_transpose_x_0, transpose_y = var_4158_transpose_y_0, x = var_4157_cast_fp16, y = state_217_cast_fp16)[name = string("op_4158_cast_fp16")];
+            tensor<int32, [4]> var_4160_begin_0 = const()[name = string("op_4160_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4160_end_0 = const()[name = string("op_4160_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_4160_end_mask_0 = const()[name = string("op_4160_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4160_squeeze_mask_0 = const()[name = string("op_4160_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4160_cast_fp16 = slice_by_index(begin = var_4160_begin_0, end = var_4160_end_0, end_mask = var_4160_end_mask_0, squeeze_mask = var_4160_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4160_cast_fp16")];
+            bool var_4161_transpose_x_0 = const()[name = string("op_4161_transpose_x_0"), val = bool(false)];
+            bool var_4161_transpose_y_0 = const()[name = string("op_4161_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4161_cast_fp16 = matmul(transpose_x = var_4161_transpose_x_0, transpose_y = var_4161_transpose_y_0, x = var_4160_cast_fp16, y = v_new_217_cast_fp16)[name = string("op_4161_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4162_cast_fp16 = add(x = var_4158_cast_fp16, y = var_4161_cast_fp16)[name = string("op_4162_cast_fp16")];
+            tensor<int32, [4]> var_4164_begin_0 = const()[name = string("op_4164_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4164_end_0 = const()[name = string("op_4164_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_4164_end_mask_0 = const()[name = string("op_4164_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4164_squeeze_mask_0 = const()[name = string("op_4164_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4164_cast_fp16 = slice_by_index(begin = var_4164_begin_0, end = var_4164_end_0, end_mask = var_4164_end_mask_0, squeeze_mask = var_4164_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4164_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4165_cast_fp16 = mul(x = state_217_cast_fp16, y = var_4164_cast_fp16)[name = string("op_4165_cast_fp16")];
+            tensor<int32, [4]> var_4167_begin_0 = const()[name = string("op_4167_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_4167_end_0 = const()[name = string("op_4167_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_4167_end_mask_0 = const()[name = string("op_4167_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4167_squeeze_mask_0 = const()[name = string("op_4167_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4167_cast_fp16 = slice_by_index(begin = var_4167_begin_0, end = var_4167_end_0, end_mask = var_4167_end_mask_0, squeeze_mask = var_4167_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4167_cast_fp16")];
+            bool var_4169_transpose_x_1 = const()[name = string("op_4169_transpose_x_1"), val = bool(true)];
+            bool var_4169_transpose_y_1 = const()[name = string("op_4169_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4169_cast_fp16 = matmul(transpose_x = var_4169_transpose_x_1, transpose_y = var_4169_transpose_y_1, x = var_4167_cast_fp16, y = v_new_217_cast_fp16)[name = string("op_4169_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_219_cast_fp16 = add(x = var_4165_cast_fp16, y = var_4169_cast_fp16)[name = string("state_219_cast_fp16")];
+            tensor<int32, [4]> var_4172_begin_0 = const()[name = string("op_4172_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4172_end_0 = const()[name = string("op_4172_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4172_end_mask_0 = const()[name = string("op_4172_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4172_squeeze_mask_0 = const()[name = string("op_4172_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4172_cast_fp16 = slice_by_index(begin = var_4172_begin_0, end = var_4172_end_0, end_mask = var_4172_end_mask_0, squeeze_mask = var_4172_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4172_cast_fp16")];
+            tensor<int32, [4]> var_4174_begin_0 = const()[name = string("op_4174_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4174_end_0 = const()[name = string("op_4174_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4174_end_mask_0 = const()[name = string("op_4174_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4174_squeeze_mask_0 = const()[name = string("op_4174_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4174_cast_fp16 = slice_by_index(begin = var_4174_begin_0, end = var_4174_end_0, end_mask = var_4174_end_mask_0, squeeze_mask = var_4174_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4174_cast_fp16")];
+            bool var_4175_transpose_x_0 = const()[name = string("op_4175_transpose_x_0"), val = bool(false)];
+            bool var_4175_transpose_y_0 = const()[name = string("op_4175_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4175_cast_fp16 = matmul(transpose_x = var_4175_transpose_x_0, transpose_y = var_4175_transpose_y_0, x = var_4174_cast_fp16, y = state_219_cast_fp16)[name = string("op_4175_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_219_cast_fp16 = sub(x = var_4172_cast_fp16, y = var_4175_cast_fp16)[name = string("v_new_219_cast_fp16")];
+            tensor<int32, [4]> var_4178_begin_0 = const()[name = string("op_4178_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4178_end_0 = const()[name = string("op_4178_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4178_end_mask_0 = const()[name = string("op_4178_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4178_squeeze_mask_0 = const()[name = string("op_4178_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4178_cast_fp16 = slice_by_index(begin = var_4178_begin_0, end = var_4178_end_0, end_mask = var_4178_end_mask_0, squeeze_mask = var_4178_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4178_cast_fp16")];
+            bool var_4179_transpose_x_0 = const()[name = string("op_4179_transpose_x_0"), val = bool(false)];
+            bool var_4179_transpose_y_0 = const()[name = string("op_4179_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4179_cast_fp16 = matmul(transpose_x = var_4179_transpose_x_0, transpose_y = var_4179_transpose_y_0, x = var_4178_cast_fp16, y = state_219_cast_fp16)[name = string("op_4179_cast_fp16")];
+            tensor<int32, [4]> var_4181_begin_0 = const()[name = string("op_4181_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4181_end_0 = const()[name = string("op_4181_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_4181_end_mask_0 = const()[name = string("op_4181_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4181_squeeze_mask_0 = const()[name = string("op_4181_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4181_cast_fp16 = slice_by_index(begin = var_4181_begin_0, end = var_4181_end_0, end_mask = var_4181_end_mask_0, squeeze_mask = var_4181_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4181_cast_fp16")];
+            bool var_4182_transpose_x_0 = const()[name = string("op_4182_transpose_x_0"), val = bool(false)];
+            bool var_4182_transpose_y_0 = const()[name = string("op_4182_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4182_cast_fp16 = matmul(transpose_x = var_4182_transpose_x_0, transpose_y = var_4182_transpose_y_0, x = var_4181_cast_fp16, y = v_new_219_cast_fp16)[name = string("op_4182_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4183_cast_fp16 = add(x = var_4179_cast_fp16, y = var_4182_cast_fp16)[name = string("op_4183_cast_fp16")];
+            tensor<int32, [4]> var_4185_begin_0 = const()[name = string("op_4185_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4185_end_0 = const()[name = string("op_4185_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_4185_end_mask_0 = const()[name = string("op_4185_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4185_squeeze_mask_0 = const()[name = string("op_4185_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4185_cast_fp16 = slice_by_index(begin = var_4185_begin_0, end = var_4185_end_0, end_mask = var_4185_end_mask_0, squeeze_mask = var_4185_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4185_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4186_cast_fp16 = mul(x = state_219_cast_fp16, y = var_4185_cast_fp16)[name = string("op_4186_cast_fp16")];
+            tensor<int32, [4]> var_4188_begin_0 = const()[name = string("op_4188_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_4188_end_0 = const()[name = string("op_4188_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_4188_end_mask_0 = const()[name = string("op_4188_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4188_squeeze_mask_0 = const()[name = string("op_4188_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4188_cast_fp16 = slice_by_index(begin = var_4188_begin_0, end = var_4188_end_0, end_mask = var_4188_end_mask_0, squeeze_mask = var_4188_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4188_cast_fp16")];
+            bool var_4190_transpose_x_1 = const()[name = string("op_4190_transpose_x_1"), val = bool(true)];
+            bool var_4190_transpose_y_1 = const()[name = string("op_4190_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4190_cast_fp16 = matmul(transpose_x = var_4190_transpose_x_1, transpose_y = var_4190_transpose_y_1, x = var_4188_cast_fp16, y = v_new_219_cast_fp16)[name = string("op_4190_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_221_cast_fp16 = add(x = var_4186_cast_fp16, y = var_4190_cast_fp16)[name = string("state_221_cast_fp16")];
+            tensor<int32, [4]> var_4193_begin_0 = const()[name = string("op_4193_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4193_end_0 = const()[name = string("op_4193_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4193_end_mask_0 = const()[name = string("op_4193_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4193_squeeze_mask_0 = const()[name = string("op_4193_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4193_cast_fp16 = slice_by_index(begin = var_4193_begin_0, end = var_4193_end_0, end_mask = var_4193_end_mask_0, squeeze_mask = var_4193_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4193_cast_fp16")];
+            tensor<int32, [4]> var_4195_begin_0 = const()[name = string("op_4195_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4195_end_0 = const()[name = string("op_4195_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4195_end_mask_0 = const()[name = string("op_4195_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4195_squeeze_mask_0 = const()[name = string("op_4195_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4195_cast_fp16 = slice_by_index(begin = var_4195_begin_0, end = var_4195_end_0, end_mask = var_4195_end_mask_0, squeeze_mask = var_4195_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4195_cast_fp16")];
+            bool var_4196_transpose_x_0 = const()[name = string("op_4196_transpose_x_0"), val = bool(false)];
+            bool var_4196_transpose_y_0 = const()[name = string("op_4196_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4196_cast_fp16 = matmul(transpose_x = var_4196_transpose_x_0, transpose_y = var_4196_transpose_y_0, x = var_4195_cast_fp16, y = state_221_cast_fp16)[name = string("op_4196_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_221_cast_fp16 = sub(x = var_4193_cast_fp16, y = var_4196_cast_fp16)[name = string("v_new_221_cast_fp16")];
+            tensor<int32, [4]> var_4199_begin_0 = const()[name = string("op_4199_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4199_end_0 = const()[name = string("op_4199_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4199_end_mask_0 = const()[name = string("op_4199_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4199_squeeze_mask_0 = const()[name = string("op_4199_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4199_cast_fp16 = slice_by_index(begin = var_4199_begin_0, end = var_4199_end_0, end_mask = var_4199_end_mask_0, squeeze_mask = var_4199_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4199_cast_fp16")];
+            bool var_4200_transpose_x_0 = const()[name = string("op_4200_transpose_x_0"), val = bool(false)];
+            bool var_4200_transpose_y_0 = const()[name = string("op_4200_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4200_cast_fp16 = matmul(transpose_x = var_4200_transpose_x_0, transpose_y = var_4200_transpose_y_0, x = var_4199_cast_fp16, y = state_221_cast_fp16)[name = string("op_4200_cast_fp16")];
+            tensor<int32, [4]> var_4202_begin_0 = const()[name = string("op_4202_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4202_end_0 = const()[name = string("op_4202_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_4202_end_mask_0 = const()[name = string("op_4202_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4202_squeeze_mask_0 = const()[name = string("op_4202_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4202_cast_fp16 = slice_by_index(begin = var_4202_begin_0, end = var_4202_end_0, end_mask = var_4202_end_mask_0, squeeze_mask = var_4202_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4202_cast_fp16")];
+            bool var_4203_transpose_x_0 = const()[name = string("op_4203_transpose_x_0"), val = bool(false)];
+            bool var_4203_transpose_y_0 = const()[name = string("op_4203_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4203_cast_fp16 = matmul(transpose_x = var_4203_transpose_x_0, transpose_y = var_4203_transpose_y_0, x = var_4202_cast_fp16, y = v_new_221_cast_fp16)[name = string("op_4203_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4204_cast_fp16 = add(x = var_4200_cast_fp16, y = var_4203_cast_fp16)[name = string("op_4204_cast_fp16")];
+            tensor<int32, [4]> var_4206_begin_0 = const()[name = string("op_4206_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4206_end_0 = const()[name = string("op_4206_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_4206_end_mask_0 = const()[name = string("op_4206_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4206_squeeze_mask_0 = const()[name = string("op_4206_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4206_cast_fp16 = slice_by_index(begin = var_4206_begin_0, end = var_4206_end_0, end_mask = var_4206_end_mask_0, squeeze_mask = var_4206_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4206_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4207_cast_fp16 = mul(x = state_221_cast_fp16, y = var_4206_cast_fp16)[name = string("op_4207_cast_fp16")];
+            tensor<int32, [4]> var_4209_begin_0 = const()[name = string("op_4209_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_4209_end_0 = const()[name = string("op_4209_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_4209_end_mask_0 = const()[name = string("op_4209_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4209_squeeze_mask_0 = const()[name = string("op_4209_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4209_cast_fp16 = slice_by_index(begin = var_4209_begin_0, end = var_4209_end_0, end_mask = var_4209_end_mask_0, squeeze_mask = var_4209_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4209_cast_fp16")];
+            bool var_4211_transpose_x_1 = const()[name = string("op_4211_transpose_x_1"), val = bool(true)];
+            bool var_4211_transpose_y_1 = const()[name = string("op_4211_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4211_cast_fp16 = matmul(transpose_x = var_4211_transpose_x_1, transpose_y = var_4211_transpose_y_1, x = var_4209_cast_fp16, y = v_new_221_cast_fp16)[name = string("op_4211_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_223_cast_fp16 = add(x = var_4207_cast_fp16, y = var_4211_cast_fp16)[name = string("state_223_cast_fp16")];
+            tensor<int32, [4]> var_4214_begin_0 = const()[name = string("op_4214_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4214_end_0 = const()[name = string("op_4214_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4214_end_mask_0 = const()[name = string("op_4214_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4214_squeeze_mask_0 = const()[name = string("op_4214_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4214_cast_fp16 = slice_by_index(begin = var_4214_begin_0, end = var_4214_end_0, end_mask = var_4214_end_mask_0, squeeze_mask = var_4214_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4214_cast_fp16")];
+            tensor<int32, [4]> var_4216_begin_0 = const()[name = string("op_4216_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4216_end_0 = const()[name = string("op_4216_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4216_end_mask_0 = const()[name = string("op_4216_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4216_squeeze_mask_0 = const()[name = string("op_4216_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4216_cast_fp16 = slice_by_index(begin = var_4216_begin_0, end = var_4216_end_0, end_mask = var_4216_end_mask_0, squeeze_mask = var_4216_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4216_cast_fp16")];
+            bool var_4217_transpose_x_0 = const()[name = string("op_4217_transpose_x_0"), val = bool(false)];
+            bool var_4217_transpose_y_0 = const()[name = string("op_4217_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4217_cast_fp16 = matmul(transpose_x = var_4217_transpose_x_0, transpose_y = var_4217_transpose_y_0, x = var_4216_cast_fp16, y = state_223_cast_fp16)[name = string("op_4217_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_223_cast_fp16 = sub(x = var_4214_cast_fp16, y = var_4217_cast_fp16)[name = string("v_new_223_cast_fp16")];
+            tensor<int32, [4]> var_4220_begin_0 = const()[name = string("op_4220_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4220_end_0 = const()[name = string("op_4220_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4220_end_mask_0 = const()[name = string("op_4220_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4220_squeeze_mask_0 = const()[name = string("op_4220_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4220_cast_fp16 = slice_by_index(begin = var_4220_begin_0, end = var_4220_end_0, end_mask = var_4220_end_mask_0, squeeze_mask = var_4220_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4220_cast_fp16")];
+            bool var_4221_transpose_x_0 = const()[name = string("op_4221_transpose_x_0"), val = bool(false)];
+            bool var_4221_transpose_y_0 = const()[name = string("op_4221_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4221_cast_fp16 = matmul(transpose_x = var_4221_transpose_x_0, transpose_y = var_4221_transpose_y_0, x = var_4220_cast_fp16, y = state_223_cast_fp16)[name = string("op_4221_cast_fp16")];
+            tensor<int32, [4]> var_4223_begin_0 = const()[name = string("op_4223_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4223_end_0 = const()[name = string("op_4223_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_4223_end_mask_0 = const()[name = string("op_4223_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4223_squeeze_mask_0 = const()[name = string("op_4223_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4223_cast_fp16 = slice_by_index(begin = var_4223_begin_0, end = var_4223_end_0, end_mask = var_4223_end_mask_0, squeeze_mask = var_4223_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4223_cast_fp16")];
+            bool var_4224_transpose_x_0 = const()[name = string("op_4224_transpose_x_0"), val = bool(false)];
+            bool var_4224_transpose_y_0 = const()[name = string("op_4224_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4224_cast_fp16 = matmul(transpose_x = var_4224_transpose_x_0, transpose_y = var_4224_transpose_y_0, x = var_4223_cast_fp16, y = v_new_223_cast_fp16)[name = string("op_4224_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4225_cast_fp16 = add(x = var_4221_cast_fp16, y = var_4224_cast_fp16)[name = string("op_4225_cast_fp16")];
+            tensor<int32, [4]> var_4227_begin_0 = const()[name = string("op_4227_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4227_end_0 = const()[name = string("op_4227_end_0"), val = tensor<int32, [4]>([32, 16, 1, 1])];
+            tensor<bool, [4]> var_4227_end_mask_0 = const()[name = string("op_4227_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4227_squeeze_mask_0 = const()[name = string("op_4227_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4227_cast_fp16 = slice_by_index(begin = var_4227_begin_0, end = var_4227_end_0, end_mask = var_4227_end_mask_0, squeeze_mask = var_4227_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4227_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4228_cast_fp16 = mul(x = state_223_cast_fp16, y = var_4227_cast_fp16)[name = string("op_4228_cast_fp16")];
+            tensor<int32, [4]> var_4230_begin_0 = const()[name = string("op_4230_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_4230_end_0 = const()[name = string("op_4230_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_4230_end_mask_0 = const()[name = string("op_4230_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4230_squeeze_mask_0 = const()[name = string("op_4230_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4230_cast_fp16 = slice_by_index(begin = var_4230_begin_0, end = var_4230_end_0, end_mask = var_4230_end_mask_0, squeeze_mask = var_4230_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4230_cast_fp16")];
+            bool var_4232_transpose_x_1 = const()[name = string("op_4232_transpose_x_1"), val = bool(true)];
+            bool var_4232_transpose_y_1 = const()[name = string("op_4232_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4232_cast_fp16 = matmul(transpose_x = var_4232_transpose_x_1, transpose_y = var_4232_transpose_y_1, x = var_4230_cast_fp16, y = v_new_223_cast_fp16)[name = string("op_4232_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_225_cast_fp16 = add(x = var_4228_cast_fp16, y = var_4232_cast_fp16)[name = string("state_225_cast_fp16")];
+            tensor<int32, [4]> var_4235_begin_0 = const()[name = string("op_4235_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_4235_end_0 = const()[name = string("op_4235_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_4235_end_mask_0 = const()[name = string("op_4235_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4235_squeeze_mask_0 = const()[name = string("op_4235_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4235_cast_fp16 = slice_by_index(begin = var_4235_begin_0, end = var_4235_end_0, end_mask = var_4235_end_mask_0, squeeze_mask = var_4235_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4235_cast_fp16")];
+            tensor<int32, [4]> var_4237_begin_0 = const()[name = string("op_4237_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_4237_end_0 = const()[name = string("op_4237_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_4237_end_mask_0 = const()[name = string("op_4237_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4237_squeeze_mask_0 = const()[name = string("op_4237_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4237_cast_fp16 = slice_by_index(begin = var_4237_begin_0, end = var_4237_end_0, end_mask = var_4237_end_mask_0, squeeze_mask = var_4237_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4237_cast_fp16")];
+            bool var_4238_transpose_x_0 = const()[name = string("op_4238_transpose_x_0"), val = bool(false)];
+            bool var_4238_transpose_y_0 = const()[name = string("op_4238_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4238_cast_fp16 = matmul(transpose_x = var_4238_transpose_x_0, transpose_y = var_4238_transpose_y_0, x = var_4237_cast_fp16, y = state_225_cast_fp16)[name = string("op_4238_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_225_cast_fp16 = sub(x = var_4235_cast_fp16, y = var_4238_cast_fp16)[name = string("v_new_225_cast_fp16")];
+            tensor<int32, [4]> var_4241_begin_0 = const()[name = string("op_4241_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_4241_end_0 = const()[name = string("op_4241_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_4241_end_mask_0 = const()[name = string("op_4241_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4241_squeeze_mask_0 = const()[name = string("op_4241_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4241_cast_fp16 = slice_by_index(begin = var_4241_begin_0, end = var_4241_end_0, end_mask = var_4241_end_mask_0, squeeze_mask = var_4241_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4241_cast_fp16")];
+            bool var_4242_transpose_x_0 = const()[name = string("op_4242_transpose_x_0"), val = bool(false)];
+            bool var_4242_transpose_y_0 = const()[name = string("op_4242_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4242_cast_fp16 = matmul(transpose_x = var_4242_transpose_x_0, transpose_y = var_4242_transpose_y_0, x = var_4241_cast_fp16, y = state_225_cast_fp16)[name = string("op_4242_cast_fp16")];
+            tensor<int32, [4]> var_4244_begin_0 = const()[name = string("op_4244_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_4244_end_0 = const()[name = string("op_4244_end_0"), val = tensor<int32, [4]>([32, 17, 64, 64])];
+            tensor<bool, [4]> var_4244_end_mask_0 = const()[name = string("op_4244_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4244_squeeze_mask_0 = const()[name = string("op_4244_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4244_cast_fp16 = slice_by_index(begin = var_4244_begin_0, end = var_4244_end_0, end_mask = var_4244_end_mask_0, squeeze_mask = var_4244_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4244_cast_fp16")];
+            bool var_4245_transpose_x_0 = const()[name = string("op_4245_transpose_x_0"), val = bool(false)];
+            bool var_4245_transpose_y_0 = const()[name = string("op_4245_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4245_cast_fp16 = matmul(transpose_x = var_4245_transpose_x_0, transpose_y = var_4245_transpose_y_0, x = var_4244_cast_fp16, y = v_new_225_cast_fp16)[name = string("op_4245_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4246_cast_fp16 = add(x = var_4242_cast_fp16, y = var_4245_cast_fp16)[name = string("op_4246_cast_fp16")];
+            tensor<int32, [4]> var_4248_begin_0 = const()[name = string("op_4248_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_4248_end_0 = const()[name = string("op_4248_end_0"), val = tensor<int32, [4]>([32, 17, 1, 1])];
+            tensor<bool, [4]> var_4248_end_mask_0 = const()[name = string("op_4248_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4248_squeeze_mask_0 = const()[name = string("op_4248_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4248_cast_fp16 = slice_by_index(begin = var_4248_begin_0, end = var_4248_end_0, end_mask = var_4248_end_mask_0, squeeze_mask = var_4248_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4248_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4249_cast_fp16 = mul(x = state_225_cast_fp16, y = var_4248_cast_fp16)[name = string("op_4249_cast_fp16")];
+            tensor<int32, [4]> var_4251_begin_0 = const()[name = string("op_4251_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_4251_end_0 = const()[name = string("op_4251_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_4251_end_mask_0 = const()[name = string("op_4251_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4251_squeeze_mask_0 = const()[name = string("op_4251_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4251_cast_fp16 = slice_by_index(begin = var_4251_begin_0, end = var_4251_end_0, end_mask = var_4251_end_mask_0, squeeze_mask = var_4251_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4251_cast_fp16")];
+            bool var_4253_transpose_x_1 = const()[name = string("op_4253_transpose_x_1"), val = bool(true)];
+            bool var_4253_transpose_y_1 = const()[name = string("op_4253_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4253_cast_fp16 = matmul(transpose_x = var_4253_transpose_x_1, transpose_y = var_4253_transpose_y_1, x = var_4251_cast_fp16, y = v_new_225_cast_fp16)[name = string("op_4253_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_227_cast_fp16 = add(x = var_4249_cast_fp16, y = var_4253_cast_fp16)[name = string("state_227_cast_fp16")];
+            tensor<int32, [4]> var_4256_begin_0 = const()[name = string("op_4256_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_4256_end_0 = const()[name = string("op_4256_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_4256_end_mask_0 = const()[name = string("op_4256_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4256_squeeze_mask_0 = const()[name = string("op_4256_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4256_cast_fp16 = slice_by_index(begin = var_4256_begin_0, end = var_4256_end_0, end_mask = var_4256_end_mask_0, squeeze_mask = var_4256_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4256_cast_fp16")];
+            tensor<int32, [4]> var_4258_begin_0 = const()[name = string("op_4258_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_4258_end_0 = const()[name = string("op_4258_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_4258_end_mask_0 = const()[name = string("op_4258_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4258_squeeze_mask_0 = const()[name = string("op_4258_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4258_cast_fp16 = slice_by_index(begin = var_4258_begin_0, end = var_4258_end_0, end_mask = var_4258_end_mask_0, squeeze_mask = var_4258_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4258_cast_fp16")];
+            bool var_4259_transpose_x_0 = const()[name = string("op_4259_transpose_x_0"), val = bool(false)];
+            bool var_4259_transpose_y_0 = const()[name = string("op_4259_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4259_cast_fp16 = matmul(transpose_x = var_4259_transpose_x_0, transpose_y = var_4259_transpose_y_0, x = var_4258_cast_fp16, y = state_227_cast_fp16)[name = string("op_4259_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_227_cast_fp16 = sub(x = var_4256_cast_fp16, y = var_4259_cast_fp16)[name = string("v_new_227_cast_fp16")];
+            tensor<int32, [4]> var_4262_begin_0 = const()[name = string("op_4262_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_4262_end_0 = const()[name = string("op_4262_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_4262_end_mask_0 = const()[name = string("op_4262_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4262_squeeze_mask_0 = const()[name = string("op_4262_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4262_cast_fp16 = slice_by_index(begin = var_4262_begin_0, end = var_4262_end_0, end_mask = var_4262_end_mask_0, squeeze_mask = var_4262_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4262_cast_fp16")];
+            bool var_4263_transpose_x_0 = const()[name = string("op_4263_transpose_x_0"), val = bool(false)];
+            bool var_4263_transpose_y_0 = const()[name = string("op_4263_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4263_cast_fp16 = matmul(transpose_x = var_4263_transpose_x_0, transpose_y = var_4263_transpose_y_0, x = var_4262_cast_fp16, y = state_227_cast_fp16)[name = string("op_4263_cast_fp16")];
+            tensor<int32, [4]> var_4265_begin_0 = const()[name = string("op_4265_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_4265_end_0 = const()[name = string("op_4265_end_0"), val = tensor<int32, [4]>([32, 18, 64, 64])];
+            tensor<bool, [4]> var_4265_end_mask_0 = const()[name = string("op_4265_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4265_squeeze_mask_0 = const()[name = string("op_4265_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4265_cast_fp16 = slice_by_index(begin = var_4265_begin_0, end = var_4265_end_0, end_mask = var_4265_end_mask_0, squeeze_mask = var_4265_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4265_cast_fp16")];
+            bool var_4266_transpose_x_0 = const()[name = string("op_4266_transpose_x_0"), val = bool(false)];
+            bool var_4266_transpose_y_0 = const()[name = string("op_4266_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4266_cast_fp16 = matmul(transpose_x = var_4266_transpose_x_0, transpose_y = var_4266_transpose_y_0, x = var_4265_cast_fp16, y = v_new_227_cast_fp16)[name = string("op_4266_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4267_cast_fp16 = add(x = var_4263_cast_fp16, y = var_4266_cast_fp16)[name = string("op_4267_cast_fp16")];
+            tensor<int32, [4]> var_4269_begin_0 = const()[name = string("op_4269_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_4269_end_0 = const()[name = string("op_4269_end_0"), val = tensor<int32, [4]>([32, 18, 1, 1])];
+            tensor<bool, [4]> var_4269_end_mask_0 = const()[name = string("op_4269_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4269_squeeze_mask_0 = const()[name = string("op_4269_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4269_cast_fp16 = slice_by_index(begin = var_4269_begin_0, end = var_4269_end_0, end_mask = var_4269_end_mask_0, squeeze_mask = var_4269_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4269_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4270_cast_fp16 = mul(x = state_227_cast_fp16, y = var_4269_cast_fp16)[name = string("op_4270_cast_fp16")];
+            tensor<int32, [4]> var_4272_begin_0 = const()[name = string("op_4272_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_4272_end_0 = const()[name = string("op_4272_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_4272_end_mask_0 = const()[name = string("op_4272_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4272_squeeze_mask_0 = const()[name = string("op_4272_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4272_cast_fp16 = slice_by_index(begin = var_4272_begin_0, end = var_4272_end_0, end_mask = var_4272_end_mask_0, squeeze_mask = var_4272_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4272_cast_fp16")];
+            bool var_4274_transpose_x_1 = const()[name = string("op_4274_transpose_x_1"), val = bool(true)];
+            bool var_4274_transpose_y_1 = const()[name = string("op_4274_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4274_cast_fp16 = matmul(transpose_x = var_4274_transpose_x_1, transpose_y = var_4274_transpose_y_1, x = var_4272_cast_fp16, y = v_new_227_cast_fp16)[name = string("op_4274_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_229_cast_fp16 = add(x = var_4270_cast_fp16, y = var_4274_cast_fp16)[name = string("state_229_cast_fp16")];
+            tensor<int32, [4]> var_4277_begin_0 = const()[name = string("op_4277_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_4277_end_0 = const()[name = string("op_4277_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_4277_end_mask_0 = const()[name = string("op_4277_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4277_squeeze_mask_0 = const()[name = string("op_4277_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4277_cast_fp16 = slice_by_index(begin = var_4277_begin_0, end = var_4277_end_0, end_mask = var_4277_end_mask_0, squeeze_mask = var_4277_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4277_cast_fp16")];
+            tensor<int32, [4]> var_4279_begin_0 = const()[name = string("op_4279_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_4279_end_0 = const()[name = string("op_4279_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_4279_end_mask_0 = const()[name = string("op_4279_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4279_squeeze_mask_0 = const()[name = string("op_4279_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4279_cast_fp16 = slice_by_index(begin = var_4279_begin_0, end = var_4279_end_0, end_mask = var_4279_end_mask_0, squeeze_mask = var_4279_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4279_cast_fp16")];
+            bool var_4280_transpose_x_0 = const()[name = string("op_4280_transpose_x_0"), val = bool(false)];
+            bool var_4280_transpose_y_0 = const()[name = string("op_4280_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4280_cast_fp16 = matmul(transpose_x = var_4280_transpose_x_0, transpose_y = var_4280_transpose_y_0, x = var_4279_cast_fp16, y = state_229_cast_fp16)[name = string("op_4280_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_229_cast_fp16 = sub(x = var_4277_cast_fp16, y = var_4280_cast_fp16)[name = string("v_new_229_cast_fp16")];
+            tensor<int32, [4]> var_4283_begin_0 = const()[name = string("op_4283_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_4283_end_0 = const()[name = string("op_4283_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_4283_end_mask_0 = const()[name = string("op_4283_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4283_squeeze_mask_0 = const()[name = string("op_4283_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4283_cast_fp16 = slice_by_index(begin = var_4283_begin_0, end = var_4283_end_0, end_mask = var_4283_end_mask_0, squeeze_mask = var_4283_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4283_cast_fp16")];
+            bool var_4284_transpose_x_0 = const()[name = string("op_4284_transpose_x_0"), val = bool(false)];
+            bool var_4284_transpose_y_0 = const()[name = string("op_4284_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4284_cast_fp16 = matmul(transpose_x = var_4284_transpose_x_0, transpose_y = var_4284_transpose_y_0, x = var_4283_cast_fp16, y = state_229_cast_fp16)[name = string("op_4284_cast_fp16")];
+            tensor<int32, [4]> var_4286_begin_0 = const()[name = string("op_4286_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_4286_end_0 = const()[name = string("op_4286_end_0"), val = tensor<int32, [4]>([32, 19, 64, 64])];
+            tensor<bool, [4]> var_4286_end_mask_0 = const()[name = string("op_4286_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4286_squeeze_mask_0 = const()[name = string("op_4286_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4286_cast_fp16 = slice_by_index(begin = var_4286_begin_0, end = var_4286_end_0, end_mask = var_4286_end_mask_0, squeeze_mask = var_4286_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4286_cast_fp16")];
+            bool var_4287_transpose_x_0 = const()[name = string("op_4287_transpose_x_0"), val = bool(false)];
+            bool var_4287_transpose_y_0 = const()[name = string("op_4287_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4287_cast_fp16 = matmul(transpose_x = var_4287_transpose_x_0, transpose_y = var_4287_transpose_y_0, x = var_4286_cast_fp16, y = v_new_229_cast_fp16)[name = string("op_4287_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4288_cast_fp16 = add(x = var_4284_cast_fp16, y = var_4287_cast_fp16)[name = string("op_4288_cast_fp16")];
+            tensor<int32, [4]> var_4290_begin_0 = const()[name = string("op_4290_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_4290_end_0 = const()[name = string("op_4290_end_0"), val = tensor<int32, [4]>([32, 19, 1, 1])];
+            tensor<bool, [4]> var_4290_end_mask_0 = const()[name = string("op_4290_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4290_squeeze_mask_0 = const()[name = string("op_4290_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4290_cast_fp16 = slice_by_index(begin = var_4290_begin_0, end = var_4290_end_0, end_mask = var_4290_end_mask_0, squeeze_mask = var_4290_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4290_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4291_cast_fp16 = mul(x = state_229_cast_fp16, y = var_4290_cast_fp16)[name = string("op_4291_cast_fp16")];
+            tensor<int32, [4]> var_4293_begin_0 = const()[name = string("op_4293_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_4293_end_0 = const()[name = string("op_4293_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_4293_end_mask_0 = const()[name = string("op_4293_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4293_squeeze_mask_0 = const()[name = string("op_4293_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4293_cast_fp16 = slice_by_index(begin = var_4293_begin_0, end = var_4293_end_0, end_mask = var_4293_end_mask_0, squeeze_mask = var_4293_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4293_cast_fp16")];
+            bool var_4295_transpose_x_1 = const()[name = string("op_4295_transpose_x_1"), val = bool(true)];
+            bool var_4295_transpose_y_1 = const()[name = string("op_4295_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4295_cast_fp16 = matmul(transpose_x = var_4295_transpose_x_1, transpose_y = var_4295_transpose_y_1, x = var_4293_cast_fp16, y = v_new_229_cast_fp16)[name = string("op_4295_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_231_cast_fp16 = add(x = var_4291_cast_fp16, y = var_4295_cast_fp16)[name = string("state_231_cast_fp16")];
+            tensor<int32, [4]> var_4298_begin_0 = const()[name = string("op_4298_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_4298_end_0 = const()[name = string("op_4298_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_4298_end_mask_0 = const()[name = string("op_4298_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4298_squeeze_mask_0 = const()[name = string("op_4298_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4298_cast_fp16 = slice_by_index(begin = var_4298_begin_0, end = var_4298_end_0, end_mask = var_4298_end_mask_0, squeeze_mask = var_4298_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4298_cast_fp16")];
+            tensor<int32, [4]> var_4300_begin_0 = const()[name = string("op_4300_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_4300_end_0 = const()[name = string("op_4300_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_4300_end_mask_0 = const()[name = string("op_4300_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4300_squeeze_mask_0 = const()[name = string("op_4300_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4300_cast_fp16 = slice_by_index(begin = var_4300_begin_0, end = var_4300_end_0, end_mask = var_4300_end_mask_0, squeeze_mask = var_4300_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4300_cast_fp16")];
+            bool var_4301_transpose_x_0 = const()[name = string("op_4301_transpose_x_0"), val = bool(false)];
+            bool var_4301_transpose_y_0 = const()[name = string("op_4301_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4301_cast_fp16 = matmul(transpose_x = var_4301_transpose_x_0, transpose_y = var_4301_transpose_y_0, x = var_4300_cast_fp16, y = state_231_cast_fp16)[name = string("op_4301_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_231_cast_fp16 = sub(x = var_4298_cast_fp16, y = var_4301_cast_fp16)[name = string("v_new_231_cast_fp16")];
+            tensor<int32, [4]> var_4304_begin_0 = const()[name = string("op_4304_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_4304_end_0 = const()[name = string("op_4304_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_4304_end_mask_0 = const()[name = string("op_4304_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4304_squeeze_mask_0 = const()[name = string("op_4304_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4304_cast_fp16 = slice_by_index(begin = var_4304_begin_0, end = var_4304_end_0, end_mask = var_4304_end_mask_0, squeeze_mask = var_4304_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4304_cast_fp16")];
+            bool var_4305_transpose_x_0 = const()[name = string("op_4305_transpose_x_0"), val = bool(false)];
+            bool var_4305_transpose_y_0 = const()[name = string("op_4305_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4305_cast_fp16 = matmul(transpose_x = var_4305_transpose_x_0, transpose_y = var_4305_transpose_y_0, x = var_4304_cast_fp16, y = state_231_cast_fp16)[name = string("op_4305_cast_fp16")];
+            tensor<int32, [4]> var_4307_begin_0 = const()[name = string("op_4307_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_4307_end_0 = const()[name = string("op_4307_end_0"), val = tensor<int32, [4]>([32, 20, 64, 64])];
+            tensor<bool, [4]> var_4307_end_mask_0 = const()[name = string("op_4307_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4307_squeeze_mask_0 = const()[name = string("op_4307_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4307_cast_fp16 = slice_by_index(begin = var_4307_begin_0, end = var_4307_end_0, end_mask = var_4307_end_mask_0, squeeze_mask = var_4307_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4307_cast_fp16")];
+            bool var_4308_transpose_x_0 = const()[name = string("op_4308_transpose_x_0"), val = bool(false)];
+            bool var_4308_transpose_y_0 = const()[name = string("op_4308_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4308_cast_fp16 = matmul(transpose_x = var_4308_transpose_x_0, transpose_y = var_4308_transpose_y_0, x = var_4307_cast_fp16, y = v_new_231_cast_fp16)[name = string("op_4308_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4309_cast_fp16 = add(x = var_4305_cast_fp16, y = var_4308_cast_fp16)[name = string("op_4309_cast_fp16")];
+            tensor<int32, [4]> var_4311_begin_0 = const()[name = string("op_4311_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_4311_end_0 = const()[name = string("op_4311_end_0"), val = tensor<int32, [4]>([32, 20, 1, 1])];
+            tensor<bool, [4]> var_4311_end_mask_0 = const()[name = string("op_4311_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4311_squeeze_mask_0 = const()[name = string("op_4311_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4311_cast_fp16 = slice_by_index(begin = var_4311_begin_0, end = var_4311_end_0, end_mask = var_4311_end_mask_0, squeeze_mask = var_4311_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4311_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4312_cast_fp16 = mul(x = state_231_cast_fp16, y = var_4311_cast_fp16)[name = string("op_4312_cast_fp16")];
+            tensor<int32, [4]> var_4314_begin_0 = const()[name = string("op_4314_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_4314_end_0 = const()[name = string("op_4314_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_4314_end_mask_0 = const()[name = string("op_4314_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4314_squeeze_mask_0 = const()[name = string("op_4314_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4314_cast_fp16 = slice_by_index(begin = var_4314_begin_0, end = var_4314_end_0, end_mask = var_4314_end_mask_0, squeeze_mask = var_4314_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4314_cast_fp16")];
+            bool var_4316_transpose_x_1 = const()[name = string("op_4316_transpose_x_1"), val = bool(true)];
+            bool var_4316_transpose_y_1 = const()[name = string("op_4316_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4316_cast_fp16 = matmul(transpose_x = var_4316_transpose_x_1, transpose_y = var_4316_transpose_y_1, x = var_4314_cast_fp16, y = v_new_231_cast_fp16)[name = string("op_4316_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_233_cast_fp16 = add(x = var_4312_cast_fp16, y = var_4316_cast_fp16)[name = string("state_233_cast_fp16")];
+            tensor<int32, [4]> var_4319_begin_0 = const()[name = string("op_4319_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_4319_end_0 = const()[name = string("op_4319_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_4319_end_mask_0 = const()[name = string("op_4319_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4319_squeeze_mask_0 = const()[name = string("op_4319_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4319_cast_fp16 = slice_by_index(begin = var_4319_begin_0, end = var_4319_end_0, end_mask = var_4319_end_mask_0, squeeze_mask = var_4319_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4319_cast_fp16")];
+            tensor<int32, [4]> var_4321_begin_0 = const()[name = string("op_4321_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_4321_end_0 = const()[name = string("op_4321_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_4321_end_mask_0 = const()[name = string("op_4321_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4321_squeeze_mask_0 = const()[name = string("op_4321_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4321_cast_fp16 = slice_by_index(begin = var_4321_begin_0, end = var_4321_end_0, end_mask = var_4321_end_mask_0, squeeze_mask = var_4321_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4321_cast_fp16")];
+            bool var_4322_transpose_x_0 = const()[name = string("op_4322_transpose_x_0"), val = bool(false)];
+            bool var_4322_transpose_y_0 = const()[name = string("op_4322_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4322_cast_fp16 = matmul(transpose_x = var_4322_transpose_x_0, transpose_y = var_4322_transpose_y_0, x = var_4321_cast_fp16, y = state_233_cast_fp16)[name = string("op_4322_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_233_cast_fp16 = sub(x = var_4319_cast_fp16, y = var_4322_cast_fp16)[name = string("v_new_233_cast_fp16")];
+            tensor<int32, [4]> var_4325_begin_0 = const()[name = string("op_4325_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_4325_end_0 = const()[name = string("op_4325_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_4325_end_mask_0 = const()[name = string("op_4325_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4325_squeeze_mask_0 = const()[name = string("op_4325_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4325_cast_fp16 = slice_by_index(begin = var_4325_begin_0, end = var_4325_end_0, end_mask = var_4325_end_mask_0, squeeze_mask = var_4325_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4325_cast_fp16")];
+            bool var_4326_transpose_x_0 = const()[name = string("op_4326_transpose_x_0"), val = bool(false)];
+            bool var_4326_transpose_y_0 = const()[name = string("op_4326_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4326_cast_fp16 = matmul(transpose_x = var_4326_transpose_x_0, transpose_y = var_4326_transpose_y_0, x = var_4325_cast_fp16, y = state_233_cast_fp16)[name = string("op_4326_cast_fp16")];
+            tensor<int32, [4]> var_4328_begin_0 = const()[name = string("op_4328_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_4328_end_0 = const()[name = string("op_4328_end_0"), val = tensor<int32, [4]>([32, 21, 64, 64])];
+            tensor<bool, [4]> var_4328_end_mask_0 = const()[name = string("op_4328_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4328_squeeze_mask_0 = const()[name = string("op_4328_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4328_cast_fp16 = slice_by_index(begin = var_4328_begin_0, end = var_4328_end_0, end_mask = var_4328_end_mask_0, squeeze_mask = var_4328_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4328_cast_fp16")];
+            bool var_4329_transpose_x_0 = const()[name = string("op_4329_transpose_x_0"), val = bool(false)];
+            bool var_4329_transpose_y_0 = const()[name = string("op_4329_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4329_cast_fp16 = matmul(transpose_x = var_4329_transpose_x_0, transpose_y = var_4329_transpose_y_0, x = var_4328_cast_fp16, y = v_new_233_cast_fp16)[name = string("op_4329_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4330_cast_fp16 = add(x = var_4326_cast_fp16, y = var_4329_cast_fp16)[name = string("op_4330_cast_fp16")];
+            tensor<int32, [4]> var_4332_begin_0 = const()[name = string("op_4332_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_4332_end_0 = const()[name = string("op_4332_end_0"), val = tensor<int32, [4]>([32, 21, 1, 1])];
+            tensor<bool, [4]> var_4332_end_mask_0 = const()[name = string("op_4332_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4332_squeeze_mask_0 = const()[name = string("op_4332_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4332_cast_fp16 = slice_by_index(begin = var_4332_begin_0, end = var_4332_end_0, end_mask = var_4332_end_mask_0, squeeze_mask = var_4332_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4332_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4333_cast_fp16 = mul(x = state_233_cast_fp16, y = var_4332_cast_fp16)[name = string("op_4333_cast_fp16")];
+            tensor<int32, [4]> var_4335_begin_0 = const()[name = string("op_4335_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_4335_end_0 = const()[name = string("op_4335_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_4335_end_mask_0 = const()[name = string("op_4335_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4335_squeeze_mask_0 = const()[name = string("op_4335_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4335_cast_fp16 = slice_by_index(begin = var_4335_begin_0, end = var_4335_end_0, end_mask = var_4335_end_mask_0, squeeze_mask = var_4335_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4335_cast_fp16")];
+            bool var_4337_transpose_x_1 = const()[name = string("op_4337_transpose_x_1"), val = bool(true)];
+            bool var_4337_transpose_y_1 = const()[name = string("op_4337_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4337_cast_fp16 = matmul(transpose_x = var_4337_transpose_x_1, transpose_y = var_4337_transpose_y_1, x = var_4335_cast_fp16, y = v_new_233_cast_fp16)[name = string("op_4337_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_235_cast_fp16 = add(x = var_4333_cast_fp16, y = var_4337_cast_fp16)[name = string("state_235_cast_fp16")];
+            tensor<int32, [4]> var_4340_begin_0 = const()[name = string("op_4340_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_4340_end_0 = const()[name = string("op_4340_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_4340_end_mask_0 = const()[name = string("op_4340_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4340_squeeze_mask_0 = const()[name = string("op_4340_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4340_cast_fp16 = slice_by_index(begin = var_4340_begin_0, end = var_4340_end_0, end_mask = var_4340_end_mask_0, squeeze_mask = var_4340_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4340_cast_fp16")];
+            tensor<int32, [4]> var_4342_begin_0 = const()[name = string("op_4342_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_4342_end_0 = const()[name = string("op_4342_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_4342_end_mask_0 = const()[name = string("op_4342_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4342_squeeze_mask_0 = const()[name = string("op_4342_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4342_cast_fp16 = slice_by_index(begin = var_4342_begin_0, end = var_4342_end_0, end_mask = var_4342_end_mask_0, squeeze_mask = var_4342_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4342_cast_fp16")];
+            bool var_4343_transpose_x_0 = const()[name = string("op_4343_transpose_x_0"), val = bool(false)];
+            bool var_4343_transpose_y_0 = const()[name = string("op_4343_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4343_cast_fp16 = matmul(transpose_x = var_4343_transpose_x_0, transpose_y = var_4343_transpose_y_0, x = var_4342_cast_fp16, y = state_235_cast_fp16)[name = string("op_4343_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_235_cast_fp16 = sub(x = var_4340_cast_fp16, y = var_4343_cast_fp16)[name = string("v_new_235_cast_fp16")];
+            tensor<int32, [4]> var_4346_begin_0 = const()[name = string("op_4346_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_4346_end_0 = const()[name = string("op_4346_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_4346_end_mask_0 = const()[name = string("op_4346_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4346_squeeze_mask_0 = const()[name = string("op_4346_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4346_cast_fp16 = slice_by_index(begin = var_4346_begin_0, end = var_4346_end_0, end_mask = var_4346_end_mask_0, squeeze_mask = var_4346_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4346_cast_fp16")];
+            bool var_4347_transpose_x_0 = const()[name = string("op_4347_transpose_x_0"), val = bool(false)];
+            bool var_4347_transpose_y_0 = const()[name = string("op_4347_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4347_cast_fp16 = matmul(transpose_x = var_4347_transpose_x_0, transpose_y = var_4347_transpose_y_0, x = var_4346_cast_fp16, y = state_235_cast_fp16)[name = string("op_4347_cast_fp16")];
+            tensor<int32, [4]> var_4349_begin_0 = const()[name = string("op_4349_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_4349_end_0 = const()[name = string("op_4349_end_0"), val = tensor<int32, [4]>([32, 22, 64, 64])];
+            tensor<bool, [4]> var_4349_end_mask_0 = const()[name = string("op_4349_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4349_squeeze_mask_0 = const()[name = string("op_4349_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4349_cast_fp16 = slice_by_index(begin = var_4349_begin_0, end = var_4349_end_0, end_mask = var_4349_end_mask_0, squeeze_mask = var_4349_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4349_cast_fp16")];
+            bool var_4350_transpose_x_0 = const()[name = string("op_4350_transpose_x_0"), val = bool(false)];
+            bool var_4350_transpose_y_0 = const()[name = string("op_4350_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4350_cast_fp16 = matmul(transpose_x = var_4350_transpose_x_0, transpose_y = var_4350_transpose_y_0, x = var_4349_cast_fp16, y = v_new_235_cast_fp16)[name = string("op_4350_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4351_cast_fp16 = add(x = var_4347_cast_fp16, y = var_4350_cast_fp16)[name = string("op_4351_cast_fp16")];
+            tensor<int32, [4]> var_4353_begin_0 = const()[name = string("op_4353_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_4353_end_0 = const()[name = string("op_4353_end_0"), val = tensor<int32, [4]>([32, 22, 1, 1])];
+            tensor<bool, [4]> var_4353_end_mask_0 = const()[name = string("op_4353_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4353_squeeze_mask_0 = const()[name = string("op_4353_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4353_cast_fp16 = slice_by_index(begin = var_4353_begin_0, end = var_4353_end_0, end_mask = var_4353_end_mask_0, squeeze_mask = var_4353_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4353_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4354_cast_fp16 = mul(x = state_235_cast_fp16, y = var_4353_cast_fp16)[name = string("op_4354_cast_fp16")];
+            tensor<int32, [4]> var_4356_begin_0 = const()[name = string("op_4356_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_4356_end_0 = const()[name = string("op_4356_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_4356_end_mask_0 = const()[name = string("op_4356_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4356_squeeze_mask_0 = const()[name = string("op_4356_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4356_cast_fp16 = slice_by_index(begin = var_4356_begin_0, end = var_4356_end_0, end_mask = var_4356_end_mask_0, squeeze_mask = var_4356_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4356_cast_fp16")];
+            bool var_4358_transpose_x_1 = const()[name = string("op_4358_transpose_x_1"), val = bool(true)];
+            bool var_4358_transpose_y_1 = const()[name = string("op_4358_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4358_cast_fp16 = matmul(transpose_x = var_4358_transpose_x_1, transpose_y = var_4358_transpose_y_1, x = var_4356_cast_fp16, y = v_new_235_cast_fp16)[name = string("op_4358_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_237_cast_fp16 = add(x = var_4354_cast_fp16, y = var_4358_cast_fp16)[name = string("state_237_cast_fp16")];
+            tensor<int32, [4]> var_4361_begin_0 = const()[name = string("op_4361_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_4361_end_0 = const()[name = string("op_4361_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_4361_end_mask_0 = const()[name = string("op_4361_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4361_squeeze_mask_0 = const()[name = string("op_4361_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4361_cast_fp16 = slice_by_index(begin = var_4361_begin_0, end = var_4361_end_0, end_mask = var_4361_end_mask_0, squeeze_mask = var_4361_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4361_cast_fp16")];
+            tensor<int32, [4]> var_4363_begin_0 = const()[name = string("op_4363_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_4363_end_0 = const()[name = string("op_4363_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_4363_end_mask_0 = const()[name = string("op_4363_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4363_squeeze_mask_0 = const()[name = string("op_4363_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4363_cast_fp16 = slice_by_index(begin = var_4363_begin_0, end = var_4363_end_0, end_mask = var_4363_end_mask_0, squeeze_mask = var_4363_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4363_cast_fp16")];
+            bool var_4364_transpose_x_0 = const()[name = string("op_4364_transpose_x_0"), val = bool(false)];
+            bool var_4364_transpose_y_0 = const()[name = string("op_4364_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4364_cast_fp16 = matmul(transpose_x = var_4364_transpose_x_0, transpose_y = var_4364_transpose_y_0, x = var_4363_cast_fp16, y = state_237_cast_fp16)[name = string("op_4364_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_237_cast_fp16 = sub(x = var_4361_cast_fp16, y = var_4364_cast_fp16)[name = string("v_new_237_cast_fp16")];
+            tensor<int32, [4]> var_4367_begin_0 = const()[name = string("op_4367_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_4367_end_0 = const()[name = string("op_4367_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_4367_end_mask_0 = const()[name = string("op_4367_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4367_squeeze_mask_0 = const()[name = string("op_4367_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4367_cast_fp16 = slice_by_index(begin = var_4367_begin_0, end = var_4367_end_0, end_mask = var_4367_end_mask_0, squeeze_mask = var_4367_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4367_cast_fp16")];
+            bool var_4368_transpose_x_0 = const()[name = string("op_4368_transpose_x_0"), val = bool(false)];
+            bool var_4368_transpose_y_0 = const()[name = string("op_4368_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4368_cast_fp16 = matmul(transpose_x = var_4368_transpose_x_0, transpose_y = var_4368_transpose_y_0, x = var_4367_cast_fp16, y = state_237_cast_fp16)[name = string("op_4368_cast_fp16")];
+            tensor<int32, [4]> var_4370_begin_0 = const()[name = string("op_4370_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_4370_end_0 = const()[name = string("op_4370_end_0"), val = tensor<int32, [4]>([32, 23, 64, 64])];
+            tensor<bool, [4]> var_4370_end_mask_0 = const()[name = string("op_4370_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4370_squeeze_mask_0 = const()[name = string("op_4370_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4370_cast_fp16 = slice_by_index(begin = var_4370_begin_0, end = var_4370_end_0, end_mask = var_4370_end_mask_0, squeeze_mask = var_4370_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4370_cast_fp16")];
+            bool var_4371_transpose_x_0 = const()[name = string("op_4371_transpose_x_0"), val = bool(false)];
+            bool var_4371_transpose_y_0 = const()[name = string("op_4371_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4371_cast_fp16 = matmul(transpose_x = var_4371_transpose_x_0, transpose_y = var_4371_transpose_y_0, x = var_4370_cast_fp16, y = v_new_237_cast_fp16)[name = string("op_4371_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4372_cast_fp16 = add(x = var_4368_cast_fp16, y = var_4371_cast_fp16)[name = string("op_4372_cast_fp16")];
+            tensor<int32, [4]> var_4374_begin_0 = const()[name = string("op_4374_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_4374_end_0 = const()[name = string("op_4374_end_0"), val = tensor<int32, [4]>([32, 23, 1, 1])];
+            tensor<bool, [4]> var_4374_end_mask_0 = const()[name = string("op_4374_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4374_squeeze_mask_0 = const()[name = string("op_4374_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4374_cast_fp16 = slice_by_index(begin = var_4374_begin_0, end = var_4374_end_0, end_mask = var_4374_end_mask_0, squeeze_mask = var_4374_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4374_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4375_cast_fp16 = mul(x = state_237_cast_fp16, y = var_4374_cast_fp16)[name = string("op_4375_cast_fp16")];
+            tensor<int32, [4]> var_4377_begin_0 = const()[name = string("op_4377_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_4377_end_0 = const()[name = string("op_4377_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_4377_end_mask_0 = const()[name = string("op_4377_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4377_squeeze_mask_0 = const()[name = string("op_4377_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4377_cast_fp16 = slice_by_index(begin = var_4377_begin_0, end = var_4377_end_0, end_mask = var_4377_end_mask_0, squeeze_mask = var_4377_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4377_cast_fp16")];
+            bool var_4379_transpose_x_1 = const()[name = string("op_4379_transpose_x_1"), val = bool(true)];
+            bool var_4379_transpose_y_1 = const()[name = string("op_4379_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4379_cast_fp16 = matmul(transpose_x = var_4379_transpose_x_1, transpose_y = var_4379_transpose_y_1, x = var_4377_cast_fp16, y = v_new_237_cast_fp16)[name = string("op_4379_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_239_cast_fp16 = add(x = var_4375_cast_fp16, y = var_4379_cast_fp16)[name = string("state_239_cast_fp16")];
+            tensor<int32, [4]> var_4382_begin_0 = const()[name = string("op_4382_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_4382_end_0 = const()[name = string("op_4382_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_4382_end_mask_0 = const()[name = string("op_4382_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4382_squeeze_mask_0 = const()[name = string("op_4382_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4382_cast_fp16 = slice_by_index(begin = var_4382_begin_0, end = var_4382_end_0, end_mask = var_4382_end_mask_0, squeeze_mask = var_4382_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4382_cast_fp16")];
+            tensor<int32, [4]> var_4384_begin_0 = const()[name = string("op_4384_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_4384_end_0 = const()[name = string("op_4384_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_4384_end_mask_0 = const()[name = string("op_4384_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4384_squeeze_mask_0 = const()[name = string("op_4384_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4384_cast_fp16 = slice_by_index(begin = var_4384_begin_0, end = var_4384_end_0, end_mask = var_4384_end_mask_0, squeeze_mask = var_4384_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4384_cast_fp16")];
+            bool var_4385_transpose_x_0 = const()[name = string("op_4385_transpose_x_0"), val = bool(false)];
+            bool var_4385_transpose_y_0 = const()[name = string("op_4385_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4385_cast_fp16 = matmul(transpose_x = var_4385_transpose_x_0, transpose_y = var_4385_transpose_y_0, x = var_4384_cast_fp16, y = state_239_cast_fp16)[name = string("op_4385_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_239_cast_fp16 = sub(x = var_4382_cast_fp16, y = var_4385_cast_fp16)[name = string("v_new_239_cast_fp16")];
+            tensor<int32, [4]> var_4388_begin_0 = const()[name = string("op_4388_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_4388_end_0 = const()[name = string("op_4388_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_4388_end_mask_0 = const()[name = string("op_4388_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4388_squeeze_mask_0 = const()[name = string("op_4388_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4388_cast_fp16 = slice_by_index(begin = var_4388_begin_0, end = var_4388_end_0, end_mask = var_4388_end_mask_0, squeeze_mask = var_4388_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4388_cast_fp16")];
+            bool var_4389_transpose_x_0 = const()[name = string("op_4389_transpose_x_0"), val = bool(false)];
+            bool var_4389_transpose_y_0 = const()[name = string("op_4389_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4389_cast_fp16 = matmul(transpose_x = var_4389_transpose_x_0, transpose_y = var_4389_transpose_y_0, x = var_4388_cast_fp16, y = state_239_cast_fp16)[name = string("op_4389_cast_fp16")];
+            tensor<int32, [4]> var_4391_begin_0 = const()[name = string("op_4391_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_4391_end_0 = const()[name = string("op_4391_end_0"), val = tensor<int32, [4]>([32, 24, 64, 64])];
+            tensor<bool, [4]> var_4391_end_mask_0 = const()[name = string("op_4391_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4391_squeeze_mask_0 = const()[name = string("op_4391_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4391_cast_fp16 = slice_by_index(begin = var_4391_begin_0, end = var_4391_end_0, end_mask = var_4391_end_mask_0, squeeze_mask = var_4391_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4391_cast_fp16")];
+            bool var_4392_transpose_x_0 = const()[name = string("op_4392_transpose_x_0"), val = bool(false)];
+            bool var_4392_transpose_y_0 = const()[name = string("op_4392_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4392_cast_fp16 = matmul(transpose_x = var_4392_transpose_x_0, transpose_y = var_4392_transpose_y_0, x = var_4391_cast_fp16, y = v_new_239_cast_fp16)[name = string("op_4392_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4393_cast_fp16 = add(x = var_4389_cast_fp16, y = var_4392_cast_fp16)[name = string("op_4393_cast_fp16")];
+            tensor<int32, [4]> var_4395_begin_0 = const()[name = string("op_4395_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_4395_end_0 = const()[name = string("op_4395_end_0"), val = tensor<int32, [4]>([32, 24, 1, 1])];
+            tensor<bool, [4]> var_4395_end_mask_0 = const()[name = string("op_4395_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4395_squeeze_mask_0 = const()[name = string("op_4395_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4395_cast_fp16 = slice_by_index(begin = var_4395_begin_0, end = var_4395_end_0, end_mask = var_4395_end_mask_0, squeeze_mask = var_4395_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4395_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4396_cast_fp16 = mul(x = state_239_cast_fp16, y = var_4395_cast_fp16)[name = string("op_4396_cast_fp16")];
+            tensor<int32, [4]> var_4398_begin_0 = const()[name = string("op_4398_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_4398_end_0 = const()[name = string("op_4398_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_4398_end_mask_0 = const()[name = string("op_4398_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4398_squeeze_mask_0 = const()[name = string("op_4398_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4398_cast_fp16 = slice_by_index(begin = var_4398_begin_0, end = var_4398_end_0, end_mask = var_4398_end_mask_0, squeeze_mask = var_4398_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4398_cast_fp16")];
+            bool var_4400_transpose_x_1 = const()[name = string("op_4400_transpose_x_1"), val = bool(true)];
+            bool var_4400_transpose_y_1 = const()[name = string("op_4400_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4400_cast_fp16 = matmul(transpose_x = var_4400_transpose_x_1, transpose_y = var_4400_transpose_y_1, x = var_4398_cast_fp16, y = v_new_239_cast_fp16)[name = string("op_4400_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_241_cast_fp16 = add(x = var_4396_cast_fp16, y = var_4400_cast_fp16)[name = string("state_241_cast_fp16")];
+            tensor<int32, [4]> var_4403_begin_0 = const()[name = string("op_4403_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_4403_end_0 = const()[name = string("op_4403_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_4403_end_mask_0 = const()[name = string("op_4403_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4403_squeeze_mask_0 = const()[name = string("op_4403_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4403_cast_fp16 = slice_by_index(begin = var_4403_begin_0, end = var_4403_end_0, end_mask = var_4403_end_mask_0, squeeze_mask = var_4403_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4403_cast_fp16")];
+            tensor<int32, [4]> var_4405_begin_0 = const()[name = string("op_4405_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_4405_end_0 = const()[name = string("op_4405_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_4405_end_mask_0 = const()[name = string("op_4405_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4405_squeeze_mask_0 = const()[name = string("op_4405_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4405_cast_fp16 = slice_by_index(begin = var_4405_begin_0, end = var_4405_end_0, end_mask = var_4405_end_mask_0, squeeze_mask = var_4405_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4405_cast_fp16")];
+            bool var_4406_transpose_x_0 = const()[name = string("op_4406_transpose_x_0"), val = bool(false)];
+            bool var_4406_transpose_y_0 = const()[name = string("op_4406_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4406_cast_fp16 = matmul(transpose_x = var_4406_transpose_x_0, transpose_y = var_4406_transpose_y_0, x = var_4405_cast_fp16, y = state_241_cast_fp16)[name = string("op_4406_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_241_cast_fp16 = sub(x = var_4403_cast_fp16, y = var_4406_cast_fp16)[name = string("v_new_241_cast_fp16")];
+            tensor<int32, [4]> var_4409_begin_0 = const()[name = string("op_4409_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_4409_end_0 = const()[name = string("op_4409_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_4409_end_mask_0 = const()[name = string("op_4409_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4409_squeeze_mask_0 = const()[name = string("op_4409_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4409_cast_fp16 = slice_by_index(begin = var_4409_begin_0, end = var_4409_end_0, end_mask = var_4409_end_mask_0, squeeze_mask = var_4409_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4409_cast_fp16")];
+            bool var_4410_transpose_x_0 = const()[name = string("op_4410_transpose_x_0"), val = bool(false)];
+            bool var_4410_transpose_y_0 = const()[name = string("op_4410_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4410_cast_fp16 = matmul(transpose_x = var_4410_transpose_x_0, transpose_y = var_4410_transpose_y_0, x = var_4409_cast_fp16, y = state_241_cast_fp16)[name = string("op_4410_cast_fp16")];
+            tensor<int32, [4]> var_4412_begin_0 = const()[name = string("op_4412_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_4412_end_0 = const()[name = string("op_4412_end_0"), val = tensor<int32, [4]>([32, 25, 64, 64])];
+            tensor<bool, [4]> var_4412_end_mask_0 = const()[name = string("op_4412_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4412_squeeze_mask_0 = const()[name = string("op_4412_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4412_cast_fp16 = slice_by_index(begin = var_4412_begin_0, end = var_4412_end_0, end_mask = var_4412_end_mask_0, squeeze_mask = var_4412_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4412_cast_fp16")];
+            bool var_4413_transpose_x_0 = const()[name = string("op_4413_transpose_x_0"), val = bool(false)];
+            bool var_4413_transpose_y_0 = const()[name = string("op_4413_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4413_cast_fp16 = matmul(transpose_x = var_4413_transpose_x_0, transpose_y = var_4413_transpose_y_0, x = var_4412_cast_fp16, y = v_new_241_cast_fp16)[name = string("op_4413_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4414_cast_fp16 = add(x = var_4410_cast_fp16, y = var_4413_cast_fp16)[name = string("op_4414_cast_fp16")];
+            tensor<int32, [4]> var_4416_begin_0 = const()[name = string("op_4416_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_4416_end_0 = const()[name = string("op_4416_end_0"), val = tensor<int32, [4]>([32, 25, 1, 1])];
+            tensor<bool, [4]> var_4416_end_mask_0 = const()[name = string("op_4416_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4416_squeeze_mask_0 = const()[name = string("op_4416_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4416_cast_fp16 = slice_by_index(begin = var_4416_begin_0, end = var_4416_end_0, end_mask = var_4416_end_mask_0, squeeze_mask = var_4416_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4416_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4417_cast_fp16 = mul(x = state_241_cast_fp16, y = var_4416_cast_fp16)[name = string("op_4417_cast_fp16")];
+            tensor<int32, [4]> var_4419_begin_0 = const()[name = string("op_4419_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_4419_end_0 = const()[name = string("op_4419_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_4419_end_mask_0 = const()[name = string("op_4419_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4419_squeeze_mask_0 = const()[name = string("op_4419_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4419_cast_fp16 = slice_by_index(begin = var_4419_begin_0, end = var_4419_end_0, end_mask = var_4419_end_mask_0, squeeze_mask = var_4419_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4419_cast_fp16")];
+            bool var_4421_transpose_x_1 = const()[name = string("op_4421_transpose_x_1"), val = bool(true)];
+            bool var_4421_transpose_y_1 = const()[name = string("op_4421_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4421_cast_fp16 = matmul(transpose_x = var_4421_transpose_x_1, transpose_y = var_4421_transpose_y_1, x = var_4419_cast_fp16, y = v_new_241_cast_fp16)[name = string("op_4421_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_243_cast_fp16 = add(x = var_4417_cast_fp16, y = var_4421_cast_fp16)[name = string("state_243_cast_fp16")];
+            tensor<int32, [4]> var_4424_begin_0 = const()[name = string("op_4424_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_4424_end_0 = const()[name = string("op_4424_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_4424_end_mask_0 = const()[name = string("op_4424_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4424_squeeze_mask_0 = const()[name = string("op_4424_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4424_cast_fp16 = slice_by_index(begin = var_4424_begin_0, end = var_4424_end_0, end_mask = var_4424_end_mask_0, squeeze_mask = var_4424_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4424_cast_fp16")];
+            tensor<int32, [4]> var_4426_begin_0 = const()[name = string("op_4426_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_4426_end_0 = const()[name = string("op_4426_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_4426_end_mask_0 = const()[name = string("op_4426_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4426_squeeze_mask_0 = const()[name = string("op_4426_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4426_cast_fp16 = slice_by_index(begin = var_4426_begin_0, end = var_4426_end_0, end_mask = var_4426_end_mask_0, squeeze_mask = var_4426_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4426_cast_fp16")];
+            bool var_4427_transpose_x_0 = const()[name = string("op_4427_transpose_x_0"), val = bool(false)];
+            bool var_4427_transpose_y_0 = const()[name = string("op_4427_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4427_cast_fp16 = matmul(transpose_x = var_4427_transpose_x_0, transpose_y = var_4427_transpose_y_0, x = var_4426_cast_fp16, y = state_243_cast_fp16)[name = string("op_4427_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_243_cast_fp16 = sub(x = var_4424_cast_fp16, y = var_4427_cast_fp16)[name = string("v_new_243_cast_fp16")];
+            tensor<int32, [4]> var_4430_begin_0 = const()[name = string("op_4430_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_4430_end_0 = const()[name = string("op_4430_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_4430_end_mask_0 = const()[name = string("op_4430_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4430_squeeze_mask_0 = const()[name = string("op_4430_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4430_cast_fp16 = slice_by_index(begin = var_4430_begin_0, end = var_4430_end_0, end_mask = var_4430_end_mask_0, squeeze_mask = var_4430_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4430_cast_fp16")];
+            bool var_4431_transpose_x_0 = const()[name = string("op_4431_transpose_x_0"), val = bool(false)];
+            bool var_4431_transpose_y_0 = const()[name = string("op_4431_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4431_cast_fp16 = matmul(transpose_x = var_4431_transpose_x_0, transpose_y = var_4431_transpose_y_0, x = var_4430_cast_fp16, y = state_243_cast_fp16)[name = string("op_4431_cast_fp16")];
+            tensor<int32, [4]> var_4433_begin_0 = const()[name = string("op_4433_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_4433_end_0 = const()[name = string("op_4433_end_0"), val = tensor<int32, [4]>([32, 26, 64, 64])];
+            tensor<bool, [4]> var_4433_end_mask_0 = const()[name = string("op_4433_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4433_squeeze_mask_0 = const()[name = string("op_4433_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4433_cast_fp16 = slice_by_index(begin = var_4433_begin_0, end = var_4433_end_0, end_mask = var_4433_end_mask_0, squeeze_mask = var_4433_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4433_cast_fp16")];
+            bool var_4434_transpose_x_0 = const()[name = string("op_4434_transpose_x_0"), val = bool(false)];
+            bool var_4434_transpose_y_0 = const()[name = string("op_4434_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4434_cast_fp16 = matmul(transpose_x = var_4434_transpose_x_0, transpose_y = var_4434_transpose_y_0, x = var_4433_cast_fp16, y = v_new_243_cast_fp16)[name = string("op_4434_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4435_cast_fp16 = add(x = var_4431_cast_fp16, y = var_4434_cast_fp16)[name = string("op_4435_cast_fp16")];
+            tensor<int32, [4]> var_4437_begin_0 = const()[name = string("op_4437_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_4437_end_0 = const()[name = string("op_4437_end_0"), val = tensor<int32, [4]>([32, 26, 1, 1])];
+            tensor<bool, [4]> var_4437_end_mask_0 = const()[name = string("op_4437_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4437_squeeze_mask_0 = const()[name = string("op_4437_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4437_cast_fp16 = slice_by_index(begin = var_4437_begin_0, end = var_4437_end_0, end_mask = var_4437_end_mask_0, squeeze_mask = var_4437_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4437_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4438_cast_fp16 = mul(x = state_243_cast_fp16, y = var_4437_cast_fp16)[name = string("op_4438_cast_fp16")];
+            tensor<int32, [4]> var_4440_begin_0 = const()[name = string("op_4440_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_4440_end_0 = const()[name = string("op_4440_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_4440_end_mask_0 = const()[name = string("op_4440_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4440_squeeze_mask_0 = const()[name = string("op_4440_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4440_cast_fp16 = slice_by_index(begin = var_4440_begin_0, end = var_4440_end_0, end_mask = var_4440_end_mask_0, squeeze_mask = var_4440_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4440_cast_fp16")];
+            bool var_4442_transpose_x_1 = const()[name = string("op_4442_transpose_x_1"), val = bool(true)];
+            bool var_4442_transpose_y_1 = const()[name = string("op_4442_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4442_cast_fp16 = matmul(transpose_x = var_4442_transpose_x_1, transpose_y = var_4442_transpose_y_1, x = var_4440_cast_fp16, y = v_new_243_cast_fp16)[name = string("op_4442_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_245_cast_fp16 = add(x = var_4438_cast_fp16, y = var_4442_cast_fp16)[name = string("state_245_cast_fp16")];
+            tensor<int32, [4]> var_4445_begin_0 = const()[name = string("op_4445_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_4445_end_0 = const()[name = string("op_4445_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_4445_end_mask_0 = const()[name = string("op_4445_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4445_squeeze_mask_0 = const()[name = string("op_4445_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4445_cast_fp16 = slice_by_index(begin = var_4445_begin_0, end = var_4445_end_0, end_mask = var_4445_end_mask_0, squeeze_mask = var_4445_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4445_cast_fp16")];
+            tensor<int32, [4]> var_4447_begin_0 = const()[name = string("op_4447_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_4447_end_0 = const()[name = string("op_4447_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_4447_end_mask_0 = const()[name = string("op_4447_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4447_squeeze_mask_0 = const()[name = string("op_4447_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4447_cast_fp16 = slice_by_index(begin = var_4447_begin_0, end = var_4447_end_0, end_mask = var_4447_end_mask_0, squeeze_mask = var_4447_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4447_cast_fp16")];
+            bool var_4448_transpose_x_0 = const()[name = string("op_4448_transpose_x_0"), val = bool(false)];
+            bool var_4448_transpose_y_0 = const()[name = string("op_4448_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4448_cast_fp16 = matmul(transpose_x = var_4448_transpose_x_0, transpose_y = var_4448_transpose_y_0, x = var_4447_cast_fp16, y = state_245_cast_fp16)[name = string("op_4448_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_245_cast_fp16 = sub(x = var_4445_cast_fp16, y = var_4448_cast_fp16)[name = string("v_new_245_cast_fp16")];
+            tensor<int32, [4]> var_4451_begin_0 = const()[name = string("op_4451_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_4451_end_0 = const()[name = string("op_4451_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_4451_end_mask_0 = const()[name = string("op_4451_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4451_squeeze_mask_0 = const()[name = string("op_4451_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4451_cast_fp16 = slice_by_index(begin = var_4451_begin_0, end = var_4451_end_0, end_mask = var_4451_end_mask_0, squeeze_mask = var_4451_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4451_cast_fp16")];
+            bool var_4452_transpose_x_0 = const()[name = string("op_4452_transpose_x_0"), val = bool(false)];
+            bool var_4452_transpose_y_0 = const()[name = string("op_4452_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4452_cast_fp16 = matmul(transpose_x = var_4452_transpose_x_0, transpose_y = var_4452_transpose_y_0, x = var_4451_cast_fp16, y = state_245_cast_fp16)[name = string("op_4452_cast_fp16")];
+            tensor<int32, [4]> var_4454_begin_0 = const()[name = string("op_4454_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_4454_end_0 = const()[name = string("op_4454_end_0"), val = tensor<int32, [4]>([32, 27, 64, 64])];
+            tensor<bool, [4]> var_4454_end_mask_0 = const()[name = string("op_4454_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4454_squeeze_mask_0 = const()[name = string("op_4454_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4454_cast_fp16 = slice_by_index(begin = var_4454_begin_0, end = var_4454_end_0, end_mask = var_4454_end_mask_0, squeeze_mask = var_4454_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4454_cast_fp16")];
+            bool var_4455_transpose_x_0 = const()[name = string("op_4455_transpose_x_0"), val = bool(false)];
+            bool var_4455_transpose_y_0 = const()[name = string("op_4455_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4455_cast_fp16 = matmul(transpose_x = var_4455_transpose_x_0, transpose_y = var_4455_transpose_y_0, x = var_4454_cast_fp16, y = v_new_245_cast_fp16)[name = string("op_4455_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4456_cast_fp16 = add(x = var_4452_cast_fp16, y = var_4455_cast_fp16)[name = string("op_4456_cast_fp16")];
+            tensor<int32, [4]> var_4458_begin_0 = const()[name = string("op_4458_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_4458_end_0 = const()[name = string("op_4458_end_0"), val = tensor<int32, [4]>([32, 27, 1, 1])];
+            tensor<bool, [4]> var_4458_end_mask_0 = const()[name = string("op_4458_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4458_squeeze_mask_0 = const()[name = string("op_4458_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4458_cast_fp16 = slice_by_index(begin = var_4458_begin_0, end = var_4458_end_0, end_mask = var_4458_end_mask_0, squeeze_mask = var_4458_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4458_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4459_cast_fp16 = mul(x = state_245_cast_fp16, y = var_4458_cast_fp16)[name = string("op_4459_cast_fp16")];
+            tensor<int32, [4]> var_4461_begin_0 = const()[name = string("op_4461_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_4461_end_0 = const()[name = string("op_4461_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_4461_end_mask_0 = const()[name = string("op_4461_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4461_squeeze_mask_0 = const()[name = string("op_4461_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4461_cast_fp16 = slice_by_index(begin = var_4461_begin_0, end = var_4461_end_0, end_mask = var_4461_end_mask_0, squeeze_mask = var_4461_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4461_cast_fp16")];
+            bool var_4463_transpose_x_1 = const()[name = string("op_4463_transpose_x_1"), val = bool(true)];
+            bool var_4463_transpose_y_1 = const()[name = string("op_4463_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4463_cast_fp16 = matmul(transpose_x = var_4463_transpose_x_1, transpose_y = var_4463_transpose_y_1, x = var_4461_cast_fp16, y = v_new_245_cast_fp16)[name = string("op_4463_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_247_cast_fp16 = add(x = var_4459_cast_fp16, y = var_4463_cast_fp16)[name = string("state_247_cast_fp16")];
+            tensor<int32, [4]> var_4466_begin_0 = const()[name = string("op_4466_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_4466_end_0 = const()[name = string("op_4466_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_4466_end_mask_0 = const()[name = string("op_4466_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4466_squeeze_mask_0 = const()[name = string("op_4466_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4466_cast_fp16 = slice_by_index(begin = var_4466_begin_0, end = var_4466_end_0, end_mask = var_4466_end_mask_0, squeeze_mask = var_4466_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4466_cast_fp16")];
+            tensor<int32, [4]> var_4468_begin_0 = const()[name = string("op_4468_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_4468_end_0 = const()[name = string("op_4468_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_4468_end_mask_0 = const()[name = string("op_4468_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4468_squeeze_mask_0 = const()[name = string("op_4468_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4468_cast_fp16 = slice_by_index(begin = var_4468_begin_0, end = var_4468_end_0, end_mask = var_4468_end_mask_0, squeeze_mask = var_4468_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4468_cast_fp16")];
+            bool var_4469_transpose_x_0 = const()[name = string("op_4469_transpose_x_0"), val = bool(false)];
+            bool var_4469_transpose_y_0 = const()[name = string("op_4469_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4469_cast_fp16 = matmul(transpose_x = var_4469_transpose_x_0, transpose_y = var_4469_transpose_y_0, x = var_4468_cast_fp16, y = state_247_cast_fp16)[name = string("op_4469_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_247_cast_fp16 = sub(x = var_4466_cast_fp16, y = var_4469_cast_fp16)[name = string("v_new_247_cast_fp16")];
+            tensor<int32, [4]> var_4472_begin_0 = const()[name = string("op_4472_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_4472_end_0 = const()[name = string("op_4472_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_4472_end_mask_0 = const()[name = string("op_4472_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4472_squeeze_mask_0 = const()[name = string("op_4472_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4472_cast_fp16 = slice_by_index(begin = var_4472_begin_0, end = var_4472_end_0, end_mask = var_4472_end_mask_0, squeeze_mask = var_4472_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4472_cast_fp16")];
+            bool var_4473_transpose_x_0 = const()[name = string("op_4473_transpose_x_0"), val = bool(false)];
+            bool var_4473_transpose_y_0 = const()[name = string("op_4473_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4473_cast_fp16 = matmul(transpose_x = var_4473_transpose_x_0, transpose_y = var_4473_transpose_y_0, x = var_4472_cast_fp16, y = state_247_cast_fp16)[name = string("op_4473_cast_fp16")];
+            tensor<int32, [4]> var_4475_begin_0 = const()[name = string("op_4475_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_4475_end_0 = const()[name = string("op_4475_end_0"), val = tensor<int32, [4]>([32, 28, 64, 64])];
+            tensor<bool, [4]> var_4475_end_mask_0 = const()[name = string("op_4475_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4475_squeeze_mask_0 = const()[name = string("op_4475_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4475_cast_fp16 = slice_by_index(begin = var_4475_begin_0, end = var_4475_end_0, end_mask = var_4475_end_mask_0, squeeze_mask = var_4475_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4475_cast_fp16")];
+            bool var_4476_transpose_x_0 = const()[name = string("op_4476_transpose_x_0"), val = bool(false)];
+            bool var_4476_transpose_y_0 = const()[name = string("op_4476_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4476_cast_fp16 = matmul(transpose_x = var_4476_transpose_x_0, transpose_y = var_4476_transpose_y_0, x = var_4475_cast_fp16, y = v_new_247_cast_fp16)[name = string("op_4476_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4477_cast_fp16 = add(x = var_4473_cast_fp16, y = var_4476_cast_fp16)[name = string("op_4477_cast_fp16")];
+            tensor<int32, [4]> var_4479_begin_0 = const()[name = string("op_4479_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_4479_end_0 = const()[name = string("op_4479_end_0"), val = tensor<int32, [4]>([32, 28, 1, 1])];
+            tensor<bool, [4]> var_4479_end_mask_0 = const()[name = string("op_4479_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4479_squeeze_mask_0 = const()[name = string("op_4479_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4479_cast_fp16 = slice_by_index(begin = var_4479_begin_0, end = var_4479_end_0, end_mask = var_4479_end_mask_0, squeeze_mask = var_4479_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4479_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4480_cast_fp16 = mul(x = state_247_cast_fp16, y = var_4479_cast_fp16)[name = string("op_4480_cast_fp16")];
+            tensor<int32, [4]> var_4482_begin_0 = const()[name = string("op_4482_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_4482_end_0 = const()[name = string("op_4482_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_4482_end_mask_0 = const()[name = string("op_4482_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4482_squeeze_mask_0 = const()[name = string("op_4482_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4482_cast_fp16 = slice_by_index(begin = var_4482_begin_0, end = var_4482_end_0, end_mask = var_4482_end_mask_0, squeeze_mask = var_4482_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4482_cast_fp16")];
+            bool var_4484_transpose_x_1 = const()[name = string("op_4484_transpose_x_1"), val = bool(true)];
+            bool var_4484_transpose_y_1 = const()[name = string("op_4484_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4484_cast_fp16 = matmul(transpose_x = var_4484_transpose_x_1, transpose_y = var_4484_transpose_y_1, x = var_4482_cast_fp16, y = v_new_247_cast_fp16)[name = string("op_4484_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_249_cast_fp16 = add(x = var_4480_cast_fp16, y = var_4484_cast_fp16)[name = string("state_249_cast_fp16")];
+            tensor<int32, [4]> var_4487_begin_0 = const()[name = string("op_4487_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_4487_end_0 = const()[name = string("op_4487_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_4487_end_mask_0 = const()[name = string("op_4487_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4487_squeeze_mask_0 = const()[name = string("op_4487_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4487_cast_fp16 = slice_by_index(begin = var_4487_begin_0, end = var_4487_end_0, end_mask = var_4487_end_mask_0, squeeze_mask = var_4487_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4487_cast_fp16")];
+            tensor<int32, [4]> var_4489_begin_0 = const()[name = string("op_4489_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_4489_end_0 = const()[name = string("op_4489_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_4489_end_mask_0 = const()[name = string("op_4489_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4489_squeeze_mask_0 = const()[name = string("op_4489_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4489_cast_fp16 = slice_by_index(begin = var_4489_begin_0, end = var_4489_end_0, end_mask = var_4489_end_mask_0, squeeze_mask = var_4489_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4489_cast_fp16")];
+            bool var_4490_transpose_x_0 = const()[name = string("op_4490_transpose_x_0"), val = bool(false)];
+            bool var_4490_transpose_y_0 = const()[name = string("op_4490_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4490_cast_fp16 = matmul(transpose_x = var_4490_transpose_x_0, transpose_y = var_4490_transpose_y_0, x = var_4489_cast_fp16, y = state_249_cast_fp16)[name = string("op_4490_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_249_cast_fp16 = sub(x = var_4487_cast_fp16, y = var_4490_cast_fp16)[name = string("v_new_249_cast_fp16")];
+            tensor<int32, [4]> var_4493_begin_0 = const()[name = string("op_4493_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_4493_end_0 = const()[name = string("op_4493_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_4493_end_mask_0 = const()[name = string("op_4493_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4493_squeeze_mask_0 = const()[name = string("op_4493_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4493_cast_fp16 = slice_by_index(begin = var_4493_begin_0, end = var_4493_end_0, end_mask = var_4493_end_mask_0, squeeze_mask = var_4493_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4493_cast_fp16")];
+            bool var_4494_transpose_x_0 = const()[name = string("op_4494_transpose_x_0"), val = bool(false)];
+            bool var_4494_transpose_y_0 = const()[name = string("op_4494_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4494_cast_fp16 = matmul(transpose_x = var_4494_transpose_x_0, transpose_y = var_4494_transpose_y_0, x = var_4493_cast_fp16, y = state_249_cast_fp16)[name = string("op_4494_cast_fp16")];
+            tensor<int32, [4]> var_4496_begin_0 = const()[name = string("op_4496_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_4496_end_0 = const()[name = string("op_4496_end_0"), val = tensor<int32, [4]>([32, 29, 64, 64])];
+            tensor<bool, [4]> var_4496_end_mask_0 = const()[name = string("op_4496_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4496_squeeze_mask_0 = const()[name = string("op_4496_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4496_cast_fp16 = slice_by_index(begin = var_4496_begin_0, end = var_4496_end_0, end_mask = var_4496_end_mask_0, squeeze_mask = var_4496_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4496_cast_fp16")];
+            bool var_4497_transpose_x_0 = const()[name = string("op_4497_transpose_x_0"), val = bool(false)];
+            bool var_4497_transpose_y_0 = const()[name = string("op_4497_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4497_cast_fp16 = matmul(transpose_x = var_4497_transpose_x_0, transpose_y = var_4497_transpose_y_0, x = var_4496_cast_fp16, y = v_new_249_cast_fp16)[name = string("op_4497_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4498_cast_fp16 = add(x = var_4494_cast_fp16, y = var_4497_cast_fp16)[name = string("op_4498_cast_fp16")];
+            tensor<int32, [4]> var_4500_begin_0 = const()[name = string("op_4500_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_4500_end_0 = const()[name = string("op_4500_end_0"), val = tensor<int32, [4]>([32, 29, 1, 1])];
+            tensor<bool, [4]> var_4500_end_mask_0 = const()[name = string("op_4500_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4500_squeeze_mask_0 = const()[name = string("op_4500_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4500_cast_fp16 = slice_by_index(begin = var_4500_begin_0, end = var_4500_end_0, end_mask = var_4500_end_mask_0, squeeze_mask = var_4500_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4500_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4501_cast_fp16 = mul(x = state_249_cast_fp16, y = var_4500_cast_fp16)[name = string("op_4501_cast_fp16")];
+            tensor<int32, [4]> var_4503_begin_0 = const()[name = string("op_4503_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_4503_end_0 = const()[name = string("op_4503_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_4503_end_mask_0 = const()[name = string("op_4503_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4503_squeeze_mask_0 = const()[name = string("op_4503_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4503_cast_fp16 = slice_by_index(begin = var_4503_begin_0, end = var_4503_end_0, end_mask = var_4503_end_mask_0, squeeze_mask = var_4503_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4503_cast_fp16")];
+            bool var_4505_transpose_x_1 = const()[name = string("op_4505_transpose_x_1"), val = bool(true)];
+            bool var_4505_transpose_y_1 = const()[name = string("op_4505_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4505_cast_fp16 = matmul(transpose_x = var_4505_transpose_x_1, transpose_y = var_4505_transpose_y_1, x = var_4503_cast_fp16, y = v_new_249_cast_fp16)[name = string("op_4505_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_251_cast_fp16 = add(x = var_4501_cast_fp16, y = var_4505_cast_fp16)[name = string("state_251_cast_fp16")];
+            tensor<int32, [4]> var_4508_begin_0 = const()[name = string("op_4508_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_4508_end_0 = const()[name = string("op_4508_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_4508_end_mask_0 = const()[name = string("op_4508_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4508_squeeze_mask_0 = const()[name = string("op_4508_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4508_cast_fp16 = slice_by_index(begin = var_4508_begin_0, end = var_4508_end_0, end_mask = var_4508_end_mask_0, squeeze_mask = var_4508_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4508_cast_fp16")];
+            tensor<int32, [4]> var_4510_begin_0 = const()[name = string("op_4510_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_4510_end_0 = const()[name = string("op_4510_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_4510_end_mask_0 = const()[name = string("op_4510_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4510_squeeze_mask_0 = const()[name = string("op_4510_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4510_cast_fp16 = slice_by_index(begin = var_4510_begin_0, end = var_4510_end_0, end_mask = var_4510_end_mask_0, squeeze_mask = var_4510_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4510_cast_fp16")];
+            bool var_4511_transpose_x_0 = const()[name = string("op_4511_transpose_x_0"), val = bool(false)];
+            bool var_4511_transpose_y_0 = const()[name = string("op_4511_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4511_cast_fp16 = matmul(transpose_x = var_4511_transpose_x_0, transpose_y = var_4511_transpose_y_0, x = var_4510_cast_fp16, y = state_251_cast_fp16)[name = string("op_4511_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_251_cast_fp16 = sub(x = var_4508_cast_fp16, y = var_4511_cast_fp16)[name = string("v_new_251_cast_fp16")];
+            tensor<int32, [4]> var_4514_begin_0 = const()[name = string("op_4514_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_4514_end_0 = const()[name = string("op_4514_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_4514_end_mask_0 = const()[name = string("op_4514_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4514_squeeze_mask_0 = const()[name = string("op_4514_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4514_cast_fp16 = slice_by_index(begin = var_4514_begin_0, end = var_4514_end_0, end_mask = var_4514_end_mask_0, squeeze_mask = var_4514_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4514_cast_fp16")];
+            bool var_4515_transpose_x_0 = const()[name = string("op_4515_transpose_x_0"), val = bool(false)];
+            bool var_4515_transpose_y_0 = const()[name = string("op_4515_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4515_cast_fp16 = matmul(transpose_x = var_4515_transpose_x_0, transpose_y = var_4515_transpose_y_0, x = var_4514_cast_fp16, y = state_251_cast_fp16)[name = string("op_4515_cast_fp16")];
+            tensor<int32, [4]> var_4517_begin_0 = const()[name = string("op_4517_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_4517_end_0 = const()[name = string("op_4517_end_0"), val = tensor<int32, [4]>([32, 30, 64, 64])];
+            tensor<bool, [4]> var_4517_end_mask_0 = const()[name = string("op_4517_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4517_squeeze_mask_0 = const()[name = string("op_4517_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4517_cast_fp16 = slice_by_index(begin = var_4517_begin_0, end = var_4517_end_0, end_mask = var_4517_end_mask_0, squeeze_mask = var_4517_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4517_cast_fp16")];
+            bool var_4518_transpose_x_0 = const()[name = string("op_4518_transpose_x_0"), val = bool(false)];
+            bool var_4518_transpose_y_0 = const()[name = string("op_4518_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4518_cast_fp16 = matmul(transpose_x = var_4518_transpose_x_0, transpose_y = var_4518_transpose_y_0, x = var_4517_cast_fp16, y = v_new_251_cast_fp16)[name = string("op_4518_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4519_cast_fp16 = add(x = var_4515_cast_fp16, y = var_4518_cast_fp16)[name = string("op_4519_cast_fp16")];
+            tensor<int32, [4]> var_4521_begin_0 = const()[name = string("op_4521_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_4521_end_0 = const()[name = string("op_4521_end_0"), val = tensor<int32, [4]>([32, 30, 1, 1])];
+            tensor<bool, [4]> var_4521_end_mask_0 = const()[name = string("op_4521_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4521_squeeze_mask_0 = const()[name = string("op_4521_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4521_cast_fp16 = slice_by_index(begin = var_4521_begin_0, end = var_4521_end_0, end_mask = var_4521_end_mask_0, squeeze_mask = var_4521_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4521_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4522_cast_fp16 = mul(x = state_251_cast_fp16, y = var_4521_cast_fp16)[name = string("op_4522_cast_fp16")];
+            tensor<int32, [4]> var_4524_begin_0 = const()[name = string("op_4524_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_4524_end_0 = const()[name = string("op_4524_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_4524_end_mask_0 = const()[name = string("op_4524_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4524_squeeze_mask_0 = const()[name = string("op_4524_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4524_cast_fp16 = slice_by_index(begin = var_4524_begin_0, end = var_4524_end_0, end_mask = var_4524_end_mask_0, squeeze_mask = var_4524_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4524_cast_fp16")];
+            bool var_4526_transpose_x_1 = const()[name = string("op_4526_transpose_x_1"), val = bool(true)];
+            bool var_4526_transpose_y_1 = const()[name = string("op_4526_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4526_cast_fp16 = matmul(transpose_x = var_4526_transpose_x_1, transpose_y = var_4526_transpose_y_1, x = var_4524_cast_fp16, y = v_new_251_cast_fp16)[name = string("op_4526_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_253_cast_fp16 = add(x = var_4522_cast_fp16, y = var_4526_cast_fp16)[name = string("state_253_cast_fp16")];
+            tensor<int32, [4]> var_4529_begin_0 = const()[name = string("op_4529_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_4529_end_0 = const()[name = string("op_4529_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_4529_end_mask_0 = const()[name = string("op_4529_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4529_squeeze_mask_0 = const()[name = string("op_4529_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4529_cast_fp16 = slice_by_index(begin = var_4529_begin_0, end = var_4529_end_0, end_mask = var_4529_end_mask_0, squeeze_mask = var_4529_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4529_cast_fp16")];
+            tensor<int32, [4]> var_4531_begin_0 = const()[name = string("op_4531_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_4531_end_0 = const()[name = string("op_4531_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_4531_end_mask_0 = const()[name = string("op_4531_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4531_squeeze_mask_0 = const()[name = string("op_4531_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4531_cast_fp16 = slice_by_index(begin = var_4531_begin_0, end = var_4531_end_0, end_mask = var_4531_end_mask_0, squeeze_mask = var_4531_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4531_cast_fp16")];
+            bool var_4532_transpose_x_0 = const()[name = string("op_4532_transpose_x_0"), val = bool(false)];
+            bool var_4532_transpose_y_0 = const()[name = string("op_4532_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4532_cast_fp16 = matmul(transpose_x = var_4532_transpose_x_0, transpose_y = var_4532_transpose_y_0, x = var_4531_cast_fp16, y = state_253_cast_fp16)[name = string("op_4532_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_253_cast_fp16 = sub(x = var_4529_cast_fp16, y = var_4532_cast_fp16)[name = string("v_new_253_cast_fp16")];
+            tensor<int32, [4]> var_4535_begin_0 = const()[name = string("op_4535_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_4535_end_0 = const()[name = string("op_4535_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_4535_end_mask_0 = const()[name = string("op_4535_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4535_squeeze_mask_0 = const()[name = string("op_4535_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4535_cast_fp16 = slice_by_index(begin = var_4535_begin_0, end = var_4535_end_0, end_mask = var_4535_end_mask_0, squeeze_mask = var_4535_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4535_cast_fp16")];
+            bool var_4536_transpose_x_0 = const()[name = string("op_4536_transpose_x_0"), val = bool(false)];
+            bool var_4536_transpose_y_0 = const()[name = string("op_4536_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4536_cast_fp16 = matmul(transpose_x = var_4536_transpose_x_0, transpose_y = var_4536_transpose_y_0, x = var_4535_cast_fp16, y = state_253_cast_fp16)[name = string("op_4536_cast_fp16")];
+            tensor<int32, [4]> var_4538_begin_0 = const()[name = string("op_4538_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_4538_end_0 = const()[name = string("op_4538_end_0"), val = tensor<int32, [4]>([32, 31, 64, 64])];
+            tensor<bool, [4]> var_4538_end_mask_0 = const()[name = string("op_4538_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4538_squeeze_mask_0 = const()[name = string("op_4538_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4538_cast_fp16 = slice_by_index(begin = var_4538_begin_0, end = var_4538_end_0, end_mask = var_4538_end_mask_0, squeeze_mask = var_4538_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4538_cast_fp16")];
+            bool var_4539_transpose_x_0 = const()[name = string("op_4539_transpose_x_0"), val = bool(false)];
+            bool var_4539_transpose_y_0 = const()[name = string("op_4539_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4539_cast_fp16 = matmul(transpose_x = var_4539_transpose_x_0, transpose_y = var_4539_transpose_y_0, x = var_4538_cast_fp16, y = v_new_253_cast_fp16)[name = string("op_4539_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4540_cast_fp16 = add(x = var_4536_cast_fp16, y = var_4539_cast_fp16)[name = string("op_4540_cast_fp16")];
+            tensor<int32, [4]> var_4542_begin_0 = const()[name = string("op_4542_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_4542_end_0 = const()[name = string("op_4542_end_0"), val = tensor<int32, [4]>([32, 31, 1, 1])];
+            tensor<bool, [4]> var_4542_end_mask_0 = const()[name = string("op_4542_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4542_squeeze_mask_0 = const()[name = string("op_4542_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_4542_cast_fp16 = slice_by_index(begin = var_4542_begin_0, end = var_4542_end_0, end_mask = var_4542_end_mask_0, squeeze_mask = var_4542_squeeze_mask_0, x = chunk_decay_7_cast_fp16)[name = string("op_4542_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_4543_cast_fp16 = mul(x = state_253_cast_fp16, y = var_4542_cast_fp16)[name = string("op_4543_cast_fp16")];
+            tensor<int32, [4]> var_4545_begin_0 = const()[name = string("op_4545_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_4545_end_0 = const()[name = string("op_4545_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_4545_end_mask_0 = const()[name = string("op_4545_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4545_squeeze_mask_0 = const()[name = string("op_4545_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4545_cast_fp16 = slice_by_index(begin = var_4545_begin_0, end = var_4545_end_0, end_mask = var_4545_end_mask_0, squeeze_mask = var_4545_squeeze_mask_0, x = k_dec_7_cast_fp16)[name = string("op_4545_cast_fp16")];
+            bool var_4547_transpose_x_1 = const()[name = string("op_4547_transpose_x_1"), val = bool(true)];
+            bool var_4547_transpose_y_1 = const()[name = string("op_4547_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_4547_cast_fp16 = matmul(transpose_x = var_4547_transpose_x_1, transpose_y = var_4547_transpose_y_1, x = var_4545_cast_fp16, y = v_new_253_cast_fp16)[name = string("op_4547_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_255_cast_fp16 = add(x = var_4543_cast_fp16, y = var_4547_cast_fp16)[name = string("state_255_cast_fp16")];
+            tensor<int32, [4]> var_4550_begin_0 = const()[name = string("op_4550_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_4550_end_0 = const()[name = string("op_4550_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_4550_end_mask_0 = const()[name = string("op_4550_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4550_squeeze_mask_0 = const()[name = string("op_4550_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4550_cast_fp16 = slice_by_index(begin = var_4550_begin_0, end = var_4550_end_0, end_mask = var_4550_end_mask_0, squeeze_mask = var_4550_squeeze_mask_0, x = new_v_7_cast_fp16)[name = string("op_4550_cast_fp16")];
+            tensor<int32, [4]> var_4552_begin_0 = const()[name = string("op_4552_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_4552_end_0 = const()[name = string("op_4552_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_4552_end_mask_0 = const()[name = string("op_4552_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4552_squeeze_mask_0 = const()[name = string("op_4552_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4552_cast_fp16 = slice_by_index(begin = var_4552_begin_0, end = var_4552_end_0, end_mask = var_4552_end_mask_0, squeeze_mask = var_4552_squeeze_mask_0, x = k_cumdecay_7_cast_fp16)[name = string("op_4552_cast_fp16")];
+            bool var_4553_transpose_x_0 = const()[name = string("op_4553_transpose_x_0"), val = bool(false)];
+            bool var_4553_transpose_y_0 = const()[name = string("op_4553_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4553_cast_fp16 = matmul(transpose_x = var_4553_transpose_x_0, transpose_y = var_4553_transpose_y_0, x = var_4552_cast_fp16, y = state_255_cast_fp16)[name = string("op_4553_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_255_cast_fp16 = sub(x = var_4550_cast_fp16, y = var_4553_cast_fp16)[name = string("v_new_255_cast_fp16")];
+            tensor<int32, [4]> var_4556_begin_0 = const()[name = string("op_4556_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_4556_end_0 = const()[name = string("op_4556_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_4556_end_mask_0 = const()[name = string("op_4556_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4556_squeeze_mask_0 = const()[name = string("op_4556_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_4556_cast_fp16 = slice_by_index(begin = var_4556_begin_0, end = var_4556_end_0, end_mask = var_4556_end_mask_0, squeeze_mask = var_4556_squeeze_mask_0, x = q_dec_7_cast_fp16)[name = string("op_4556_cast_fp16")];
+            bool var_4557_transpose_x_0 = const()[name = string("op_4557_transpose_x_0"), val = bool(false)];
+            bool var_4557_transpose_y_0 = const()[name = string("op_4557_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4557_cast_fp16 = matmul(transpose_x = var_4557_transpose_x_0, transpose_y = var_4557_transpose_y_0, x = var_4556_cast_fp16, y = state_255_cast_fp16)[name = string("op_4557_cast_fp16")];
+            tensor<int32, [4]> var_4559_begin_0 = const()[name = string("op_4559_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_4559_end_0 = const()[name = string("op_4559_end_0"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<bool, [4]> var_4559_end_mask_0 = const()[name = string("op_4559_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_4559_squeeze_mask_0 = const()[name = string("op_4559_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_4559_cast_fp16 = slice_by_index(begin = var_4559_begin_0, end = var_4559_end_0, end_mask = var_4559_end_mask_0, squeeze_mask = var_4559_squeeze_mask_0, x = attn_intra_7_cast_fp16)[name = string("op_4559_cast_fp16")];
+            bool var_4560_transpose_x_0 = const()[name = string("op_4560_transpose_x_0"), val = bool(false)];
+            bool var_4560_transpose_y_0 = const()[name = string("op_4560_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_4560_cast_fp16 = matmul(transpose_x = var_4560_transpose_x_0, transpose_y = var_4560_transpose_y_0, x = var_4559_cast_fp16, y = v_new_255_cast_fp16)[name = string("op_4560_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_4561_cast_fp16 = add(x = var_4557_cast_fp16, y = var_4560_cast_fp16)[name = string("op_4561_cast_fp16")];
+            int32 var_4563_axis_0 = const()[name = string("op_4563_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 32, 64, 128]> var_4563_cast_fp16 = stack(axis = var_4563_axis_0, values = (var_3910_cast_fp16, var_3931_cast_fp16, var_3952_cast_fp16, var_3973_cast_fp16, var_3994_cast_fp16, var_4015_cast_fp16, var_4036_cast_fp16, var_4057_cast_fp16, var_4078_cast_fp16, var_4099_cast_fp16, var_4120_cast_fp16, var_4141_cast_fp16, var_4162_cast_fp16, var_4183_cast_fp16, var_4204_cast_fp16, var_4225_cast_fp16, var_4246_cast_fp16, var_4267_cast_fp16, var_4288_cast_fp16, var_4309_cast_fp16, var_4330_cast_fp16, var_4351_cast_fp16, var_4372_cast_fp16, var_4393_cast_fp16, var_4414_cast_fp16, var_4435_cast_fp16, var_4456_cast_fp16, var_4477_cast_fp16, var_4498_cast_fp16, var_4519_cast_fp16, var_4540_cast_fp16, var_4561_cast_fp16))[name = string("op_4563_cast_fp16")];
+            tensor<int32, [3]> var_4564 = const()[name = string("op_4564"), val = tensor<int32, [3]>([32, 2048, 128])];
+            tensor<fp16, [32, 2048, 128]> var_4565_cast_fp16 = reshape(shape = var_4564, x = var_4563_cast_fp16)[name = string("op_4565_cast_fp16")];
+            tensor<int32, [3]> var_4566 = const()[name = string("op_4566"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_63_begin_0 = const()[name = string("input_63_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_63_end_0 = const()[name = string("input_63_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> input_63_end_mask_0 = const()[name = string("input_63_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_63_squeeze_mask_0 = const()[name = string("input_63_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> input_63_cast_fp16 = slice_by_index(begin = input_63_begin_0, end = input_63_end_0, end_mask = input_63_end_mask_0, squeeze_mask = input_63_squeeze_mask_0, x = z_7_cast_fp16)[name = string("input_63_cast_fp16")];
+            fp16 var_3546_promoted_1_to_fp16 = const()[name = string("op_3546_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [2048, 32, 128]> x_117_cast_fp16 = transpose(perm = var_4566, x = var_4565_cast_fp16)[name = string("transpose_20")];
+            tensor<fp16, [2048, 32, 128]> var_4570_cast_fp16 = pow(x = x_117_cast_fp16, y = var_3546_promoted_1_to_fp16)[name = string("op_4570_cast_fp16")];
+            tensor<int32, [1]> var_4572_axes_0 = const()[name = string("op_4572_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4572_keep_dims_0 = const()[name = string("op_4572_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [2048, 32, 1]> var_4572_cast_fp16 = reduce_mean(axes = var_4572_axes_0, keep_dims = var_4572_keep_dims_0, x = var_4570_cast_fp16)[name = string("op_4572_cast_fp16")];
+            fp16 var_4573_to_fp16 = const()[name = string("op_4573_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [2048, 32, 1]> var_4574_cast_fp16 = add(x = var_4572_cast_fp16, y = var_4573_to_fp16)[name = string("op_4574_cast_fp16")];
+            fp32 var_4575_epsilon_0 = const()[name = string("op_4575_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [2048, 32, 1]> var_4575_cast_fp16 = rsqrt(epsilon = var_4575_epsilon_0, x = var_4574_cast_fp16)[name = string("op_4575_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> x_119_cast_fp16 = mul(x = x_117_cast_fp16, y = var_4575_cast_fp16)[name = string("x_119_cast_fp16")];
+            tensor<fp16, [128]> layers_4_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_4_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(970139328)))];
+            tensor<fp16, [2048, 32, 128]> var_4577_cast_fp16 = mul(x = layers_4_linear_attn_norm_weight_to_fp16, y = x_119_cast_fp16)[name = string("op_4577_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> var_4578_cast_fp16 = silu(x = input_63_cast_fp16)[name = string("op_4578_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> core_7_cast_fp16 = mul(x = var_4577_cast_fp16, y = var_4578_cast_fp16)[name = string("core_7_cast_fp16")];
+            tensor<int32, [3]> var_4580 = const()[name = string("op_4580"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> input_65_cast_fp16 = reshape(shape = var_4580, x = core_7_cast_fp16)[name = string("input_65_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_4_linear_attn_out_proj_weight_to_fp16 = const()[name = string("layers_4_linear_attn_out_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(970139648)))];
+            tensor<fp16, [1, 2048, 2560]> linear_35_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_4_linear_attn_out_proj_weight_to_fp16, x = input_65_cast_fp16)[name = string("linear_35_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_121_cast_fp16 = add(x = x_99_cast_fp16, y = linear_35_cast_fp16)[name = string("x_121_cast_fp16")];
+            fp16 var_3546_promoted_2_to_fp16 = const()[name = string("op_3546_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_4586_cast_fp16 = pow(x = x_121_cast_fp16, y = var_3546_promoted_2_to_fp16)[name = string("op_4586_cast_fp16")];
+            tensor<int32, [1]> var_4588_axes_0 = const()[name = string("op_4588_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4588_keep_dims_0 = const()[name = string("op_4588_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_4588_cast_fp16 = reduce_mean(axes = var_4588_axes_0, keep_dims = var_4588_keep_dims_0, x = var_4586_cast_fp16)[name = string("op_4588_cast_fp16")];
+            fp16 var_4589_to_fp16 = const()[name = string("op_4589_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_4590_cast_fp16 = add(x = var_4588_cast_fp16, y = var_4589_to_fp16)[name = string("op_4590_cast_fp16")];
+            fp32 var_4591_epsilon_0 = const()[name = string("op_4591_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_4591_cast_fp16 = rsqrt(epsilon = var_4591_epsilon_0, x = var_4590_cast_fp16)[name = string("op_4591_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_123_cast_fp16 = mul(x = x_121_cast_fp16, y = var_4591_cast_fp16)[name = string("x_123_cast_fp16")];
+            tensor<fp16, [2560]> var_4594_to_fp16 = const()[name = string("op_4594_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(991111232)))];
+            tensor<fp16, [1, 2048, 2560]> input_67_cast_fp16 = mul(x = x_123_cast_fp16, y = var_4594_to_fp16)[name = string("input_67_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_4_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_4_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(991116416)))];
+            tensor<fp16, [1, 2048, 9216]> linear_36_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_4_mlp_gate_proj_weight_to_fp16, x = input_67_cast_fp16)[name = string("linear_36_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_4601_cast_fp16 = silu(x = linear_36_cast_fp16)[name = string("op_4601_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_4_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_4_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1038302400)))];
+            tensor<fp16, [1, 2048, 9216]> linear_37_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_4_mlp_up_proj_weight_to_fp16, x = input_67_cast_fp16)[name = string("linear_37_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_71_cast_fp16 = mul(x = var_4601_cast_fp16, y = linear_37_cast_fp16)[name = string("input_71_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_4_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_4_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1085488384)))];
+            tensor<fp16, [1, 2048, 2560]> linear_38_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_4_mlp_down_proj_weight_to_fp16, x = input_71_cast_fp16)[name = string("linear_38_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_125_cast_fp16 = add(x = x_121_cast_fp16, y = linear_38_cast_fp16)[name = string("x_125_cast_fp16")];
+            int32 var_4621 = const()[name = string("op_4621"), val = int32(-2)];
+            int32 var_4650 = const()[name = string("op_4650"), val = int32(-1)];
+            fp16 var_4649_promoted_to_fp16 = const()[name = string("op_4649_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_4659_cast_fp16 = pow(x = x_125_cast_fp16, y = var_4649_promoted_to_fp16)[name = string("op_4659_cast_fp16")];
+            tensor<int32, [1]> var_4661_axes_0 = const()[name = string("op_4661_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4661_keep_dims_0 = const()[name = string("op_4661_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_4661_cast_fp16 = reduce_mean(axes = var_4661_axes_0, keep_dims = var_4661_keep_dims_0, x = var_4659_cast_fp16)[name = string("op_4661_cast_fp16")];
+            fp16 var_4662_to_fp16 = const()[name = string("op_4662_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_4663_cast_fp16 = add(x = var_4661_cast_fp16, y = var_4662_to_fp16)[name = string("op_4663_cast_fp16")];
+            fp32 var_4664_epsilon_0 = const()[name = string("op_4664_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_4664_cast_fp16 = rsqrt(epsilon = var_4664_epsilon_0, x = var_4663_cast_fp16)[name = string("op_4664_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_127_cast_fp16 = mul(x = x_125_cast_fp16, y = var_4664_cast_fp16)[name = string("x_127_cast_fp16")];
+            tensor<fp16, [2560]> var_4667_to_fp16 = const()[name = string("op_4667_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1132674368)))];
+            tensor<fp16, [1, 2048, 2560]> x_129_cast_fp16 = mul(x = x_127_cast_fp16, y = var_4667_to_fp16)[name = string("x_129_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_5_linear_attn_in_proj_qkv_weight_to_fp16 = const()[name = string("layers_5_linear_attn_in_proj_qkv_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1132679552)))];
+            tensor<fp16, [1, 2048, 8192]> linear_39_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_qkv_weight_to_fp16, x = x_129_cast_fp16)[name = string("linear_39_cast_fp16")];
+            tensor<int32, [3]> input_73_perm_0 = const()[name = string("input_73_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_75_pad_0 = const()[name = string("input_75_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_75_mode_0 = const()[name = string("input_75_mode_0"), val = string("constant")];
+            fp16 const_54_to_fp16 = const()[name = string("const_54_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 2048]> input_73_cast_fp16 = transpose(perm = input_73_perm_0, x = linear_39_cast_fp16)[name = string("transpose_19")];
+            tensor<fp16, [1, 8192, 2051]> input_75_cast_fp16 = pad(constant_val = const_54_to_fp16, mode = input_75_mode_0, pad = input_75_pad_0, x = input_73_cast_fp16)[name = string("input_75_cast_fp16")];
+            string input_77_pad_type_0 = const()[name = string("input_77_pad_type_0"), val = string("valid")];
+            int32 input_77_groups_0 = const()[name = string("input_77_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_77_strides_0 = const()[name = string("input_77_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_77_pad_0 = const()[name = string("input_77_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_77_dilations_0 = const()[name = string("input_77_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_5_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_5_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1174622656)))];
+            tensor<fp16, [1, 8192, 2048]> input_77_cast_fp16 = conv(dilations = input_77_dilations_0, groups = input_77_groups_0, pad = input_77_pad_0, pad_type = input_77_pad_type_0, strides = input_77_strides_0, weight = layers_5_linear_attn_conv1d_weight_to_fp16, x = input_75_cast_fp16)[name = string("input_77_cast_fp16")];
+            tensor<fp16, [1, 8192, 2048]> var_4702_cast_fp16 = silu(x = input_77_cast_fp16)[name = string("op_4702_cast_fp16")];
+            tensor<int32, [3]> qkv_9_perm_0 = const()[name = string("qkv_9_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_4704_begin_0 = const()[name = string("op_4704_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_4704_end_0 = const()[name = string("op_4704_end_0"), val = tensor<int32, [3]>([1, 2048, 2048])];
+            tensor<bool, [3]> var_4704_end_mask_0 = const()[name = string("op_4704_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 8192]> qkv_9_cast_fp16 = transpose(perm = qkv_9_perm_0, x = var_4702_cast_fp16)[name = string("transpose_18")];
+            tensor<fp16, [1, 2048, 2048]> var_4704_cast_fp16 = slice_by_index(begin = var_4704_begin_0, end = var_4704_end_0, end_mask = var_4704_end_mask_0, x = qkv_9_cast_fp16)[name = string("op_4704_cast_fp16")];
+            tensor<int32, [4]> var_4705 = const()[name = string("op_4705"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> q_45_cast_fp16 = reshape(shape = var_4705, x = var_4704_cast_fp16)[name = string("q_45_cast_fp16")];
+            tensor<int32, [3]> var_4707_begin_0 = const()[name = string("op_4707_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_4707_end_0 = const()[name = string("op_4707_end_0"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<bool, [3]> var_4707_end_mask_0 = const()[name = string("op_4707_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 2048]> var_4707_cast_fp16 = slice_by_index(begin = var_4707_begin_0, end = var_4707_end_0, end_mask = var_4707_end_mask_0, x = qkv_9_cast_fp16)[name = string("op_4707_cast_fp16")];
+            tensor<int32, [4]> var_4708 = const()[name = string("op_4708"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> k_39_cast_fp16 = reshape(shape = var_4708, x = var_4707_cast_fp16)[name = string("k_39_cast_fp16")];
+            tensor<int32, [3]> var_4710_begin_0 = const()[name = string("op_4710_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_4710_end_0 = const()[name = string("op_4710_end_0"), val = tensor<int32, [3]>([1, 2048, 8192])];
+            tensor<bool, [3]> var_4710_end_mask_0 = const()[name = string("op_4710_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 2048, 4096]> var_4710_cast_fp16 = slice_by_index(begin = var_4710_begin_0, end = var_4710_end_0, end_mask = var_4710_end_mask_0, x = qkv_9_cast_fp16)[name = string("op_4710_cast_fp16")];
+            tensor<int32, [4]> var_4711 = const()[name = string("op_4711"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> v_21_cast_fp16 = reshape(shape = var_4711, x = var_4710_cast_fp16)[name = string("v_21_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_5_linear_attn_in_proj_z_weight_to_fp16 = const()[name = string("layers_5_linear_attn_in_proj_z_weight_to_fp16"), val = tensor<fp16, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1174688256)))];
+            tensor<fp16, [1, 2048, 4096]> linear_40_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_z_weight_to_fp16, x = x_129_cast_fp16)[name = string("linear_40_cast_fp16")];
+            tensor<int32, [4]> var_4715 = const()[name = string("op_4715"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> z_9_cast_fp16 = reshape(shape = var_4715, x = linear_40_cast_fp16)[name = string("z_9_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_5_linear_attn_in_proj_b_weight_to_fp16 = const()[name = string("layers_5_linear_attn_in_proj_b_weight_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195659840)))];
+            tensor<fp16, [32]> linear_41_bias_0_to_fp16 = const()[name = string("linear_41_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195823744)))];
+            tensor<fp16, [1, 2048, 32]> linear_41_cast_fp16 = linear(bias = linear_41_bias_0_to_fp16, weight = layers_5_linear_attn_in_proj_b_weight_to_fp16, x = x_129_cast_fp16)[name = string("linear_41_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> beta_17_cast_fp16 = sigmoid(x = linear_41_cast_fp16)[name = string("beta_17_cast_fp16")];
+            tensor<fp16, [32, 2560]> var_4724_weight_0_to_fp16 = const()[name = string("op_4724_weight_0_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195823872)))];
+            tensor<fp16, [32]> var_4724_bias_0_to_fp16 = const()[name = string("op_4724_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195987776)))];
+            tensor<fp16, [1, 2048, 32]> var_4724_cast_fp16 = linear(bias = var_4724_bias_0_to_fp16, weight = var_4724_weight_0_to_fp16, x = x_129_cast_fp16)[name = string("op_4724_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> var_4725_cast_fp16 = softplus(x = var_4724_cast_fp16)[name = string("op_4725_cast_fp16")];
+            tensor<fp16, [32]> var_4721_to_fp16 = const()[name = string("op_4721_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195987904)))];
+            tensor<fp16, [1, 2048, 32]> g_17_cast_fp16 = mul(x = var_4721_to_fp16, y = var_4725_cast_fp16)[name = string("g_17_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_4727_cast_fp16 = mul(x = q_45_cast_fp16, y = q_45_cast_fp16)[name = string("op_4727_cast_fp16")];
+            tensor<int32, [1]> var_4729_axes_0 = const()[name = string("op_4729_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4729_keep_dims_0 = const()[name = string("op_4729_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_4729_cast_fp16 = reduce_sum(axes = var_4729_axes_0, keep_dims = var_4729_keep_dims_0, x = var_4727_cast_fp16)[name = string("op_4729_cast_fp16")];
+            fp16 var_4730_to_fp16 = const()[name = string("op_4730_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_4731_cast_fp16 = add(x = var_4729_cast_fp16, y = var_4730_to_fp16)[name = string("op_4731_cast_fp16")];
+            fp32 var_4732_epsilon_0 = const()[name = string("op_4732_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_4732_cast_fp16 = rsqrt(epsilon = var_4732_epsilon_0, x = var_4731_cast_fp16)[name = string("op_4732_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> q_47_cast_fp16 = mul(x = q_45_cast_fp16, y = var_4732_cast_fp16)[name = string("q_47_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_4734_cast_fp16 = mul(x = k_39_cast_fp16, y = k_39_cast_fp16)[name = string("op_4734_cast_fp16")];
+            tensor<int32, [1]> var_4736_axes_0 = const()[name = string("op_4736_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_4736_keep_dims_0 = const()[name = string("op_4736_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_4736_cast_fp16 = reduce_sum(axes = var_4736_axes_0, keep_dims = var_4736_keep_dims_0, x = var_4734_cast_fp16)[name = string("op_4736_cast_fp16")];
+            fp16 var_4737_to_fp16 = const()[name = string("op_4737_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_4738_cast_fp16 = add(x = var_4736_cast_fp16, y = var_4737_to_fp16)[name = string("op_4738_cast_fp16")];
+            fp32 var_4739_epsilon_0 = const()[name = string("op_4739_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_4739_cast_fp16 = rsqrt(epsilon = var_4739_epsilon_0, x = var_4738_cast_fp16)[name = string("op_4739_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> k_41_cast_fp16 = mul(x = k_39_cast_fp16, y = var_4739_cast_fp16)[name = string("k_41_cast_fp16")];
+            tensor<int32, [1]> var_4744_axes_0 = const()[name = string("op_4744_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_4744_cast_fp16 = expand_dims(axes = var_4744_axes_0, x = q_47_cast_fp16)[name = string("op_4744_cast_fp16")];
+            tensor<int32, [5]> var_4746_reps_0 = const()[name = string("op_4746_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_4746_cast_fp16 = tile(reps = var_4746_reps_0, x = var_4744_cast_fp16)[name = string("op_4746_cast_fp16")];
+            tensor<int32, [4]> var_4747 = const()[name = string("op_4747"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> q_49_cast_fp16 = reshape(shape = var_4747, x = var_4746_cast_fp16)[name = string("q_49_cast_fp16")];
+            tensor<int32, [1]> var_4752_axes_0 = const()[name = string("op_4752_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_4752_cast_fp16 = expand_dims(axes = var_4752_axes_0, x = k_41_cast_fp16)[name = string("op_4752_cast_fp16")];
+            tensor<int32, [5]> var_4754_reps_0 = const()[name = string("op_4754_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_4754_cast_fp16 = tile(reps = var_4754_reps_0, x = var_4752_cast_fp16)[name = string("op_4754_cast_fp16")];
+            tensor<int32, [4]> var_4755 = const()[name = string("op_4755"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> k_43_cast_fp16 = reshape(shape = var_4755, x = var_4754_cast_fp16)[name = string("k_43_cast_fp16")];
+            fp16 var_4757_to_fp16 = const()[name = string("op_4757_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 2048, 32, 128]> q_51_cast_fp16 = mul(x = q_49_cast_fp16, y = var_4757_to_fp16)[name = string("q_51_cast_fp16")];
+            tensor<int32, [4]> var_4759_begin_0 = const()[name = string("op_4759_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4759_end_0 = const()[name = string("op_4759_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_4759_end_mask_0 = const()[name = string("op_4759_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_4759_squeeze_mask_0 = const()[name = string("op_4759_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_4759_cast_fp16 = slice_by_index(begin = var_4759_begin_0, end = var_4759_end_0, end_mask = var_4759_end_mask_0, squeeze_mask = var_4759_squeeze_mask_0, x = q_51_cast_fp16)[name = string("op_4759_cast_fp16")];
+            tensor<int32, [3]> var_4760 = const()[name = string("op_4760"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_4762 = const()[name = string("op_4762"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_4761_cast_fp16 = transpose(perm = var_4760, x = var_4759_cast_fp16)[name = string("transpose_17")];
+            tensor<fp16, [32, 32, 64, 128]> q_53_cast_fp16 = reshape(shape = var_4762, x = var_4761_cast_fp16)[name = string("q_53_cast_fp16")];
+            tensor<int32, [4]> var_4764_begin_0 = const()[name = string("op_4764_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4764_end_0 = const()[name = string("op_4764_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_4764_end_mask_0 = const()[name = string("op_4764_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_4764_squeeze_mask_0 = const()[name = string("op_4764_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_4764_cast_fp16 = slice_by_index(begin = var_4764_begin_0, end = var_4764_end_0, end_mask = var_4764_end_mask_0, squeeze_mask = var_4764_squeeze_mask_0, x = k_43_cast_fp16)[name = string("op_4764_cast_fp16")];
+            tensor<int32, [3]> var_4765 = const()[name = string("op_4765"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_4767 = const()[name = string("op_4767"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_4766_cast_fp16 = transpose(perm = var_4765, x = var_4764_cast_fp16)[name = string("transpose_16")];
+            tensor<fp16, [32, 32, 64, 128]> k_45_cast_fp16 = reshape(shape = var_4767, x = var_4766_cast_fp16)[name = string("k_45_cast_fp16")];
+            tensor<int32, [4]> var_4769_begin_0 = const()[name = string("op_4769_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_4769_end_0 = const()[name = string("op_4769_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_4769_end_mask_0 = const()[name = string("op_4769_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_4769_squeeze_mask_0 = const()[name = string("op_4769_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_4769_cast_fp16 = slice_by_index(begin = var_4769_begin_0, end = var_4769_end_0, end_mask = var_4769_end_mask_0, squeeze_mask = var_4769_squeeze_mask_0, x = v_21_cast_fp16)[name = string("op_4769_cast_fp16")];
+            tensor<int32, [3]> var_4770 = const()[name = string("op_4770"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_4772 = const()[name = string("op_4772"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_4771_cast_fp16 = transpose(perm = var_4770, x = var_4769_cast_fp16)[name = string("transpose_15")];
+            tensor<fp16, [32, 32, 64, 128]> v_23_cast_fp16 = reshape(shape = var_4772, x = var_4771_cast_fp16)[name = string("v_23_cast_fp16")];
+            tensor<int32, [3]> var_4774_begin_0 = const()[name = string("op_4774_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_4774_end_0 = const()[name = string("op_4774_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_4774_end_mask_0 = const()[name = string("op_4774_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_4774_squeeze_mask_0 = const()[name = string("op_4774_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_4774_cast_fp16 = slice_by_index(begin = var_4774_begin_0, end = var_4774_end_0, end_mask = var_4774_end_mask_0, squeeze_mask = var_4774_squeeze_mask_0, x = beta_17_cast_fp16)[name = string("op_4774_cast_fp16")];
+            tensor<int32, [2]> var_4775_perm_0 = const()[name = string("op_4775_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_4776 = const()[name = string("op_4776"), val = tensor<int32, [4]>([32, 32, 64, 1])];
+            tensor<fp16, [32, 2048]> var_4775_cast_fp16 = transpose(perm = var_4775_perm_0, x = var_4774_cast_fp16)[name = string("transpose_14")];
+            tensor<fp16, [32, 32, 64, 1]> beta_19_cast_fp16 = reshape(shape = var_4776, x = var_4775_cast_fp16)[name = string("beta_19_cast_fp16")];
+            tensor<int32, [3]> var_4778_begin_0 = const()[name = string("op_4778_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_4778_end_0 = const()[name = string("op_4778_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_4778_end_mask_0 = const()[name = string("op_4778_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_4778_squeeze_mask_0 = const()[name = string("op_4778_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_4778_cast_fp16 = slice_by_index(begin = var_4778_begin_0, end = var_4778_end_0, end_mask = var_4778_end_mask_0, squeeze_mask = var_4778_squeeze_mask_0, x = g_17_cast_fp16)[name = string("op_4778_cast_fp16")];
+            tensor<int32, [2]> var_4779_perm_0 = const()[name = string("op_4779_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_4780 = const()[name = string("op_4780"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<fp16, [32, 2048]> var_4779_cast_fp16 = transpose(perm = var_4779_perm_0, x = var_4778_cast_fp16)[name = string("transpose_13")];
+            tensor<fp16, [32, 32, 64]> g_19_cast_fp16 = reshape(shape = var_4780, x = var_4779_cast_fp16)[name = string("g_19_cast_fp16")];
+            tensor<int32, [1]> g_col_9_axes_0 = const()[name = string("g_col_9_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64, 1]> g_col_9_cast_fp16 = expand_dims(axes = g_col_9_axes_0, x = g_19_cast_fp16)[name = string("g_col_9_cast_fp16")];
+            bool var_4783_transpose_x_0 = const()[name = string("op_4783_transpose_x_0"), val = bool(false)];
+            bool var_4783_transpose_y_0 = const()[name = string("op_4783_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_4783_cast_fp16 = matmul(transpose_x = var_4783_transpose_x_0, transpose_y = var_4783_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_9_cast_fp16)[name = string("op_4783_cast_fp16")];
+            tensor<int32, [1]> cum_9_axes_0 = const()[name = string("cum_9_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> cum_9_cast_fp16 = squeeze(axes = cum_9_axes_0, x = var_4783_cast_fp16)[name = string("cum_9_cast_fp16")];
+            bool var_4785_transpose_x_0 = const()[name = string("op_4785_transpose_x_0"), val = bool(false)];
+            bool var_4785_transpose_y_0 = const()[name = string("op_4785_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_4785_cast_fp16 = matmul(transpose_x = var_4785_transpose_x_0, transpose_y = var_4785_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_9_cast_fp16)[name = string("op_4785_cast_fp16")];
+            tensor<int32, [1]> to_end_9_axes_0 = const()[name = string("to_end_9_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> to_end_9_cast_fp16 = squeeze(axes = to_end_9_axes_0, x = var_4785_cast_fp16)[name = string("to_end_9_cast_fp16")];
+            bool var_4787_transpose_x_0 = const()[name = string("op_4787_transpose_x_0"), val = bool(false)];
+            bool var_4787_transpose_y_0 = const()[name = string("op_4787_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 4096, 1]> var_4787_cast_fp16 = matmul(transpose_x = var_4787_transpose_x_0, transpose_y = var_4787_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_9_cast_fp16)[name = string("op_4787_cast_fp16")];
+            tensor<int32, [4]> var_4788 = const()[name = string("op_4788"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> pair_57_cast_fp16 = reshape(shape = var_4788, x = var_4787_cast_fp16)[name = string("pair_57_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_4790_cast_fp16 = exp(x = pair_57_cast_fp16)[name = string("op_4790_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> pair_decay_9_cast_fp16 = mul(x = var_4790_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_9_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_beta_9_cast_fp16 = mul(x = k_45_cast_fp16, y = beta_19_cast_fp16)[name = string("k_beta_9_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> v_beta_9_cast_fp16 = mul(x = v_23_cast_fp16, y = beta_19_cast_fp16)[name = string("v_beta_9_cast_fp16")];
+            bool var_4795_transpose_x_1 = const()[name = string("op_4795_transpose_x_1"), val = bool(false)];
+            bool var_4795_transpose_y_1 = const()[name = string("op_4795_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_4795_cast_fp16 = matmul(transpose_x = var_4795_transpose_x_1, transpose_y = var_4795_transpose_y_1, x = k_beta_9_cast_fp16, y = k_45_cast_fp16)[name = string("op_4795_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> kkt_9_cast_fp16 = mul(x = var_4795_cast_fp16, y = pair_decay_9_cast_fp16)[name = string("kkt_9_cast_fp16")];
+            bool var_4798_transpose_x_1 = const()[name = string("op_4798_transpose_x_1"), val = bool(false)];
+            bool var_4798_transpose_y_1 = const()[name = string("op_4798_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_4798_cast_fp16 = matmul(transpose_x = var_4798_transpose_x_1, transpose_y = var_4798_transpose_y_1, x = q_53_cast_fp16, y = k_45_cast_fp16)[name = string("op_4798_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> attn_intra_9_cast_fp16 = mul(x = var_4798_cast_fp16, y = pair_decay_9_cast_fp16)[name = string("attn_intra_9_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_4800_cast_fp16 = mul(x = kkt_9_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_4800_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> t_17_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_4800_cast_fp16)[name = string("t_17_cast_fp16")];
+            tensor<int32, [3]> var_4804 = const()[name = string("op_4804"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [1024, 64, 64]> t_19_cast_fp16 = reshape(shape = var_4804, x = t_17_cast_fp16)[name = string("t_19_cast_fp16")];
+            tensor<int32, [5]> var_4809 = const()[name = string("op_4809"), val = tensor<int32, [5]>([1024, 32, 2, 32, 2])];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_4810_cast_fp16 = reshape(shape = var_4809, x = t_19_cast_fp16)[name = string("op_4810_cast_fp16")];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_4815_cast_fp16 = mul(x = var_4810_cast_fp16, y = var_230_to_fp16)[name = string("op_4815_cast_fp16")];
+            tensor<int32, [1]> blocks_49_axes_0 = const()[name = string("blocks_49_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_49_keep_dims_0 = const()[name = string("blocks_49_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> blocks_49_cast_fp16 = reduce_sum(axes = blocks_49_axes_0, keep_dims = blocks_49_keep_dims_0, x = var_4815_cast_fp16)[name = string("blocks_49_cast_fp16")];
+            tensor<int32, [4]> var_4818_begin_0 = const()[name = string("op_4818_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_4818_end_0 = const()[name = string("op_4818_end_0"), val = tensor<int32, [4]>([1024, 32, 2, 2])];
+            tensor<bool, [4]> var_4818_end_mask_0 = const()[name = string("op_4818_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 32, 1, 2]> var_4818_cast_fp16 = slice_by_index(begin = var_4818_begin_0, end = var_4818_end_0, end_mask = var_4818_end_mask_0, x = blocks_49_cast_fp16)[name = string("op_4818_cast_fp16")];
+            tensor<int32, [4]> x_131_begin_0 = const()[name = string("x_131_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_131_end_0 = const()[name = string("x_131_end_0"), val = tensor<int32, [4]>([1024, 32, 1, 1])];
+            tensor<bool, [4]> x_131_end_mask_0 = const()[name = string("x_131_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 32, 1, 1]> x_131_cast_fp16 = slice_by_index(begin = x_131_begin_0, end = x_131_end_0, end_mask = x_131_end_mask_0, x = var_4818_cast_fp16)[name = string("x_131_cast_fp16")];
+            bool var_4828_transpose_x_0 = const()[name = string("op_4828_transpose_x_0"), val = bool(false)];
+            bool var_4828_transpose_y_0 = const()[name = string("op_4828_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_4828_cast_fp16 = matmul(transpose_x = var_4828_transpose_x_0, transpose_y = var_4828_transpose_y_0, x = x_131_cast_fp16, y = var_238_to_fp16)[name = string("op_4828_cast_fp16")];
+            bool var_4829_transpose_x_0 = const()[name = string("op_4829_transpose_x_0"), val = bool(false)];
+            bool var_4829_transpose_y_0 = const()[name = string("op_4829_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_4829_cast_fp16 = matmul(transpose_x = var_4829_transpose_x_0, transpose_y = var_4829_transpose_y_0, x = var_238_to_fp16, y = var_4828_cast_fp16)[name = string("op_4829_cast_fp16")];
+            fp16 const_59_promoted_to_fp16 = const()[name = string("const_59_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 32, 1, 1]> lower_49_cast_fp16 = mul(x = var_4829_cast_fp16, y = const_59_promoted_to_fp16)[name = string("lower_49_cast_fp16")];
+            bool bottom_49_interleave_0 = const()[name = string("bottom_49_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> bottom_49_cast_fp16 = concat(axis = var_4650, interleave = bottom_49_interleave_0, values = (lower_49_cast_fp16, var_238_to_fp16))[name = string("bottom_49_cast_fp16")];
+            bool inv_67_interleave_0 = const()[name = string("inv_67_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> inv_67_cast_fp16 = concat(axis = var_4621, interleave = inv_67_interleave_0, values = (top_1_to_fp16, bottom_49_cast_fp16))[name = string("inv_67_cast_fp16")];
+            tensor<int32, [5]> var_4838 = const()[name = string("op_4838"), val = tensor<int32, [5]>([1024, 16, 4, 16, 4])];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_4839_cast_fp16 = reshape(shape = var_4838, x = t_19_cast_fp16)[name = string("op_4839_cast_fp16")];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_4844_cast_fp16 = mul(x = var_4839_cast_fp16, y = var_259_to_fp16)[name = string("op_4844_cast_fp16")];
+            tensor<int32, [1]> blocks_51_axes_0 = const()[name = string("blocks_51_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_51_keep_dims_0 = const()[name = string("blocks_51_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> blocks_51_cast_fp16 = reduce_sum(axes = blocks_51_axes_0, keep_dims = blocks_51_keep_dims_0, x = var_4844_cast_fp16)[name = string("blocks_51_cast_fp16")];
+            tensor<int32, [4]> var_4847_begin_0 = const()[name = string("op_4847_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_4847_end_0 = const()[name = string("op_4847_end_0"), val = tensor<int32, [4]>([1024, 16, 4, 4])];
+            tensor<bool, [4]> var_4847_end_mask_0 = const()[name = string("op_4847_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 16, 2, 4]> var_4847_cast_fp16 = slice_by_index(begin = var_4847_begin_0, end = var_4847_end_0, end_mask = var_4847_end_mask_0, x = blocks_51_cast_fp16)[name = string("op_4847_cast_fp16")];
+            tensor<int32, [4]> x_133_begin_0 = const()[name = string("x_133_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_133_end_0 = const()[name = string("x_133_end_0"), val = tensor<int32, [4]>([1024, 16, 2, 2])];
+            tensor<bool, [4]> x_133_end_mask_0 = const()[name = string("x_133_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 16, 2, 2]> x_133_cast_fp16 = slice_by_index(begin = x_133_begin_0, end = x_133_end_0, end_mask = x_133_end_mask_0, x = var_4847_cast_fp16)[name = string("x_133_cast_fp16")];
+            tensor<int32, [5]> var_4849 = const()[name = string("op_4849"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<fp16, [1024, 16, 2, 2, 2]> pair_61_cast_fp16 = reshape(shape = var_4849, x = inv_67_cast_fp16)[name = string("pair_61_cast_fp16")];
+            tensor<int32, [5]> var_4851_begin_0 = const()[name = string("op_4851_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4851_end_0 = const()[name = string("op_4851_end_0"), val = tensor<int32, [5]>([1024, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_4851_end_mask_0 = const()[name = string("op_4851_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4851_squeeze_mask_0 = const()[name = string("op_4851_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_4851_cast_fp16 = slice_by_index(begin = var_4851_begin_0, end = var_4851_end_0, end_mask = var_4851_end_mask_0, squeeze_mask = var_4851_squeeze_mask_0, x = pair_61_cast_fp16)[name = string("op_4851_cast_fp16")];
+            tensor<int32, [5]> var_4854_begin_0 = const()[name = string("op_4854_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4854_end_0 = const()[name = string("op_4854_end_0"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_4854_end_mask_0 = const()[name = string("op_4854_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4854_squeeze_mask_0 = const()[name = string("op_4854_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_4854_cast_fp16 = slice_by_index(begin = var_4854_begin_0, end = var_4854_end_0, end_mask = var_4854_end_mask_0, squeeze_mask = var_4854_squeeze_mask_0, x = pair_61_cast_fp16)[name = string("op_4854_cast_fp16")];
+            bool var_4857_transpose_x_0 = const()[name = string("op_4857_transpose_x_0"), val = bool(false)];
+            bool var_4857_transpose_y_0 = const()[name = string("op_4857_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_4857_cast_fp16 = matmul(transpose_x = var_4857_transpose_x_0, transpose_y = var_4857_transpose_y_0, x = x_133_cast_fp16, y = var_4851_cast_fp16)[name = string("op_4857_cast_fp16")];
+            bool var_4858_transpose_x_0 = const()[name = string("op_4858_transpose_x_0"), val = bool(false)];
+            bool var_4858_transpose_y_0 = const()[name = string("op_4858_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_4858_cast_fp16 = matmul(transpose_x = var_4858_transpose_x_0, transpose_y = var_4858_transpose_y_0, x = var_4854_cast_fp16, y = var_4857_cast_fp16)[name = string("op_4858_cast_fp16")];
+            fp16 const_60_promoted_to_fp16 = const()[name = string("const_60_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 16, 2, 2]> lower_51_cast_fp16 = mul(x = var_4858_cast_fp16, y = const_60_promoted_to_fp16)[name = string("lower_51_cast_fp16")];
+            bool top_51_interleave_0 = const()[name = string("top_51_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> top_51_cast_fp16 = concat(axis = var_4650, interleave = top_51_interleave_0, values = (var_4851_cast_fp16, var_276_to_fp16))[name = string("top_51_cast_fp16")];
+            bool bottom_51_interleave_0 = const()[name = string("bottom_51_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> bottom_51_cast_fp16 = concat(axis = var_4650, interleave = bottom_51_interleave_0, values = (lower_51_cast_fp16, var_4854_cast_fp16))[name = string("bottom_51_cast_fp16")];
+            bool inv_69_interleave_0 = const()[name = string("inv_69_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> inv_69_cast_fp16 = concat(axis = var_4621, interleave = inv_69_interleave_0, values = (top_51_cast_fp16, bottom_51_cast_fp16))[name = string("inv_69_cast_fp16")];
+            tensor<int32, [5]> var_4867 = const()[name = string("op_4867"), val = tensor<int32, [5]>([1024, 8, 8, 8, 8])];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_4868_cast_fp16 = reshape(shape = var_4867, x = t_19_cast_fp16)[name = string("op_4868_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_4872_to_fp16 = const()[name = string("op_4872_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195988032)))];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_4873_cast_fp16 = mul(x = var_4868_cast_fp16, y = var_4872_to_fp16)[name = string("op_4873_cast_fp16")];
+            tensor<int32, [1]> blocks_53_axes_0 = const()[name = string("blocks_53_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_53_keep_dims_0 = const()[name = string("blocks_53_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> blocks_53_cast_fp16 = reduce_sum(axes = blocks_53_axes_0, keep_dims = blocks_53_keep_dims_0, x = var_4873_cast_fp16)[name = string("blocks_53_cast_fp16")];
+            tensor<int32, [4]> var_4876_begin_0 = const()[name = string("op_4876_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_4876_end_0 = const()[name = string("op_4876_end_0"), val = tensor<int32, [4]>([1024, 8, 8, 8])];
+            tensor<bool, [4]> var_4876_end_mask_0 = const()[name = string("op_4876_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 8, 4, 8]> var_4876_cast_fp16 = slice_by_index(begin = var_4876_begin_0, end = var_4876_end_0, end_mask = var_4876_end_mask_0, x = blocks_53_cast_fp16)[name = string("op_4876_cast_fp16")];
+            tensor<int32, [4]> x_135_begin_0 = const()[name = string("x_135_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_135_end_0 = const()[name = string("x_135_end_0"), val = tensor<int32, [4]>([1024, 8, 4, 4])];
+            tensor<bool, [4]> x_135_end_mask_0 = const()[name = string("x_135_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 8, 4, 4]> x_135_cast_fp16 = slice_by_index(begin = x_135_begin_0, end = x_135_end_0, end_mask = x_135_end_mask_0, x = var_4876_cast_fp16)[name = string("x_135_cast_fp16")];
+            tensor<int32, [5]> var_4878 = const()[name = string("op_4878"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<fp16, [1024, 8, 2, 4, 4]> pair_63_cast_fp16 = reshape(shape = var_4878, x = inv_69_cast_fp16)[name = string("pair_63_cast_fp16")];
+            tensor<int32, [5]> var_4880_begin_0 = const()[name = string("op_4880_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4880_end_0 = const()[name = string("op_4880_end_0"), val = tensor<int32, [5]>([1024, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_4880_end_mask_0 = const()[name = string("op_4880_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4880_squeeze_mask_0 = const()[name = string("op_4880_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_4880_cast_fp16 = slice_by_index(begin = var_4880_begin_0, end = var_4880_end_0, end_mask = var_4880_end_mask_0, squeeze_mask = var_4880_squeeze_mask_0, x = pair_63_cast_fp16)[name = string("op_4880_cast_fp16")];
+            tensor<int32, [5]> var_4883_begin_0 = const()[name = string("op_4883_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4883_end_0 = const()[name = string("op_4883_end_0"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_4883_end_mask_0 = const()[name = string("op_4883_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4883_squeeze_mask_0 = const()[name = string("op_4883_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_4883_cast_fp16 = slice_by_index(begin = var_4883_begin_0, end = var_4883_end_0, end_mask = var_4883_end_mask_0, squeeze_mask = var_4883_squeeze_mask_0, x = pair_63_cast_fp16)[name = string("op_4883_cast_fp16")];
+            bool var_4886_transpose_x_0 = const()[name = string("op_4886_transpose_x_0"), val = bool(false)];
+            bool var_4886_transpose_y_0 = const()[name = string("op_4886_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_4886_cast_fp16 = matmul(transpose_x = var_4886_transpose_x_0, transpose_y = var_4886_transpose_y_0, x = x_135_cast_fp16, y = var_4880_cast_fp16)[name = string("op_4886_cast_fp16")];
+            bool var_4887_transpose_x_0 = const()[name = string("op_4887_transpose_x_0"), val = bool(false)];
+            bool var_4887_transpose_y_0 = const()[name = string("op_4887_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_4887_cast_fp16 = matmul(transpose_x = var_4887_transpose_x_0, transpose_y = var_4887_transpose_y_0, x = var_4883_cast_fp16, y = var_4886_cast_fp16)[name = string("op_4887_cast_fp16")];
+            fp16 const_61_promoted_to_fp16 = const()[name = string("const_61_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 8, 4, 4]> lower_53_cast_fp16 = mul(x = var_4887_cast_fp16, y = const_61_promoted_to_fp16)[name = string("lower_53_cast_fp16")];
+            bool top_53_interleave_0 = const()[name = string("top_53_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> top_53_cast_fp16 = concat(axis = var_4650, interleave = top_53_interleave_0, values = (var_4880_cast_fp16, var_305_to_fp16))[name = string("top_53_cast_fp16")];
+            bool bottom_53_interleave_0 = const()[name = string("bottom_53_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> bottom_53_cast_fp16 = concat(axis = var_4650, interleave = bottom_53_interleave_0, values = (lower_53_cast_fp16, var_4883_cast_fp16))[name = string("bottom_53_cast_fp16")];
+            bool inv_71_interleave_0 = const()[name = string("inv_71_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> inv_71_cast_fp16 = concat(axis = var_4621, interleave = inv_71_interleave_0, values = (top_53_cast_fp16, bottom_53_cast_fp16))[name = string("inv_71_cast_fp16")];
+            tensor<int32, [5]> var_4896 = const()[name = string("op_4896"), val = tensor<int32, [5]>([1024, 4, 16, 4, 16])];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_4897_cast_fp16 = reshape(shape = var_4896, x = t_19_cast_fp16)[name = string("op_4897_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_4901_to_fp16 = const()[name = string("op_4901_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195988224)))];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_4902_cast_fp16 = mul(x = var_4897_cast_fp16, y = var_4901_to_fp16)[name = string("op_4902_cast_fp16")];
+            tensor<int32, [1]> blocks_55_axes_0 = const()[name = string("blocks_55_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_55_keep_dims_0 = const()[name = string("blocks_55_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> blocks_55_cast_fp16 = reduce_sum(axes = blocks_55_axes_0, keep_dims = blocks_55_keep_dims_0, x = var_4902_cast_fp16)[name = string("blocks_55_cast_fp16")];
+            tensor<int32, [4]> var_4905_begin_0 = const()[name = string("op_4905_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_4905_end_0 = const()[name = string("op_4905_end_0"), val = tensor<int32, [4]>([1024, 4, 16, 16])];
+            tensor<bool, [4]> var_4905_end_mask_0 = const()[name = string("op_4905_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 4, 8, 16]> var_4905_cast_fp16 = slice_by_index(begin = var_4905_begin_0, end = var_4905_end_0, end_mask = var_4905_end_mask_0, x = blocks_55_cast_fp16)[name = string("op_4905_cast_fp16")];
+            tensor<int32, [4]> x_137_begin_0 = const()[name = string("x_137_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_137_end_0 = const()[name = string("x_137_end_0"), val = tensor<int32, [4]>([1024, 4, 8, 8])];
+            tensor<bool, [4]> x_137_end_mask_0 = const()[name = string("x_137_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 4, 8, 8]> x_137_cast_fp16 = slice_by_index(begin = x_137_begin_0, end = x_137_end_0, end_mask = x_137_end_mask_0, x = var_4905_cast_fp16)[name = string("x_137_cast_fp16")];
+            tensor<int32, [5]> var_4907 = const()[name = string("op_4907"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<fp16, [1024, 4, 2, 8, 8]> pair_65_cast_fp16 = reshape(shape = var_4907, x = inv_71_cast_fp16)[name = string("pair_65_cast_fp16")];
+            tensor<int32, [5]> var_4909_begin_0 = const()[name = string("op_4909_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4909_end_0 = const()[name = string("op_4909_end_0"), val = tensor<int32, [5]>([1024, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_4909_end_mask_0 = const()[name = string("op_4909_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4909_squeeze_mask_0 = const()[name = string("op_4909_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_4909_cast_fp16 = slice_by_index(begin = var_4909_begin_0, end = var_4909_end_0, end_mask = var_4909_end_mask_0, squeeze_mask = var_4909_squeeze_mask_0, x = pair_65_cast_fp16)[name = string("op_4909_cast_fp16")];
+            tensor<int32, [5]> var_4912_begin_0 = const()[name = string("op_4912_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4912_end_0 = const()[name = string("op_4912_end_0"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_4912_end_mask_0 = const()[name = string("op_4912_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4912_squeeze_mask_0 = const()[name = string("op_4912_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_4912_cast_fp16 = slice_by_index(begin = var_4912_begin_0, end = var_4912_end_0, end_mask = var_4912_end_mask_0, squeeze_mask = var_4912_squeeze_mask_0, x = pair_65_cast_fp16)[name = string("op_4912_cast_fp16")];
+            bool var_4915_transpose_x_0 = const()[name = string("op_4915_transpose_x_0"), val = bool(false)];
+            bool var_4915_transpose_y_0 = const()[name = string("op_4915_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_4915_cast_fp16 = matmul(transpose_x = var_4915_transpose_x_0, transpose_y = var_4915_transpose_y_0, x = x_137_cast_fp16, y = var_4909_cast_fp16)[name = string("op_4915_cast_fp16")];
+            bool var_4916_transpose_x_0 = const()[name = string("op_4916_transpose_x_0"), val = bool(false)];
+            bool var_4916_transpose_y_0 = const()[name = string("op_4916_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_4916_cast_fp16 = matmul(transpose_x = var_4916_transpose_x_0, transpose_y = var_4916_transpose_y_0, x = var_4912_cast_fp16, y = var_4915_cast_fp16)[name = string("op_4916_cast_fp16")];
+            fp16 const_62_promoted_to_fp16 = const()[name = string("const_62_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 4, 8, 8]> lower_55_cast_fp16 = mul(x = var_4916_cast_fp16, y = const_62_promoted_to_fp16)[name = string("lower_55_cast_fp16")];
+            bool top_55_interleave_0 = const()[name = string("top_55_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> top_55_cast_fp16 = concat(axis = var_4650, interleave = top_55_interleave_0, values = (var_4909_cast_fp16, var_334_to_fp16))[name = string("top_55_cast_fp16")];
+            bool bottom_55_interleave_0 = const()[name = string("bottom_55_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> bottom_55_cast_fp16 = concat(axis = var_4650, interleave = bottom_55_interleave_0, values = (lower_55_cast_fp16, var_4912_cast_fp16))[name = string("bottom_55_cast_fp16")];
+            bool inv_73_interleave_0 = const()[name = string("inv_73_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> inv_73_cast_fp16 = concat(axis = var_4621, interleave = inv_73_interleave_0, values = (top_55_cast_fp16, bottom_55_cast_fp16))[name = string("inv_73_cast_fp16")];
+            tensor<int32, [5]> var_4925 = const()[name = string("op_4925"), val = tensor<int32, [5]>([1024, 2, 32, 2, 32])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_4926_cast_fp16 = reshape(shape = var_4925, x = t_19_cast_fp16)[name = string("op_4926_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_4930_to_fp16 = const()[name = string("op_4930_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_4931_cast_fp16 = mul(x = var_4926_cast_fp16, y = var_4930_to_fp16)[name = string("op_4931_cast_fp16")];
+            tensor<int32, [1]> blocks_57_axes_0 = const()[name = string("blocks_57_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_57_keep_dims_0 = const()[name = string("blocks_57_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> blocks_57_cast_fp16 = reduce_sum(axes = blocks_57_axes_0, keep_dims = blocks_57_keep_dims_0, x = var_4931_cast_fp16)[name = string("blocks_57_cast_fp16")];
+            tensor<int32, [4]> var_4934_begin_0 = const()[name = string("op_4934_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_4934_end_0 = const()[name = string("op_4934_end_0"), val = tensor<int32, [4]>([1024, 2, 32, 32])];
+            tensor<bool, [4]> var_4934_end_mask_0 = const()[name = string("op_4934_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 2, 16, 32]> var_4934_cast_fp16 = slice_by_index(begin = var_4934_begin_0, end = var_4934_end_0, end_mask = var_4934_end_mask_0, x = blocks_57_cast_fp16)[name = string("op_4934_cast_fp16")];
+            tensor<int32, [4]> x_139_begin_0 = const()[name = string("x_139_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_139_end_0 = const()[name = string("x_139_end_0"), val = tensor<int32, [4]>([1024, 2, 16, 16])];
+            tensor<bool, [4]> x_139_end_mask_0 = const()[name = string("x_139_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 2, 16, 16]> x_139_cast_fp16 = slice_by_index(begin = x_139_begin_0, end = x_139_end_0, end_mask = x_139_end_mask_0, x = var_4934_cast_fp16)[name = string("x_139_cast_fp16")];
+            tensor<int32, [5]> var_4936 = const()[name = string("op_4936"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<fp16, [1024, 2, 2, 16, 16]> pair_67_cast_fp16 = reshape(shape = var_4936, x = inv_73_cast_fp16)[name = string("pair_67_cast_fp16")];
+            tensor<int32, [5]> var_4938_begin_0 = const()[name = string("op_4938_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4938_end_0 = const()[name = string("op_4938_end_0"), val = tensor<int32, [5]>([1024, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_4938_end_mask_0 = const()[name = string("op_4938_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4938_squeeze_mask_0 = const()[name = string("op_4938_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_4938_cast_fp16 = slice_by_index(begin = var_4938_begin_0, end = var_4938_end_0, end_mask = var_4938_end_mask_0, squeeze_mask = var_4938_squeeze_mask_0, x = pair_67_cast_fp16)[name = string("op_4938_cast_fp16")];
+            tensor<int32, [5]> var_4941_begin_0 = const()[name = string("op_4941_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4941_end_0 = const()[name = string("op_4941_end_0"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_4941_end_mask_0 = const()[name = string("op_4941_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4941_squeeze_mask_0 = const()[name = string("op_4941_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_4941_cast_fp16 = slice_by_index(begin = var_4941_begin_0, end = var_4941_end_0, end_mask = var_4941_end_mask_0, squeeze_mask = var_4941_squeeze_mask_0, x = pair_67_cast_fp16)[name = string("op_4941_cast_fp16")];
+            bool var_4944_transpose_x_0 = const()[name = string("op_4944_transpose_x_0"), val = bool(false)];
+            bool var_4944_transpose_y_0 = const()[name = string("op_4944_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_4944_cast_fp16 = matmul(transpose_x = var_4944_transpose_x_0, transpose_y = var_4944_transpose_y_0, x = x_139_cast_fp16, y = var_4938_cast_fp16)[name = string("op_4944_cast_fp16")];
+            bool var_4945_transpose_x_0 = const()[name = string("op_4945_transpose_x_0"), val = bool(false)];
+            bool var_4945_transpose_y_0 = const()[name = string("op_4945_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_4945_cast_fp16 = matmul(transpose_x = var_4945_transpose_x_0, transpose_y = var_4945_transpose_y_0, x = var_4941_cast_fp16, y = var_4944_cast_fp16)[name = string("op_4945_cast_fp16")];
+            fp16 const_63_promoted_to_fp16 = const()[name = string("const_63_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 2, 16, 16]> lower_57_cast_fp16 = mul(x = var_4945_cast_fp16, y = const_63_promoted_to_fp16)[name = string("lower_57_cast_fp16")];
+            bool top_57_interleave_0 = const()[name = string("top_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> top_57_cast_fp16 = concat(axis = var_4650, interleave = top_57_interleave_0, values = (var_4938_cast_fp16, var_363_to_fp16))[name = string("top_57_cast_fp16")];
+            bool bottom_57_interleave_0 = const()[name = string("bottom_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> bottom_57_cast_fp16 = concat(axis = var_4650, interleave = bottom_57_interleave_0, values = (lower_57_cast_fp16, var_4941_cast_fp16))[name = string("bottom_57_cast_fp16")];
+            bool inv_75_interleave_0 = const()[name = string("inv_75_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> inv_75_cast_fp16 = concat(axis = var_4621, interleave = inv_75_interleave_0, values = (top_57_cast_fp16, bottom_57_cast_fp16))[name = string("inv_75_cast_fp16")];
+            tensor<int32, [5]> var_4954 = const()[name = string("op_4954"), val = tensor<int32, [5]>([1024, 1, 64, 1, 64])];
+            tensor<fp16, [1024, 1, 64, 1, 64]> var_4955_cast_fp16 = reshape(shape = var_4954, x = t_19_cast_fp16)[name = string("op_4955_cast_fp16")];
+            tensor<int32, [1]> blocks_59_axes_0 = const()[name = string("blocks_59_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_59_keep_dims_0 = const()[name = string("blocks_59_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> blocks_59_cast_fp16 = reduce_sum(axes = blocks_59_axes_0, keep_dims = blocks_59_keep_dims_0, x = var_4955_cast_fp16)[name = string("blocks_59_cast_fp16")];
+            tensor<int32, [4]> var_4963_begin_0 = const()[name = string("op_4963_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_4963_end_0 = const()[name = string("op_4963_end_0"), val = tensor<int32, [4]>([1024, 1, 64, 64])];
+            tensor<bool, [4]> var_4963_end_mask_0 = const()[name = string("op_4963_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 1, 32, 64]> var_4963_cast_fp16 = slice_by_index(begin = var_4963_begin_0, end = var_4963_end_0, end_mask = var_4963_end_mask_0, x = blocks_59_cast_fp16)[name = string("op_4963_cast_fp16")];
+            tensor<int32, [4]> x_141_begin_0 = const()[name = string("x_141_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_141_end_0 = const()[name = string("x_141_end_0"), val = tensor<int32, [4]>([1024, 1, 32, 32])];
+            tensor<bool, [4]> x_141_end_mask_0 = const()[name = string("x_141_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 1, 32, 32]> x_141_cast_fp16 = slice_by_index(begin = x_141_begin_0, end = x_141_end_0, end_mask = x_141_end_mask_0, x = var_4963_cast_fp16)[name = string("x_141_cast_fp16")];
+            tensor<int32, [5]> var_4965 = const()[name = string("op_4965"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<fp16, [1024, 1, 2, 32, 32]> pair_69_cast_fp16 = reshape(shape = var_4965, x = inv_75_cast_fp16)[name = string("pair_69_cast_fp16")];
+            tensor<int32, [5]> var_4967_begin_0 = const()[name = string("op_4967_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_4967_end_0 = const()[name = string("op_4967_end_0"), val = tensor<int32, [5]>([1024, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_4967_end_mask_0 = const()[name = string("op_4967_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4967_squeeze_mask_0 = const()[name = string("op_4967_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_4967_cast_fp16 = slice_by_index(begin = var_4967_begin_0, end = var_4967_end_0, end_mask = var_4967_end_mask_0, squeeze_mask = var_4967_squeeze_mask_0, x = pair_69_cast_fp16)[name = string("op_4967_cast_fp16")];
+            tensor<int32, [5]> var_4970_begin_0 = const()[name = string("op_4970_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_4970_end_0 = const()[name = string("op_4970_end_0"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_4970_end_mask_0 = const()[name = string("op_4970_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_4970_squeeze_mask_0 = const()[name = string("op_4970_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_4970_cast_fp16 = slice_by_index(begin = var_4970_begin_0, end = var_4970_end_0, end_mask = var_4970_end_mask_0, squeeze_mask = var_4970_squeeze_mask_0, x = pair_69_cast_fp16)[name = string("op_4970_cast_fp16")];
+            bool var_4973_transpose_x_0 = const()[name = string("op_4973_transpose_x_0"), val = bool(false)];
+            bool var_4973_transpose_y_0 = const()[name = string("op_4973_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_4973_cast_fp16 = matmul(transpose_x = var_4973_transpose_x_0, transpose_y = var_4973_transpose_y_0, x = x_141_cast_fp16, y = var_4967_cast_fp16)[name = string("op_4973_cast_fp16")];
+            bool var_4974_transpose_x_0 = const()[name = string("op_4974_transpose_x_0"), val = bool(false)];
+            bool var_4974_transpose_y_0 = const()[name = string("op_4974_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_4974_cast_fp16 = matmul(transpose_x = var_4974_transpose_x_0, transpose_y = var_4974_transpose_y_0, x = var_4970_cast_fp16, y = var_4973_cast_fp16)[name = string("op_4974_cast_fp16")];
+            fp16 const_64_promoted_to_fp16 = const()[name = string("const_64_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 1, 32, 32]> lower_59_cast_fp16 = mul(x = var_4974_cast_fp16, y = const_64_promoted_to_fp16)[name = string("lower_59_cast_fp16")];
+            bool top_59_interleave_0 = const()[name = string("top_59_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> top_59_cast_fp16 = concat(axis = var_4650, interleave = top_59_interleave_0, values = (var_4967_cast_fp16, var_392_to_fp16))[name = string("top_59_cast_fp16")];
+            bool bottom_59_interleave_0 = const()[name = string("bottom_59_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> bottom_59_cast_fp16 = concat(axis = var_4650, interleave = bottom_59_interleave_0, values = (lower_59_cast_fp16, var_4970_cast_fp16))[name = string("bottom_59_cast_fp16")];
+            bool inv_77_interleave_0 = const()[name = string("inv_77_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> inv_77_cast_fp16 = concat(axis = var_4621, interleave = inv_77_interleave_0, values = (top_59_cast_fp16, bottom_59_cast_fp16))[name = string("inv_77_cast_fp16")];
+            tensor<int32, [4]> var_4983 = const()[name = string("op_4983"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> inv_79_cast_fp16 = reshape(shape = var_4983, x = inv_77_cast_fp16)[name = string("inv_79_cast_fp16")];
+            bool new_v_9_transpose_x_0 = const()[name = string("new_v_9_transpose_x_0"), val = bool(false)];
+            bool new_v_9_transpose_y_0 = const()[name = string("new_v_9_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> new_v_9_cast_fp16 = matmul(transpose_x = new_v_9_transpose_x_0, transpose_y = new_v_9_transpose_y_0, x = inv_79_cast_fp16, y = v_beta_9_cast_fp16)[name = string("new_v_9_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_4986_cast_fp16 = exp(x = cum_9_cast_fp16)[name = string("op_4986_cast_fp16")];
+            tensor<int32, [1]> var_4987_axes_0 = const()[name = string("op_4987_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_4987_cast_fp16 = expand_dims(axes = var_4987_axes_0, x = var_4986_cast_fp16)[name = string("op_4987_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> var_4988_cast_fp16 = mul(x = k_beta_9_cast_fp16, y = var_4987_cast_fp16)[name = string("op_4988_cast_fp16")];
+            bool k_cumdecay_9_transpose_x_0 = const()[name = string("k_cumdecay_9_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_9_transpose_y_0 = const()[name = string("k_cumdecay_9_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> k_cumdecay_9_cast_fp16 = matmul(transpose_x = k_cumdecay_9_transpose_x_0, transpose_y = k_cumdecay_9_transpose_y_0, x = inv_79_cast_fp16, y = var_4988_cast_fp16)[name = string("k_cumdecay_9_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> q_dec_9_cast_fp16 = mul(x = q_53_cast_fp16, y = var_4987_cast_fp16)[name = string("q_dec_9_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_4993_cast_fp16 = exp(x = to_end_9_cast_fp16)[name = string("op_4993_cast_fp16")];
+            tensor<int32, [1]> var_4994_axes_0 = const()[name = string("op_4994_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_4994_cast_fp16 = expand_dims(axes = var_4994_axes_0, x = var_4993_cast_fp16)[name = string("op_4994_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_dec_9_cast_fp16 = mul(x = k_45_cast_fp16, y = var_4994_cast_fp16)[name = string("k_dec_9_cast_fp16")];
+            tensor<int32, [3]> var_4996_begin_0 = const()[name = string("op_4996_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_4996_end_0 = const()[name = string("op_4996_end_0"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<bool, [3]> var_4996_end_mask_0 = const()[name = string("op_4996_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 32, 1]> var_4996_cast_fp16 = slice_by_index(begin = var_4996_begin_0, end = var_4996_end_0, end_mask = var_4996_end_mask_0, x = cum_9_cast_fp16)[name = string("op_4996_cast_fp16")];
+            tensor<fp16, [32, 32, 1]> var_4997_cast_fp16 = exp(x = var_4996_cast_fp16)[name = string("op_4997_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_9_axes_0 = const()[name = string("chunk_decay_9_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 1, 1]> chunk_decay_9_cast_fp16 = expand_dims(axes = chunk_decay_9_axes_0, x = var_4997_cast_fp16)[name = string("chunk_decay_9_cast_fp16")];
+            tensor<int32, [4]> var_5002_begin_0 = const()[name = string("op_5002_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5002_end_0 = const()[name = string("op_5002_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_5002_end_mask_0 = const()[name = string("op_5002_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5002_squeeze_mask_0 = const()[name = string("op_5002_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5002_cast_fp16 = slice_by_index(begin = var_5002_begin_0, end = var_5002_end_0, end_mask = var_5002_end_mask_0, squeeze_mask = var_5002_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5002_cast_fp16")];
+            tensor<int32, [4]> var_5004_begin_0 = const()[name = string("op_5004_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5004_end_0 = const()[name = string("op_5004_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_5004_end_mask_0 = const()[name = string("op_5004_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5004_squeeze_mask_0 = const()[name = string("op_5004_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5004_cast_fp16 = slice_by_index(begin = var_5004_begin_0, end = var_5004_end_0, end_mask = var_5004_end_mask_0, squeeze_mask = var_5004_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5004_cast_fp16")];
+            bool var_5005_transpose_x_0 = const()[name = string("op_5005_transpose_x_0"), val = bool(false)];
+            bool var_5005_transpose_y_0 = const()[name = string("op_5005_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5005_cast_fp16 = matmul(transpose_x = var_5005_transpose_x_0, transpose_y = var_5005_transpose_y_0, x = var_5004_cast_fp16, y = state_1_to_fp16)[name = string("op_5005_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_257_cast_fp16 = sub(x = var_5002_cast_fp16, y = var_5005_cast_fp16)[name = string("v_new_257_cast_fp16")];
+            tensor<int32, [4]> var_5008_begin_0 = const()[name = string("op_5008_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5008_end_0 = const()[name = string("op_5008_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_5008_end_mask_0 = const()[name = string("op_5008_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5008_squeeze_mask_0 = const()[name = string("op_5008_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5008_cast_fp16 = slice_by_index(begin = var_5008_begin_0, end = var_5008_end_0, end_mask = var_5008_end_mask_0, squeeze_mask = var_5008_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5008_cast_fp16")];
+            bool var_5009_transpose_x_0 = const()[name = string("op_5009_transpose_x_0"), val = bool(false)];
+            bool var_5009_transpose_y_0 = const()[name = string("op_5009_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5009_cast_fp16 = matmul(transpose_x = var_5009_transpose_x_0, transpose_y = var_5009_transpose_y_0, x = var_5008_cast_fp16, y = state_1_to_fp16)[name = string("op_5009_cast_fp16")];
+            tensor<int32, [4]> var_5011_begin_0 = const()[name = string("op_5011_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5011_end_0 = const()[name = string("op_5011_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_5011_end_mask_0 = const()[name = string("op_5011_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5011_squeeze_mask_0 = const()[name = string("op_5011_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5011_cast_fp16 = slice_by_index(begin = var_5011_begin_0, end = var_5011_end_0, end_mask = var_5011_end_mask_0, squeeze_mask = var_5011_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5011_cast_fp16")];
+            bool var_5012_transpose_x_0 = const()[name = string("op_5012_transpose_x_0"), val = bool(false)];
+            bool var_5012_transpose_y_0 = const()[name = string("op_5012_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5012_cast_fp16 = matmul(transpose_x = var_5012_transpose_x_0, transpose_y = var_5012_transpose_y_0, x = var_5011_cast_fp16, y = v_new_257_cast_fp16)[name = string("op_5012_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5013_cast_fp16 = add(x = var_5009_cast_fp16, y = var_5012_cast_fp16)[name = string("op_5013_cast_fp16")];
+            tensor<int32, [4]> var_5015_begin_0 = const()[name = string("op_5015_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5015_end_0 = const()[name = string("op_5015_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_5015_end_mask_0 = const()[name = string("op_5015_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5015_squeeze_mask_0 = const()[name = string("op_5015_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5015_cast_fp16 = slice_by_index(begin = var_5015_begin_0, end = var_5015_end_0, end_mask = var_5015_end_mask_0, squeeze_mask = var_5015_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5015_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5016_cast_fp16 = mul(x = state_1_to_fp16, y = var_5015_cast_fp16)[name = string("op_5016_cast_fp16")];
+            tensor<int32, [4]> var_5018_begin_0 = const()[name = string("op_5018_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5018_end_0 = const()[name = string("op_5018_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_5018_end_mask_0 = const()[name = string("op_5018_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5018_squeeze_mask_0 = const()[name = string("op_5018_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5018_cast_fp16 = slice_by_index(begin = var_5018_begin_0, end = var_5018_end_0, end_mask = var_5018_end_mask_0, squeeze_mask = var_5018_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5018_cast_fp16")];
+            bool var_5020_transpose_x_1 = const()[name = string("op_5020_transpose_x_1"), val = bool(true)];
+            bool var_5020_transpose_y_1 = const()[name = string("op_5020_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5020_cast_fp16 = matmul(transpose_x = var_5020_transpose_x_1, transpose_y = var_5020_transpose_y_1, x = var_5018_cast_fp16, y = v_new_257_cast_fp16)[name = string("op_5020_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_259_cast_fp16 = add(x = var_5016_cast_fp16, y = var_5020_cast_fp16)[name = string("state_259_cast_fp16")];
+            tensor<int32, [4]> var_5023_begin_0 = const()[name = string("op_5023_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_5023_end_0 = const()[name = string("op_5023_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_5023_end_mask_0 = const()[name = string("op_5023_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5023_squeeze_mask_0 = const()[name = string("op_5023_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5023_cast_fp16 = slice_by_index(begin = var_5023_begin_0, end = var_5023_end_0, end_mask = var_5023_end_mask_0, squeeze_mask = var_5023_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5023_cast_fp16")];
+            tensor<int32, [4]> var_5025_begin_0 = const()[name = string("op_5025_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_5025_end_0 = const()[name = string("op_5025_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_5025_end_mask_0 = const()[name = string("op_5025_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5025_squeeze_mask_0 = const()[name = string("op_5025_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5025_cast_fp16 = slice_by_index(begin = var_5025_begin_0, end = var_5025_end_0, end_mask = var_5025_end_mask_0, squeeze_mask = var_5025_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5025_cast_fp16")];
+            bool var_5026_transpose_x_0 = const()[name = string("op_5026_transpose_x_0"), val = bool(false)];
+            bool var_5026_transpose_y_0 = const()[name = string("op_5026_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5026_cast_fp16 = matmul(transpose_x = var_5026_transpose_x_0, transpose_y = var_5026_transpose_y_0, x = var_5025_cast_fp16, y = state_259_cast_fp16)[name = string("op_5026_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_259_cast_fp16 = sub(x = var_5023_cast_fp16, y = var_5026_cast_fp16)[name = string("v_new_259_cast_fp16")];
+            tensor<int32, [4]> var_5029_begin_0 = const()[name = string("op_5029_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_5029_end_0 = const()[name = string("op_5029_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_5029_end_mask_0 = const()[name = string("op_5029_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5029_squeeze_mask_0 = const()[name = string("op_5029_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5029_cast_fp16 = slice_by_index(begin = var_5029_begin_0, end = var_5029_end_0, end_mask = var_5029_end_mask_0, squeeze_mask = var_5029_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5029_cast_fp16")];
+            bool var_5030_transpose_x_0 = const()[name = string("op_5030_transpose_x_0"), val = bool(false)];
+            bool var_5030_transpose_y_0 = const()[name = string("op_5030_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5030_cast_fp16 = matmul(transpose_x = var_5030_transpose_x_0, transpose_y = var_5030_transpose_y_0, x = var_5029_cast_fp16, y = state_259_cast_fp16)[name = string("op_5030_cast_fp16")];
+            tensor<int32, [4]> var_5032_begin_0 = const()[name = string("op_5032_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_5032_end_0 = const()[name = string("op_5032_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_5032_end_mask_0 = const()[name = string("op_5032_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5032_squeeze_mask_0 = const()[name = string("op_5032_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5032_cast_fp16 = slice_by_index(begin = var_5032_begin_0, end = var_5032_end_0, end_mask = var_5032_end_mask_0, squeeze_mask = var_5032_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5032_cast_fp16")];
+            bool var_5033_transpose_x_0 = const()[name = string("op_5033_transpose_x_0"), val = bool(false)];
+            bool var_5033_transpose_y_0 = const()[name = string("op_5033_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5033_cast_fp16 = matmul(transpose_x = var_5033_transpose_x_0, transpose_y = var_5033_transpose_y_0, x = var_5032_cast_fp16, y = v_new_259_cast_fp16)[name = string("op_5033_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5034_cast_fp16 = add(x = var_5030_cast_fp16, y = var_5033_cast_fp16)[name = string("op_5034_cast_fp16")];
+            tensor<int32, [4]> var_5036_begin_0 = const()[name = string("op_5036_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_5036_end_0 = const()[name = string("op_5036_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_5036_end_mask_0 = const()[name = string("op_5036_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5036_squeeze_mask_0 = const()[name = string("op_5036_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5036_cast_fp16 = slice_by_index(begin = var_5036_begin_0, end = var_5036_end_0, end_mask = var_5036_end_mask_0, squeeze_mask = var_5036_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5036_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5037_cast_fp16 = mul(x = state_259_cast_fp16, y = var_5036_cast_fp16)[name = string("op_5037_cast_fp16")];
+            tensor<int32, [4]> var_5039_begin_0 = const()[name = string("op_5039_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_5039_end_0 = const()[name = string("op_5039_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_5039_end_mask_0 = const()[name = string("op_5039_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5039_squeeze_mask_0 = const()[name = string("op_5039_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5039_cast_fp16 = slice_by_index(begin = var_5039_begin_0, end = var_5039_end_0, end_mask = var_5039_end_mask_0, squeeze_mask = var_5039_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5039_cast_fp16")];
+            bool var_5041_transpose_x_1 = const()[name = string("op_5041_transpose_x_1"), val = bool(true)];
+            bool var_5041_transpose_y_1 = const()[name = string("op_5041_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5041_cast_fp16 = matmul(transpose_x = var_5041_transpose_x_1, transpose_y = var_5041_transpose_y_1, x = var_5039_cast_fp16, y = v_new_259_cast_fp16)[name = string("op_5041_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_261_cast_fp16 = add(x = var_5037_cast_fp16, y = var_5041_cast_fp16)[name = string("state_261_cast_fp16")];
+            tensor<int32, [4]> var_5044_begin_0 = const()[name = string("op_5044_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_5044_end_0 = const()[name = string("op_5044_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_5044_end_mask_0 = const()[name = string("op_5044_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5044_squeeze_mask_0 = const()[name = string("op_5044_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5044_cast_fp16 = slice_by_index(begin = var_5044_begin_0, end = var_5044_end_0, end_mask = var_5044_end_mask_0, squeeze_mask = var_5044_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5044_cast_fp16")];
+            tensor<int32, [4]> var_5046_begin_0 = const()[name = string("op_5046_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_5046_end_0 = const()[name = string("op_5046_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_5046_end_mask_0 = const()[name = string("op_5046_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5046_squeeze_mask_0 = const()[name = string("op_5046_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5046_cast_fp16 = slice_by_index(begin = var_5046_begin_0, end = var_5046_end_0, end_mask = var_5046_end_mask_0, squeeze_mask = var_5046_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5046_cast_fp16")];
+            bool var_5047_transpose_x_0 = const()[name = string("op_5047_transpose_x_0"), val = bool(false)];
+            bool var_5047_transpose_y_0 = const()[name = string("op_5047_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5047_cast_fp16 = matmul(transpose_x = var_5047_transpose_x_0, transpose_y = var_5047_transpose_y_0, x = var_5046_cast_fp16, y = state_261_cast_fp16)[name = string("op_5047_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_261_cast_fp16 = sub(x = var_5044_cast_fp16, y = var_5047_cast_fp16)[name = string("v_new_261_cast_fp16")];
+            tensor<int32, [4]> var_5050_begin_0 = const()[name = string("op_5050_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_5050_end_0 = const()[name = string("op_5050_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_5050_end_mask_0 = const()[name = string("op_5050_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5050_squeeze_mask_0 = const()[name = string("op_5050_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5050_cast_fp16 = slice_by_index(begin = var_5050_begin_0, end = var_5050_end_0, end_mask = var_5050_end_mask_0, squeeze_mask = var_5050_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5050_cast_fp16")];
+            bool var_5051_transpose_x_0 = const()[name = string("op_5051_transpose_x_0"), val = bool(false)];
+            bool var_5051_transpose_y_0 = const()[name = string("op_5051_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5051_cast_fp16 = matmul(transpose_x = var_5051_transpose_x_0, transpose_y = var_5051_transpose_y_0, x = var_5050_cast_fp16, y = state_261_cast_fp16)[name = string("op_5051_cast_fp16")];
+            tensor<int32, [4]> var_5053_begin_0 = const()[name = string("op_5053_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_5053_end_0 = const()[name = string("op_5053_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_5053_end_mask_0 = const()[name = string("op_5053_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5053_squeeze_mask_0 = const()[name = string("op_5053_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5053_cast_fp16 = slice_by_index(begin = var_5053_begin_0, end = var_5053_end_0, end_mask = var_5053_end_mask_0, squeeze_mask = var_5053_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5053_cast_fp16")];
+            bool var_5054_transpose_x_0 = const()[name = string("op_5054_transpose_x_0"), val = bool(false)];
+            bool var_5054_transpose_y_0 = const()[name = string("op_5054_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5054_cast_fp16 = matmul(transpose_x = var_5054_transpose_x_0, transpose_y = var_5054_transpose_y_0, x = var_5053_cast_fp16, y = v_new_261_cast_fp16)[name = string("op_5054_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5055_cast_fp16 = add(x = var_5051_cast_fp16, y = var_5054_cast_fp16)[name = string("op_5055_cast_fp16")];
+            tensor<int32, [4]> var_5057_begin_0 = const()[name = string("op_5057_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_5057_end_0 = const()[name = string("op_5057_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_5057_end_mask_0 = const()[name = string("op_5057_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5057_squeeze_mask_0 = const()[name = string("op_5057_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5057_cast_fp16 = slice_by_index(begin = var_5057_begin_0, end = var_5057_end_0, end_mask = var_5057_end_mask_0, squeeze_mask = var_5057_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5057_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5058_cast_fp16 = mul(x = state_261_cast_fp16, y = var_5057_cast_fp16)[name = string("op_5058_cast_fp16")];
+            tensor<int32, [4]> var_5060_begin_0 = const()[name = string("op_5060_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_5060_end_0 = const()[name = string("op_5060_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_5060_end_mask_0 = const()[name = string("op_5060_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5060_squeeze_mask_0 = const()[name = string("op_5060_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5060_cast_fp16 = slice_by_index(begin = var_5060_begin_0, end = var_5060_end_0, end_mask = var_5060_end_mask_0, squeeze_mask = var_5060_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5060_cast_fp16")];
+            bool var_5062_transpose_x_1 = const()[name = string("op_5062_transpose_x_1"), val = bool(true)];
+            bool var_5062_transpose_y_1 = const()[name = string("op_5062_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5062_cast_fp16 = matmul(transpose_x = var_5062_transpose_x_1, transpose_y = var_5062_transpose_y_1, x = var_5060_cast_fp16, y = v_new_261_cast_fp16)[name = string("op_5062_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_263_cast_fp16 = add(x = var_5058_cast_fp16, y = var_5062_cast_fp16)[name = string("state_263_cast_fp16")];
+            tensor<int32, [4]> var_5065_begin_0 = const()[name = string("op_5065_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_5065_end_0 = const()[name = string("op_5065_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_5065_end_mask_0 = const()[name = string("op_5065_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5065_squeeze_mask_0 = const()[name = string("op_5065_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5065_cast_fp16 = slice_by_index(begin = var_5065_begin_0, end = var_5065_end_0, end_mask = var_5065_end_mask_0, squeeze_mask = var_5065_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5065_cast_fp16")];
+            tensor<int32, [4]> var_5067_begin_0 = const()[name = string("op_5067_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_5067_end_0 = const()[name = string("op_5067_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_5067_end_mask_0 = const()[name = string("op_5067_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5067_squeeze_mask_0 = const()[name = string("op_5067_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5067_cast_fp16 = slice_by_index(begin = var_5067_begin_0, end = var_5067_end_0, end_mask = var_5067_end_mask_0, squeeze_mask = var_5067_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5067_cast_fp16")];
+            bool var_5068_transpose_x_0 = const()[name = string("op_5068_transpose_x_0"), val = bool(false)];
+            bool var_5068_transpose_y_0 = const()[name = string("op_5068_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5068_cast_fp16 = matmul(transpose_x = var_5068_transpose_x_0, transpose_y = var_5068_transpose_y_0, x = var_5067_cast_fp16, y = state_263_cast_fp16)[name = string("op_5068_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_263_cast_fp16 = sub(x = var_5065_cast_fp16, y = var_5068_cast_fp16)[name = string("v_new_263_cast_fp16")];
+            tensor<int32, [4]> var_5071_begin_0 = const()[name = string("op_5071_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_5071_end_0 = const()[name = string("op_5071_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_5071_end_mask_0 = const()[name = string("op_5071_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5071_squeeze_mask_0 = const()[name = string("op_5071_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5071_cast_fp16 = slice_by_index(begin = var_5071_begin_0, end = var_5071_end_0, end_mask = var_5071_end_mask_0, squeeze_mask = var_5071_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5071_cast_fp16")];
+            bool var_5072_transpose_x_0 = const()[name = string("op_5072_transpose_x_0"), val = bool(false)];
+            bool var_5072_transpose_y_0 = const()[name = string("op_5072_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5072_cast_fp16 = matmul(transpose_x = var_5072_transpose_x_0, transpose_y = var_5072_transpose_y_0, x = var_5071_cast_fp16, y = state_263_cast_fp16)[name = string("op_5072_cast_fp16")];
+            tensor<int32, [4]> var_5074_begin_0 = const()[name = string("op_5074_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_5074_end_0 = const()[name = string("op_5074_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_5074_end_mask_0 = const()[name = string("op_5074_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5074_squeeze_mask_0 = const()[name = string("op_5074_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5074_cast_fp16 = slice_by_index(begin = var_5074_begin_0, end = var_5074_end_0, end_mask = var_5074_end_mask_0, squeeze_mask = var_5074_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5074_cast_fp16")];
+            bool var_5075_transpose_x_0 = const()[name = string("op_5075_transpose_x_0"), val = bool(false)];
+            bool var_5075_transpose_y_0 = const()[name = string("op_5075_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5075_cast_fp16 = matmul(transpose_x = var_5075_transpose_x_0, transpose_y = var_5075_transpose_y_0, x = var_5074_cast_fp16, y = v_new_263_cast_fp16)[name = string("op_5075_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5076_cast_fp16 = add(x = var_5072_cast_fp16, y = var_5075_cast_fp16)[name = string("op_5076_cast_fp16")];
+            tensor<int32, [4]> var_5078_begin_0 = const()[name = string("op_5078_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_5078_end_0 = const()[name = string("op_5078_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_5078_end_mask_0 = const()[name = string("op_5078_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5078_squeeze_mask_0 = const()[name = string("op_5078_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5078_cast_fp16 = slice_by_index(begin = var_5078_begin_0, end = var_5078_end_0, end_mask = var_5078_end_mask_0, squeeze_mask = var_5078_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5078_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5079_cast_fp16 = mul(x = state_263_cast_fp16, y = var_5078_cast_fp16)[name = string("op_5079_cast_fp16")];
+            tensor<int32, [4]> var_5081_begin_0 = const()[name = string("op_5081_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_5081_end_0 = const()[name = string("op_5081_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_5081_end_mask_0 = const()[name = string("op_5081_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5081_squeeze_mask_0 = const()[name = string("op_5081_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5081_cast_fp16 = slice_by_index(begin = var_5081_begin_0, end = var_5081_end_0, end_mask = var_5081_end_mask_0, squeeze_mask = var_5081_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5081_cast_fp16")];
+            bool var_5083_transpose_x_1 = const()[name = string("op_5083_transpose_x_1"), val = bool(true)];
+            bool var_5083_transpose_y_1 = const()[name = string("op_5083_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5083_cast_fp16 = matmul(transpose_x = var_5083_transpose_x_1, transpose_y = var_5083_transpose_y_1, x = var_5081_cast_fp16, y = v_new_263_cast_fp16)[name = string("op_5083_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_265_cast_fp16 = add(x = var_5079_cast_fp16, y = var_5083_cast_fp16)[name = string("state_265_cast_fp16")];
+            tensor<int32, [4]> var_5086_begin_0 = const()[name = string("op_5086_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_5086_end_0 = const()[name = string("op_5086_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_5086_end_mask_0 = const()[name = string("op_5086_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5086_squeeze_mask_0 = const()[name = string("op_5086_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5086_cast_fp16 = slice_by_index(begin = var_5086_begin_0, end = var_5086_end_0, end_mask = var_5086_end_mask_0, squeeze_mask = var_5086_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5086_cast_fp16")];
+            tensor<int32, [4]> var_5088_begin_0 = const()[name = string("op_5088_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_5088_end_0 = const()[name = string("op_5088_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_5088_end_mask_0 = const()[name = string("op_5088_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5088_squeeze_mask_0 = const()[name = string("op_5088_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5088_cast_fp16 = slice_by_index(begin = var_5088_begin_0, end = var_5088_end_0, end_mask = var_5088_end_mask_0, squeeze_mask = var_5088_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5088_cast_fp16")];
+            bool var_5089_transpose_x_0 = const()[name = string("op_5089_transpose_x_0"), val = bool(false)];
+            bool var_5089_transpose_y_0 = const()[name = string("op_5089_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5089_cast_fp16 = matmul(transpose_x = var_5089_transpose_x_0, transpose_y = var_5089_transpose_y_0, x = var_5088_cast_fp16, y = state_265_cast_fp16)[name = string("op_5089_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_265_cast_fp16 = sub(x = var_5086_cast_fp16, y = var_5089_cast_fp16)[name = string("v_new_265_cast_fp16")];
+            tensor<int32, [4]> var_5092_begin_0 = const()[name = string("op_5092_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_5092_end_0 = const()[name = string("op_5092_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_5092_end_mask_0 = const()[name = string("op_5092_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5092_squeeze_mask_0 = const()[name = string("op_5092_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5092_cast_fp16 = slice_by_index(begin = var_5092_begin_0, end = var_5092_end_0, end_mask = var_5092_end_mask_0, squeeze_mask = var_5092_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5092_cast_fp16")];
+            bool var_5093_transpose_x_0 = const()[name = string("op_5093_transpose_x_0"), val = bool(false)];
+            bool var_5093_transpose_y_0 = const()[name = string("op_5093_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5093_cast_fp16 = matmul(transpose_x = var_5093_transpose_x_0, transpose_y = var_5093_transpose_y_0, x = var_5092_cast_fp16, y = state_265_cast_fp16)[name = string("op_5093_cast_fp16")];
+            tensor<int32, [4]> var_5095_begin_0 = const()[name = string("op_5095_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_5095_end_0 = const()[name = string("op_5095_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_5095_end_mask_0 = const()[name = string("op_5095_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5095_squeeze_mask_0 = const()[name = string("op_5095_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5095_cast_fp16 = slice_by_index(begin = var_5095_begin_0, end = var_5095_end_0, end_mask = var_5095_end_mask_0, squeeze_mask = var_5095_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5095_cast_fp16")];
+            bool var_5096_transpose_x_0 = const()[name = string("op_5096_transpose_x_0"), val = bool(false)];
+            bool var_5096_transpose_y_0 = const()[name = string("op_5096_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5096_cast_fp16 = matmul(transpose_x = var_5096_transpose_x_0, transpose_y = var_5096_transpose_y_0, x = var_5095_cast_fp16, y = v_new_265_cast_fp16)[name = string("op_5096_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5097_cast_fp16 = add(x = var_5093_cast_fp16, y = var_5096_cast_fp16)[name = string("op_5097_cast_fp16")];
+            tensor<int32, [4]> var_5099_begin_0 = const()[name = string("op_5099_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_5099_end_0 = const()[name = string("op_5099_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_5099_end_mask_0 = const()[name = string("op_5099_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5099_squeeze_mask_0 = const()[name = string("op_5099_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5099_cast_fp16 = slice_by_index(begin = var_5099_begin_0, end = var_5099_end_0, end_mask = var_5099_end_mask_0, squeeze_mask = var_5099_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5099_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5100_cast_fp16 = mul(x = state_265_cast_fp16, y = var_5099_cast_fp16)[name = string("op_5100_cast_fp16")];
+            tensor<int32, [4]> var_5102_begin_0 = const()[name = string("op_5102_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_5102_end_0 = const()[name = string("op_5102_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_5102_end_mask_0 = const()[name = string("op_5102_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5102_squeeze_mask_0 = const()[name = string("op_5102_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5102_cast_fp16 = slice_by_index(begin = var_5102_begin_0, end = var_5102_end_0, end_mask = var_5102_end_mask_0, squeeze_mask = var_5102_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5102_cast_fp16")];
+            bool var_5104_transpose_x_1 = const()[name = string("op_5104_transpose_x_1"), val = bool(true)];
+            bool var_5104_transpose_y_1 = const()[name = string("op_5104_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5104_cast_fp16 = matmul(transpose_x = var_5104_transpose_x_1, transpose_y = var_5104_transpose_y_1, x = var_5102_cast_fp16, y = v_new_265_cast_fp16)[name = string("op_5104_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_267_cast_fp16 = add(x = var_5100_cast_fp16, y = var_5104_cast_fp16)[name = string("state_267_cast_fp16")];
+            tensor<int32, [4]> var_5107_begin_0 = const()[name = string("op_5107_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_5107_end_0 = const()[name = string("op_5107_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_5107_end_mask_0 = const()[name = string("op_5107_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5107_squeeze_mask_0 = const()[name = string("op_5107_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5107_cast_fp16 = slice_by_index(begin = var_5107_begin_0, end = var_5107_end_0, end_mask = var_5107_end_mask_0, squeeze_mask = var_5107_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5107_cast_fp16")];
+            tensor<int32, [4]> var_5109_begin_0 = const()[name = string("op_5109_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_5109_end_0 = const()[name = string("op_5109_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_5109_end_mask_0 = const()[name = string("op_5109_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5109_squeeze_mask_0 = const()[name = string("op_5109_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5109_cast_fp16 = slice_by_index(begin = var_5109_begin_0, end = var_5109_end_0, end_mask = var_5109_end_mask_0, squeeze_mask = var_5109_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5109_cast_fp16")];
+            bool var_5110_transpose_x_0 = const()[name = string("op_5110_transpose_x_0"), val = bool(false)];
+            bool var_5110_transpose_y_0 = const()[name = string("op_5110_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5110_cast_fp16 = matmul(transpose_x = var_5110_transpose_x_0, transpose_y = var_5110_transpose_y_0, x = var_5109_cast_fp16, y = state_267_cast_fp16)[name = string("op_5110_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_267_cast_fp16 = sub(x = var_5107_cast_fp16, y = var_5110_cast_fp16)[name = string("v_new_267_cast_fp16")];
+            tensor<int32, [4]> var_5113_begin_0 = const()[name = string("op_5113_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_5113_end_0 = const()[name = string("op_5113_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_5113_end_mask_0 = const()[name = string("op_5113_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5113_squeeze_mask_0 = const()[name = string("op_5113_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5113_cast_fp16 = slice_by_index(begin = var_5113_begin_0, end = var_5113_end_0, end_mask = var_5113_end_mask_0, squeeze_mask = var_5113_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5113_cast_fp16")];
+            bool var_5114_transpose_x_0 = const()[name = string("op_5114_transpose_x_0"), val = bool(false)];
+            bool var_5114_transpose_y_0 = const()[name = string("op_5114_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5114_cast_fp16 = matmul(transpose_x = var_5114_transpose_x_0, transpose_y = var_5114_transpose_y_0, x = var_5113_cast_fp16, y = state_267_cast_fp16)[name = string("op_5114_cast_fp16")];
+            tensor<int32, [4]> var_5116_begin_0 = const()[name = string("op_5116_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_5116_end_0 = const()[name = string("op_5116_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_5116_end_mask_0 = const()[name = string("op_5116_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5116_squeeze_mask_0 = const()[name = string("op_5116_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5116_cast_fp16 = slice_by_index(begin = var_5116_begin_0, end = var_5116_end_0, end_mask = var_5116_end_mask_0, squeeze_mask = var_5116_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5116_cast_fp16")];
+            bool var_5117_transpose_x_0 = const()[name = string("op_5117_transpose_x_0"), val = bool(false)];
+            bool var_5117_transpose_y_0 = const()[name = string("op_5117_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5117_cast_fp16 = matmul(transpose_x = var_5117_transpose_x_0, transpose_y = var_5117_transpose_y_0, x = var_5116_cast_fp16, y = v_new_267_cast_fp16)[name = string("op_5117_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5118_cast_fp16 = add(x = var_5114_cast_fp16, y = var_5117_cast_fp16)[name = string("op_5118_cast_fp16")];
+            tensor<int32, [4]> var_5120_begin_0 = const()[name = string("op_5120_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_5120_end_0 = const()[name = string("op_5120_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_5120_end_mask_0 = const()[name = string("op_5120_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5120_squeeze_mask_0 = const()[name = string("op_5120_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5120_cast_fp16 = slice_by_index(begin = var_5120_begin_0, end = var_5120_end_0, end_mask = var_5120_end_mask_0, squeeze_mask = var_5120_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5120_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5121_cast_fp16 = mul(x = state_267_cast_fp16, y = var_5120_cast_fp16)[name = string("op_5121_cast_fp16")];
+            tensor<int32, [4]> var_5123_begin_0 = const()[name = string("op_5123_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_5123_end_0 = const()[name = string("op_5123_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_5123_end_mask_0 = const()[name = string("op_5123_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5123_squeeze_mask_0 = const()[name = string("op_5123_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5123_cast_fp16 = slice_by_index(begin = var_5123_begin_0, end = var_5123_end_0, end_mask = var_5123_end_mask_0, squeeze_mask = var_5123_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5123_cast_fp16")];
+            bool var_5125_transpose_x_1 = const()[name = string("op_5125_transpose_x_1"), val = bool(true)];
+            bool var_5125_transpose_y_1 = const()[name = string("op_5125_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5125_cast_fp16 = matmul(transpose_x = var_5125_transpose_x_1, transpose_y = var_5125_transpose_y_1, x = var_5123_cast_fp16, y = v_new_267_cast_fp16)[name = string("op_5125_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_269_cast_fp16 = add(x = var_5121_cast_fp16, y = var_5125_cast_fp16)[name = string("state_269_cast_fp16")];
+            tensor<int32, [4]> var_5128_begin_0 = const()[name = string("op_5128_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_5128_end_0 = const()[name = string("op_5128_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_5128_end_mask_0 = const()[name = string("op_5128_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5128_squeeze_mask_0 = const()[name = string("op_5128_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5128_cast_fp16 = slice_by_index(begin = var_5128_begin_0, end = var_5128_end_0, end_mask = var_5128_end_mask_0, squeeze_mask = var_5128_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5128_cast_fp16")];
+            tensor<int32, [4]> var_5130_begin_0 = const()[name = string("op_5130_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_5130_end_0 = const()[name = string("op_5130_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_5130_end_mask_0 = const()[name = string("op_5130_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5130_squeeze_mask_0 = const()[name = string("op_5130_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5130_cast_fp16 = slice_by_index(begin = var_5130_begin_0, end = var_5130_end_0, end_mask = var_5130_end_mask_0, squeeze_mask = var_5130_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5130_cast_fp16")];
+            bool var_5131_transpose_x_0 = const()[name = string("op_5131_transpose_x_0"), val = bool(false)];
+            bool var_5131_transpose_y_0 = const()[name = string("op_5131_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5131_cast_fp16 = matmul(transpose_x = var_5131_transpose_x_0, transpose_y = var_5131_transpose_y_0, x = var_5130_cast_fp16, y = state_269_cast_fp16)[name = string("op_5131_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_269_cast_fp16 = sub(x = var_5128_cast_fp16, y = var_5131_cast_fp16)[name = string("v_new_269_cast_fp16")];
+            tensor<int32, [4]> var_5134_begin_0 = const()[name = string("op_5134_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_5134_end_0 = const()[name = string("op_5134_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_5134_end_mask_0 = const()[name = string("op_5134_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5134_squeeze_mask_0 = const()[name = string("op_5134_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5134_cast_fp16 = slice_by_index(begin = var_5134_begin_0, end = var_5134_end_0, end_mask = var_5134_end_mask_0, squeeze_mask = var_5134_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5134_cast_fp16")];
+            bool var_5135_transpose_x_0 = const()[name = string("op_5135_transpose_x_0"), val = bool(false)];
+            bool var_5135_transpose_y_0 = const()[name = string("op_5135_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5135_cast_fp16 = matmul(transpose_x = var_5135_transpose_x_0, transpose_y = var_5135_transpose_y_0, x = var_5134_cast_fp16, y = state_269_cast_fp16)[name = string("op_5135_cast_fp16")];
+            tensor<int32, [4]> var_5137_begin_0 = const()[name = string("op_5137_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_5137_end_0 = const()[name = string("op_5137_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_5137_end_mask_0 = const()[name = string("op_5137_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5137_squeeze_mask_0 = const()[name = string("op_5137_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5137_cast_fp16 = slice_by_index(begin = var_5137_begin_0, end = var_5137_end_0, end_mask = var_5137_end_mask_0, squeeze_mask = var_5137_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5137_cast_fp16")];
+            bool var_5138_transpose_x_0 = const()[name = string("op_5138_transpose_x_0"), val = bool(false)];
+            bool var_5138_transpose_y_0 = const()[name = string("op_5138_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5138_cast_fp16 = matmul(transpose_x = var_5138_transpose_x_0, transpose_y = var_5138_transpose_y_0, x = var_5137_cast_fp16, y = v_new_269_cast_fp16)[name = string("op_5138_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5139_cast_fp16 = add(x = var_5135_cast_fp16, y = var_5138_cast_fp16)[name = string("op_5139_cast_fp16")];
+            tensor<int32, [4]> var_5141_begin_0 = const()[name = string("op_5141_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_5141_end_0 = const()[name = string("op_5141_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_5141_end_mask_0 = const()[name = string("op_5141_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5141_squeeze_mask_0 = const()[name = string("op_5141_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5141_cast_fp16 = slice_by_index(begin = var_5141_begin_0, end = var_5141_end_0, end_mask = var_5141_end_mask_0, squeeze_mask = var_5141_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5141_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5142_cast_fp16 = mul(x = state_269_cast_fp16, y = var_5141_cast_fp16)[name = string("op_5142_cast_fp16")];
+            tensor<int32, [4]> var_5144_begin_0 = const()[name = string("op_5144_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_5144_end_0 = const()[name = string("op_5144_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_5144_end_mask_0 = const()[name = string("op_5144_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5144_squeeze_mask_0 = const()[name = string("op_5144_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5144_cast_fp16 = slice_by_index(begin = var_5144_begin_0, end = var_5144_end_0, end_mask = var_5144_end_mask_0, squeeze_mask = var_5144_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5144_cast_fp16")];
+            bool var_5146_transpose_x_1 = const()[name = string("op_5146_transpose_x_1"), val = bool(true)];
+            bool var_5146_transpose_y_1 = const()[name = string("op_5146_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5146_cast_fp16 = matmul(transpose_x = var_5146_transpose_x_1, transpose_y = var_5146_transpose_y_1, x = var_5144_cast_fp16, y = v_new_269_cast_fp16)[name = string("op_5146_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_271_cast_fp16 = add(x = var_5142_cast_fp16, y = var_5146_cast_fp16)[name = string("state_271_cast_fp16")];
+            tensor<int32, [4]> var_5149_begin_0 = const()[name = string("op_5149_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_5149_end_0 = const()[name = string("op_5149_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_5149_end_mask_0 = const()[name = string("op_5149_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5149_squeeze_mask_0 = const()[name = string("op_5149_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5149_cast_fp16 = slice_by_index(begin = var_5149_begin_0, end = var_5149_end_0, end_mask = var_5149_end_mask_0, squeeze_mask = var_5149_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5149_cast_fp16")];
+            tensor<int32, [4]> var_5151_begin_0 = const()[name = string("op_5151_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_5151_end_0 = const()[name = string("op_5151_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_5151_end_mask_0 = const()[name = string("op_5151_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5151_squeeze_mask_0 = const()[name = string("op_5151_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5151_cast_fp16 = slice_by_index(begin = var_5151_begin_0, end = var_5151_end_0, end_mask = var_5151_end_mask_0, squeeze_mask = var_5151_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5151_cast_fp16")];
+            bool var_5152_transpose_x_0 = const()[name = string("op_5152_transpose_x_0"), val = bool(false)];
+            bool var_5152_transpose_y_0 = const()[name = string("op_5152_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5152_cast_fp16 = matmul(transpose_x = var_5152_transpose_x_0, transpose_y = var_5152_transpose_y_0, x = var_5151_cast_fp16, y = state_271_cast_fp16)[name = string("op_5152_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_271_cast_fp16 = sub(x = var_5149_cast_fp16, y = var_5152_cast_fp16)[name = string("v_new_271_cast_fp16")];
+            tensor<int32, [4]> var_5155_begin_0 = const()[name = string("op_5155_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_5155_end_0 = const()[name = string("op_5155_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_5155_end_mask_0 = const()[name = string("op_5155_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5155_squeeze_mask_0 = const()[name = string("op_5155_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5155_cast_fp16 = slice_by_index(begin = var_5155_begin_0, end = var_5155_end_0, end_mask = var_5155_end_mask_0, squeeze_mask = var_5155_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5155_cast_fp16")];
+            bool var_5156_transpose_x_0 = const()[name = string("op_5156_transpose_x_0"), val = bool(false)];
+            bool var_5156_transpose_y_0 = const()[name = string("op_5156_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5156_cast_fp16 = matmul(transpose_x = var_5156_transpose_x_0, transpose_y = var_5156_transpose_y_0, x = var_5155_cast_fp16, y = state_271_cast_fp16)[name = string("op_5156_cast_fp16")];
+            tensor<int32, [4]> var_5158_begin_0 = const()[name = string("op_5158_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_5158_end_0 = const()[name = string("op_5158_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_5158_end_mask_0 = const()[name = string("op_5158_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5158_squeeze_mask_0 = const()[name = string("op_5158_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5158_cast_fp16 = slice_by_index(begin = var_5158_begin_0, end = var_5158_end_0, end_mask = var_5158_end_mask_0, squeeze_mask = var_5158_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5158_cast_fp16")];
+            bool var_5159_transpose_x_0 = const()[name = string("op_5159_transpose_x_0"), val = bool(false)];
+            bool var_5159_transpose_y_0 = const()[name = string("op_5159_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5159_cast_fp16 = matmul(transpose_x = var_5159_transpose_x_0, transpose_y = var_5159_transpose_y_0, x = var_5158_cast_fp16, y = v_new_271_cast_fp16)[name = string("op_5159_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5160_cast_fp16 = add(x = var_5156_cast_fp16, y = var_5159_cast_fp16)[name = string("op_5160_cast_fp16")];
+            tensor<int32, [4]> var_5162_begin_0 = const()[name = string("op_5162_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_5162_end_0 = const()[name = string("op_5162_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_5162_end_mask_0 = const()[name = string("op_5162_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5162_squeeze_mask_0 = const()[name = string("op_5162_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5162_cast_fp16 = slice_by_index(begin = var_5162_begin_0, end = var_5162_end_0, end_mask = var_5162_end_mask_0, squeeze_mask = var_5162_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5162_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5163_cast_fp16 = mul(x = state_271_cast_fp16, y = var_5162_cast_fp16)[name = string("op_5163_cast_fp16")];
+            tensor<int32, [4]> var_5165_begin_0 = const()[name = string("op_5165_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_5165_end_0 = const()[name = string("op_5165_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_5165_end_mask_0 = const()[name = string("op_5165_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5165_squeeze_mask_0 = const()[name = string("op_5165_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5165_cast_fp16 = slice_by_index(begin = var_5165_begin_0, end = var_5165_end_0, end_mask = var_5165_end_mask_0, squeeze_mask = var_5165_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5165_cast_fp16")];
+            bool var_5167_transpose_x_1 = const()[name = string("op_5167_transpose_x_1"), val = bool(true)];
+            bool var_5167_transpose_y_1 = const()[name = string("op_5167_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5167_cast_fp16 = matmul(transpose_x = var_5167_transpose_x_1, transpose_y = var_5167_transpose_y_1, x = var_5165_cast_fp16, y = v_new_271_cast_fp16)[name = string("op_5167_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_273_cast_fp16 = add(x = var_5163_cast_fp16, y = var_5167_cast_fp16)[name = string("state_273_cast_fp16")];
+            tensor<int32, [4]> var_5170_begin_0 = const()[name = string("op_5170_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_5170_end_0 = const()[name = string("op_5170_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_5170_end_mask_0 = const()[name = string("op_5170_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5170_squeeze_mask_0 = const()[name = string("op_5170_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5170_cast_fp16 = slice_by_index(begin = var_5170_begin_0, end = var_5170_end_0, end_mask = var_5170_end_mask_0, squeeze_mask = var_5170_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5170_cast_fp16")];
+            tensor<int32, [4]> var_5172_begin_0 = const()[name = string("op_5172_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_5172_end_0 = const()[name = string("op_5172_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_5172_end_mask_0 = const()[name = string("op_5172_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5172_squeeze_mask_0 = const()[name = string("op_5172_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5172_cast_fp16 = slice_by_index(begin = var_5172_begin_0, end = var_5172_end_0, end_mask = var_5172_end_mask_0, squeeze_mask = var_5172_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5172_cast_fp16")];
+            bool var_5173_transpose_x_0 = const()[name = string("op_5173_transpose_x_0"), val = bool(false)];
+            bool var_5173_transpose_y_0 = const()[name = string("op_5173_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5173_cast_fp16 = matmul(transpose_x = var_5173_transpose_x_0, transpose_y = var_5173_transpose_y_0, x = var_5172_cast_fp16, y = state_273_cast_fp16)[name = string("op_5173_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_273_cast_fp16 = sub(x = var_5170_cast_fp16, y = var_5173_cast_fp16)[name = string("v_new_273_cast_fp16")];
+            tensor<int32, [4]> var_5176_begin_0 = const()[name = string("op_5176_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_5176_end_0 = const()[name = string("op_5176_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_5176_end_mask_0 = const()[name = string("op_5176_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5176_squeeze_mask_0 = const()[name = string("op_5176_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5176_cast_fp16 = slice_by_index(begin = var_5176_begin_0, end = var_5176_end_0, end_mask = var_5176_end_mask_0, squeeze_mask = var_5176_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5176_cast_fp16")];
+            bool var_5177_transpose_x_0 = const()[name = string("op_5177_transpose_x_0"), val = bool(false)];
+            bool var_5177_transpose_y_0 = const()[name = string("op_5177_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5177_cast_fp16 = matmul(transpose_x = var_5177_transpose_x_0, transpose_y = var_5177_transpose_y_0, x = var_5176_cast_fp16, y = state_273_cast_fp16)[name = string("op_5177_cast_fp16")];
+            tensor<int32, [4]> var_5179_begin_0 = const()[name = string("op_5179_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_5179_end_0 = const()[name = string("op_5179_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_5179_end_mask_0 = const()[name = string("op_5179_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5179_squeeze_mask_0 = const()[name = string("op_5179_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5179_cast_fp16 = slice_by_index(begin = var_5179_begin_0, end = var_5179_end_0, end_mask = var_5179_end_mask_0, squeeze_mask = var_5179_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5179_cast_fp16")];
+            bool var_5180_transpose_x_0 = const()[name = string("op_5180_transpose_x_0"), val = bool(false)];
+            bool var_5180_transpose_y_0 = const()[name = string("op_5180_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5180_cast_fp16 = matmul(transpose_x = var_5180_transpose_x_0, transpose_y = var_5180_transpose_y_0, x = var_5179_cast_fp16, y = v_new_273_cast_fp16)[name = string("op_5180_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5181_cast_fp16 = add(x = var_5177_cast_fp16, y = var_5180_cast_fp16)[name = string("op_5181_cast_fp16")];
+            tensor<int32, [4]> var_5183_begin_0 = const()[name = string("op_5183_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_5183_end_0 = const()[name = string("op_5183_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_5183_end_mask_0 = const()[name = string("op_5183_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5183_squeeze_mask_0 = const()[name = string("op_5183_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5183_cast_fp16 = slice_by_index(begin = var_5183_begin_0, end = var_5183_end_0, end_mask = var_5183_end_mask_0, squeeze_mask = var_5183_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5183_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5184_cast_fp16 = mul(x = state_273_cast_fp16, y = var_5183_cast_fp16)[name = string("op_5184_cast_fp16")];
+            tensor<int32, [4]> var_5186_begin_0 = const()[name = string("op_5186_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_5186_end_0 = const()[name = string("op_5186_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_5186_end_mask_0 = const()[name = string("op_5186_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5186_squeeze_mask_0 = const()[name = string("op_5186_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5186_cast_fp16 = slice_by_index(begin = var_5186_begin_0, end = var_5186_end_0, end_mask = var_5186_end_mask_0, squeeze_mask = var_5186_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5186_cast_fp16")];
+            bool var_5188_transpose_x_1 = const()[name = string("op_5188_transpose_x_1"), val = bool(true)];
+            bool var_5188_transpose_y_1 = const()[name = string("op_5188_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5188_cast_fp16 = matmul(transpose_x = var_5188_transpose_x_1, transpose_y = var_5188_transpose_y_1, x = var_5186_cast_fp16, y = v_new_273_cast_fp16)[name = string("op_5188_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_275_cast_fp16 = add(x = var_5184_cast_fp16, y = var_5188_cast_fp16)[name = string("state_275_cast_fp16")];
+            tensor<int32, [4]> var_5191_begin_0 = const()[name = string("op_5191_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_5191_end_0 = const()[name = string("op_5191_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_5191_end_mask_0 = const()[name = string("op_5191_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5191_squeeze_mask_0 = const()[name = string("op_5191_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5191_cast_fp16 = slice_by_index(begin = var_5191_begin_0, end = var_5191_end_0, end_mask = var_5191_end_mask_0, squeeze_mask = var_5191_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5191_cast_fp16")];
+            tensor<int32, [4]> var_5193_begin_0 = const()[name = string("op_5193_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_5193_end_0 = const()[name = string("op_5193_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_5193_end_mask_0 = const()[name = string("op_5193_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5193_squeeze_mask_0 = const()[name = string("op_5193_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5193_cast_fp16 = slice_by_index(begin = var_5193_begin_0, end = var_5193_end_0, end_mask = var_5193_end_mask_0, squeeze_mask = var_5193_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5193_cast_fp16")];
+            bool var_5194_transpose_x_0 = const()[name = string("op_5194_transpose_x_0"), val = bool(false)];
+            bool var_5194_transpose_y_0 = const()[name = string("op_5194_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5194_cast_fp16 = matmul(transpose_x = var_5194_transpose_x_0, transpose_y = var_5194_transpose_y_0, x = var_5193_cast_fp16, y = state_275_cast_fp16)[name = string("op_5194_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_275_cast_fp16 = sub(x = var_5191_cast_fp16, y = var_5194_cast_fp16)[name = string("v_new_275_cast_fp16")];
+            tensor<int32, [4]> var_5197_begin_0 = const()[name = string("op_5197_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_5197_end_0 = const()[name = string("op_5197_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_5197_end_mask_0 = const()[name = string("op_5197_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5197_squeeze_mask_0 = const()[name = string("op_5197_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5197_cast_fp16 = slice_by_index(begin = var_5197_begin_0, end = var_5197_end_0, end_mask = var_5197_end_mask_0, squeeze_mask = var_5197_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5197_cast_fp16")];
+            bool var_5198_transpose_x_0 = const()[name = string("op_5198_transpose_x_0"), val = bool(false)];
+            bool var_5198_transpose_y_0 = const()[name = string("op_5198_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5198_cast_fp16 = matmul(transpose_x = var_5198_transpose_x_0, transpose_y = var_5198_transpose_y_0, x = var_5197_cast_fp16, y = state_275_cast_fp16)[name = string("op_5198_cast_fp16")];
+            tensor<int32, [4]> var_5200_begin_0 = const()[name = string("op_5200_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_5200_end_0 = const()[name = string("op_5200_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_5200_end_mask_0 = const()[name = string("op_5200_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5200_squeeze_mask_0 = const()[name = string("op_5200_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5200_cast_fp16 = slice_by_index(begin = var_5200_begin_0, end = var_5200_end_0, end_mask = var_5200_end_mask_0, squeeze_mask = var_5200_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5200_cast_fp16")];
+            bool var_5201_transpose_x_0 = const()[name = string("op_5201_transpose_x_0"), val = bool(false)];
+            bool var_5201_transpose_y_0 = const()[name = string("op_5201_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5201_cast_fp16 = matmul(transpose_x = var_5201_transpose_x_0, transpose_y = var_5201_transpose_y_0, x = var_5200_cast_fp16, y = v_new_275_cast_fp16)[name = string("op_5201_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5202_cast_fp16 = add(x = var_5198_cast_fp16, y = var_5201_cast_fp16)[name = string("op_5202_cast_fp16")];
+            tensor<int32, [4]> var_5204_begin_0 = const()[name = string("op_5204_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_5204_end_0 = const()[name = string("op_5204_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_5204_end_mask_0 = const()[name = string("op_5204_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5204_squeeze_mask_0 = const()[name = string("op_5204_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5204_cast_fp16 = slice_by_index(begin = var_5204_begin_0, end = var_5204_end_0, end_mask = var_5204_end_mask_0, squeeze_mask = var_5204_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5204_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5205_cast_fp16 = mul(x = state_275_cast_fp16, y = var_5204_cast_fp16)[name = string("op_5205_cast_fp16")];
+            tensor<int32, [4]> var_5207_begin_0 = const()[name = string("op_5207_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_5207_end_0 = const()[name = string("op_5207_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_5207_end_mask_0 = const()[name = string("op_5207_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5207_squeeze_mask_0 = const()[name = string("op_5207_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5207_cast_fp16 = slice_by_index(begin = var_5207_begin_0, end = var_5207_end_0, end_mask = var_5207_end_mask_0, squeeze_mask = var_5207_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5207_cast_fp16")];
+            bool var_5209_transpose_x_1 = const()[name = string("op_5209_transpose_x_1"), val = bool(true)];
+            bool var_5209_transpose_y_1 = const()[name = string("op_5209_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5209_cast_fp16 = matmul(transpose_x = var_5209_transpose_x_1, transpose_y = var_5209_transpose_y_1, x = var_5207_cast_fp16, y = v_new_275_cast_fp16)[name = string("op_5209_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_277_cast_fp16 = add(x = var_5205_cast_fp16, y = var_5209_cast_fp16)[name = string("state_277_cast_fp16")];
+            tensor<int32, [4]> var_5212_begin_0 = const()[name = string("op_5212_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_5212_end_0 = const()[name = string("op_5212_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_5212_end_mask_0 = const()[name = string("op_5212_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5212_squeeze_mask_0 = const()[name = string("op_5212_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5212_cast_fp16 = slice_by_index(begin = var_5212_begin_0, end = var_5212_end_0, end_mask = var_5212_end_mask_0, squeeze_mask = var_5212_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5212_cast_fp16")];
+            tensor<int32, [4]> var_5214_begin_0 = const()[name = string("op_5214_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_5214_end_0 = const()[name = string("op_5214_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_5214_end_mask_0 = const()[name = string("op_5214_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5214_squeeze_mask_0 = const()[name = string("op_5214_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5214_cast_fp16 = slice_by_index(begin = var_5214_begin_0, end = var_5214_end_0, end_mask = var_5214_end_mask_0, squeeze_mask = var_5214_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5214_cast_fp16")];
+            bool var_5215_transpose_x_0 = const()[name = string("op_5215_transpose_x_0"), val = bool(false)];
+            bool var_5215_transpose_y_0 = const()[name = string("op_5215_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5215_cast_fp16 = matmul(transpose_x = var_5215_transpose_x_0, transpose_y = var_5215_transpose_y_0, x = var_5214_cast_fp16, y = state_277_cast_fp16)[name = string("op_5215_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_277_cast_fp16 = sub(x = var_5212_cast_fp16, y = var_5215_cast_fp16)[name = string("v_new_277_cast_fp16")];
+            tensor<int32, [4]> var_5218_begin_0 = const()[name = string("op_5218_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_5218_end_0 = const()[name = string("op_5218_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_5218_end_mask_0 = const()[name = string("op_5218_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5218_squeeze_mask_0 = const()[name = string("op_5218_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5218_cast_fp16 = slice_by_index(begin = var_5218_begin_0, end = var_5218_end_0, end_mask = var_5218_end_mask_0, squeeze_mask = var_5218_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5218_cast_fp16")];
+            bool var_5219_transpose_x_0 = const()[name = string("op_5219_transpose_x_0"), val = bool(false)];
+            bool var_5219_transpose_y_0 = const()[name = string("op_5219_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5219_cast_fp16 = matmul(transpose_x = var_5219_transpose_x_0, transpose_y = var_5219_transpose_y_0, x = var_5218_cast_fp16, y = state_277_cast_fp16)[name = string("op_5219_cast_fp16")];
+            tensor<int32, [4]> var_5221_begin_0 = const()[name = string("op_5221_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_5221_end_0 = const()[name = string("op_5221_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_5221_end_mask_0 = const()[name = string("op_5221_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5221_squeeze_mask_0 = const()[name = string("op_5221_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5221_cast_fp16 = slice_by_index(begin = var_5221_begin_0, end = var_5221_end_0, end_mask = var_5221_end_mask_0, squeeze_mask = var_5221_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5221_cast_fp16")];
+            bool var_5222_transpose_x_0 = const()[name = string("op_5222_transpose_x_0"), val = bool(false)];
+            bool var_5222_transpose_y_0 = const()[name = string("op_5222_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5222_cast_fp16 = matmul(transpose_x = var_5222_transpose_x_0, transpose_y = var_5222_transpose_y_0, x = var_5221_cast_fp16, y = v_new_277_cast_fp16)[name = string("op_5222_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5223_cast_fp16 = add(x = var_5219_cast_fp16, y = var_5222_cast_fp16)[name = string("op_5223_cast_fp16")];
+            tensor<int32, [4]> var_5225_begin_0 = const()[name = string("op_5225_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_5225_end_0 = const()[name = string("op_5225_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_5225_end_mask_0 = const()[name = string("op_5225_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5225_squeeze_mask_0 = const()[name = string("op_5225_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5225_cast_fp16 = slice_by_index(begin = var_5225_begin_0, end = var_5225_end_0, end_mask = var_5225_end_mask_0, squeeze_mask = var_5225_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5225_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5226_cast_fp16 = mul(x = state_277_cast_fp16, y = var_5225_cast_fp16)[name = string("op_5226_cast_fp16")];
+            tensor<int32, [4]> var_5228_begin_0 = const()[name = string("op_5228_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_5228_end_0 = const()[name = string("op_5228_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_5228_end_mask_0 = const()[name = string("op_5228_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5228_squeeze_mask_0 = const()[name = string("op_5228_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5228_cast_fp16 = slice_by_index(begin = var_5228_begin_0, end = var_5228_end_0, end_mask = var_5228_end_mask_0, squeeze_mask = var_5228_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5228_cast_fp16")];
+            bool var_5230_transpose_x_1 = const()[name = string("op_5230_transpose_x_1"), val = bool(true)];
+            bool var_5230_transpose_y_1 = const()[name = string("op_5230_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5230_cast_fp16 = matmul(transpose_x = var_5230_transpose_x_1, transpose_y = var_5230_transpose_y_1, x = var_5228_cast_fp16, y = v_new_277_cast_fp16)[name = string("op_5230_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_279_cast_fp16 = add(x = var_5226_cast_fp16, y = var_5230_cast_fp16)[name = string("state_279_cast_fp16")];
+            tensor<int32, [4]> var_5233_begin_0 = const()[name = string("op_5233_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_5233_end_0 = const()[name = string("op_5233_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_5233_end_mask_0 = const()[name = string("op_5233_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5233_squeeze_mask_0 = const()[name = string("op_5233_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5233_cast_fp16 = slice_by_index(begin = var_5233_begin_0, end = var_5233_end_0, end_mask = var_5233_end_mask_0, squeeze_mask = var_5233_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5233_cast_fp16")];
+            tensor<int32, [4]> var_5235_begin_0 = const()[name = string("op_5235_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_5235_end_0 = const()[name = string("op_5235_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_5235_end_mask_0 = const()[name = string("op_5235_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5235_squeeze_mask_0 = const()[name = string("op_5235_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5235_cast_fp16 = slice_by_index(begin = var_5235_begin_0, end = var_5235_end_0, end_mask = var_5235_end_mask_0, squeeze_mask = var_5235_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5235_cast_fp16")];
+            bool var_5236_transpose_x_0 = const()[name = string("op_5236_transpose_x_0"), val = bool(false)];
+            bool var_5236_transpose_y_0 = const()[name = string("op_5236_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5236_cast_fp16 = matmul(transpose_x = var_5236_transpose_x_0, transpose_y = var_5236_transpose_y_0, x = var_5235_cast_fp16, y = state_279_cast_fp16)[name = string("op_5236_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_279_cast_fp16 = sub(x = var_5233_cast_fp16, y = var_5236_cast_fp16)[name = string("v_new_279_cast_fp16")];
+            tensor<int32, [4]> var_5239_begin_0 = const()[name = string("op_5239_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_5239_end_0 = const()[name = string("op_5239_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_5239_end_mask_0 = const()[name = string("op_5239_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5239_squeeze_mask_0 = const()[name = string("op_5239_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5239_cast_fp16 = slice_by_index(begin = var_5239_begin_0, end = var_5239_end_0, end_mask = var_5239_end_mask_0, squeeze_mask = var_5239_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5239_cast_fp16")];
+            bool var_5240_transpose_x_0 = const()[name = string("op_5240_transpose_x_0"), val = bool(false)];
+            bool var_5240_transpose_y_0 = const()[name = string("op_5240_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5240_cast_fp16 = matmul(transpose_x = var_5240_transpose_x_0, transpose_y = var_5240_transpose_y_0, x = var_5239_cast_fp16, y = state_279_cast_fp16)[name = string("op_5240_cast_fp16")];
+            tensor<int32, [4]> var_5242_begin_0 = const()[name = string("op_5242_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_5242_end_0 = const()[name = string("op_5242_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_5242_end_mask_0 = const()[name = string("op_5242_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5242_squeeze_mask_0 = const()[name = string("op_5242_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5242_cast_fp16 = slice_by_index(begin = var_5242_begin_0, end = var_5242_end_0, end_mask = var_5242_end_mask_0, squeeze_mask = var_5242_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5242_cast_fp16")];
+            bool var_5243_transpose_x_0 = const()[name = string("op_5243_transpose_x_0"), val = bool(false)];
+            bool var_5243_transpose_y_0 = const()[name = string("op_5243_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5243_cast_fp16 = matmul(transpose_x = var_5243_transpose_x_0, transpose_y = var_5243_transpose_y_0, x = var_5242_cast_fp16, y = v_new_279_cast_fp16)[name = string("op_5243_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5244_cast_fp16 = add(x = var_5240_cast_fp16, y = var_5243_cast_fp16)[name = string("op_5244_cast_fp16")];
+            tensor<int32, [4]> var_5246_begin_0 = const()[name = string("op_5246_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_5246_end_0 = const()[name = string("op_5246_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_5246_end_mask_0 = const()[name = string("op_5246_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5246_squeeze_mask_0 = const()[name = string("op_5246_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5246_cast_fp16 = slice_by_index(begin = var_5246_begin_0, end = var_5246_end_0, end_mask = var_5246_end_mask_0, squeeze_mask = var_5246_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5246_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5247_cast_fp16 = mul(x = state_279_cast_fp16, y = var_5246_cast_fp16)[name = string("op_5247_cast_fp16")];
+            tensor<int32, [4]> var_5249_begin_0 = const()[name = string("op_5249_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_5249_end_0 = const()[name = string("op_5249_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_5249_end_mask_0 = const()[name = string("op_5249_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5249_squeeze_mask_0 = const()[name = string("op_5249_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5249_cast_fp16 = slice_by_index(begin = var_5249_begin_0, end = var_5249_end_0, end_mask = var_5249_end_mask_0, squeeze_mask = var_5249_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5249_cast_fp16")];
+            bool var_5251_transpose_x_1 = const()[name = string("op_5251_transpose_x_1"), val = bool(true)];
+            bool var_5251_transpose_y_1 = const()[name = string("op_5251_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5251_cast_fp16 = matmul(transpose_x = var_5251_transpose_x_1, transpose_y = var_5251_transpose_y_1, x = var_5249_cast_fp16, y = v_new_279_cast_fp16)[name = string("op_5251_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_281_cast_fp16 = add(x = var_5247_cast_fp16, y = var_5251_cast_fp16)[name = string("state_281_cast_fp16")];
+            tensor<int32, [4]> var_5254_begin_0 = const()[name = string("op_5254_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_5254_end_0 = const()[name = string("op_5254_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_5254_end_mask_0 = const()[name = string("op_5254_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5254_squeeze_mask_0 = const()[name = string("op_5254_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5254_cast_fp16 = slice_by_index(begin = var_5254_begin_0, end = var_5254_end_0, end_mask = var_5254_end_mask_0, squeeze_mask = var_5254_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5254_cast_fp16")];
+            tensor<int32, [4]> var_5256_begin_0 = const()[name = string("op_5256_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_5256_end_0 = const()[name = string("op_5256_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_5256_end_mask_0 = const()[name = string("op_5256_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5256_squeeze_mask_0 = const()[name = string("op_5256_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5256_cast_fp16 = slice_by_index(begin = var_5256_begin_0, end = var_5256_end_0, end_mask = var_5256_end_mask_0, squeeze_mask = var_5256_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5256_cast_fp16")];
+            bool var_5257_transpose_x_0 = const()[name = string("op_5257_transpose_x_0"), val = bool(false)];
+            bool var_5257_transpose_y_0 = const()[name = string("op_5257_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5257_cast_fp16 = matmul(transpose_x = var_5257_transpose_x_0, transpose_y = var_5257_transpose_y_0, x = var_5256_cast_fp16, y = state_281_cast_fp16)[name = string("op_5257_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_281_cast_fp16 = sub(x = var_5254_cast_fp16, y = var_5257_cast_fp16)[name = string("v_new_281_cast_fp16")];
+            tensor<int32, [4]> var_5260_begin_0 = const()[name = string("op_5260_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_5260_end_0 = const()[name = string("op_5260_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_5260_end_mask_0 = const()[name = string("op_5260_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5260_squeeze_mask_0 = const()[name = string("op_5260_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5260_cast_fp16 = slice_by_index(begin = var_5260_begin_0, end = var_5260_end_0, end_mask = var_5260_end_mask_0, squeeze_mask = var_5260_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5260_cast_fp16")];
+            bool var_5261_transpose_x_0 = const()[name = string("op_5261_transpose_x_0"), val = bool(false)];
+            bool var_5261_transpose_y_0 = const()[name = string("op_5261_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5261_cast_fp16 = matmul(transpose_x = var_5261_transpose_x_0, transpose_y = var_5261_transpose_y_0, x = var_5260_cast_fp16, y = state_281_cast_fp16)[name = string("op_5261_cast_fp16")];
+            tensor<int32, [4]> var_5263_begin_0 = const()[name = string("op_5263_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_5263_end_0 = const()[name = string("op_5263_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_5263_end_mask_0 = const()[name = string("op_5263_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5263_squeeze_mask_0 = const()[name = string("op_5263_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5263_cast_fp16 = slice_by_index(begin = var_5263_begin_0, end = var_5263_end_0, end_mask = var_5263_end_mask_0, squeeze_mask = var_5263_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5263_cast_fp16")];
+            bool var_5264_transpose_x_0 = const()[name = string("op_5264_transpose_x_0"), val = bool(false)];
+            bool var_5264_transpose_y_0 = const()[name = string("op_5264_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5264_cast_fp16 = matmul(transpose_x = var_5264_transpose_x_0, transpose_y = var_5264_transpose_y_0, x = var_5263_cast_fp16, y = v_new_281_cast_fp16)[name = string("op_5264_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5265_cast_fp16 = add(x = var_5261_cast_fp16, y = var_5264_cast_fp16)[name = string("op_5265_cast_fp16")];
+            tensor<int32, [4]> var_5267_begin_0 = const()[name = string("op_5267_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_5267_end_0 = const()[name = string("op_5267_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_5267_end_mask_0 = const()[name = string("op_5267_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5267_squeeze_mask_0 = const()[name = string("op_5267_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5267_cast_fp16 = slice_by_index(begin = var_5267_begin_0, end = var_5267_end_0, end_mask = var_5267_end_mask_0, squeeze_mask = var_5267_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5267_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5268_cast_fp16 = mul(x = state_281_cast_fp16, y = var_5267_cast_fp16)[name = string("op_5268_cast_fp16")];
+            tensor<int32, [4]> var_5270_begin_0 = const()[name = string("op_5270_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_5270_end_0 = const()[name = string("op_5270_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_5270_end_mask_0 = const()[name = string("op_5270_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5270_squeeze_mask_0 = const()[name = string("op_5270_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5270_cast_fp16 = slice_by_index(begin = var_5270_begin_0, end = var_5270_end_0, end_mask = var_5270_end_mask_0, squeeze_mask = var_5270_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5270_cast_fp16")];
+            bool var_5272_transpose_x_1 = const()[name = string("op_5272_transpose_x_1"), val = bool(true)];
+            bool var_5272_transpose_y_1 = const()[name = string("op_5272_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5272_cast_fp16 = matmul(transpose_x = var_5272_transpose_x_1, transpose_y = var_5272_transpose_y_1, x = var_5270_cast_fp16, y = v_new_281_cast_fp16)[name = string("op_5272_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_283_cast_fp16 = add(x = var_5268_cast_fp16, y = var_5272_cast_fp16)[name = string("state_283_cast_fp16")];
+            tensor<int32, [4]> var_5275_begin_0 = const()[name = string("op_5275_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_5275_end_0 = const()[name = string("op_5275_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_5275_end_mask_0 = const()[name = string("op_5275_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5275_squeeze_mask_0 = const()[name = string("op_5275_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5275_cast_fp16 = slice_by_index(begin = var_5275_begin_0, end = var_5275_end_0, end_mask = var_5275_end_mask_0, squeeze_mask = var_5275_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5275_cast_fp16")];
+            tensor<int32, [4]> var_5277_begin_0 = const()[name = string("op_5277_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_5277_end_0 = const()[name = string("op_5277_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_5277_end_mask_0 = const()[name = string("op_5277_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5277_squeeze_mask_0 = const()[name = string("op_5277_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5277_cast_fp16 = slice_by_index(begin = var_5277_begin_0, end = var_5277_end_0, end_mask = var_5277_end_mask_0, squeeze_mask = var_5277_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5277_cast_fp16")];
+            bool var_5278_transpose_x_0 = const()[name = string("op_5278_transpose_x_0"), val = bool(false)];
+            bool var_5278_transpose_y_0 = const()[name = string("op_5278_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5278_cast_fp16 = matmul(transpose_x = var_5278_transpose_x_0, transpose_y = var_5278_transpose_y_0, x = var_5277_cast_fp16, y = state_283_cast_fp16)[name = string("op_5278_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_283_cast_fp16 = sub(x = var_5275_cast_fp16, y = var_5278_cast_fp16)[name = string("v_new_283_cast_fp16")];
+            tensor<int32, [4]> var_5281_begin_0 = const()[name = string("op_5281_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_5281_end_0 = const()[name = string("op_5281_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_5281_end_mask_0 = const()[name = string("op_5281_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5281_squeeze_mask_0 = const()[name = string("op_5281_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5281_cast_fp16 = slice_by_index(begin = var_5281_begin_0, end = var_5281_end_0, end_mask = var_5281_end_mask_0, squeeze_mask = var_5281_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5281_cast_fp16")];
+            bool var_5282_transpose_x_0 = const()[name = string("op_5282_transpose_x_0"), val = bool(false)];
+            bool var_5282_transpose_y_0 = const()[name = string("op_5282_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5282_cast_fp16 = matmul(transpose_x = var_5282_transpose_x_0, transpose_y = var_5282_transpose_y_0, x = var_5281_cast_fp16, y = state_283_cast_fp16)[name = string("op_5282_cast_fp16")];
+            tensor<int32, [4]> var_5284_begin_0 = const()[name = string("op_5284_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_5284_end_0 = const()[name = string("op_5284_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_5284_end_mask_0 = const()[name = string("op_5284_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5284_squeeze_mask_0 = const()[name = string("op_5284_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5284_cast_fp16 = slice_by_index(begin = var_5284_begin_0, end = var_5284_end_0, end_mask = var_5284_end_mask_0, squeeze_mask = var_5284_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5284_cast_fp16")];
+            bool var_5285_transpose_x_0 = const()[name = string("op_5285_transpose_x_0"), val = bool(false)];
+            bool var_5285_transpose_y_0 = const()[name = string("op_5285_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5285_cast_fp16 = matmul(transpose_x = var_5285_transpose_x_0, transpose_y = var_5285_transpose_y_0, x = var_5284_cast_fp16, y = v_new_283_cast_fp16)[name = string("op_5285_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5286_cast_fp16 = add(x = var_5282_cast_fp16, y = var_5285_cast_fp16)[name = string("op_5286_cast_fp16")];
+            tensor<int32, [4]> var_5288_begin_0 = const()[name = string("op_5288_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_5288_end_0 = const()[name = string("op_5288_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_5288_end_mask_0 = const()[name = string("op_5288_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5288_squeeze_mask_0 = const()[name = string("op_5288_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5288_cast_fp16 = slice_by_index(begin = var_5288_begin_0, end = var_5288_end_0, end_mask = var_5288_end_mask_0, squeeze_mask = var_5288_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5288_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5289_cast_fp16 = mul(x = state_283_cast_fp16, y = var_5288_cast_fp16)[name = string("op_5289_cast_fp16")];
+            tensor<int32, [4]> var_5291_begin_0 = const()[name = string("op_5291_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_5291_end_0 = const()[name = string("op_5291_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_5291_end_mask_0 = const()[name = string("op_5291_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5291_squeeze_mask_0 = const()[name = string("op_5291_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5291_cast_fp16 = slice_by_index(begin = var_5291_begin_0, end = var_5291_end_0, end_mask = var_5291_end_mask_0, squeeze_mask = var_5291_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5291_cast_fp16")];
+            bool var_5293_transpose_x_1 = const()[name = string("op_5293_transpose_x_1"), val = bool(true)];
+            bool var_5293_transpose_y_1 = const()[name = string("op_5293_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5293_cast_fp16 = matmul(transpose_x = var_5293_transpose_x_1, transpose_y = var_5293_transpose_y_1, x = var_5291_cast_fp16, y = v_new_283_cast_fp16)[name = string("op_5293_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_285_cast_fp16 = add(x = var_5289_cast_fp16, y = var_5293_cast_fp16)[name = string("state_285_cast_fp16")];
+            tensor<int32, [4]> var_5296_begin_0 = const()[name = string("op_5296_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_5296_end_0 = const()[name = string("op_5296_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_5296_end_mask_0 = const()[name = string("op_5296_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5296_squeeze_mask_0 = const()[name = string("op_5296_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5296_cast_fp16 = slice_by_index(begin = var_5296_begin_0, end = var_5296_end_0, end_mask = var_5296_end_mask_0, squeeze_mask = var_5296_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5296_cast_fp16")];
+            tensor<int32, [4]> var_5298_begin_0 = const()[name = string("op_5298_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_5298_end_0 = const()[name = string("op_5298_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_5298_end_mask_0 = const()[name = string("op_5298_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5298_squeeze_mask_0 = const()[name = string("op_5298_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5298_cast_fp16 = slice_by_index(begin = var_5298_begin_0, end = var_5298_end_0, end_mask = var_5298_end_mask_0, squeeze_mask = var_5298_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5298_cast_fp16")];
+            bool var_5299_transpose_x_0 = const()[name = string("op_5299_transpose_x_0"), val = bool(false)];
+            bool var_5299_transpose_y_0 = const()[name = string("op_5299_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5299_cast_fp16 = matmul(transpose_x = var_5299_transpose_x_0, transpose_y = var_5299_transpose_y_0, x = var_5298_cast_fp16, y = state_285_cast_fp16)[name = string("op_5299_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_285_cast_fp16 = sub(x = var_5296_cast_fp16, y = var_5299_cast_fp16)[name = string("v_new_285_cast_fp16")];
+            tensor<int32, [4]> var_5302_begin_0 = const()[name = string("op_5302_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_5302_end_0 = const()[name = string("op_5302_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_5302_end_mask_0 = const()[name = string("op_5302_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5302_squeeze_mask_0 = const()[name = string("op_5302_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5302_cast_fp16 = slice_by_index(begin = var_5302_begin_0, end = var_5302_end_0, end_mask = var_5302_end_mask_0, squeeze_mask = var_5302_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5302_cast_fp16")];
+            bool var_5303_transpose_x_0 = const()[name = string("op_5303_transpose_x_0"), val = bool(false)];
+            bool var_5303_transpose_y_0 = const()[name = string("op_5303_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5303_cast_fp16 = matmul(transpose_x = var_5303_transpose_x_0, transpose_y = var_5303_transpose_y_0, x = var_5302_cast_fp16, y = state_285_cast_fp16)[name = string("op_5303_cast_fp16")];
+            tensor<int32, [4]> var_5305_begin_0 = const()[name = string("op_5305_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_5305_end_0 = const()[name = string("op_5305_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_5305_end_mask_0 = const()[name = string("op_5305_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5305_squeeze_mask_0 = const()[name = string("op_5305_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5305_cast_fp16 = slice_by_index(begin = var_5305_begin_0, end = var_5305_end_0, end_mask = var_5305_end_mask_0, squeeze_mask = var_5305_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5305_cast_fp16")];
+            bool var_5306_transpose_x_0 = const()[name = string("op_5306_transpose_x_0"), val = bool(false)];
+            bool var_5306_transpose_y_0 = const()[name = string("op_5306_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5306_cast_fp16 = matmul(transpose_x = var_5306_transpose_x_0, transpose_y = var_5306_transpose_y_0, x = var_5305_cast_fp16, y = v_new_285_cast_fp16)[name = string("op_5306_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5307_cast_fp16 = add(x = var_5303_cast_fp16, y = var_5306_cast_fp16)[name = string("op_5307_cast_fp16")];
+            tensor<int32, [4]> var_5309_begin_0 = const()[name = string("op_5309_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_5309_end_0 = const()[name = string("op_5309_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_5309_end_mask_0 = const()[name = string("op_5309_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5309_squeeze_mask_0 = const()[name = string("op_5309_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5309_cast_fp16 = slice_by_index(begin = var_5309_begin_0, end = var_5309_end_0, end_mask = var_5309_end_mask_0, squeeze_mask = var_5309_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5309_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5310_cast_fp16 = mul(x = state_285_cast_fp16, y = var_5309_cast_fp16)[name = string("op_5310_cast_fp16")];
+            tensor<int32, [4]> var_5312_begin_0 = const()[name = string("op_5312_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_5312_end_0 = const()[name = string("op_5312_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_5312_end_mask_0 = const()[name = string("op_5312_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5312_squeeze_mask_0 = const()[name = string("op_5312_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5312_cast_fp16 = slice_by_index(begin = var_5312_begin_0, end = var_5312_end_0, end_mask = var_5312_end_mask_0, squeeze_mask = var_5312_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5312_cast_fp16")];
+            bool var_5314_transpose_x_1 = const()[name = string("op_5314_transpose_x_1"), val = bool(true)];
+            bool var_5314_transpose_y_1 = const()[name = string("op_5314_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5314_cast_fp16 = matmul(transpose_x = var_5314_transpose_x_1, transpose_y = var_5314_transpose_y_1, x = var_5312_cast_fp16, y = v_new_285_cast_fp16)[name = string("op_5314_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_287_cast_fp16 = add(x = var_5310_cast_fp16, y = var_5314_cast_fp16)[name = string("state_287_cast_fp16")];
+            tensor<int32, [4]> var_5317_begin_0 = const()[name = string("op_5317_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_5317_end_0 = const()[name = string("op_5317_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_5317_end_mask_0 = const()[name = string("op_5317_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5317_squeeze_mask_0 = const()[name = string("op_5317_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5317_cast_fp16 = slice_by_index(begin = var_5317_begin_0, end = var_5317_end_0, end_mask = var_5317_end_mask_0, squeeze_mask = var_5317_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5317_cast_fp16")];
+            tensor<int32, [4]> var_5319_begin_0 = const()[name = string("op_5319_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_5319_end_0 = const()[name = string("op_5319_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_5319_end_mask_0 = const()[name = string("op_5319_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5319_squeeze_mask_0 = const()[name = string("op_5319_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5319_cast_fp16 = slice_by_index(begin = var_5319_begin_0, end = var_5319_end_0, end_mask = var_5319_end_mask_0, squeeze_mask = var_5319_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5319_cast_fp16")];
+            bool var_5320_transpose_x_0 = const()[name = string("op_5320_transpose_x_0"), val = bool(false)];
+            bool var_5320_transpose_y_0 = const()[name = string("op_5320_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5320_cast_fp16 = matmul(transpose_x = var_5320_transpose_x_0, transpose_y = var_5320_transpose_y_0, x = var_5319_cast_fp16, y = state_287_cast_fp16)[name = string("op_5320_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_287_cast_fp16 = sub(x = var_5317_cast_fp16, y = var_5320_cast_fp16)[name = string("v_new_287_cast_fp16")];
+            tensor<int32, [4]> var_5323_begin_0 = const()[name = string("op_5323_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_5323_end_0 = const()[name = string("op_5323_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_5323_end_mask_0 = const()[name = string("op_5323_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5323_squeeze_mask_0 = const()[name = string("op_5323_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5323_cast_fp16 = slice_by_index(begin = var_5323_begin_0, end = var_5323_end_0, end_mask = var_5323_end_mask_0, squeeze_mask = var_5323_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5323_cast_fp16")];
+            bool var_5324_transpose_x_0 = const()[name = string("op_5324_transpose_x_0"), val = bool(false)];
+            bool var_5324_transpose_y_0 = const()[name = string("op_5324_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5324_cast_fp16 = matmul(transpose_x = var_5324_transpose_x_0, transpose_y = var_5324_transpose_y_0, x = var_5323_cast_fp16, y = state_287_cast_fp16)[name = string("op_5324_cast_fp16")];
+            tensor<int32, [4]> var_5326_begin_0 = const()[name = string("op_5326_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_5326_end_0 = const()[name = string("op_5326_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_5326_end_mask_0 = const()[name = string("op_5326_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5326_squeeze_mask_0 = const()[name = string("op_5326_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5326_cast_fp16 = slice_by_index(begin = var_5326_begin_0, end = var_5326_end_0, end_mask = var_5326_end_mask_0, squeeze_mask = var_5326_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5326_cast_fp16")];
+            bool var_5327_transpose_x_0 = const()[name = string("op_5327_transpose_x_0"), val = bool(false)];
+            bool var_5327_transpose_y_0 = const()[name = string("op_5327_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5327_cast_fp16 = matmul(transpose_x = var_5327_transpose_x_0, transpose_y = var_5327_transpose_y_0, x = var_5326_cast_fp16, y = v_new_287_cast_fp16)[name = string("op_5327_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5328_cast_fp16 = add(x = var_5324_cast_fp16, y = var_5327_cast_fp16)[name = string("op_5328_cast_fp16")];
+            tensor<int32, [4]> var_5330_begin_0 = const()[name = string("op_5330_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_5330_end_0 = const()[name = string("op_5330_end_0"), val = tensor<int32, [4]>([32, 16, 1, 1])];
+            tensor<bool, [4]> var_5330_end_mask_0 = const()[name = string("op_5330_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5330_squeeze_mask_0 = const()[name = string("op_5330_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5330_cast_fp16 = slice_by_index(begin = var_5330_begin_0, end = var_5330_end_0, end_mask = var_5330_end_mask_0, squeeze_mask = var_5330_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5330_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5331_cast_fp16 = mul(x = state_287_cast_fp16, y = var_5330_cast_fp16)[name = string("op_5331_cast_fp16")];
+            tensor<int32, [4]> var_5333_begin_0 = const()[name = string("op_5333_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_5333_end_0 = const()[name = string("op_5333_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_5333_end_mask_0 = const()[name = string("op_5333_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5333_squeeze_mask_0 = const()[name = string("op_5333_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5333_cast_fp16 = slice_by_index(begin = var_5333_begin_0, end = var_5333_end_0, end_mask = var_5333_end_mask_0, squeeze_mask = var_5333_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5333_cast_fp16")];
+            bool var_5335_transpose_x_1 = const()[name = string("op_5335_transpose_x_1"), val = bool(true)];
+            bool var_5335_transpose_y_1 = const()[name = string("op_5335_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5335_cast_fp16 = matmul(transpose_x = var_5335_transpose_x_1, transpose_y = var_5335_transpose_y_1, x = var_5333_cast_fp16, y = v_new_287_cast_fp16)[name = string("op_5335_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_289_cast_fp16 = add(x = var_5331_cast_fp16, y = var_5335_cast_fp16)[name = string("state_289_cast_fp16")];
+            tensor<int32, [4]> var_5338_begin_0 = const()[name = string("op_5338_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_5338_end_0 = const()[name = string("op_5338_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_5338_end_mask_0 = const()[name = string("op_5338_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5338_squeeze_mask_0 = const()[name = string("op_5338_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5338_cast_fp16 = slice_by_index(begin = var_5338_begin_0, end = var_5338_end_0, end_mask = var_5338_end_mask_0, squeeze_mask = var_5338_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5338_cast_fp16")];
+            tensor<int32, [4]> var_5340_begin_0 = const()[name = string("op_5340_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_5340_end_0 = const()[name = string("op_5340_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_5340_end_mask_0 = const()[name = string("op_5340_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5340_squeeze_mask_0 = const()[name = string("op_5340_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5340_cast_fp16 = slice_by_index(begin = var_5340_begin_0, end = var_5340_end_0, end_mask = var_5340_end_mask_0, squeeze_mask = var_5340_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5340_cast_fp16")];
+            bool var_5341_transpose_x_0 = const()[name = string("op_5341_transpose_x_0"), val = bool(false)];
+            bool var_5341_transpose_y_0 = const()[name = string("op_5341_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5341_cast_fp16 = matmul(transpose_x = var_5341_transpose_x_0, transpose_y = var_5341_transpose_y_0, x = var_5340_cast_fp16, y = state_289_cast_fp16)[name = string("op_5341_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_289_cast_fp16 = sub(x = var_5338_cast_fp16, y = var_5341_cast_fp16)[name = string("v_new_289_cast_fp16")];
+            tensor<int32, [4]> var_5344_begin_0 = const()[name = string("op_5344_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_5344_end_0 = const()[name = string("op_5344_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_5344_end_mask_0 = const()[name = string("op_5344_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5344_squeeze_mask_0 = const()[name = string("op_5344_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5344_cast_fp16 = slice_by_index(begin = var_5344_begin_0, end = var_5344_end_0, end_mask = var_5344_end_mask_0, squeeze_mask = var_5344_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5344_cast_fp16")];
+            bool var_5345_transpose_x_0 = const()[name = string("op_5345_transpose_x_0"), val = bool(false)];
+            bool var_5345_transpose_y_0 = const()[name = string("op_5345_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5345_cast_fp16 = matmul(transpose_x = var_5345_transpose_x_0, transpose_y = var_5345_transpose_y_0, x = var_5344_cast_fp16, y = state_289_cast_fp16)[name = string("op_5345_cast_fp16")];
+            tensor<int32, [4]> var_5347_begin_0 = const()[name = string("op_5347_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_5347_end_0 = const()[name = string("op_5347_end_0"), val = tensor<int32, [4]>([32, 17, 64, 64])];
+            tensor<bool, [4]> var_5347_end_mask_0 = const()[name = string("op_5347_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5347_squeeze_mask_0 = const()[name = string("op_5347_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5347_cast_fp16 = slice_by_index(begin = var_5347_begin_0, end = var_5347_end_0, end_mask = var_5347_end_mask_0, squeeze_mask = var_5347_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5347_cast_fp16")];
+            bool var_5348_transpose_x_0 = const()[name = string("op_5348_transpose_x_0"), val = bool(false)];
+            bool var_5348_transpose_y_0 = const()[name = string("op_5348_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5348_cast_fp16 = matmul(transpose_x = var_5348_transpose_x_0, transpose_y = var_5348_transpose_y_0, x = var_5347_cast_fp16, y = v_new_289_cast_fp16)[name = string("op_5348_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5349_cast_fp16 = add(x = var_5345_cast_fp16, y = var_5348_cast_fp16)[name = string("op_5349_cast_fp16")];
+            tensor<int32, [4]> var_5351_begin_0 = const()[name = string("op_5351_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_5351_end_0 = const()[name = string("op_5351_end_0"), val = tensor<int32, [4]>([32, 17, 1, 1])];
+            tensor<bool, [4]> var_5351_end_mask_0 = const()[name = string("op_5351_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5351_squeeze_mask_0 = const()[name = string("op_5351_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5351_cast_fp16 = slice_by_index(begin = var_5351_begin_0, end = var_5351_end_0, end_mask = var_5351_end_mask_0, squeeze_mask = var_5351_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5351_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5352_cast_fp16 = mul(x = state_289_cast_fp16, y = var_5351_cast_fp16)[name = string("op_5352_cast_fp16")];
+            tensor<int32, [4]> var_5354_begin_0 = const()[name = string("op_5354_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_5354_end_0 = const()[name = string("op_5354_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_5354_end_mask_0 = const()[name = string("op_5354_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5354_squeeze_mask_0 = const()[name = string("op_5354_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5354_cast_fp16 = slice_by_index(begin = var_5354_begin_0, end = var_5354_end_0, end_mask = var_5354_end_mask_0, squeeze_mask = var_5354_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5354_cast_fp16")];
+            bool var_5356_transpose_x_1 = const()[name = string("op_5356_transpose_x_1"), val = bool(true)];
+            bool var_5356_transpose_y_1 = const()[name = string("op_5356_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5356_cast_fp16 = matmul(transpose_x = var_5356_transpose_x_1, transpose_y = var_5356_transpose_y_1, x = var_5354_cast_fp16, y = v_new_289_cast_fp16)[name = string("op_5356_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_291_cast_fp16 = add(x = var_5352_cast_fp16, y = var_5356_cast_fp16)[name = string("state_291_cast_fp16")];
+            tensor<int32, [4]> var_5359_begin_0 = const()[name = string("op_5359_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_5359_end_0 = const()[name = string("op_5359_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_5359_end_mask_0 = const()[name = string("op_5359_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5359_squeeze_mask_0 = const()[name = string("op_5359_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5359_cast_fp16 = slice_by_index(begin = var_5359_begin_0, end = var_5359_end_0, end_mask = var_5359_end_mask_0, squeeze_mask = var_5359_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5359_cast_fp16")];
+            tensor<int32, [4]> var_5361_begin_0 = const()[name = string("op_5361_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_5361_end_0 = const()[name = string("op_5361_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_5361_end_mask_0 = const()[name = string("op_5361_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5361_squeeze_mask_0 = const()[name = string("op_5361_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5361_cast_fp16 = slice_by_index(begin = var_5361_begin_0, end = var_5361_end_0, end_mask = var_5361_end_mask_0, squeeze_mask = var_5361_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5361_cast_fp16")];
+            bool var_5362_transpose_x_0 = const()[name = string("op_5362_transpose_x_0"), val = bool(false)];
+            bool var_5362_transpose_y_0 = const()[name = string("op_5362_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5362_cast_fp16 = matmul(transpose_x = var_5362_transpose_x_0, transpose_y = var_5362_transpose_y_0, x = var_5361_cast_fp16, y = state_291_cast_fp16)[name = string("op_5362_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_291_cast_fp16 = sub(x = var_5359_cast_fp16, y = var_5362_cast_fp16)[name = string("v_new_291_cast_fp16")];
+            tensor<int32, [4]> var_5365_begin_0 = const()[name = string("op_5365_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_5365_end_0 = const()[name = string("op_5365_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_5365_end_mask_0 = const()[name = string("op_5365_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5365_squeeze_mask_0 = const()[name = string("op_5365_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5365_cast_fp16 = slice_by_index(begin = var_5365_begin_0, end = var_5365_end_0, end_mask = var_5365_end_mask_0, squeeze_mask = var_5365_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5365_cast_fp16")];
+            bool var_5366_transpose_x_0 = const()[name = string("op_5366_transpose_x_0"), val = bool(false)];
+            bool var_5366_transpose_y_0 = const()[name = string("op_5366_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5366_cast_fp16 = matmul(transpose_x = var_5366_transpose_x_0, transpose_y = var_5366_transpose_y_0, x = var_5365_cast_fp16, y = state_291_cast_fp16)[name = string("op_5366_cast_fp16")];
+            tensor<int32, [4]> var_5368_begin_0 = const()[name = string("op_5368_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_5368_end_0 = const()[name = string("op_5368_end_0"), val = tensor<int32, [4]>([32, 18, 64, 64])];
+            tensor<bool, [4]> var_5368_end_mask_0 = const()[name = string("op_5368_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5368_squeeze_mask_0 = const()[name = string("op_5368_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5368_cast_fp16 = slice_by_index(begin = var_5368_begin_0, end = var_5368_end_0, end_mask = var_5368_end_mask_0, squeeze_mask = var_5368_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5368_cast_fp16")];
+            bool var_5369_transpose_x_0 = const()[name = string("op_5369_transpose_x_0"), val = bool(false)];
+            bool var_5369_transpose_y_0 = const()[name = string("op_5369_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5369_cast_fp16 = matmul(transpose_x = var_5369_transpose_x_0, transpose_y = var_5369_transpose_y_0, x = var_5368_cast_fp16, y = v_new_291_cast_fp16)[name = string("op_5369_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5370_cast_fp16 = add(x = var_5366_cast_fp16, y = var_5369_cast_fp16)[name = string("op_5370_cast_fp16")];
+            tensor<int32, [4]> var_5372_begin_0 = const()[name = string("op_5372_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_5372_end_0 = const()[name = string("op_5372_end_0"), val = tensor<int32, [4]>([32, 18, 1, 1])];
+            tensor<bool, [4]> var_5372_end_mask_0 = const()[name = string("op_5372_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5372_squeeze_mask_0 = const()[name = string("op_5372_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5372_cast_fp16 = slice_by_index(begin = var_5372_begin_0, end = var_5372_end_0, end_mask = var_5372_end_mask_0, squeeze_mask = var_5372_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5372_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5373_cast_fp16 = mul(x = state_291_cast_fp16, y = var_5372_cast_fp16)[name = string("op_5373_cast_fp16")];
+            tensor<int32, [4]> var_5375_begin_0 = const()[name = string("op_5375_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_5375_end_0 = const()[name = string("op_5375_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_5375_end_mask_0 = const()[name = string("op_5375_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5375_squeeze_mask_0 = const()[name = string("op_5375_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5375_cast_fp16 = slice_by_index(begin = var_5375_begin_0, end = var_5375_end_0, end_mask = var_5375_end_mask_0, squeeze_mask = var_5375_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5375_cast_fp16")];
+            bool var_5377_transpose_x_1 = const()[name = string("op_5377_transpose_x_1"), val = bool(true)];
+            bool var_5377_transpose_y_1 = const()[name = string("op_5377_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5377_cast_fp16 = matmul(transpose_x = var_5377_transpose_x_1, transpose_y = var_5377_transpose_y_1, x = var_5375_cast_fp16, y = v_new_291_cast_fp16)[name = string("op_5377_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_293_cast_fp16 = add(x = var_5373_cast_fp16, y = var_5377_cast_fp16)[name = string("state_293_cast_fp16")];
+            tensor<int32, [4]> var_5380_begin_0 = const()[name = string("op_5380_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_5380_end_0 = const()[name = string("op_5380_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_5380_end_mask_0 = const()[name = string("op_5380_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5380_squeeze_mask_0 = const()[name = string("op_5380_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5380_cast_fp16 = slice_by_index(begin = var_5380_begin_0, end = var_5380_end_0, end_mask = var_5380_end_mask_0, squeeze_mask = var_5380_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5380_cast_fp16")];
+            tensor<int32, [4]> var_5382_begin_0 = const()[name = string("op_5382_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_5382_end_0 = const()[name = string("op_5382_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_5382_end_mask_0 = const()[name = string("op_5382_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5382_squeeze_mask_0 = const()[name = string("op_5382_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5382_cast_fp16 = slice_by_index(begin = var_5382_begin_0, end = var_5382_end_0, end_mask = var_5382_end_mask_0, squeeze_mask = var_5382_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5382_cast_fp16")];
+            bool var_5383_transpose_x_0 = const()[name = string("op_5383_transpose_x_0"), val = bool(false)];
+            bool var_5383_transpose_y_0 = const()[name = string("op_5383_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5383_cast_fp16 = matmul(transpose_x = var_5383_transpose_x_0, transpose_y = var_5383_transpose_y_0, x = var_5382_cast_fp16, y = state_293_cast_fp16)[name = string("op_5383_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_293_cast_fp16 = sub(x = var_5380_cast_fp16, y = var_5383_cast_fp16)[name = string("v_new_293_cast_fp16")];
+            tensor<int32, [4]> var_5386_begin_0 = const()[name = string("op_5386_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_5386_end_0 = const()[name = string("op_5386_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_5386_end_mask_0 = const()[name = string("op_5386_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5386_squeeze_mask_0 = const()[name = string("op_5386_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5386_cast_fp16 = slice_by_index(begin = var_5386_begin_0, end = var_5386_end_0, end_mask = var_5386_end_mask_0, squeeze_mask = var_5386_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5386_cast_fp16")];
+            bool var_5387_transpose_x_0 = const()[name = string("op_5387_transpose_x_0"), val = bool(false)];
+            bool var_5387_transpose_y_0 = const()[name = string("op_5387_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5387_cast_fp16 = matmul(transpose_x = var_5387_transpose_x_0, transpose_y = var_5387_transpose_y_0, x = var_5386_cast_fp16, y = state_293_cast_fp16)[name = string("op_5387_cast_fp16")];
+            tensor<int32, [4]> var_5389_begin_0 = const()[name = string("op_5389_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_5389_end_0 = const()[name = string("op_5389_end_0"), val = tensor<int32, [4]>([32, 19, 64, 64])];
+            tensor<bool, [4]> var_5389_end_mask_0 = const()[name = string("op_5389_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5389_squeeze_mask_0 = const()[name = string("op_5389_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5389_cast_fp16 = slice_by_index(begin = var_5389_begin_0, end = var_5389_end_0, end_mask = var_5389_end_mask_0, squeeze_mask = var_5389_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5389_cast_fp16")];
+            bool var_5390_transpose_x_0 = const()[name = string("op_5390_transpose_x_0"), val = bool(false)];
+            bool var_5390_transpose_y_0 = const()[name = string("op_5390_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5390_cast_fp16 = matmul(transpose_x = var_5390_transpose_x_0, transpose_y = var_5390_transpose_y_0, x = var_5389_cast_fp16, y = v_new_293_cast_fp16)[name = string("op_5390_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5391_cast_fp16 = add(x = var_5387_cast_fp16, y = var_5390_cast_fp16)[name = string("op_5391_cast_fp16")];
+            tensor<int32, [4]> var_5393_begin_0 = const()[name = string("op_5393_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_5393_end_0 = const()[name = string("op_5393_end_0"), val = tensor<int32, [4]>([32, 19, 1, 1])];
+            tensor<bool, [4]> var_5393_end_mask_0 = const()[name = string("op_5393_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5393_squeeze_mask_0 = const()[name = string("op_5393_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5393_cast_fp16 = slice_by_index(begin = var_5393_begin_0, end = var_5393_end_0, end_mask = var_5393_end_mask_0, squeeze_mask = var_5393_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5393_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5394_cast_fp16 = mul(x = state_293_cast_fp16, y = var_5393_cast_fp16)[name = string("op_5394_cast_fp16")];
+            tensor<int32, [4]> var_5396_begin_0 = const()[name = string("op_5396_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_5396_end_0 = const()[name = string("op_5396_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_5396_end_mask_0 = const()[name = string("op_5396_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5396_squeeze_mask_0 = const()[name = string("op_5396_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5396_cast_fp16 = slice_by_index(begin = var_5396_begin_0, end = var_5396_end_0, end_mask = var_5396_end_mask_0, squeeze_mask = var_5396_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5396_cast_fp16")];
+            bool var_5398_transpose_x_1 = const()[name = string("op_5398_transpose_x_1"), val = bool(true)];
+            bool var_5398_transpose_y_1 = const()[name = string("op_5398_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5398_cast_fp16 = matmul(transpose_x = var_5398_transpose_x_1, transpose_y = var_5398_transpose_y_1, x = var_5396_cast_fp16, y = v_new_293_cast_fp16)[name = string("op_5398_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_295_cast_fp16 = add(x = var_5394_cast_fp16, y = var_5398_cast_fp16)[name = string("state_295_cast_fp16")];
+            tensor<int32, [4]> var_5401_begin_0 = const()[name = string("op_5401_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_5401_end_0 = const()[name = string("op_5401_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_5401_end_mask_0 = const()[name = string("op_5401_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5401_squeeze_mask_0 = const()[name = string("op_5401_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5401_cast_fp16 = slice_by_index(begin = var_5401_begin_0, end = var_5401_end_0, end_mask = var_5401_end_mask_0, squeeze_mask = var_5401_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5401_cast_fp16")];
+            tensor<int32, [4]> var_5403_begin_0 = const()[name = string("op_5403_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_5403_end_0 = const()[name = string("op_5403_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_5403_end_mask_0 = const()[name = string("op_5403_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5403_squeeze_mask_0 = const()[name = string("op_5403_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5403_cast_fp16 = slice_by_index(begin = var_5403_begin_0, end = var_5403_end_0, end_mask = var_5403_end_mask_0, squeeze_mask = var_5403_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5403_cast_fp16")];
+            bool var_5404_transpose_x_0 = const()[name = string("op_5404_transpose_x_0"), val = bool(false)];
+            bool var_5404_transpose_y_0 = const()[name = string("op_5404_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5404_cast_fp16 = matmul(transpose_x = var_5404_transpose_x_0, transpose_y = var_5404_transpose_y_0, x = var_5403_cast_fp16, y = state_295_cast_fp16)[name = string("op_5404_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_295_cast_fp16 = sub(x = var_5401_cast_fp16, y = var_5404_cast_fp16)[name = string("v_new_295_cast_fp16")];
+            tensor<int32, [4]> var_5407_begin_0 = const()[name = string("op_5407_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_5407_end_0 = const()[name = string("op_5407_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_5407_end_mask_0 = const()[name = string("op_5407_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5407_squeeze_mask_0 = const()[name = string("op_5407_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5407_cast_fp16 = slice_by_index(begin = var_5407_begin_0, end = var_5407_end_0, end_mask = var_5407_end_mask_0, squeeze_mask = var_5407_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5407_cast_fp16")];
+            bool var_5408_transpose_x_0 = const()[name = string("op_5408_transpose_x_0"), val = bool(false)];
+            bool var_5408_transpose_y_0 = const()[name = string("op_5408_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5408_cast_fp16 = matmul(transpose_x = var_5408_transpose_x_0, transpose_y = var_5408_transpose_y_0, x = var_5407_cast_fp16, y = state_295_cast_fp16)[name = string("op_5408_cast_fp16")];
+            tensor<int32, [4]> var_5410_begin_0 = const()[name = string("op_5410_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_5410_end_0 = const()[name = string("op_5410_end_0"), val = tensor<int32, [4]>([32, 20, 64, 64])];
+            tensor<bool, [4]> var_5410_end_mask_0 = const()[name = string("op_5410_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5410_squeeze_mask_0 = const()[name = string("op_5410_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5410_cast_fp16 = slice_by_index(begin = var_5410_begin_0, end = var_5410_end_0, end_mask = var_5410_end_mask_0, squeeze_mask = var_5410_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5410_cast_fp16")];
+            bool var_5411_transpose_x_0 = const()[name = string("op_5411_transpose_x_0"), val = bool(false)];
+            bool var_5411_transpose_y_0 = const()[name = string("op_5411_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5411_cast_fp16 = matmul(transpose_x = var_5411_transpose_x_0, transpose_y = var_5411_transpose_y_0, x = var_5410_cast_fp16, y = v_new_295_cast_fp16)[name = string("op_5411_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5412_cast_fp16 = add(x = var_5408_cast_fp16, y = var_5411_cast_fp16)[name = string("op_5412_cast_fp16")];
+            tensor<int32, [4]> var_5414_begin_0 = const()[name = string("op_5414_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_5414_end_0 = const()[name = string("op_5414_end_0"), val = tensor<int32, [4]>([32, 20, 1, 1])];
+            tensor<bool, [4]> var_5414_end_mask_0 = const()[name = string("op_5414_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5414_squeeze_mask_0 = const()[name = string("op_5414_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5414_cast_fp16 = slice_by_index(begin = var_5414_begin_0, end = var_5414_end_0, end_mask = var_5414_end_mask_0, squeeze_mask = var_5414_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5414_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5415_cast_fp16 = mul(x = state_295_cast_fp16, y = var_5414_cast_fp16)[name = string("op_5415_cast_fp16")];
+            tensor<int32, [4]> var_5417_begin_0 = const()[name = string("op_5417_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_5417_end_0 = const()[name = string("op_5417_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_5417_end_mask_0 = const()[name = string("op_5417_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5417_squeeze_mask_0 = const()[name = string("op_5417_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5417_cast_fp16 = slice_by_index(begin = var_5417_begin_0, end = var_5417_end_0, end_mask = var_5417_end_mask_0, squeeze_mask = var_5417_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5417_cast_fp16")];
+            bool var_5419_transpose_x_1 = const()[name = string("op_5419_transpose_x_1"), val = bool(true)];
+            bool var_5419_transpose_y_1 = const()[name = string("op_5419_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5419_cast_fp16 = matmul(transpose_x = var_5419_transpose_x_1, transpose_y = var_5419_transpose_y_1, x = var_5417_cast_fp16, y = v_new_295_cast_fp16)[name = string("op_5419_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_297_cast_fp16 = add(x = var_5415_cast_fp16, y = var_5419_cast_fp16)[name = string("state_297_cast_fp16")];
+            tensor<int32, [4]> var_5422_begin_0 = const()[name = string("op_5422_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_5422_end_0 = const()[name = string("op_5422_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_5422_end_mask_0 = const()[name = string("op_5422_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5422_squeeze_mask_0 = const()[name = string("op_5422_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5422_cast_fp16 = slice_by_index(begin = var_5422_begin_0, end = var_5422_end_0, end_mask = var_5422_end_mask_0, squeeze_mask = var_5422_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5422_cast_fp16")];
+            tensor<int32, [4]> var_5424_begin_0 = const()[name = string("op_5424_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_5424_end_0 = const()[name = string("op_5424_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_5424_end_mask_0 = const()[name = string("op_5424_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5424_squeeze_mask_0 = const()[name = string("op_5424_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5424_cast_fp16 = slice_by_index(begin = var_5424_begin_0, end = var_5424_end_0, end_mask = var_5424_end_mask_0, squeeze_mask = var_5424_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5424_cast_fp16")];
+            bool var_5425_transpose_x_0 = const()[name = string("op_5425_transpose_x_0"), val = bool(false)];
+            bool var_5425_transpose_y_0 = const()[name = string("op_5425_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5425_cast_fp16 = matmul(transpose_x = var_5425_transpose_x_0, transpose_y = var_5425_transpose_y_0, x = var_5424_cast_fp16, y = state_297_cast_fp16)[name = string("op_5425_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_297_cast_fp16 = sub(x = var_5422_cast_fp16, y = var_5425_cast_fp16)[name = string("v_new_297_cast_fp16")];
+            tensor<int32, [4]> var_5428_begin_0 = const()[name = string("op_5428_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_5428_end_0 = const()[name = string("op_5428_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_5428_end_mask_0 = const()[name = string("op_5428_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5428_squeeze_mask_0 = const()[name = string("op_5428_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5428_cast_fp16 = slice_by_index(begin = var_5428_begin_0, end = var_5428_end_0, end_mask = var_5428_end_mask_0, squeeze_mask = var_5428_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5428_cast_fp16")];
+            bool var_5429_transpose_x_0 = const()[name = string("op_5429_transpose_x_0"), val = bool(false)];
+            bool var_5429_transpose_y_0 = const()[name = string("op_5429_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5429_cast_fp16 = matmul(transpose_x = var_5429_transpose_x_0, transpose_y = var_5429_transpose_y_0, x = var_5428_cast_fp16, y = state_297_cast_fp16)[name = string("op_5429_cast_fp16")];
+            tensor<int32, [4]> var_5431_begin_0 = const()[name = string("op_5431_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_5431_end_0 = const()[name = string("op_5431_end_0"), val = tensor<int32, [4]>([32, 21, 64, 64])];
+            tensor<bool, [4]> var_5431_end_mask_0 = const()[name = string("op_5431_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5431_squeeze_mask_0 = const()[name = string("op_5431_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5431_cast_fp16 = slice_by_index(begin = var_5431_begin_0, end = var_5431_end_0, end_mask = var_5431_end_mask_0, squeeze_mask = var_5431_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5431_cast_fp16")];
+            bool var_5432_transpose_x_0 = const()[name = string("op_5432_transpose_x_0"), val = bool(false)];
+            bool var_5432_transpose_y_0 = const()[name = string("op_5432_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5432_cast_fp16 = matmul(transpose_x = var_5432_transpose_x_0, transpose_y = var_5432_transpose_y_0, x = var_5431_cast_fp16, y = v_new_297_cast_fp16)[name = string("op_5432_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5433_cast_fp16 = add(x = var_5429_cast_fp16, y = var_5432_cast_fp16)[name = string("op_5433_cast_fp16")];
+            tensor<int32, [4]> var_5435_begin_0 = const()[name = string("op_5435_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_5435_end_0 = const()[name = string("op_5435_end_0"), val = tensor<int32, [4]>([32, 21, 1, 1])];
+            tensor<bool, [4]> var_5435_end_mask_0 = const()[name = string("op_5435_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5435_squeeze_mask_0 = const()[name = string("op_5435_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5435_cast_fp16 = slice_by_index(begin = var_5435_begin_0, end = var_5435_end_0, end_mask = var_5435_end_mask_0, squeeze_mask = var_5435_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5435_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5436_cast_fp16 = mul(x = state_297_cast_fp16, y = var_5435_cast_fp16)[name = string("op_5436_cast_fp16")];
+            tensor<int32, [4]> var_5438_begin_0 = const()[name = string("op_5438_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_5438_end_0 = const()[name = string("op_5438_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_5438_end_mask_0 = const()[name = string("op_5438_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5438_squeeze_mask_0 = const()[name = string("op_5438_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5438_cast_fp16 = slice_by_index(begin = var_5438_begin_0, end = var_5438_end_0, end_mask = var_5438_end_mask_0, squeeze_mask = var_5438_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5438_cast_fp16")];
+            bool var_5440_transpose_x_1 = const()[name = string("op_5440_transpose_x_1"), val = bool(true)];
+            bool var_5440_transpose_y_1 = const()[name = string("op_5440_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5440_cast_fp16 = matmul(transpose_x = var_5440_transpose_x_1, transpose_y = var_5440_transpose_y_1, x = var_5438_cast_fp16, y = v_new_297_cast_fp16)[name = string("op_5440_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_299_cast_fp16 = add(x = var_5436_cast_fp16, y = var_5440_cast_fp16)[name = string("state_299_cast_fp16")];
+            tensor<int32, [4]> var_5443_begin_0 = const()[name = string("op_5443_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_5443_end_0 = const()[name = string("op_5443_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_5443_end_mask_0 = const()[name = string("op_5443_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5443_squeeze_mask_0 = const()[name = string("op_5443_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5443_cast_fp16 = slice_by_index(begin = var_5443_begin_0, end = var_5443_end_0, end_mask = var_5443_end_mask_0, squeeze_mask = var_5443_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5443_cast_fp16")];
+            tensor<int32, [4]> var_5445_begin_0 = const()[name = string("op_5445_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_5445_end_0 = const()[name = string("op_5445_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_5445_end_mask_0 = const()[name = string("op_5445_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5445_squeeze_mask_0 = const()[name = string("op_5445_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5445_cast_fp16 = slice_by_index(begin = var_5445_begin_0, end = var_5445_end_0, end_mask = var_5445_end_mask_0, squeeze_mask = var_5445_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5445_cast_fp16")];
+            bool var_5446_transpose_x_0 = const()[name = string("op_5446_transpose_x_0"), val = bool(false)];
+            bool var_5446_transpose_y_0 = const()[name = string("op_5446_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5446_cast_fp16 = matmul(transpose_x = var_5446_transpose_x_0, transpose_y = var_5446_transpose_y_0, x = var_5445_cast_fp16, y = state_299_cast_fp16)[name = string("op_5446_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_299_cast_fp16 = sub(x = var_5443_cast_fp16, y = var_5446_cast_fp16)[name = string("v_new_299_cast_fp16")];
+            tensor<int32, [4]> var_5449_begin_0 = const()[name = string("op_5449_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_5449_end_0 = const()[name = string("op_5449_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_5449_end_mask_0 = const()[name = string("op_5449_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5449_squeeze_mask_0 = const()[name = string("op_5449_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5449_cast_fp16 = slice_by_index(begin = var_5449_begin_0, end = var_5449_end_0, end_mask = var_5449_end_mask_0, squeeze_mask = var_5449_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5449_cast_fp16")];
+            bool var_5450_transpose_x_0 = const()[name = string("op_5450_transpose_x_0"), val = bool(false)];
+            bool var_5450_transpose_y_0 = const()[name = string("op_5450_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5450_cast_fp16 = matmul(transpose_x = var_5450_transpose_x_0, transpose_y = var_5450_transpose_y_0, x = var_5449_cast_fp16, y = state_299_cast_fp16)[name = string("op_5450_cast_fp16")];
+            tensor<int32, [4]> var_5452_begin_0 = const()[name = string("op_5452_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_5452_end_0 = const()[name = string("op_5452_end_0"), val = tensor<int32, [4]>([32, 22, 64, 64])];
+            tensor<bool, [4]> var_5452_end_mask_0 = const()[name = string("op_5452_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5452_squeeze_mask_0 = const()[name = string("op_5452_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5452_cast_fp16 = slice_by_index(begin = var_5452_begin_0, end = var_5452_end_0, end_mask = var_5452_end_mask_0, squeeze_mask = var_5452_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5452_cast_fp16")];
+            bool var_5453_transpose_x_0 = const()[name = string("op_5453_transpose_x_0"), val = bool(false)];
+            bool var_5453_transpose_y_0 = const()[name = string("op_5453_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5453_cast_fp16 = matmul(transpose_x = var_5453_transpose_x_0, transpose_y = var_5453_transpose_y_0, x = var_5452_cast_fp16, y = v_new_299_cast_fp16)[name = string("op_5453_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5454_cast_fp16 = add(x = var_5450_cast_fp16, y = var_5453_cast_fp16)[name = string("op_5454_cast_fp16")];
+            tensor<int32, [4]> var_5456_begin_0 = const()[name = string("op_5456_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_5456_end_0 = const()[name = string("op_5456_end_0"), val = tensor<int32, [4]>([32, 22, 1, 1])];
+            tensor<bool, [4]> var_5456_end_mask_0 = const()[name = string("op_5456_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5456_squeeze_mask_0 = const()[name = string("op_5456_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5456_cast_fp16 = slice_by_index(begin = var_5456_begin_0, end = var_5456_end_0, end_mask = var_5456_end_mask_0, squeeze_mask = var_5456_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5456_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5457_cast_fp16 = mul(x = state_299_cast_fp16, y = var_5456_cast_fp16)[name = string("op_5457_cast_fp16")];
+            tensor<int32, [4]> var_5459_begin_0 = const()[name = string("op_5459_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_5459_end_0 = const()[name = string("op_5459_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_5459_end_mask_0 = const()[name = string("op_5459_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5459_squeeze_mask_0 = const()[name = string("op_5459_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5459_cast_fp16 = slice_by_index(begin = var_5459_begin_0, end = var_5459_end_0, end_mask = var_5459_end_mask_0, squeeze_mask = var_5459_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5459_cast_fp16")];
+            bool var_5461_transpose_x_1 = const()[name = string("op_5461_transpose_x_1"), val = bool(true)];
+            bool var_5461_transpose_y_1 = const()[name = string("op_5461_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5461_cast_fp16 = matmul(transpose_x = var_5461_transpose_x_1, transpose_y = var_5461_transpose_y_1, x = var_5459_cast_fp16, y = v_new_299_cast_fp16)[name = string("op_5461_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_301_cast_fp16 = add(x = var_5457_cast_fp16, y = var_5461_cast_fp16)[name = string("state_301_cast_fp16")];
+            tensor<int32, [4]> var_5464_begin_0 = const()[name = string("op_5464_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_5464_end_0 = const()[name = string("op_5464_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_5464_end_mask_0 = const()[name = string("op_5464_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5464_squeeze_mask_0 = const()[name = string("op_5464_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5464_cast_fp16 = slice_by_index(begin = var_5464_begin_0, end = var_5464_end_0, end_mask = var_5464_end_mask_0, squeeze_mask = var_5464_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5464_cast_fp16")];
+            tensor<int32, [4]> var_5466_begin_0 = const()[name = string("op_5466_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_5466_end_0 = const()[name = string("op_5466_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_5466_end_mask_0 = const()[name = string("op_5466_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5466_squeeze_mask_0 = const()[name = string("op_5466_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5466_cast_fp16 = slice_by_index(begin = var_5466_begin_0, end = var_5466_end_0, end_mask = var_5466_end_mask_0, squeeze_mask = var_5466_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5466_cast_fp16")];
+            bool var_5467_transpose_x_0 = const()[name = string("op_5467_transpose_x_0"), val = bool(false)];
+            bool var_5467_transpose_y_0 = const()[name = string("op_5467_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5467_cast_fp16 = matmul(transpose_x = var_5467_transpose_x_0, transpose_y = var_5467_transpose_y_0, x = var_5466_cast_fp16, y = state_301_cast_fp16)[name = string("op_5467_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_301_cast_fp16 = sub(x = var_5464_cast_fp16, y = var_5467_cast_fp16)[name = string("v_new_301_cast_fp16")];
+            tensor<int32, [4]> var_5470_begin_0 = const()[name = string("op_5470_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_5470_end_0 = const()[name = string("op_5470_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_5470_end_mask_0 = const()[name = string("op_5470_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5470_squeeze_mask_0 = const()[name = string("op_5470_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5470_cast_fp16 = slice_by_index(begin = var_5470_begin_0, end = var_5470_end_0, end_mask = var_5470_end_mask_0, squeeze_mask = var_5470_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5470_cast_fp16")];
+            bool var_5471_transpose_x_0 = const()[name = string("op_5471_transpose_x_0"), val = bool(false)];
+            bool var_5471_transpose_y_0 = const()[name = string("op_5471_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5471_cast_fp16 = matmul(transpose_x = var_5471_transpose_x_0, transpose_y = var_5471_transpose_y_0, x = var_5470_cast_fp16, y = state_301_cast_fp16)[name = string("op_5471_cast_fp16")];
+            tensor<int32, [4]> var_5473_begin_0 = const()[name = string("op_5473_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_5473_end_0 = const()[name = string("op_5473_end_0"), val = tensor<int32, [4]>([32, 23, 64, 64])];
+            tensor<bool, [4]> var_5473_end_mask_0 = const()[name = string("op_5473_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5473_squeeze_mask_0 = const()[name = string("op_5473_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5473_cast_fp16 = slice_by_index(begin = var_5473_begin_0, end = var_5473_end_0, end_mask = var_5473_end_mask_0, squeeze_mask = var_5473_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5473_cast_fp16")];
+            bool var_5474_transpose_x_0 = const()[name = string("op_5474_transpose_x_0"), val = bool(false)];
+            bool var_5474_transpose_y_0 = const()[name = string("op_5474_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5474_cast_fp16 = matmul(transpose_x = var_5474_transpose_x_0, transpose_y = var_5474_transpose_y_0, x = var_5473_cast_fp16, y = v_new_301_cast_fp16)[name = string("op_5474_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5475_cast_fp16 = add(x = var_5471_cast_fp16, y = var_5474_cast_fp16)[name = string("op_5475_cast_fp16")];
+            tensor<int32, [4]> var_5477_begin_0 = const()[name = string("op_5477_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_5477_end_0 = const()[name = string("op_5477_end_0"), val = tensor<int32, [4]>([32, 23, 1, 1])];
+            tensor<bool, [4]> var_5477_end_mask_0 = const()[name = string("op_5477_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5477_squeeze_mask_0 = const()[name = string("op_5477_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5477_cast_fp16 = slice_by_index(begin = var_5477_begin_0, end = var_5477_end_0, end_mask = var_5477_end_mask_0, squeeze_mask = var_5477_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5477_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5478_cast_fp16 = mul(x = state_301_cast_fp16, y = var_5477_cast_fp16)[name = string("op_5478_cast_fp16")];
+            tensor<int32, [4]> var_5480_begin_0 = const()[name = string("op_5480_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_5480_end_0 = const()[name = string("op_5480_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_5480_end_mask_0 = const()[name = string("op_5480_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5480_squeeze_mask_0 = const()[name = string("op_5480_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5480_cast_fp16 = slice_by_index(begin = var_5480_begin_0, end = var_5480_end_0, end_mask = var_5480_end_mask_0, squeeze_mask = var_5480_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5480_cast_fp16")];
+            bool var_5482_transpose_x_1 = const()[name = string("op_5482_transpose_x_1"), val = bool(true)];
+            bool var_5482_transpose_y_1 = const()[name = string("op_5482_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5482_cast_fp16 = matmul(transpose_x = var_5482_transpose_x_1, transpose_y = var_5482_transpose_y_1, x = var_5480_cast_fp16, y = v_new_301_cast_fp16)[name = string("op_5482_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_303_cast_fp16 = add(x = var_5478_cast_fp16, y = var_5482_cast_fp16)[name = string("state_303_cast_fp16")];
+            tensor<int32, [4]> var_5485_begin_0 = const()[name = string("op_5485_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_5485_end_0 = const()[name = string("op_5485_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_5485_end_mask_0 = const()[name = string("op_5485_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5485_squeeze_mask_0 = const()[name = string("op_5485_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5485_cast_fp16 = slice_by_index(begin = var_5485_begin_0, end = var_5485_end_0, end_mask = var_5485_end_mask_0, squeeze_mask = var_5485_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5485_cast_fp16")];
+            tensor<int32, [4]> var_5487_begin_0 = const()[name = string("op_5487_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_5487_end_0 = const()[name = string("op_5487_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_5487_end_mask_0 = const()[name = string("op_5487_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5487_squeeze_mask_0 = const()[name = string("op_5487_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5487_cast_fp16 = slice_by_index(begin = var_5487_begin_0, end = var_5487_end_0, end_mask = var_5487_end_mask_0, squeeze_mask = var_5487_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5487_cast_fp16")];
+            bool var_5488_transpose_x_0 = const()[name = string("op_5488_transpose_x_0"), val = bool(false)];
+            bool var_5488_transpose_y_0 = const()[name = string("op_5488_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5488_cast_fp16 = matmul(transpose_x = var_5488_transpose_x_0, transpose_y = var_5488_transpose_y_0, x = var_5487_cast_fp16, y = state_303_cast_fp16)[name = string("op_5488_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_303_cast_fp16 = sub(x = var_5485_cast_fp16, y = var_5488_cast_fp16)[name = string("v_new_303_cast_fp16")];
+            tensor<int32, [4]> var_5491_begin_0 = const()[name = string("op_5491_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_5491_end_0 = const()[name = string("op_5491_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_5491_end_mask_0 = const()[name = string("op_5491_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5491_squeeze_mask_0 = const()[name = string("op_5491_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5491_cast_fp16 = slice_by_index(begin = var_5491_begin_0, end = var_5491_end_0, end_mask = var_5491_end_mask_0, squeeze_mask = var_5491_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5491_cast_fp16")];
+            bool var_5492_transpose_x_0 = const()[name = string("op_5492_transpose_x_0"), val = bool(false)];
+            bool var_5492_transpose_y_0 = const()[name = string("op_5492_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5492_cast_fp16 = matmul(transpose_x = var_5492_transpose_x_0, transpose_y = var_5492_transpose_y_0, x = var_5491_cast_fp16, y = state_303_cast_fp16)[name = string("op_5492_cast_fp16")];
+            tensor<int32, [4]> var_5494_begin_0 = const()[name = string("op_5494_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_5494_end_0 = const()[name = string("op_5494_end_0"), val = tensor<int32, [4]>([32, 24, 64, 64])];
+            tensor<bool, [4]> var_5494_end_mask_0 = const()[name = string("op_5494_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5494_squeeze_mask_0 = const()[name = string("op_5494_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5494_cast_fp16 = slice_by_index(begin = var_5494_begin_0, end = var_5494_end_0, end_mask = var_5494_end_mask_0, squeeze_mask = var_5494_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5494_cast_fp16")];
+            bool var_5495_transpose_x_0 = const()[name = string("op_5495_transpose_x_0"), val = bool(false)];
+            bool var_5495_transpose_y_0 = const()[name = string("op_5495_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5495_cast_fp16 = matmul(transpose_x = var_5495_transpose_x_0, transpose_y = var_5495_transpose_y_0, x = var_5494_cast_fp16, y = v_new_303_cast_fp16)[name = string("op_5495_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5496_cast_fp16 = add(x = var_5492_cast_fp16, y = var_5495_cast_fp16)[name = string("op_5496_cast_fp16")];
+            tensor<int32, [4]> var_5498_begin_0 = const()[name = string("op_5498_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_5498_end_0 = const()[name = string("op_5498_end_0"), val = tensor<int32, [4]>([32, 24, 1, 1])];
+            tensor<bool, [4]> var_5498_end_mask_0 = const()[name = string("op_5498_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5498_squeeze_mask_0 = const()[name = string("op_5498_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5498_cast_fp16 = slice_by_index(begin = var_5498_begin_0, end = var_5498_end_0, end_mask = var_5498_end_mask_0, squeeze_mask = var_5498_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5498_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5499_cast_fp16 = mul(x = state_303_cast_fp16, y = var_5498_cast_fp16)[name = string("op_5499_cast_fp16")];
+            tensor<int32, [4]> var_5501_begin_0 = const()[name = string("op_5501_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_5501_end_0 = const()[name = string("op_5501_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_5501_end_mask_0 = const()[name = string("op_5501_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5501_squeeze_mask_0 = const()[name = string("op_5501_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5501_cast_fp16 = slice_by_index(begin = var_5501_begin_0, end = var_5501_end_0, end_mask = var_5501_end_mask_0, squeeze_mask = var_5501_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5501_cast_fp16")];
+            bool var_5503_transpose_x_1 = const()[name = string("op_5503_transpose_x_1"), val = bool(true)];
+            bool var_5503_transpose_y_1 = const()[name = string("op_5503_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5503_cast_fp16 = matmul(transpose_x = var_5503_transpose_x_1, transpose_y = var_5503_transpose_y_1, x = var_5501_cast_fp16, y = v_new_303_cast_fp16)[name = string("op_5503_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_305_cast_fp16 = add(x = var_5499_cast_fp16, y = var_5503_cast_fp16)[name = string("state_305_cast_fp16")];
+            tensor<int32, [4]> var_5506_begin_0 = const()[name = string("op_5506_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_5506_end_0 = const()[name = string("op_5506_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_5506_end_mask_0 = const()[name = string("op_5506_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5506_squeeze_mask_0 = const()[name = string("op_5506_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5506_cast_fp16 = slice_by_index(begin = var_5506_begin_0, end = var_5506_end_0, end_mask = var_5506_end_mask_0, squeeze_mask = var_5506_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5506_cast_fp16")];
+            tensor<int32, [4]> var_5508_begin_0 = const()[name = string("op_5508_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_5508_end_0 = const()[name = string("op_5508_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_5508_end_mask_0 = const()[name = string("op_5508_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5508_squeeze_mask_0 = const()[name = string("op_5508_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5508_cast_fp16 = slice_by_index(begin = var_5508_begin_0, end = var_5508_end_0, end_mask = var_5508_end_mask_0, squeeze_mask = var_5508_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5508_cast_fp16")];
+            bool var_5509_transpose_x_0 = const()[name = string("op_5509_transpose_x_0"), val = bool(false)];
+            bool var_5509_transpose_y_0 = const()[name = string("op_5509_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5509_cast_fp16 = matmul(transpose_x = var_5509_transpose_x_0, transpose_y = var_5509_transpose_y_0, x = var_5508_cast_fp16, y = state_305_cast_fp16)[name = string("op_5509_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_305_cast_fp16 = sub(x = var_5506_cast_fp16, y = var_5509_cast_fp16)[name = string("v_new_305_cast_fp16")];
+            tensor<int32, [4]> var_5512_begin_0 = const()[name = string("op_5512_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_5512_end_0 = const()[name = string("op_5512_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_5512_end_mask_0 = const()[name = string("op_5512_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5512_squeeze_mask_0 = const()[name = string("op_5512_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5512_cast_fp16 = slice_by_index(begin = var_5512_begin_0, end = var_5512_end_0, end_mask = var_5512_end_mask_0, squeeze_mask = var_5512_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5512_cast_fp16")];
+            bool var_5513_transpose_x_0 = const()[name = string("op_5513_transpose_x_0"), val = bool(false)];
+            bool var_5513_transpose_y_0 = const()[name = string("op_5513_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5513_cast_fp16 = matmul(transpose_x = var_5513_transpose_x_0, transpose_y = var_5513_transpose_y_0, x = var_5512_cast_fp16, y = state_305_cast_fp16)[name = string("op_5513_cast_fp16")];
+            tensor<int32, [4]> var_5515_begin_0 = const()[name = string("op_5515_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_5515_end_0 = const()[name = string("op_5515_end_0"), val = tensor<int32, [4]>([32, 25, 64, 64])];
+            tensor<bool, [4]> var_5515_end_mask_0 = const()[name = string("op_5515_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5515_squeeze_mask_0 = const()[name = string("op_5515_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5515_cast_fp16 = slice_by_index(begin = var_5515_begin_0, end = var_5515_end_0, end_mask = var_5515_end_mask_0, squeeze_mask = var_5515_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5515_cast_fp16")];
+            bool var_5516_transpose_x_0 = const()[name = string("op_5516_transpose_x_0"), val = bool(false)];
+            bool var_5516_transpose_y_0 = const()[name = string("op_5516_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5516_cast_fp16 = matmul(transpose_x = var_5516_transpose_x_0, transpose_y = var_5516_transpose_y_0, x = var_5515_cast_fp16, y = v_new_305_cast_fp16)[name = string("op_5516_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5517_cast_fp16 = add(x = var_5513_cast_fp16, y = var_5516_cast_fp16)[name = string("op_5517_cast_fp16")];
+            tensor<int32, [4]> var_5519_begin_0 = const()[name = string("op_5519_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_5519_end_0 = const()[name = string("op_5519_end_0"), val = tensor<int32, [4]>([32, 25, 1, 1])];
+            tensor<bool, [4]> var_5519_end_mask_0 = const()[name = string("op_5519_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5519_squeeze_mask_0 = const()[name = string("op_5519_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5519_cast_fp16 = slice_by_index(begin = var_5519_begin_0, end = var_5519_end_0, end_mask = var_5519_end_mask_0, squeeze_mask = var_5519_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5519_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5520_cast_fp16 = mul(x = state_305_cast_fp16, y = var_5519_cast_fp16)[name = string("op_5520_cast_fp16")];
+            tensor<int32, [4]> var_5522_begin_0 = const()[name = string("op_5522_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_5522_end_0 = const()[name = string("op_5522_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_5522_end_mask_0 = const()[name = string("op_5522_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5522_squeeze_mask_0 = const()[name = string("op_5522_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5522_cast_fp16 = slice_by_index(begin = var_5522_begin_0, end = var_5522_end_0, end_mask = var_5522_end_mask_0, squeeze_mask = var_5522_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5522_cast_fp16")];
+            bool var_5524_transpose_x_1 = const()[name = string("op_5524_transpose_x_1"), val = bool(true)];
+            bool var_5524_transpose_y_1 = const()[name = string("op_5524_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5524_cast_fp16 = matmul(transpose_x = var_5524_transpose_x_1, transpose_y = var_5524_transpose_y_1, x = var_5522_cast_fp16, y = v_new_305_cast_fp16)[name = string("op_5524_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_307_cast_fp16 = add(x = var_5520_cast_fp16, y = var_5524_cast_fp16)[name = string("state_307_cast_fp16")];
+            tensor<int32, [4]> var_5527_begin_0 = const()[name = string("op_5527_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_5527_end_0 = const()[name = string("op_5527_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_5527_end_mask_0 = const()[name = string("op_5527_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5527_squeeze_mask_0 = const()[name = string("op_5527_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5527_cast_fp16 = slice_by_index(begin = var_5527_begin_0, end = var_5527_end_0, end_mask = var_5527_end_mask_0, squeeze_mask = var_5527_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5527_cast_fp16")];
+            tensor<int32, [4]> var_5529_begin_0 = const()[name = string("op_5529_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_5529_end_0 = const()[name = string("op_5529_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_5529_end_mask_0 = const()[name = string("op_5529_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5529_squeeze_mask_0 = const()[name = string("op_5529_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5529_cast_fp16 = slice_by_index(begin = var_5529_begin_0, end = var_5529_end_0, end_mask = var_5529_end_mask_0, squeeze_mask = var_5529_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5529_cast_fp16")];
+            bool var_5530_transpose_x_0 = const()[name = string("op_5530_transpose_x_0"), val = bool(false)];
+            bool var_5530_transpose_y_0 = const()[name = string("op_5530_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5530_cast_fp16 = matmul(transpose_x = var_5530_transpose_x_0, transpose_y = var_5530_transpose_y_0, x = var_5529_cast_fp16, y = state_307_cast_fp16)[name = string("op_5530_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_307_cast_fp16 = sub(x = var_5527_cast_fp16, y = var_5530_cast_fp16)[name = string("v_new_307_cast_fp16")];
+            tensor<int32, [4]> var_5533_begin_0 = const()[name = string("op_5533_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_5533_end_0 = const()[name = string("op_5533_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_5533_end_mask_0 = const()[name = string("op_5533_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5533_squeeze_mask_0 = const()[name = string("op_5533_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5533_cast_fp16 = slice_by_index(begin = var_5533_begin_0, end = var_5533_end_0, end_mask = var_5533_end_mask_0, squeeze_mask = var_5533_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5533_cast_fp16")];
+            bool var_5534_transpose_x_0 = const()[name = string("op_5534_transpose_x_0"), val = bool(false)];
+            bool var_5534_transpose_y_0 = const()[name = string("op_5534_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5534_cast_fp16 = matmul(transpose_x = var_5534_transpose_x_0, transpose_y = var_5534_transpose_y_0, x = var_5533_cast_fp16, y = state_307_cast_fp16)[name = string("op_5534_cast_fp16")];
+            tensor<int32, [4]> var_5536_begin_0 = const()[name = string("op_5536_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_5536_end_0 = const()[name = string("op_5536_end_0"), val = tensor<int32, [4]>([32, 26, 64, 64])];
+            tensor<bool, [4]> var_5536_end_mask_0 = const()[name = string("op_5536_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5536_squeeze_mask_0 = const()[name = string("op_5536_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5536_cast_fp16 = slice_by_index(begin = var_5536_begin_0, end = var_5536_end_0, end_mask = var_5536_end_mask_0, squeeze_mask = var_5536_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5536_cast_fp16")];
+            bool var_5537_transpose_x_0 = const()[name = string("op_5537_transpose_x_0"), val = bool(false)];
+            bool var_5537_transpose_y_0 = const()[name = string("op_5537_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5537_cast_fp16 = matmul(transpose_x = var_5537_transpose_x_0, transpose_y = var_5537_transpose_y_0, x = var_5536_cast_fp16, y = v_new_307_cast_fp16)[name = string("op_5537_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5538_cast_fp16 = add(x = var_5534_cast_fp16, y = var_5537_cast_fp16)[name = string("op_5538_cast_fp16")];
+            tensor<int32, [4]> var_5540_begin_0 = const()[name = string("op_5540_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_5540_end_0 = const()[name = string("op_5540_end_0"), val = tensor<int32, [4]>([32, 26, 1, 1])];
+            tensor<bool, [4]> var_5540_end_mask_0 = const()[name = string("op_5540_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5540_squeeze_mask_0 = const()[name = string("op_5540_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5540_cast_fp16 = slice_by_index(begin = var_5540_begin_0, end = var_5540_end_0, end_mask = var_5540_end_mask_0, squeeze_mask = var_5540_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5540_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5541_cast_fp16 = mul(x = state_307_cast_fp16, y = var_5540_cast_fp16)[name = string("op_5541_cast_fp16")];
+            tensor<int32, [4]> var_5543_begin_0 = const()[name = string("op_5543_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_5543_end_0 = const()[name = string("op_5543_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_5543_end_mask_0 = const()[name = string("op_5543_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5543_squeeze_mask_0 = const()[name = string("op_5543_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5543_cast_fp16 = slice_by_index(begin = var_5543_begin_0, end = var_5543_end_0, end_mask = var_5543_end_mask_0, squeeze_mask = var_5543_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5543_cast_fp16")];
+            bool var_5545_transpose_x_1 = const()[name = string("op_5545_transpose_x_1"), val = bool(true)];
+            bool var_5545_transpose_y_1 = const()[name = string("op_5545_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5545_cast_fp16 = matmul(transpose_x = var_5545_transpose_x_1, transpose_y = var_5545_transpose_y_1, x = var_5543_cast_fp16, y = v_new_307_cast_fp16)[name = string("op_5545_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_309_cast_fp16 = add(x = var_5541_cast_fp16, y = var_5545_cast_fp16)[name = string("state_309_cast_fp16")];
+            tensor<int32, [4]> var_5548_begin_0 = const()[name = string("op_5548_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_5548_end_0 = const()[name = string("op_5548_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_5548_end_mask_0 = const()[name = string("op_5548_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5548_squeeze_mask_0 = const()[name = string("op_5548_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5548_cast_fp16 = slice_by_index(begin = var_5548_begin_0, end = var_5548_end_0, end_mask = var_5548_end_mask_0, squeeze_mask = var_5548_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5548_cast_fp16")];
+            tensor<int32, [4]> var_5550_begin_0 = const()[name = string("op_5550_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_5550_end_0 = const()[name = string("op_5550_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_5550_end_mask_0 = const()[name = string("op_5550_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5550_squeeze_mask_0 = const()[name = string("op_5550_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5550_cast_fp16 = slice_by_index(begin = var_5550_begin_0, end = var_5550_end_0, end_mask = var_5550_end_mask_0, squeeze_mask = var_5550_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5550_cast_fp16")];
+            bool var_5551_transpose_x_0 = const()[name = string("op_5551_transpose_x_0"), val = bool(false)];
+            bool var_5551_transpose_y_0 = const()[name = string("op_5551_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5551_cast_fp16 = matmul(transpose_x = var_5551_transpose_x_0, transpose_y = var_5551_transpose_y_0, x = var_5550_cast_fp16, y = state_309_cast_fp16)[name = string("op_5551_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_309_cast_fp16 = sub(x = var_5548_cast_fp16, y = var_5551_cast_fp16)[name = string("v_new_309_cast_fp16")];
+            tensor<int32, [4]> var_5554_begin_0 = const()[name = string("op_5554_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_5554_end_0 = const()[name = string("op_5554_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_5554_end_mask_0 = const()[name = string("op_5554_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5554_squeeze_mask_0 = const()[name = string("op_5554_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5554_cast_fp16 = slice_by_index(begin = var_5554_begin_0, end = var_5554_end_0, end_mask = var_5554_end_mask_0, squeeze_mask = var_5554_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5554_cast_fp16")];
+            bool var_5555_transpose_x_0 = const()[name = string("op_5555_transpose_x_0"), val = bool(false)];
+            bool var_5555_transpose_y_0 = const()[name = string("op_5555_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5555_cast_fp16 = matmul(transpose_x = var_5555_transpose_x_0, transpose_y = var_5555_transpose_y_0, x = var_5554_cast_fp16, y = state_309_cast_fp16)[name = string("op_5555_cast_fp16")];
+            tensor<int32, [4]> var_5557_begin_0 = const()[name = string("op_5557_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_5557_end_0 = const()[name = string("op_5557_end_0"), val = tensor<int32, [4]>([32, 27, 64, 64])];
+            tensor<bool, [4]> var_5557_end_mask_0 = const()[name = string("op_5557_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5557_squeeze_mask_0 = const()[name = string("op_5557_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5557_cast_fp16 = slice_by_index(begin = var_5557_begin_0, end = var_5557_end_0, end_mask = var_5557_end_mask_0, squeeze_mask = var_5557_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5557_cast_fp16")];
+            bool var_5558_transpose_x_0 = const()[name = string("op_5558_transpose_x_0"), val = bool(false)];
+            bool var_5558_transpose_y_0 = const()[name = string("op_5558_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5558_cast_fp16 = matmul(transpose_x = var_5558_transpose_x_0, transpose_y = var_5558_transpose_y_0, x = var_5557_cast_fp16, y = v_new_309_cast_fp16)[name = string("op_5558_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5559_cast_fp16 = add(x = var_5555_cast_fp16, y = var_5558_cast_fp16)[name = string("op_5559_cast_fp16")];
+            tensor<int32, [4]> var_5561_begin_0 = const()[name = string("op_5561_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_5561_end_0 = const()[name = string("op_5561_end_0"), val = tensor<int32, [4]>([32, 27, 1, 1])];
+            tensor<bool, [4]> var_5561_end_mask_0 = const()[name = string("op_5561_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5561_squeeze_mask_0 = const()[name = string("op_5561_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5561_cast_fp16 = slice_by_index(begin = var_5561_begin_0, end = var_5561_end_0, end_mask = var_5561_end_mask_0, squeeze_mask = var_5561_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5561_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5562_cast_fp16 = mul(x = state_309_cast_fp16, y = var_5561_cast_fp16)[name = string("op_5562_cast_fp16")];
+            tensor<int32, [4]> var_5564_begin_0 = const()[name = string("op_5564_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_5564_end_0 = const()[name = string("op_5564_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_5564_end_mask_0 = const()[name = string("op_5564_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5564_squeeze_mask_0 = const()[name = string("op_5564_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5564_cast_fp16 = slice_by_index(begin = var_5564_begin_0, end = var_5564_end_0, end_mask = var_5564_end_mask_0, squeeze_mask = var_5564_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5564_cast_fp16")];
+            bool var_5566_transpose_x_1 = const()[name = string("op_5566_transpose_x_1"), val = bool(true)];
+            bool var_5566_transpose_y_1 = const()[name = string("op_5566_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5566_cast_fp16 = matmul(transpose_x = var_5566_transpose_x_1, transpose_y = var_5566_transpose_y_1, x = var_5564_cast_fp16, y = v_new_309_cast_fp16)[name = string("op_5566_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_311_cast_fp16 = add(x = var_5562_cast_fp16, y = var_5566_cast_fp16)[name = string("state_311_cast_fp16")];
+            tensor<int32, [4]> var_5569_begin_0 = const()[name = string("op_5569_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_5569_end_0 = const()[name = string("op_5569_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_5569_end_mask_0 = const()[name = string("op_5569_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5569_squeeze_mask_0 = const()[name = string("op_5569_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5569_cast_fp16 = slice_by_index(begin = var_5569_begin_0, end = var_5569_end_0, end_mask = var_5569_end_mask_0, squeeze_mask = var_5569_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5569_cast_fp16")];
+            tensor<int32, [4]> var_5571_begin_0 = const()[name = string("op_5571_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_5571_end_0 = const()[name = string("op_5571_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_5571_end_mask_0 = const()[name = string("op_5571_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5571_squeeze_mask_0 = const()[name = string("op_5571_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5571_cast_fp16 = slice_by_index(begin = var_5571_begin_0, end = var_5571_end_0, end_mask = var_5571_end_mask_0, squeeze_mask = var_5571_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5571_cast_fp16")];
+            bool var_5572_transpose_x_0 = const()[name = string("op_5572_transpose_x_0"), val = bool(false)];
+            bool var_5572_transpose_y_0 = const()[name = string("op_5572_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5572_cast_fp16 = matmul(transpose_x = var_5572_transpose_x_0, transpose_y = var_5572_transpose_y_0, x = var_5571_cast_fp16, y = state_311_cast_fp16)[name = string("op_5572_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_311_cast_fp16 = sub(x = var_5569_cast_fp16, y = var_5572_cast_fp16)[name = string("v_new_311_cast_fp16")];
+            tensor<int32, [4]> var_5575_begin_0 = const()[name = string("op_5575_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_5575_end_0 = const()[name = string("op_5575_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_5575_end_mask_0 = const()[name = string("op_5575_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5575_squeeze_mask_0 = const()[name = string("op_5575_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5575_cast_fp16 = slice_by_index(begin = var_5575_begin_0, end = var_5575_end_0, end_mask = var_5575_end_mask_0, squeeze_mask = var_5575_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5575_cast_fp16")];
+            bool var_5576_transpose_x_0 = const()[name = string("op_5576_transpose_x_0"), val = bool(false)];
+            bool var_5576_transpose_y_0 = const()[name = string("op_5576_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5576_cast_fp16 = matmul(transpose_x = var_5576_transpose_x_0, transpose_y = var_5576_transpose_y_0, x = var_5575_cast_fp16, y = state_311_cast_fp16)[name = string("op_5576_cast_fp16")];
+            tensor<int32, [4]> var_5578_begin_0 = const()[name = string("op_5578_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_5578_end_0 = const()[name = string("op_5578_end_0"), val = tensor<int32, [4]>([32, 28, 64, 64])];
+            tensor<bool, [4]> var_5578_end_mask_0 = const()[name = string("op_5578_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5578_squeeze_mask_0 = const()[name = string("op_5578_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5578_cast_fp16 = slice_by_index(begin = var_5578_begin_0, end = var_5578_end_0, end_mask = var_5578_end_mask_0, squeeze_mask = var_5578_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5578_cast_fp16")];
+            bool var_5579_transpose_x_0 = const()[name = string("op_5579_transpose_x_0"), val = bool(false)];
+            bool var_5579_transpose_y_0 = const()[name = string("op_5579_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5579_cast_fp16 = matmul(transpose_x = var_5579_transpose_x_0, transpose_y = var_5579_transpose_y_0, x = var_5578_cast_fp16, y = v_new_311_cast_fp16)[name = string("op_5579_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5580_cast_fp16 = add(x = var_5576_cast_fp16, y = var_5579_cast_fp16)[name = string("op_5580_cast_fp16")];
+            tensor<int32, [4]> var_5582_begin_0 = const()[name = string("op_5582_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_5582_end_0 = const()[name = string("op_5582_end_0"), val = tensor<int32, [4]>([32, 28, 1, 1])];
+            tensor<bool, [4]> var_5582_end_mask_0 = const()[name = string("op_5582_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5582_squeeze_mask_0 = const()[name = string("op_5582_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5582_cast_fp16 = slice_by_index(begin = var_5582_begin_0, end = var_5582_end_0, end_mask = var_5582_end_mask_0, squeeze_mask = var_5582_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5582_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5583_cast_fp16 = mul(x = state_311_cast_fp16, y = var_5582_cast_fp16)[name = string("op_5583_cast_fp16")];
+            tensor<int32, [4]> var_5585_begin_0 = const()[name = string("op_5585_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_5585_end_0 = const()[name = string("op_5585_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_5585_end_mask_0 = const()[name = string("op_5585_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5585_squeeze_mask_0 = const()[name = string("op_5585_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5585_cast_fp16 = slice_by_index(begin = var_5585_begin_0, end = var_5585_end_0, end_mask = var_5585_end_mask_0, squeeze_mask = var_5585_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5585_cast_fp16")];
+            bool var_5587_transpose_x_1 = const()[name = string("op_5587_transpose_x_1"), val = bool(true)];
+            bool var_5587_transpose_y_1 = const()[name = string("op_5587_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5587_cast_fp16 = matmul(transpose_x = var_5587_transpose_x_1, transpose_y = var_5587_transpose_y_1, x = var_5585_cast_fp16, y = v_new_311_cast_fp16)[name = string("op_5587_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_313_cast_fp16 = add(x = var_5583_cast_fp16, y = var_5587_cast_fp16)[name = string("state_313_cast_fp16")];
+            tensor<int32, [4]> var_5590_begin_0 = const()[name = string("op_5590_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_5590_end_0 = const()[name = string("op_5590_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_5590_end_mask_0 = const()[name = string("op_5590_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5590_squeeze_mask_0 = const()[name = string("op_5590_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5590_cast_fp16 = slice_by_index(begin = var_5590_begin_0, end = var_5590_end_0, end_mask = var_5590_end_mask_0, squeeze_mask = var_5590_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5590_cast_fp16")];
+            tensor<int32, [4]> var_5592_begin_0 = const()[name = string("op_5592_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_5592_end_0 = const()[name = string("op_5592_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_5592_end_mask_0 = const()[name = string("op_5592_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5592_squeeze_mask_0 = const()[name = string("op_5592_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5592_cast_fp16 = slice_by_index(begin = var_5592_begin_0, end = var_5592_end_0, end_mask = var_5592_end_mask_0, squeeze_mask = var_5592_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5592_cast_fp16")];
+            bool var_5593_transpose_x_0 = const()[name = string("op_5593_transpose_x_0"), val = bool(false)];
+            bool var_5593_transpose_y_0 = const()[name = string("op_5593_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5593_cast_fp16 = matmul(transpose_x = var_5593_transpose_x_0, transpose_y = var_5593_transpose_y_0, x = var_5592_cast_fp16, y = state_313_cast_fp16)[name = string("op_5593_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_313_cast_fp16 = sub(x = var_5590_cast_fp16, y = var_5593_cast_fp16)[name = string("v_new_313_cast_fp16")];
+            tensor<int32, [4]> var_5596_begin_0 = const()[name = string("op_5596_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_5596_end_0 = const()[name = string("op_5596_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_5596_end_mask_0 = const()[name = string("op_5596_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5596_squeeze_mask_0 = const()[name = string("op_5596_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5596_cast_fp16 = slice_by_index(begin = var_5596_begin_0, end = var_5596_end_0, end_mask = var_5596_end_mask_0, squeeze_mask = var_5596_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5596_cast_fp16")];
+            bool var_5597_transpose_x_0 = const()[name = string("op_5597_transpose_x_0"), val = bool(false)];
+            bool var_5597_transpose_y_0 = const()[name = string("op_5597_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5597_cast_fp16 = matmul(transpose_x = var_5597_transpose_x_0, transpose_y = var_5597_transpose_y_0, x = var_5596_cast_fp16, y = state_313_cast_fp16)[name = string("op_5597_cast_fp16")];
+            tensor<int32, [4]> var_5599_begin_0 = const()[name = string("op_5599_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_5599_end_0 = const()[name = string("op_5599_end_0"), val = tensor<int32, [4]>([32, 29, 64, 64])];
+            tensor<bool, [4]> var_5599_end_mask_0 = const()[name = string("op_5599_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5599_squeeze_mask_0 = const()[name = string("op_5599_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5599_cast_fp16 = slice_by_index(begin = var_5599_begin_0, end = var_5599_end_0, end_mask = var_5599_end_mask_0, squeeze_mask = var_5599_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5599_cast_fp16")];
+            bool var_5600_transpose_x_0 = const()[name = string("op_5600_transpose_x_0"), val = bool(false)];
+            bool var_5600_transpose_y_0 = const()[name = string("op_5600_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5600_cast_fp16 = matmul(transpose_x = var_5600_transpose_x_0, transpose_y = var_5600_transpose_y_0, x = var_5599_cast_fp16, y = v_new_313_cast_fp16)[name = string("op_5600_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5601_cast_fp16 = add(x = var_5597_cast_fp16, y = var_5600_cast_fp16)[name = string("op_5601_cast_fp16")];
+            tensor<int32, [4]> var_5603_begin_0 = const()[name = string("op_5603_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_5603_end_0 = const()[name = string("op_5603_end_0"), val = tensor<int32, [4]>([32, 29, 1, 1])];
+            tensor<bool, [4]> var_5603_end_mask_0 = const()[name = string("op_5603_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5603_squeeze_mask_0 = const()[name = string("op_5603_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5603_cast_fp16 = slice_by_index(begin = var_5603_begin_0, end = var_5603_end_0, end_mask = var_5603_end_mask_0, squeeze_mask = var_5603_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5603_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5604_cast_fp16 = mul(x = state_313_cast_fp16, y = var_5603_cast_fp16)[name = string("op_5604_cast_fp16")];
+            tensor<int32, [4]> var_5606_begin_0 = const()[name = string("op_5606_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_5606_end_0 = const()[name = string("op_5606_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_5606_end_mask_0 = const()[name = string("op_5606_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5606_squeeze_mask_0 = const()[name = string("op_5606_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5606_cast_fp16 = slice_by_index(begin = var_5606_begin_0, end = var_5606_end_0, end_mask = var_5606_end_mask_0, squeeze_mask = var_5606_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5606_cast_fp16")];
+            bool var_5608_transpose_x_1 = const()[name = string("op_5608_transpose_x_1"), val = bool(true)];
+            bool var_5608_transpose_y_1 = const()[name = string("op_5608_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5608_cast_fp16 = matmul(transpose_x = var_5608_transpose_x_1, transpose_y = var_5608_transpose_y_1, x = var_5606_cast_fp16, y = v_new_313_cast_fp16)[name = string("op_5608_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_315_cast_fp16 = add(x = var_5604_cast_fp16, y = var_5608_cast_fp16)[name = string("state_315_cast_fp16")];
+            tensor<int32, [4]> var_5611_begin_0 = const()[name = string("op_5611_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_5611_end_0 = const()[name = string("op_5611_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_5611_end_mask_0 = const()[name = string("op_5611_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5611_squeeze_mask_0 = const()[name = string("op_5611_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5611_cast_fp16 = slice_by_index(begin = var_5611_begin_0, end = var_5611_end_0, end_mask = var_5611_end_mask_0, squeeze_mask = var_5611_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5611_cast_fp16")];
+            tensor<int32, [4]> var_5613_begin_0 = const()[name = string("op_5613_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_5613_end_0 = const()[name = string("op_5613_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_5613_end_mask_0 = const()[name = string("op_5613_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5613_squeeze_mask_0 = const()[name = string("op_5613_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5613_cast_fp16 = slice_by_index(begin = var_5613_begin_0, end = var_5613_end_0, end_mask = var_5613_end_mask_0, squeeze_mask = var_5613_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5613_cast_fp16")];
+            bool var_5614_transpose_x_0 = const()[name = string("op_5614_transpose_x_0"), val = bool(false)];
+            bool var_5614_transpose_y_0 = const()[name = string("op_5614_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5614_cast_fp16 = matmul(transpose_x = var_5614_transpose_x_0, transpose_y = var_5614_transpose_y_0, x = var_5613_cast_fp16, y = state_315_cast_fp16)[name = string("op_5614_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_315_cast_fp16 = sub(x = var_5611_cast_fp16, y = var_5614_cast_fp16)[name = string("v_new_315_cast_fp16")];
+            tensor<int32, [4]> var_5617_begin_0 = const()[name = string("op_5617_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_5617_end_0 = const()[name = string("op_5617_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_5617_end_mask_0 = const()[name = string("op_5617_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5617_squeeze_mask_0 = const()[name = string("op_5617_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5617_cast_fp16 = slice_by_index(begin = var_5617_begin_0, end = var_5617_end_0, end_mask = var_5617_end_mask_0, squeeze_mask = var_5617_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5617_cast_fp16")];
+            bool var_5618_transpose_x_0 = const()[name = string("op_5618_transpose_x_0"), val = bool(false)];
+            bool var_5618_transpose_y_0 = const()[name = string("op_5618_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5618_cast_fp16 = matmul(transpose_x = var_5618_transpose_x_0, transpose_y = var_5618_transpose_y_0, x = var_5617_cast_fp16, y = state_315_cast_fp16)[name = string("op_5618_cast_fp16")];
+            tensor<int32, [4]> var_5620_begin_0 = const()[name = string("op_5620_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_5620_end_0 = const()[name = string("op_5620_end_0"), val = tensor<int32, [4]>([32, 30, 64, 64])];
+            tensor<bool, [4]> var_5620_end_mask_0 = const()[name = string("op_5620_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5620_squeeze_mask_0 = const()[name = string("op_5620_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5620_cast_fp16 = slice_by_index(begin = var_5620_begin_0, end = var_5620_end_0, end_mask = var_5620_end_mask_0, squeeze_mask = var_5620_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5620_cast_fp16")];
+            bool var_5621_transpose_x_0 = const()[name = string("op_5621_transpose_x_0"), val = bool(false)];
+            bool var_5621_transpose_y_0 = const()[name = string("op_5621_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5621_cast_fp16 = matmul(transpose_x = var_5621_transpose_x_0, transpose_y = var_5621_transpose_y_0, x = var_5620_cast_fp16, y = v_new_315_cast_fp16)[name = string("op_5621_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5622_cast_fp16 = add(x = var_5618_cast_fp16, y = var_5621_cast_fp16)[name = string("op_5622_cast_fp16")];
+            tensor<int32, [4]> var_5624_begin_0 = const()[name = string("op_5624_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_5624_end_0 = const()[name = string("op_5624_end_0"), val = tensor<int32, [4]>([32, 30, 1, 1])];
+            tensor<bool, [4]> var_5624_end_mask_0 = const()[name = string("op_5624_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5624_squeeze_mask_0 = const()[name = string("op_5624_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5624_cast_fp16 = slice_by_index(begin = var_5624_begin_0, end = var_5624_end_0, end_mask = var_5624_end_mask_0, squeeze_mask = var_5624_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5624_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5625_cast_fp16 = mul(x = state_315_cast_fp16, y = var_5624_cast_fp16)[name = string("op_5625_cast_fp16")];
+            tensor<int32, [4]> var_5627_begin_0 = const()[name = string("op_5627_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_5627_end_0 = const()[name = string("op_5627_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_5627_end_mask_0 = const()[name = string("op_5627_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5627_squeeze_mask_0 = const()[name = string("op_5627_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5627_cast_fp16 = slice_by_index(begin = var_5627_begin_0, end = var_5627_end_0, end_mask = var_5627_end_mask_0, squeeze_mask = var_5627_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5627_cast_fp16")];
+            bool var_5629_transpose_x_1 = const()[name = string("op_5629_transpose_x_1"), val = bool(true)];
+            bool var_5629_transpose_y_1 = const()[name = string("op_5629_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5629_cast_fp16 = matmul(transpose_x = var_5629_transpose_x_1, transpose_y = var_5629_transpose_y_1, x = var_5627_cast_fp16, y = v_new_315_cast_fp16)[name = string("op_5629_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_317_cast_fp16 = add(x = var_5625_cast_fp16, y = var_5629_cast_fp16)[name = string("state_317_cast_fp16")];
+            tensor<int32, [4]> var_5632_begin_0 = const()[name = string("op_5632_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_5632_end_0 = const()[name = string("op_5632_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_5632_end_mask_0 = const()[name = string("op_5632_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5632_squeeze_mask_0 = const()[name = string("op_5632_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5632_cast_fp16 = slice_by_index(begin = var_5632_begin_0, end = var_5632_end_0, end_mask = var_5632_end_mask_0, squeeze_mask = var_5632_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5632_cast_fp16")];
+            tensor<int32, [4]> var_5634_begin_0 = const()[name = string("op_5634_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_5634_end_0 = const()[name = string("op_5634_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_5634_end_mask_0 = const()[name = string("op_5634_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5634_squeeze_mask_0 = const()[name = string("op_5634_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5634_cast_fp16 = slice_by_index(begin = var_5634_begin_0, end = var_5634_end_0, end_mask = var_5634_end_mask_0, squeeze_mask = var_5634_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5634_cast_fp16")];
+            bool var_5635_transpose_x_0 = const()[name = string("op_5635_transpose_x_0"), val = bool(false)];
+            bool var_5635_transpose_y_0 = const()[name = string("op_5635_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5635_cast_fp16 = matmul(transpose_x = var_5635_transpose_x_0, transpose_y = var_5635_transpose_y_0, x = var_5634_cast_fp16, y = state_317_cast_fp16)[name = string("op_5635_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_317_cast_fp16 = sub(x = var_5632_cast_fp16, y = var_5635_cast_fp16)[name = string("v_new_317_cast_fp16")];
+            tensor<int32, [4]> var_5638_begin_0 = const()[name = string("op_5638_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_5638_end_0 = const()[name = string("op_5638_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_5638_end_mask_0 = const()[name = string("op_5638_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5638_squeeze_mask_0 = const()[name = string("op_5638_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5638_cast_fp16 = slice_by_index(begin = var_5638_begin_0, end = var_5638_end_0, end_mask = var_5638_end_mask_0, squeeze_mask = var_5638_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5638_cast_fp16")];
+            bool var_5639_transpose_x_0 = const()[name = string("op_5639_transpose_x_0"), val = bool(false)];
+            bool var_5639_transpose_y_0 = const()[name = string("op_5639_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5639_cast_fp16 = matmul(transpose_x = var_5639_transpose_x_0, transpose_y = var_5639_transpose_y_0, x = var_5638_cast_fp16, y = state_317_cast_fp16)[name = string("op_5639_cast_fp16")];
+            tensor<int32, [4]> var_5641_begin_0 = const()[name = string("op_5641_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_5641_end_0 = const()[name = string("op_5641_end_0"), val = tensor<int32, [4]>([32, 31, 64, 64])];
+            tensor<bool, [4]> var_5641_end_mask_0 = const()[name = string("op_5641_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5641_squeeze_mask_0 = const()[name = string("op_5641_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5641_cast_fp16 = slice_by_index(begin = var_5641_begin_0, end = var_5641_end_0, end_mask = var_5641_end_mask_0, squeeze_mask = var_5641_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5641_cast_fp16")];
+            bool var_5642_transpose_x_0 = const()[name = string("op_5642_transpose_x_0"), val = bool(false)];
+            bool var_5642_transpose_y_0 = const()[name = string("op_5642_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5642_cast_fp16 = matmul(transpose_x = var_5642_transpose_x_0, transpose_y = var_5642_transpose_y_0, x = var_5641_cast_fp16, y = v_new_317_cast_fp16)[name = string("op_5642_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5643_cast_fp16 = add(x = var_5639_cast_fp16, y = var_5642_cast_fp16)[name = string("op_5643_cast_fp16")];
+            tensor<int32, [4]> var_5645_begin_0 = const()[name = string("op_5645_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_5645_end_0 = const()[name = string("op_5645_end_0"), val = tensor<int32, [4]>([32, 31, 1, 1])];
+            tensor<bool, [4]> var_5645_end_mask_0 = const()[name = string("op_5645_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5645_squeeze_mask_0 = const()[name = string("op_5645_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_5645_cast_fp16 = slice_by_index(begin = var_5645_begin_0, end = var_5645_end_0, end_mask = var_5645_end_mask_0, squeeze_mask = var_5645_squeeze_mask_0, x = chunk_decay_9_cast_fp16)[name = string("op_5645_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_5646_cast_fp16 = mul(x = state_317_cast_fp16, y = var_5645_cast_fp16)[name = string("op_5646_cast_fp16")];
+            tensor<int32, [4]> var_5648_begin_0 = const()[name = string("op_5648_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_5648_end_0 = const()[name = string("op_5648_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_5648_end_mask_0 = const()[name = string("op_5648_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5648_squeeze_mask_0 = const()[name = string("op_5648_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5648_cast_fp16 = slice_by_index(begin = var_5648_begin_0, end = var_5648_end_0, end_mask = var_5648_end_mask_0, squeeze_mask = var_5648_squeeze_mask_0, x = k_dec_9_cast_fp16)[name = string("op_5648_cast_fp16")];
+            bool var_5650_transpose_x_1 = const()[name = string("op_5650_transpose_x_1"), val = bool(true)];
+            bool var_5650_transpose_y_1 = const()[name = string("op_5650_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_5650_cast_fp16 = matmul(transpose_x = var_5650_transpose_x_1, transpose_y = var_5650_transpose_y_1, x = var_5648_cast_fp16, y = v_new_317_cast_fp16)[name = string("op_5650_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_319_cast_fp16 = add(x = var_5646_cast_fp16, y = var_5650_cast_fp16)[name = string("state_319_cast_fp16")];
+            tensor<int32, [4]> var_5653_begin_0 = const()[name = string("op_5653_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_5653_end_0 = const()[name = string("op_5653_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_5653_end_mask_0 = const()[name = string("op_5653_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5653_squeeze_mask_0 = const()[name = string("op_5653_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5653_cast_fp16 = slice_by_index(begin = var_5653_begin_0, end = var_5653_end_0, end_mask = var_5653_end_mask_0, squeeze_mask = var_5653_squeeze_mask_0, x = new_v_9_cast_fp16)[name = string("op_5653_cast_fp16")];
+            tensor<int32, [4]> var_5655_begin_0 = const()[name = string("op_5655_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_5655_end_0 = const()[name = string("op_5655_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_5655_end_mask_0 = const()[name = string("op_5655_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5655_squeeze_mask_0 = const()[name = string("op_5655_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5655_cast_fp16 = slice_by_index(begin = var_5655_begin_0, end = var_5655_end_0, end_mask = var_5655_end_mask_0, squeeze_mask = var_5655_squeeze_mask_0, x = k_cumdecay_9_cast_fp16)[name = string("op_5655_cast_fp16")];
+            bool var_5656_transpose_x_0 = const()[name = string("op_5656_transpose_x_0"), val = bool(false)];
+            bool var_5656_transpose_y_0 = const()[name = string("op_5656_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5656_cast_fp16 = matmul(transpose_x = var_5656_transpose_x_0, transpose_y = var_5656_transpose_y_0, x = var_5655_cast_fp16, y = state_319_cast_fp16)[name = string("op_5656_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_319_cast_fp16 = sub(x = var_5653_cast_fp16, y = var_5656_cast_fp16)[name = string("v_new_319_cast_fp16")];
+            tensor<int32, [4]> var_5659_begin_0 = const()[name = string("op_5659_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_5659_end_0 = const()[name = string("op_5659_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_5659_end_mask_0 = const()[name = string("op_5659_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5659_squeeze_mask_0 = const()[name = string("op_5659_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_5659_cast_fp16 = slice_by_index(begin = var_5659_begin_0, end = var_5659_end_0, end_mask = var_5659_end_mask_0, squeeze_mask = var_5659_squeeze_mask_0, x = q_dec_9_cast_fp16)[name = string("op_5659_cast_fp16")];
+            bool var_5660_transpose_x_0 = const()[name = string("op_5660_transpose_x_0"), val = bool(false)];
+            bool var_5660_transpose_y_0 = const()[name = string("op_5660_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5660_cast_fp16 = matmul(transpose_x = var_5660_transpose_x_0, transpose_y = var_5660_transpose_y_0, x = var_5659_cast_fp16, y = state_319_cast_fp16)[name = string("op_5660_cast_fp16")];
+            tensor<int32, [4]> var_5662_begin_0 = const()[name = string("op_5662_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_5662_end_0 = const()[name = string("op_5662_end_0"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<bool, [4]> var_5662_end_mask_0 = const()[name = string("op_5662_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_5662_squeeze_mask_0 = const()[name = string("op_5662_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_5662_cast_fp16 = slice_by_index(begin = var_5662_begin_0, end = var_5662_end_0, end_mask = var_5662_end_mask_0, squeeze_mask = var_5662_squeeze_mask_0, x = attn_intra_9_cast_fp16)[name = string("op_5662_cast_fp16")];
+            bool var_5663_transpose_x_0 = const()[name = string("op_5663_transpose_x_0"), val = bool(false)];
+            bool var_5663_transpose_y_0 = const()[name = string("op_5663_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_5663_cast_fp16 = matmul(transpose_x = var_5663_transpose_x_0, transpose_y = var_5663_transpose_y_0, x = var_5662_cast_fp16, y = v_new_319_cast_fp16)[name = string("op_5663_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_5664_cast_fp16 = add(x = var_5660_cast_fp16, y = var_5663_cast_fp16)[name = string("op_5664_cast_fp16")];
+            int32 var_5666_axis_0 = const()[name = string("op_5666_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 32, 64, 128]> var_5666_cast_fp16 = stack(axis = var_5666_axis_0, values = (var_5013_cast_fp16, var_5034_cast_fp16, var_5055_cast_fp16, var_5076_cast_fp16, var_5097_cast_fp16, var_5118_cast_fp16, var_5139_cast_fp16, var_5160_cast_fp16, var_5181_cast_fp16, var_5202_cast_fp16, var_5223_cast_fp16, var_5244_cast_fp16, var_5265_cast_fp16, var_5286_cast_fp16, var_5307_cast_fp16, var_5328_cast_fp16, var_5349_cast_fp16, var_5370_cast_fp16, var_5391_cast_fp16, var_5412_cast_fp16, var_5433_cast_fp16, var_5454_cast_fp16, var_5475_cast_fp16, var_5496_cast_fp16, var_5517_cast_fp16, var_5538_cast_fp16, var_5559_cast_fp16, var_5580_cast_fp16, var_5601_cast_fp16, var_5622_cast_fp16, var_5643_cast_fp16, var_5664_cast_fp16))[name = string("op_5666_cast_fp16")];
+            tensor<int32, [3]> var_5667 = const()[name = string("op_5667"), val = tensor<int32, [3]>([32, 2048, 128])];
+            tensor<fp16, [32, 2048, 128]> var_5668_cast_fp16 = reshape(shape = var_5667, x = var_5666_cast_fp16)[name = string("op_5668_cast_fp16")];
+            tensor<int32, [3]> var_5669 = const()[name = string("op_5669"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_79_begin_0 = const()[name = string("input_79_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_79_end_0 = const()[name = string("input_79_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> input_79_end_mask_0 = const()[name = string("input_79_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_79_squeeze_mask_0 = const()[name = string("input_79_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> input_79_cast_fp16 = slice_by_index(begin = input_79_begin_0, end = input_79_end_0, end_mask = input_79_end_mask_0, squeeze_mask = input_79_squeeze_mask_0, x = z_9_cast_fp16)[name = string("input_79_cast_fp16")];
+            fp16 var_4649_promoted_1_to_fp16 = const()[name = string("op_4649_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [2048, 32, 128]> x_143_cast_fp16 = transpose(perm = var_5669, x = var_5668_cast_fp16)[name = string("transpose_12")];
+            tensor<fp16, [2048, 32, 128]> var_5673_cast_fp16 = pow(x = x_143_cast_fp16, y = var_4649_promoted_1_to_fp16)[name = string("op_5673_cast_fp16")];
+            tensor<int32, [1]> var_5675_axes_0 = const()[name = string("op_5675_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_5675_keep_dims_0 = const()[name = string("op_5675_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [2048, 32, 1]> var_5675_cast_fp16 = reduce_mean(axes = var_5675_axes_0, keep_dims = var_5675_keep_dims_0, x = var_5673_cast_fp16)[name = string("op_5675_cast_fp16")];
+            fp16 var_5676_to_fp16 = const()[name = string("op_5676_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [2048, 32, 1]> var_5677_cast_fp16 = add(x = var_5675_cast_fp16, y = var_5676_to_fp16)[name = string("op_5677_cast_fp16")];
+            fp32 var_5678_epsilon_0 = const()[name = string("op_5678_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [2048, 32, 1]> var_5678_cast_fp16 = rsqrt(epsilon = var_5678_epsilon_0, x = var_5677_cast_fp16)[name = string("op_5678_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> x_145_cast_fp16 = mul(x = x_143_cast_fp16, y = var_5678_cast_fp16)[name = string("x_145_cast_fp16")];
+            tensor<fp16, [128]> layers_5_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_5_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195988352)))];
+            tensor<fp16, [2048, 32, 128]> var_5680_cast_fp16 = mul(x = layers_5_linear_attn_norm_weight_to_fp16, y = x_145_cast_fp16)[name = string("op_5680_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> var_5681_cast_fp16 = silu(x = input_79_cast_fp16)[name = string("op_5681_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> core_9_cast_fp16 = mul(x = var_5680_cast_fp16, y = var_5681_cast_fp16)[name = string("core_9_cast_fp16")];
+            tensor<int32, [3]> var_5683 = const()[name = string("op_5683"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> input_81_cast_fp16 = reshape(shape = var_5683, x = core_9_cast_fp16)[name = string("input_81_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_5_linear_attn_out_proj_weight_to_fp16 = const()[name = string("layers_5_linear_attn_out_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1195988672)))];
+            tensor<fp16, [1, 2048, 2560]> linear_43_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_5_linear_attn_out_proj_weight_to_fp16, x = input_81_cast_fp16)[name = string("linear_43_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_147_cast_fp16 = add(x = x_125_cast_fp16, y = linear_43_cast_fp16)[name = string("x_147_cast_fp16")];
+            fp16 var_4649_promoted_2_to_fp16 = const()[name = string("op_4649_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_5689_cast_fp16 = pow(x = x_147_cast_fp16, y = var_4649_promoted_2_to_fp16)[name = string("op_5689_cast_fp16")];
+            tensor<int32, [1]> var_5691_axes_0 = const()[name = string("op_5691_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_5691_keep_dims_0 = const()[name = string("op_5691_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_5691_cast_fp16 = reduce_mean(axes = var_5691_axes_0, keep_dims = var_5691_keep_dims_0, x = var_5689_cast_fp16)[name = string("op_5691_cast_fp16")];
+            fp16 var_5692_to_fp16 = const()[name = string("op_5692_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_5693_cast_fp16 = add(x = var_5691_cast_fp16, y = var_5692_to_fp16)[name = string("op_5693_cast_fp16")];
+            fp32 var_5694_epsilon_0 = const()[name = string("op_5694_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_5694_cast_fp16 = rsqrt(epsilon = var_5694_epsilon_0, x = var_5693_cast_fp16)[name = string("op_5694_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_149_cast_fp16 = mul(x = x_147_cast_fp16, y = var_5694_cast_fp16)[name = string("x_149_cast_fp16")];
+            tensor<fp16, [2560]> var_5697_to_fp16 = const()[name = string("op_5697_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1216960256)))];
+            tensor<fp16, [1, 2048, 2560]> input_83_cast_fp16 = mul(x = x_149_cast_fp16, y = var_5697_to_fp16)[name = string("input_83_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_5_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_5_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1216965440)))];
+            tensor<fp16, [1, 2048, 9216]> linear_44_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_5_mlp_gate_proj_weight_to_fp16, x = input_83_cast_fp16)[name = string("linear_44_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_5704_cast_fp16 = silu(x = linear_44_cast_fp16)[name = string("op_5704_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_5_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_5_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1264151424)))];
+            tensor<fp16, [1, 2048, 9216]> linear_45_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_5_mlp_up_proj_weight_to_fp16, x = input_83_cast_fp16)[name = string("linear_45_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_87_cast_fp16 = mul(x = var_5704_cast_fp16, y = linear_45_cast_fp16)[name = string("input_87_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_5_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_5_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1311337408)))];
+            tensor<fp16, [1, 2048, 2560]> linear_46_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_5_mlp_down_proj_weight_to_fp16, x = input_87_cast_fp16)[name = string("linear_46_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_151_cast_fp16 = add(x = x_147_cast_fp16, y = linear_46_cast_fp16)[name = string("x_151_cast_fp16")];
+            int32 var_5724 = const()[name = string("op_5724"), val = int32(-2)];
+            int32 var_5753 = const()[name = string("op_5753"), val = int32(-1)];
+            fp16 var_5752_promoted_to_fp16 = const()[name = string("op_5752_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_5762_cast_fp16 = pow(x = x_151_cast_fp16, y = var_5752_promoted_to_fp16)[name = string("op_5762_cast_fp16")];
+            tensor<int32, [1]> var_5764_axes_0 = const()[name = string("op_5764_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_5764_keep_dims_0 = const()[name = string("op_5764_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_5764_cast_fp16 = reduce_mean(axes = var_5764_axes_0, keep_dims = var_5764_keep_dims_0, x = var_5762_cast_fp16)[name = string("op_5764_cast_fp16")];
+            fp16 var_5765_to_fp16 = const()[name = string("op_5765_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_5766_cast_fp16 = add(x = var_5764_cast_fp16, y = var_5765_to_fp16)[name = string("op_5766_cast_fp16")];
+            fp32 var_5767_epsilon_0 = const()[name = string("op_5767_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_5767_cast_fp16 = rsqrt(epsilon = var_5767_epsilon_0, x = var_5766_cast_fp16)[name = string("op_5767_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_153_cast_fp16 = mul(x = x_151_cast_fp16, y = var_5767_cast_fp16)[name = string("x_153_cast_fp16")];
+            tensor<fp16, [2560]> var_5770_to_fp16 = const()[name = string("op_5770_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1358523392)))];
+            tensor<fp16, [1, 2048, 2560]> x_155_cast_fp16 = mul(x = x_153_cast_fp16, y = var_5770_to_fp16)[name = string("x_155_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_6_linear_attn_in_proj_qkv_weight_to_fp16 = const()[name = string("layers_6_linear_attn_in_proj_qkv_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1358528576)))];
+            tensor<fp16, [1, 2048, 8192]> linear_47_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_qkv_weight_to_fp16, x = x_155_cast_fp16)[name = string("linear_47_cast_fp16")];
+            tensor<int32, [3]> input_89_perm_0 = const()[name = string("input_89_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [6]> input_91_pad_0 = const()[name = string("input_91_pad_0"), val = tensor<int32, [6]>([0, 0, 0, 0, 3, 0])];
+            string input_91_mode_0 = const()[name = string("input_91_mode_0"), val = string("constant")];
+            fp16 const_66_to_fp16 = const()[name = string("const_66_to_fp16"), val = fp16(0x0p+0)];
+            tensor<fp16, [1, 8192, 2048]> input_89_cast_fp16 = transpose(perm = input_89_perm_0, x = linear_47_cast_fp16)[name = string("transpose_11")];
+            tensor<fp16, [1, 8192, 2051]> input_91_cast_fp16 = pad(constant_val = const_66_to_fp16, mode = input_91_mode_0, pad = input_91_pad_0, x = input_89_cast_fp16)[name = string("input_91_cast_fp16")];
+            string input_93_pad_type_0 = const()[name = string("input_93_pad_type_0"), val = string("valid")];
+            int32 input_93_groups_0 = const()[name = string("input_93_groups_0"), val = int32(8192)];
+            tensor<int32, [1]> input_93_strides_0 = const()[name = string("input_93_strides_0"), val = tensor<int32, [1]>([1])];
+            tensor<int32, [2]> input_93_pad_0 = const()[name = string("input_93_pad_0"), val = tensor<int32, [2]>([0, 0])];
+            tensor<int32, [1]> input_93_dilations_0 = const()[name = string("input_93_dilations_0"), val = tensor<int32, [1]>([1])];
+            tensor<fp16, [8192, 1, 4]> layers_6_linear_attn_conv1d_weight_to_fp16 = const()[name = string("layers_6_linear_attn_conv1d_weight_to_fp16"), val = tensor<fp16, [8192, 1, 4]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1400471680)))];
+            tensor<fp16, [1, 8192, 2048]> input_93_cast_fp16 = conv(dilations = input_93_dilations_0, groups = input_93_groups_0, pad = input_93_pad_0, pad_type = input_93_pad_type_0, strides = input_93_strides_0, weight = layers_6_linear_attn_conv1d_weight_to_fp16, x = input_91_cast_fp16)[name = string("input_93_cast_fp16")];
+            tensor<fp16, [1, 8192, 2048]> var_5805_cast_fp16 = silu(x = input_93_cast_fp16)[name = string("op_5805_cast_fp16")];
+            tensor<int32, [3]> qkv_perm_0 = const()[name = string("qkv_perm_0"), val = tensor<int32, [3]>([0, 2, 1])];
+            tensor<int32, [3]> var_5807_begin_0 = const()[name = string("op_5807_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_5807_end_0 = const()[name = string("op_5807_end_0"), val = tensor<int32, [3]>([1, 2048, 2048])];
+            tensor<bool, [3]> var_5807_end_mask_0 = const()[name = string("op_5807_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 8192]> qkv_cast_fp16 = transpose(perm = qkv_perm_0, x = var_5805_cast_fp16)[name = string("transpose_10")];
+            tensor<fp16, [1, 2048, 2048]> var_5807_cast_fp16 = slice_by_index(begin = var_5807_begin_0, end = var_5807_end_0, end_mask = var_5807_end_mask_0, x = qkv_cast_fp16)[name = string("op_5807_cast_fp16")];
+            tensor<int32, [4]> var_5808 = const()[name = string("op_5808"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> q_55_cast_fp16 = reshape(shape = var_5808, x = var_5807_cast_fp16)[name = string("q_55_cast_fp16")];
+            tensor<int32, [3]> var_5810_begin_0 = const()[name = string("op_5810_begin_0"), val = tensor<int32, [3]>([0, 0, 2048])];
+            tensor<int32, [3]> var_5810_end_0 = const()[name = string("op_5810_end_0"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<bool, [3]> var_5810_end_mask_0 = const()[name = string("op_5810_end_mask_0"), val = tensor<bool, [3]>([true, true, false])];
+            tensor<fp16, [1, 2048, 2048]> var_5810_cast_fp16 = slice_by_index(begin = var_5810_begin_0, end = var_5810_end_0, end_mask = var_5810_end_mask_0, x = qkv_cast_fp16)[name = string("op_5810_cast_fp16")];
+            tensor<int32, [4]> var_5811 = const()[name = string("op_5811"), val = tensor<int32, [4]>([1, 2048, 16, 128])];
+            tensor<fp16, [1, 2048, 16, 128]> k_47_cast_fp16 = reshape(shape = var_5811, x = var_5810_cast_fp16)[name = string("k_47_cast_fp16")];
+            tensor<int32, [3]> var_5813_begin_0 = const()[name = string("op_5813_begin_0"), val = tensor<int32, [3]>([0, 0, 4096])];
+            tensor<int32, [3]> var_5813_end_0 = const()[name = string("op_5813_end_0"), val = tensor<int32, [3]>([1, 2048, 8192])];
+            tensor<bool, [3]> var_5813_end_mask_0 = const()[name = string("op_5813_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [1, 2048, 4096]> var_5813_cast_fp16 = slice_by_index(begin = var_5813_begin_0, end = var_5813_end_0, end_mask = var_5813_end_mask_0, x = qkv_cast_fp16)[name = string("op_5813_cast_fp16")];
+            tensor<int32, [4]> var_5814 = const()[name = string("op_5814"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> v_25_cast_fp16 = reshape(shape = var_5814, x = var_5813_cast_fp16)[name = string("v_25_cast_fp16")];
+            tensor<fp16, [4096, 2560]> layers_6_linear_attn_in_proj_z_weight_to_fp16 = const()[name = string("layers_6_linear_attn_in_proj_z_weight_to_fp16"), val = tensor<fp16, [4096, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1400537280)))];
+            tensor<fp16, [1, 2048, 4096]> linear_48_cast_fp16 = linear(bias = linear_1_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_z_weight_to_fp16, x = x_155_cast_fp16)[name = string("linear_48_cast_fp16")];
+            tensor<int32, [4]> var_5818 = const()[name = string("op_5818"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> z_cast_fp16 = reshape(shape = var_5818, x = linear_48_cast_fp16)[name = string("z_cast_fp16")];
+            tensor<fp16, [32, 2560]> layers_6_linear_attn_in_proj_b_weight_to_fp16 = const()[name = string("layers_6_linear_attn_in_proj_b_weight_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421508864)))];
+            tensor<fp16, [32]> linear_49_bias_0_to_fp16 = const()[name = string("linear_49_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421672768)))];
+            tensor<fp16, [1, 2048, 32]> linear_49_cast_fp16 = linear(bias = linear_49_bias_0_to_fp16, weight = layers_6_linear_attn_in_proj_b_weight_to_fp16, x = x_155_cast_fp16)[name = string("linear_49_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> beta_21_cast_fp16 = sigmoid(x = linear_49_cast_fp16)[name = string("beta_21_cast_fp16")];
+            tensor<fp16, [32, 2560]> var_5827_weight_0_to_fp16 = const()[name = string("op_5827_weight_0_to_fp16"), val = tensor<fp16, [32, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421672896)))];
+            tensor<fp16, [32]> var_5827_bias_0_to_fp16 = const()[name = string("op_5827_bias_0_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421836800)))];
+            tensor<fp16, [1, 2048, 32]> var_5827_cast_fp16 = linear(bias = var_5827_bias_0_to_fp16, weight = var_5827_weight_0_to_fp16, x = x_155_cast_fp16)[name = string("op_5827_cast_fp16")];
+            tensor<fp16, [1, 2048, 32]> var_5828_cast_fp16 = softplus(x = var_5827_cast_fp16)[name = string("op_5828_cast_fp16")];
+            tensor<fp16, [32]> var_5824_to_fp16 = const()[name = string("op_5824_to_fp16"), val = tensor<fp16, [32]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421836928)))];
+            tensor<fp16, [1, 2048, 32]> g_21_cast_fp16 = mul(x = var_5824_to_fp16, y = var_5828_cast_fp16)[name = string("g_21_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_5830_cast_fp16 = mul(x = q_55_cast_fp16, y = q_55_cast_fp16)[name = string("op_5830_cast_fp16")];
+            tensor<int32, [1]> var_5832_axes_0 = const()[name = string("op_5832_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_5832_keep_dims_0 = const()[name = string("op_5832_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_5832_cast_fp16 = reduce_sum(axes = var_5832_axes_0, keep_dims = var_5832_keep_dims_0, x = var_5830_cast_fp16)[name = string("op_5832_cast_fp16")];
+            fp16 var_5833_to_fp16 = const()[name = string("op_5833_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_5834_cast_fp16 = add(x = var_5832_cast_fp16, y = var_5833_to_fp16)[name = string("op_5834_cast_fp16")];
+            fp32 var_5835_epsilon_0 = const()[name = string("op_5835_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_5835_cast_fp16 = rsqrt(epsilon = var_5835_epsilon_0, x = var_5834_cast_fp16)[name = string("op_5835_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> q_57_cast_fp16 = mul(x = q_55_cast_fp16, y = var_5835_cast_fp16)[name = string("q_57_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> var_5837_cast_fp16 = mul(x = k_47_cast_fp16, y = k_47_cast_fp16)[name = string("op_5837_cast_fp16")];
+            tensor<int32, [1]> var_5839_axes_0 = const()[name = string("op_5839_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_5839_keep_dims_0 = const()[name = string("op_5839_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_5839_cast_fp16 = reduce_sum(axes = var_5839_axes_0, keep_dims = var_5839_keep_dims_0, x = var_5837_cast_fp16)[name = string("op_5839_cast_fp16")];
+            fp16 var_5840_to_fp16 = const()[name = string("op_5840_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_5841_cast_fp16 = add(x = var_5839_cast_fp16, y = var_5840_to_fp16)[name = string("op_5841_cast_fp16")];
+            fp32 var_5842_epsilon_0 = const()[name = string("op_5842_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_5842_cast_fp16 = rsqrt(epsilon = var_5842_epsilon_0, x = var_5841_cast_fp16)[name = string("op_5842_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 128]> k_49_cast_fp16 = mul(x = k_47_cast_fp16, y = var_5842_cast_fp16)[name = string("k_49_cast_fp16")];
+            tensor<int32, [1]> var_5847_axes_0 = const()[name = string("op_5847_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_5847_cast_fp16 = expand_dims(axes = var_5847_axes_0, x = q_57_cast_fp16)[name = string("op_5847_cast_fp16")];
+            tensor<int32, [5]> var_5849_reps_0 = const()[name = string("op_5849_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_5849_cast_fp16 = tile(reps = var_5849_reps_0, x = var_5847_cast_fp16)[name = string("op_5849_cast_fp16")];
+            tensor<int32, [4]> var_5850 = const()[name = string("op_5850"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> q_59_cast_fp16 = reshape(shape = var_5850, x = var_5849_cast_fp16)[name = string("q_59_cast_fp16")];
+            tensor<int32, [1]> var_5855_axes_0 = const()[name = string("op_5855_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [1, 2048, 16, 1, 128]> var_5855_cast_fp16 = expand_dims(axes = var_5855_axes_0, x = k_49_cast_fp16)[name = string("op_5855_cast_fp16")];
+            tensor<int32, [5]> var_5857_reps_0 = const()[name = string("op_5857_reps_0"), val = tensor<int32, [5]>([1, 1, 1, 2, 1])];
+            tensor<fp16, [1, 2048, 16, 2, 128]> var_5857_cast_fp16 = tile(reps = var_5857_reps_0, x = var_5855_cast_fp16)[name = string("op_5857_cast_fp16")];
+            tensor<int32, [4]> var_5858 = const()[name = string("op_5858"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<fp16, [1, 2048, 32, 128]> k_51_cast_fp16 = reshape(shape = var_5858, x = var_5857_cast_fp16)[name = string("k_51_cast_fp16")];
+            fp16 var_5860_to_fp16 = const()[name = string("op_5860_to_fp16"), val = fp16(0x1.6ap-4)];
+            tensor<fp16, [1, 2048, 32, 128]> q_61_cast_fp16 = mul(x = q_59_cast_fp16, y = var_5860_to_fp16)[name = string("q_61_cast_fp16")];
+            tensor<int32, [4]> var_5862_begin_0 = const()[name = string("op_5862_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5862_end_0 = const()[name = string("op_5862_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_5862_end_mask_0 = const()[name = string("op_5862_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_5862_squeeze_mask_0 = const()[name = string("op_5862_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_5862_cast_fp16 = slice_by_index(begin = var_5862_begin_0, end = var_5862_end_0, end_mask = var_5862_end_mask_0, squeeze_mask = var_5862_squeeze_mask_0, x = q_61_cast_fp16)[name = string("op_5862_cast_fp16")];
+            tensor<int32, [3]> var_5863 = const()[name = string("op_5863"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_5865 = const()[name = string("op_5865"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_5864_cast_fp16 = transpose(perm = var_5863, x = var_5862_cast_fp16)[name = string("transpose_9")];
+            tensor<fp16, [32, 32, 64, 128]> q_63_cast_fp16 = reshape(shape = var_5865, x = var_5864_cast_fp16)[name = string("q_63_cast_fp16")];
+            tensor<int32, [4]> var_5867_begin_0 = const()[name = string("op_5867_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5867_end_0 = const()[name = string("op_5867_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_5867_end_mask_0 = const()[name = string("op_5867_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_5867_squeeze_mask_0 = const()[name = string("op_5867_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_5867_cast_fp16 = slice_by_index(begin = var_5867_begin_0, end = var_5867_end_0, end_mask = var_5867_end_mask_0, squeeze_mask = var_5867_squeeze_mask_0, x = k_51_cast_fp16)[name = string("op_5867_cast_fp16")];
+            tensor<int32, [3]> var_5868 = const()[name = string("op_5868"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_5870 = const()[name = string("op_5870"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_5869_cast_fp16 = transpose(perm = var_5868, x = var_5867_cast_fp16)[name = string("transpose_8")];
+            tensor<fp16, [32, 32, 64, 128]> k_53_cast_fp16 = reshape(shape = var_5870, x = var_5869_cast_fp16)[name = string("k_53_cast_fp16")];
+            tensor<int32, [4]> var_5872_begin_0 = const()[name = string("op_5872_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_5872_end_0 = const()[name = string("op_5872_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> var_5872_end_mask_0 = const()[name = string("op_5872_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> var_5872_squeeze_mask_0 = const()[name = string("op_5872_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> var_5872_cast_fp16 = slice_by_index(begin = var_5872_begin_0, end = var_5872_end_0, end_mask = var_5872_end_mask_0, squeeze_mask = var_5872_squeeze_mask_0, x = v_25_cast_fp16)[name = string("op_5872_cast_fp16")];
+            tensor<int32, [3]> var_5873 = const()[name = string("op_5873"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> var_5875 = const()[name = string("op_5875"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<fp16, [32, 2048, 128]> var_5874_cast_fp16 = transpose(perm = var_5873, x = var_5872_cast_fp16)[name = string("transpose_7")];
+            tensor<fp16, [32, 32, 64, 128]> v_27_cast_fp16 = reshape(shape = var_5875, x = var_5874_cast_fp16)[name = string("v_27_cast_fp16")];
+            tensor<int32, [3]> var_5877_begin_0 = const()[name = string("op_5877_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_5877_end_0 = const()[name = string("op_5877_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_5877_end_mask_0 = const()[name = string("op_5877_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_5877_squeeze_mask_0 = const()[name = string("op_5877_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_5877_cast_fp16 = slice_by_index(begin = var_5877_begin_0, end = var_5877_end_0, end_mask = var_5877_end_mask_0, squeeze_mask = var_5877_squeeze_mask_0, x = beta_21_cast_fp16)[name = string("op_5877_cast_fp16")];
+            tensor<int32, [2]> var_5878_perm_0 = const()[name = string("op_5878_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [4]> var_5879 = const()[name = string("op_5879"), val = tensor<int32, [4]>([32, 32, 64, 1])];
+            tensor<fp16, [32, 2048]> var_5878_cast_fp16 = transpose(perm = var_5878_perm_0, x = var_5877_cast_fp16)[name = string("transpose_6")];
+            tensor<fp16, [32, 32, 64, 1]> beta_cast_fp16 = reshape(shape = var_5879, x = var_5878_cast_fp16)[name = string("beta_cast_fp16")];
+            tensor<int32, [3]> var_5881_begin_0 = const()[name = string("op_5881_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_5881_end_0 = const()[name = string("op_5881_end_0"), val = tensor<int32, [3]>([1, 2048, 32])];
+            tensor<bool, [3]> var_5881_end_mask_0 = const()[name = string("op_5881_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_5881_squeeze_mask_0 = const()[name = string("op_5881_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 32]> var_5881_cast_fp16 = slice_by_index(begin = var_5881_begin_0, end = var_5881_end_0, end_mask = var_5881_end_mask_0, squeeze_mask = var_5881_squeeze_mask_0, x = g_21_cast_fp16)[name = string("op_5881_cast_fp16")];
+            tensor<int32, [2]> var_5882_perm_0 = const()[name = string("op_5882_perm_0"), val = tensor<int32, [2]>([1, 0])];
+            tensor<int32, [3]> var_5883 = const()[name = string("op_5883"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<fp16, [32, 2048]> var_5882_cast_fp16 = transpose(perm = var_5882_perm_0, x = var_5881_cast_fp16)[name = string("transpose_5")];
+            tensor<fp16, [32, 32, 64]> g_cast_fp16 = reshape(shape = var_5883, x = var_5882_cast_fp16)[name = string("g_cast_fp16")];
+            tensor<int32, [1]> g_col_axes_0 = const()[name = string("g_col_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64, 1]> g_col_cast_fp16 = expand_dims(axes = g_col_axes_0, x = g_cast_fp16)[name = string("g_col_cast_fp16")];
+            bool var_5886_transpose_x_0 = const()[name = string("op_5886_transpose_x_0"), val = bool(false)];
+            bool var_5886_transpose_y_0 = const()[name = string("op_5886_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_5886_cast_fp16 = matmul(transpose_x = var_5886_transpose_x_0, transpose_y = var_5886_transpose_y_0, x = layers_0_linear_attn_lower_incl_to_fp16, y = g_col_cast_fp16)[name = string("op_5886_cast_fp16")];
+            tensor<int32, [1]> cum_axes_0 = const()[name = string("cum_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> cum_cast_fp16 = squeeze(axes = cum_axes_0, x = var_5886_cast_fp16)[name = string("cum_cast_fp16")];
+            bool var_5888_transpose_x_0 = const()[name = string("op_5888_transpose_x_0"), val = bool(false)];
+            bool var_5888_transpose_y_0 = const()[name = string("op_5888_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 1]> var_5888_cast_fp16 = matmul(transpose_x = var_5888_transpose_x_0, transpose_y = var_5888_transpose_y_0, x = layers_0_linear_attn_suffix_sum_t_to_fp16, y = g_col_cast_fp16)[name = string("op_5888_cast_fp16")];
+            tensor<int32, [1]> to_end_axes_0 = const()[name = string("to_end_axes_0"), val = tensor<int32, [1]>([-1])];
+            tensor<fp16, [32, 32, 64]> to_end_cast_fp16 = squeeze(axes = to_end_axes_0, x = var_5888_cast_fp16)[name = string("to_end_cast_fp16")];
+            bool var_5890_transpose_x_0 = const()[name = string("op_5890_transpose_x_0"), val = bool(false)];
+            bool var_5890_transpose_y_0 = const()[name = string("op_5890_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 4096, 1]> var_5890_cast_fp16 = matmul(transpose_x = var_5890_transpose_x_0, transpose_y = var_5890_transpose_y_0, x = layers_0_linear_attn_pair_sum_t_to_fp16, y = g_col_cast_fp16)[name = string("op_5890_cast_fp16")];
+            tensor<int32, [4]> var_5891 = const()[name = string("op_5891"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> pair_71_cast_fp16 = reshape(shape = var_5891, x = var_5890_cast_fp16)[name = string("pair_71_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_5893_cast_fp16 = exp(x = pair_71_cast_fp16)[name = string("op_5893_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> pair_decay_cast_fp16 = mul(x = var_5893_cast_fp16, y = layers_0_linear_attn_lower_incl_to_fp16)[name = string("pair_decay_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_beta_cast_fp16 = mul(x = k_53_cast_fp16, y = beta_cast_fp16)[name = string("k_beta_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> v_beta_cast_fp16 = mul(x = v_27_cast_fp16, y = beta_cast_fp16)[name = string("v_beta_cast_fp16")];
+            bool var_5898_transpose_x_1 = const()[name = string("op_5898_transpose_x_1"), val = bool(false)];
+            bool var_5898_transpose_y_1 = const()[name = string("op_5898_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_5898_cast_fp16 = matmul(transpose_x = var_5898_transpose_x_1, transpose_y = var_5898_transpose_y_1, x = k_beta_cast_fp16, y = k_53_cast_fp16)[name = string("op_5898_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> kkt_cast_fp16 = mul(x = var_5898_cast_fp16, y = pair_decay_cast_fp16)[name = string("kkt_cast_fp16")];
+            bool var_5901_transpose_x_1 = const()[name = string("op_5901_transpose_x_1"), val = bool(false)];
+            bool var_5901_transpose_y_1 = const()[name = string("op_5901_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [32, 32, 64, 64]> var_5901_cast_fp16 = matmul(transpose_x = var_5901_transpose_x_1, transpose_y = var_5901_transpose_y_1, x = q_63_cast_fp16, y = k_53_cast_fp16)[name = string("op_5901_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> attn_intra_cast_fp16 = mul(x = var_5901_cast_fp16, y = pair_decay_cast_fp16)[name = string("attn_intra_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> var_5903_cast_fp16 = mul(x = kkt_cast_fp16, y = layers_0_linear_attn_strict_lower_to_fp16)[name = string("op_5903_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 64]> t_21_cast_fp16 = add(x = layers_0_linear_attn_eye_to_fp16, y = var_5903_cast_fp16)[name = string("t_21_cast_fp16")];
+            tensor<int32, [3]> var_5907 = const()[name = string("op_5907"), val = tensor<int32, [3]>([-1, 64, 64])];
+            tensor<fp16, [1024, 64, 64]> t_cast_fp16 = reshape(shape = var_5907, x = t_21_cast_fp16)[name = string("t_cast_fp16")];
+            tensor<int32, [5]> var_5912 = const()[name = string("op_5912"), val = tensor<int32, [5]>([1024, 32, 2, 32, 2])];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_5913_cast_fp16 = reshape(shape = var_5912, x = t_cast_fp16)[name = string("op_5913_cast_fp16")];
+            tensor<fp16, [1024, 32, 2, 32, 2]> var_5918_cast_fp16 = mul(x = var_5913_cast_fp16, y = var_230_to_fp16)[name = string("op_5918_cast_fp16")];
+            tensor<int32, [1]> blocks_61_axes_0 = const()[name = string("blocks_61_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_61_keep_dims_0 = const()[name = string("blocks_61_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> blocks_61_cast_fp16 = reduce_sum(axes = blocks_61_axes_0, keep_dims = blocks_61_keep_dims_0, x = var_5918_cast_fp16)[name = string("blocks_61_cast_fp16")];
+            tensor<int32, [4]> var_5921_begin_0 = const()[name = string("op_5921_begin_0"), val = tensor<int32, [4]>([0, 0, 1, 0])];
+            tensor<int32, [4]> var_5921_end_0 = const()[name = string("op_5921_end_0"), val = tensor<int32, [4]>([1024, 32, 2, 2])];
+            tensor<bool, [4]> var_5921_end_mask_0 = const()[name = string("op_5921_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 32, 1, 2]> var_5921_cast_fp16 = slice_by_index(begin = var_5921_begin_0, end = var_5921_end_0, end_mask = var_5921_end_mask_0, x = blocks_61_cast_fp16)[name = string("op_5921_cast_fp16")];
+            tensor<int32, [4]> x_157_begin_0 = const()[name = string("x_157_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_157_end_0 = const()[name = string("x_157_end_0"), val = tensor<int32, [4]>([1024, 32, 1, 1])];
+            tensor<bool, [4]> x_157_end_mask_0 = const()[name = string("x_157_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 32, 1, 1]> x_157_cast_fp16 = slice_by_index(begin = x_157_begin_0, end = x_157_end_0, end_mask = x_157_end_mask_0, x = var_5921_cast_fp16)[name = string("x_157_cast_fp16")];
+            bool var_5931_transpose_x_0 = const()[name = string("op_5931_transpose_x_0"), val = bool(false)];
+            bool var_5931_transpose_y_0 = const()[name = string("op_5931_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_5931_cast_fp16 = matmul(transpose_x = var_5931_transpose_x_0, transpose_y = var_5931_transpose_y_0, x = x_157_cast_fp16, y = var_238_to_fp16)[name = string("op_5931_cast_fp16")];
+            bool var_5932_transpose_x_0 = const()[name = string("op_5932_transpose_x_0"), val = bool(false)];
+            bool var_5932_transpose_y_0 = const()[name = string("op_5932_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 1]> var_5932_cast_fp16 = matmul(transpose_x = var_5932_transpose_x_0, transpose_y = var_5932_transpose_y_0, x = var_238_to_fp16, y = var_5931_cast_fp16)[name = string("op_5932_cast_fp16")];
+            fp16 const_71_promoted_to_fp16 = const()[name = string("const_71_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 32, 1, 1]> lower_61_cast_fp16 = mul(x = var_5932_cast_fp16, y = const_71_promoted_to_fp16)[name = string("lower_61_cast_fp16")];
+            bool bottom_61_interleave_0 = const()[name = string("bottom_61_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 1, 2]> bottom_61_cast_fp16 = concat(axis = var_5753, interleave = bottom_61_interleave_0, values = (lower_61_cast_fp16, var_238_to_fp16))[name = string("bottom_61_cast_fp16")];
+            bool inv_83_interleave_0 = const()[name = string("inv_83_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 32, 2, 2]> inv_83_cast_fp16 = concat(axis = var_5724, interleave = inv_83_interleave_0, values = (top_1_to_fp16, bottom_61_cast_fp16))[name = string("inv_83_cast_fp16")];
+            tensor<int32, [5]> var_5941 = const()[name = string("op_5941"), val = tensor<int32, [5]>([1024, 16, 4, 16, 4])];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_5942_cast_fp16 = reshape(shape = var_5941, x = t_cast_fp16)[name = string("op_5942_cast_fp16")];
+            tensor<fp16, [1024, 16, 4, 16, 4]> var_5947_cast_fp16 = mul(x = var_5942_cast_fp16, y = var_259_to_fp16)[name = string("op_5947_cast_fp16")];
+            tensor<int32, [1]> blocks_63_axes_0 = const()[name = string("blocks_63_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_63_keep_dims_0 = const()[name = string("blocks_63_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> blocks_63_cast_fp16 = reduce_sum(axes = blocks_63_axes_0, keep_dims = blocks_63_keep_dims_0, x = var_5947_cast_fp16)[name = string("blocks_63_cast_fp16")];
+            tensor<int32, [4]> var_5950_begin_0 = const()[name = string("op_5950_begin_0"), val = tensor<int32, [4]>([0, 0, 2, 0])];
+            tensor<int32, [4]> var_5950_end_0 = const()[name = string("op_5950_end_0"), val = tensor<int32, [4]>([1024, 16, 4, 4])];
+            tensor<bool, [4]> var_5950_end_mask_0 = const()[name = string("op_5950_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 16, 2, 4]> var_5950_cast_fp16 = slice_by_index(begin = var_5950_begin_0, end = var_5950_end_0, end_mask = var_5950_end_mask_0, x = blocks_63_cast_fp16)[name = string("op_5950_cast_fp16")];
+            tensor<int32, [4]> x_159_begin_0 = const()[name = string("x_159_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_159_end_0 = const()[name = string("x_159_end_0"), val = tensor<int32, [4]>([1024, 16, 2, 2])];
+            tensor<bool, [4]> x_159_end_mask_0 = const()[name = string("x_159_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 16, 2, 2]> x_159_cast_fp16 = slice_by_index(begin = x_159_begin_0, end = x_159_end_0, end_mask = x_159_end_mask_0, x = var_5950_cast_fp16)[name = string("x_159_cast_fp16")];
+            tensor<int32, [5]> var_5952 = const()[name = string("op_5952"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<fp16, [1024, 16, 2, 2, 2]> pair_75_cast_fp16 = reshape(shape = var_5952, x = inv_83_cast_fp16)[name = string("pair_75_cast_fp16")];
+            tensor<int32, [5]> var_5954_begin_0 = const()[name = string("op_5954_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_5954_end_0 = const()[name = string("op_5954_end_0"), val = tensor<int32, [5]>([1024, 16, 1, 2, 2])];
+            tensor<bool, [5]> var_5954_end_mask_0 = const()[name = string("op_5954_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_5954_squeeze_mask_0 = const()[name = string("op_5954_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_5954_cast_fp16 = slice_by_index(begin = var_5954_begin_0, end = var_5954_end_0, end_mask = var_5954_end_mask_0, squeeze_mask = var_5954_squeeze_mask_0, x = pair_75_cast_fp16)[name = string("op_5954_cast_fp16")];
+            tensor<int32, [5]> var_5957_begin_0 = const()[name = string("op_5957_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_5957_end_0 = const()[name = string("op_5957_end_0"), val = tensor<int32, [5]>([1024, 16, 2, 2, 2])];
+            tensor<bool, [5]> var_5957_end_mask_0 = const()[name = string("op_5957_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_5957_squeeze_mask_0 = const()[name = string("op_5957_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 16, 2, 2]> var_5957_cast_fp16 = slice_by_index(begin = var_5957_begin_0, end = var_5957_end_0, end_mask = var_5957_end_mask_0, squeeze_mask = var_5957_squeeze_mask_0, x = pair_75_cast_fp16)[name = string("op_5957_cast_fp16")];
+            bool var_5960_transpose_x_0 = const()[name = string("op_5960_transpose_x_0"), val = bool(false)];
+            bool var_5960_transpose_y_0 = const()[name = string("op_5960_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_5960_cast_fp16 = matmul(transpose_x = var_5960_transpose_x_0, transpose_y = var_5960_transpose_y_0, x = x_159_cast_fp16, y = var_5954_cast_fp16)[name = string("op_5960_cast_fp16")];
+            bool var_5961_transpose_x_0 = const()[name = string("op_5961_transpose_x_0"), val = bool(false)];
+            bool var_5961_transpose_y_0 = const()[name = string("op_5961_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 2]> var_5961_cast_fp16 = matmul(transpose_x = var_5961_transpose_x_0, transpose_y = var_5961_transpose_y_0, x = var_5957_cast_fp16, y = var_5960_cast_fp16)[name = string("op_5961_cast_fp16")];
+            fp16 const_72_promoted_to_fp16 = const()[name = string("const_72_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 16, 2, 2]> lower_63_cast_fp16 = mul(x = var_5961_cast_fp16, y = const_72_promoted_to_fp16)[name = string("lower_63_cast_fp16")];
+            bool top_63_interleave_0 = const()[name = string("top_63_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> top_63_cast_fp16 = concat(axis = var_5753, interleave = top_63_interleave_0, values = (var_5954_cast_fp16, var_276_to_fp16))[name = string("top_63_cast_fp16")];
+            bool bottom_63_interleave_0 = const()[name = string("bottom_63_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 2, 4]> bottom_63_cast_fp16 = concat(axis = var_5753, interleave = bottom_63_interleave_0, values = (lower_63_cast_fp16, var_5957_cast_fp16))[name = string("bottom_63_cast_fp16")];
+            bool inv_85_interleave_0 = const()[name = string("inv_85_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 16, 4, 4]> inv_85_cast_fp16 = concat(axis = var_5724, interleave = inv_85_interleave_0, values = (top_63_cast_fp16, bottom_63_cast_fp16))[name = string("inv_85_cast_fp16")];
+            tensor<int32, [5]> var_5970 = const()[name = string("op_5970"), val = tensor<int32, [5]>([1024, 8, 8, 8, 8])];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_5971_cast_fp16 = reshape(shape = var_5970, x = t_cast_fp16)[name = string("op_5971_cast_fp16")];
+            tensor<fp16, [8, 1, 8, 1]> var_5975_to_fp16 = const()[name = string("op_5975_to_fp16"), val = tensor<fp16, [8, 1, 8, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421837056)))];
+            tensor<fp16, [1024, 8, 8, 8, 8]> var_5976_cast_fp16 = mul(x = var_5971_cast_fp16, y = var_5975_to_fp16)[name = string("op_5976_cast_fp16")];
+            tensor<int32, [1]> blocks_65_axes_0 = const()[name = string("blocks_65_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_65_keep_dims_0 = const()[name = string("blocks_65_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> blocks_65_cast_fp16 = reduce_sum(axes = blocks_65_axes_0, keep_dims = blocks_65_keep_dims_0, x = var_5976_cast_fp16)[name = string("blocks_65_cast_fp16")];
+            tensor<int32, [4]> var_5979_begin_0 = const()[name = string("op_5979_begin_0"), val = tensor<int32, [4]>([0, 0, 4, 0])];
+            tensor<int32, [4]> var_5979_end_0 = const()[name = string("op_5979_end_0"), val = tensor<int32, [4]>([1024, 8, 8, 8])];
+            tensor<bool, [4]> var_5979_end_mask_0 = const()[name = string("op_5979_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 8, 4, 8]> var_5979_cast_fp16 = slice_by_index(begin = var_5979_begin_0, end = var_5979_end_0, end_mask = var_5979_end_mask_0, x = blocks_65_cast_fp16)[name = string("op_5979_cast_fp16")];
+            tensor<int32, [4]> x_161_begin_0 = const()[name = string("x_161_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_161_end_0 = const()[name = string("x_161_end_0"), val = tensor<int32, [4]>([1024, 8, 4, 4])];
+            tensor<bool, [4]> x_161_end_mask_0 = const()[name = string("x_161_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 8, 4, 4]> x_161_cast_fp16 = slice_by_index(begin = x_161_begin_0, end = x_161_end_0, end_mask = x_161_end_mask_0, x = var_5979_cast_fp16)[name = string("x_161_cast_fp16")];
+            tensor<int32, [5]> var_5981 = const()[name = string("op_5981"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<fp16, [1024, 8, 2, 4, 4]> pair_77_cast_fp16 = reshape(shape = var_5981, x = inv_85_cast_fp16)[name = string("pair_77_cast_fp16")];
+            tensor<int32, [5]> var_5983_begin_0 = const()[name = string("op_5983_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_5983_end_0 = const()[name = string("op_5983_end_0"), val = tensor<int32, [5]>([1024, 8, 1, 4, 4])];
+            tensor<bool, [5]> var_5983_end_mask_0 = const()[name = string("op_5983_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_5983_squeeze_mask_0 = const()[name = string("op_5983_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_5983_cast_fp16 = slice_by_index(begin = var_5983_begin_0, end = var_5983_end_0, end_mask = var_5983_end_mask_0, squeeze_mask = var_5983_squeeze_mask_0, x = pair_77_cast_fp16)[name = string("op_5983_cast_fp16")];
+            tensor<int32, [5]> var_5986_begin_0 = const()[name = string("op_5986_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_5986_end_0 = const()[name = string("op_5986_end_0"), val = tensor<int32, [5]>([1024, 8, 2, 4, 4])];
+            tensor<bool, [5]> var_5986_end_mask_0 = const()[name = string("op_5986_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_5986_squeeze_mask_0 = const()[name = string("op_5986_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 8, 4, 4]> var_5986_cast_fp16 = slice_by_index(begin = var_5986_begin_0, end = var_5986_end_0, end_mask = var_5986_end_mask_0, squeeze_mask = var_5986_squeeze_mask_0, x = pair_77_cast_fp16)[name = string("op_5986_cast_fp16")];
+            bool var_5989_transpose_x_0 = const()[name = string("op_5989_transpose_x_0"), val = bool(false)];
+            bool var_5989_transpose_y_0 = const()[name = string("op_5989_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_5989_cast_fp16 = matmul(transpose_x = var_5989_transpose_x_0, transpose_y = var_5989_transpose_y_0, x = x_161_cast_fp16, y = var_5983_cast_fp16)[name = string("op_5989_cast_fp16")];
+            bool var_5990_transpose_x_0 = const()[name = string("op_5990_transpose_x_0"), val = bool(false)];
+            bool var_5990_transpose_y_0 = const()[name = string("op_5990_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 4]> var_5990_cast_fp16 = matmul(transpose_x = var_5990_transpose_x_0, transpose_y = var_5990_transpose_y_0, x = var_5986_cast_fp16, y = var_5989_cast_fp16)[name = string("op_5990_cast_fp16")];
+            fp16 const_73_promoted_to_fp16 = const()[name = string("const_73_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 8, 4, 4]> lower_65_cast_fp16 = mul(x = var_5990_cast_fp16, y = const_73_promoted_to_fp16)[name = string("lower_65_cast_fp16")];
+            bool top_65_interleave_0 = const()[name = string("top_65_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> top_65_cast_fp16 = concat(axis = var_5753, interleave = top_65_interleave_0, values = (var_5983_cast_fp16, var_305_to_fp16))[name = string("top_65_cast_fp16")];
+            bool bottom_65_interleave_0 = const()[name = string("bottom_65_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 4, 8]> bottom_65_cast_fp16 = concat(axis = var_5753, interleave = bottom_65_interleave_0, values = (lower_65_cast_fp16, var_5986_cast_fp16))[name = string("bottom_65_cast_fp16")];
+            bool inv_87_interleave_0 = const()[name = string("inv_87_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 8, 8, 8]> inv_87_cast_fp16 = concat(axis = var_5724, interleave = inv_87_interleave_0, values = (top_65_cast_fp16, bottom_65_cast_fp16))[name = string("inv_87_cast_fp16")];
+            tensor<int32, [5]> var_5999 = const()[name = string("op_5999"), val = tensor<int32, [5]>([1024, 4, 16, 4, 16])];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_6000_cast_fp16 = reshape(shape = var_5999, x = t_cast_fp16)[name = string("op_6000_cast_fp16")];
+            tensor<fp16, [4, 1, 4, 1]> var_6004_to_fp16 = const()[name = string("op_6004_to_fp16"), val = tensor<fp16, [4, 1, 4, 1]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421837248)))];
+            tensor<fp16, [1024, 4, 16, 4, 16]> var_6005_cast_fp16 = mul(x = var_6000_cast_fp16, y = var_6004_to_fp16)[name = string("op_6005_cast_fp16")];
+            tensor<int32, [1]> blocks_67_axes_0 = const()[name = string("blocks_67_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_67_keep_dims_0 = const()[name = string("blocks_67_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> blocks_67_cast_fp16 = reduce_sum(axes = blocks_67_axes_0, keep_dims = blocks_67_keep_dims_0, x = var_6005_cast_fp16)[name = string("blocks_67_cast_fp16")];
+            tensor<int32, [4]> var_6008_begin_0 = const()[name = string("op_6008_begin_0"), val = tensor<int32, [4]>([0, 0, 8, 0])];
+            tensor<int32, [4]> var_6008_end_0 = const()[name = string("op_6008_end_0"), val = tensor<int32, [4]>([1024, 4, 16, 16])];
+            tensor<bool, [4]> var_6008_end_mask_0 = const()[name = string("op_6008_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 4, 8, 16]> var_6008_cast_fp16 = slice_by_index(begin = var_6008_begin_0, end = var_6008_end_0, end_mask = var_6008_end_mask_0, x = blocks_67_cast_fp16)[name = string("op_6008_cast_fp16")];
+            tensor<int32, [4]> x_163_begin_0 = const()[name = string("x_163_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_163_end_0 = const()[name = string("x_163_end_0"), val = tensor<int32, [4]>([1024, 4, 8, 8])];
+            tensor<bool, [4]> x_163_end_mask_0 = const()[name = string("x_163_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 4, 8, 8]> x_163_cast_fp16 = slice_by_index(begin = x_163_begin_0, end = x_163_end_0, end_mask = x_163_end_mask_0, x = var_6008_cast_fp16)[name = string("x_163_cast_fp16")];
+            tensor<int32, [5]> var_6010 = const()[name = string("op_6010"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<fp16, [1024, 4, 2, 8, 8]> pair_79_cast_fp16 = reshape(shape = var_6010, x = inv_87_cast_fp16)[name = string("pair_79_cast_fp16")];
+            tensor<int32, [5]> var_6012_begin_0 = const()[name = string("op_6012_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_6012_end_0 = const()[name = string("op_6012_end_0"), val = tensor<int32, [5]>([1024, 4, 1, 8, 8])];
+            tensor<bool, [5]> var_6012_end_mask_0 = const()[name = string("op_6012_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_6012_squeeze_mask_0 = const()[name = string("op_6012_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_6012_cast_fp16 = slice_by_index(begin = var_6012_begin_0, end = var_6012_end_0, end_mask = var_6012_end_mask_0, squeeze_mask = var_6012_squeeze_mask_0, x = pair_79_cast_fp16)[name = string("op_6012_cast_fp16")];
+            tensor<int32, [5]> var_6015_begin_0 = const()[name = string("op_6015_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_6015_end_0 = const()[name = string("op_6015_end_0"), val = tensor<int32, [5]>([1024, 4, 2, 8, 8])];
+            tensor<bool, [5]> var_6015_end_mask_0 = const()[name = string("op_6015_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_6015_squeeze_mask_0 = const()[name = string("op_6015_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 4, 8, 8]> var_6015_cast_fp16 = slice_by_index(begin = var_6015_begin_0, end = var_6015_end_0, end_mask = var_6015_end_mask_0, squeeze_mask = var_6015_squeeze_mask_0, x = pair_79_cast_fp16)[name = string("op_6015_cast_fp16")];
+            bool var_6018_transpose_x_0 = const()[name = string("op_6018_transpose_x_0"), val = bool(false)];
+            bool var_6018_transpose_y_0 = const()[name = string("op_6018_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_6018_cast_fp16 = matmul(transpose_x = var_6018_transpose_x_0, transpose_y = var_6018_transpose_y_0, x = x_163_cast_fp16, y = var_6012_cast_fp16)[name = string("op_6018_cast_fp16")];
+            bool var_6019_transpose_x_0 = const()[name = string("op_6019_transpose_x_0"), val = bool(false)];
+            bool var_6019_transpose_y_0 = const()[name = string("op_6019_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 8]> var_6019_cast_fp16 = matmul(transpose_x = var_6019_transpose_x_0, transpose_y = var_6019_transpose_y_0, x = var_6015_cast_fp16, y = var_6018_cast_fp16)[name = string("op_6019_cast_fp16")];
+            fp16 const_74_promoted_to_fp16 = const()[name = string("const_74_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 4, 8, 8]> lower_67_cast_fp16 = mul(x = var_6019_cast_fp16, y = const_74_promoted_to_fp16)[name = string("lower_67_cast_fp16")];
+            bool top_67_interleave_0 = const()[name = string("top_67_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> top_67_cast_fp16 = concat(axis = var_5753, interleave = top_67_interleave_0, values = (var_6012_cast_fp16, var_334_to_fp16))[name = string("top_67_cast_fp16")];
+            bool bottom_67_interleave_0 = const()[name = string("bottom_67_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 8, 16]> bottom_67_cast_fp16 = concat(axis = var_5753, interleave = bottom_67_interleave_0, values = (lower_67_cast_fp16, var_6015_cast_fp16))[name = string("bottom_67_cast_fp16")];
+            bool inv_89_interleave_0 = const()[name = string("inv_89_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 4, 16, 16]> inv_89_cast_fp16 = concat(axis = var_5724, interleave = inv_89_interleave_0, values = (top_67_cast_fp16, bottom_67_cast_fp16))[name = string("inv_89_cast_fp16")];
+            tensor<int32, [5]> var_6028 = const()[name = string("op_6028"), val = tensor<int32, [5]>([1024, 2, 32, 2, 32])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_6029_cast_fp16 = reshape(shape = var_6028, x = t_cast_fp16)[name = string("op_6029_cast_fp16")];
+            tensor<fp16, [2, 1, 2, 1]> var_6033_to_fp16 = const()[name = string("op_6033_to_fp16"), val = tensor<fp16, [2, 1, 2, 1]>([[[[0x1p+0], [0x0p+0]]], [[[0x0p+0], [0x1p+0]]]])];
+            tensor<fp16, [1024, 2, 32, 2, 32]> var_6034_cast_fp16 = mul(x = var_6029_cast_fp16, y = var_6033_to_fp16)[name = string("op_6034_cast_fp16")];
+            tensor<int32, [1]> blocks_69_axes_0 = const()[name = string("blocks_69_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_69_keep_dims_0 = const()[name = string("blocks_69_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> blocks_69_cast_fp16 = reduce_sum(axes = blocks_69_axes_0, keep_dims = blocks_69_keep_dims_0, x = var_6034_cast_fp16)[name = string("blocks_69_cast_fp16")];
+            tensor<int32, [4]> var_6037_begin_0 = const()[name = string("op_6037_begin_0"), val = tensor<int32, [4]>([0, 0, 16, 0])];
+            tensor<int32, [4]> var_6037_end_0 = const()[name = string("op_6037_end_0"), val = tensor<int32, [4]>([1024, 2, 32, 32])];
+            tensor<bool, [4]> var_6037_end_mask_0 = const()[name = string("op_6037_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 2, 16, 32]> var_6037_cast_fp16 = slice_by_index(begin = var_6037_begin_0, end = var_6037_end_0, end_mask = var_6037_end_mask_0, x = blocks_69_cast_fp16)[name = string("op_6037_cast_fp16")];
+            tensor<int32, [4]> x_165_begin_0 = const()[name = string("x_165_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_165_end_0 = const()[name = string("x_165_end_0"), val = tensor<int32, [4]>([1024, 2, 16, 16])];
+            tensor<bool, [4]> x_165_end_mask_0 = const()[name = string("x_165_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 2, 16, 16]> x_165_cast_fp16 = slice_by_index(begin = x_165_begin_0, end = x_165_end_0, end_mask = x_165_end_mask_0, x = var_6037_cast_fp16)[name = string("x_165_cast_fp16")];
+            tensor<int32, [5]> var_6039 = const()[name = string("op_6039"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<fp16, [1024, 2, 2, 16, 16]> pair_81_cast_fp16 = reshape(shape = var_6039, x = inv_89_cast_fp16)[name = string("pair_81_cast_fp16")];
+            tensor<int32, [5]> var_6041_begin_0 = const()[name = string("op_6041_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_6041_end_0 = const()[name = string("op_6041_end_0"), val = tensor<int32, [5]>([1024, 2, 1, 16, 16])];
+            tensor<bool, [5]> var_6041_end_mask_0 = const()[name = string("op_6041_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_6041_squeeze_mask_0 = const()[name = string("op_6041_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_6041_cast_fp16 = slice_by_index(begin = var_6041_begin_0, end = var_6041_end_0, end_mask = var_6041_end_mask_0, squeeze_mask = var_6041_squeeze_mask_0, x = pair_81_cast_fp16)[name = string("op_6041_cast_fp16")];
+            tensor<int32, [5]> var_6044_begin_0 = const()[name = string("op_6044_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_6044_end_0 = const()[name = string("op_6044_end_0"), val = tensor<int32, [5]>([1024, 2, 2, 16, 16])];
+            tensor<bool, [5]> var_6044_end_mask_0 = const()[name = string("op_6044_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_6044_squeeze_mask_0 = const()[name = string("op_6044_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 2, 16, 16]> var_6044_cast_fp16 = slice_by_index(begin = var_6044_begin_0, end = var_6044_end_0, end_mask = var_6044_end_mask_0, squeeze_mask = var_6044_squeeze_mask_0, x = pair_81_cast_fp16)[name = string("op_6044_cast_fp16")];
+            bool var_6047_transpose_x_0 = const()[name = string("op_6047_transpose_x_0"), val = bool(false)];
+            bool var_6047_transpose_y_0 = const()[name = string("op_6047_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_6047_cast_fp16 = matmul(transpose_x = var_6047_transpose_x_0, transpose_y = var_6047_transpose_y_0, x = x_165_cast_fp16, y = var_6041_cast_fp16)[name = string("op_6047_cast_fp16")];
+            bool var_6048_transpose_x_0 = const()[name = string("op_6048_transpose_x_0"), val = bool(false)];
+            bool var_6048_transpose_y_0 = const()[name = string("op_6048_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 16]> var_6048_cast_fp16 = matmul(transpose_x = var_6048_transpose_x_0, transpose_y = var_6048_transpose_y_0, x = var_6044_cast_fp16, y = var_6047_cast_fp16)[name = string("op_6048_cast_fp16")];
+            fp16 const_75_promoted_to_fp16 = const()[name = string("const_75_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 2, 16, 16]> lower_69_cast_fp16 = mul(x = var_6048_cast_fp16, y = const_75_promoted_to_fp16)[name = string("lower_69_cast_fp16")];
+            bool top_69_interleave_0 = const()[name = string("top_69_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> top_69_cast_fp16 = concat(axis = var_5753, interleave = top_69_interleave_0, values = (var_6041_cast_fp16, var_363_to_fp16))[name = string("top_69_cast_fp16")];
+            bool bottom_69_interleave_0 = const()[name = string("bottom_69_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 16, 32]> bottom_69_cast_fp16 = concat(axis = var_5753, interleave = bottom_69_interleave_0, values = (lower_69_cast_fp16, var_6044_cast_fp16))[name = string("bottom_69_cast_fp16")];
+            bool inv_91_interleave_0 = const()[name = string("inv_91_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 2, 32, 32]> inv_91_cast_fp16 = concat(axis = var_5724, interleave = inv_91_interleave_0, values = (top_69_cast_fp16, bottom_69_cast_fp16))[name = string("inv_91_cast_fp16")];
+            tensor<int32, [5]> var_6057 = const()[name = string("op_6057"), val = tensor<int32, [5]>([1024, 1, 64, 1, 64])];
+            tensor<fp16, [1024, 1, 64, 1, 64]> var_6058_cast_fp16 = reshape(shape = var_6057, x = t_cast_fp16)[name = string("op_6058_cast_fp16")];
+            tensor<int32, [1]> blocks_axes_0 = const()[name = string("blocks_axes_0"), val = tensor<int32, [1]>([-2])];
+            bool blocks_keep_dims_0 = const()[name = string("blocks_keep_dims_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> blocks_cast_fp16 = reduce_sum(axes = blocks_axes_0, keep_dims = blocks_keep_dims_0, x = var_6058_cast_fp16)[name = string("blocks_cast_fp16")];
+            tensor<int32, [4]> var_6066_begin_0 = const()[name = string("op_6066_begin_0"), val = tensor<int32, [4]>([0, 0, 32, 0])];
+            tensor<int32, [4]> var_6066_end_0 = const()[name = string("op_6066_end_0"), val = tensor<int32, [4]>([1024, 1, 64, 64])];
+            tensor<bool, [4]> var_6066_end_mask_0 = const()[name = string("op_6066_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1024, 1, 32, 64]> var_6066_cast_fp16 = slice_by_index(begin = var_6066_begin_0, end = var_6066_end_0, end_mask = var_6066_end_mask_0, x = blocks_cast_fp16)[name = string("op_6066_cast_fp16")];
+            tensor<int32, [4]> x_167_begin_0 = const()[name = string("x_167_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_167_end_0 = const()[name = string("x_167_end_0"), val = tensor<int32, [4]>([1024, 1, 32, 32])];
+            tensor<bool, [4]> x_167_end_mask_0 = const()[name = string("x_167_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1024, 1, 32, 32]> x_167_cast_fp16 = slice_by_index(begin = x_167_begin_0, end = x_167_end_0, end_mask = x_167_end_mask_0, x = var_6066_cast_fp16)[name = string("x_167_cast_fp16")];
+            tensor<int32, [5]> var_6068 = const()[name = string("op_6068"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<fp16, [1024, 1, 2, 32, 32]> pair_cast_fp16 = reshape(shape = var_6068, x = inv_91_cast_fp16)[name = string("pair_cast_fp16")];
+            tensor<int32, [5]> var_6070_begin_0 = const()[name = string("op_6070_begin_0"), val = tensor<int32, [5]>([0, 0, 0, 0, 0])];
+            tensor<int32, [5]> var_6070_end_0 = const()[name = string("op_6070_end_0"), val = tensor<int32, [5]>([1024, 1, 1, 32, 32])];
+            tensor<bool, [5]> var_6070_end_mask_0 = const()[name = string("op_6070_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_6070_squeeze_mask_0 = const()[name = string("op_6070_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_6070_cast_fp16 = slice_by_index(begin = var_6070_begin_0, end = var_6070_end_0, end_mask = var_6070_end_mask_0, squeeze_mask = var_6070_squeeze_mask_0, x = pair_cast_fp16)[name = string("op_6070_cast_fp16")];
+            tensor<int32, [5]> var_6073_begin_0 = const()[name = string("op_6073_begin_0"), val = tensor<int32, [5]>([0, 0, 1, 0, 0])];
+            tensor<int32, [5]> var_6073_end_0 = const()[name = string("op_6073_end_0"), val = tensor<int32, [5]>([1024, 1, 2, 32, 32])];
+            tensor<bool, [5]> var_6073_end_mask_0 = const()[name = string("op_6073_end_mask_0"), val = tensor<bool, [5]>([true, true, false, true, true])];
+            tensor<bool, [5]> var_6073_squeeze_mask_0 = const()[name = string("op_6073_squeeze_mask_0"), val = tensor<bool, [5]>([false, false, true, false, false])];
+            tensor<fp16, [1024, 1, 32, 32]> var_6073_cast_fp16 = slice_by_index(begin = var_6073_begin_0, end = var_6073_end_0, end_mask = var_6073_end_mask_0, squeeze_mask = var_6073_squeeze_mask_0, x = pair_cast_fp16)[name = string("op_6073_cast_fp16")];
+            bool var_6076_transpose_x_0 = const()[name = string("op_6076_transpose_x_0"), val = bool(false)];
+            bool var_6076_transpose_y_0 = const()[name = string("op_6076_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_6076_cast_fp16 = matmul(transpose_x = var_6076_transpose_x_0, transpose_y = var_6076_transpose_y_0, x = x_167_cast_fp16, y = var_6070_cast_fp16)[name = string("op_6076_cast_fp16")];
+            bool var_6077_transpose_x_0 = const()[name = string("op_6077_transpose_x_0"), val = bool(false)];
+            bool var_6077_transpose_y_0 = const()[name = string("op_6077_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 32]> var_6077_cast_fp16 = matmul(transpose_x = var_6077_transpose_x_0, transpose_y = var_6077_transpose_y_0, x = var_6073_cast_fp16, y = var_6076_cast_fp16)[name = string("op_6077_cast_fp16")];
+            fp16 const_76_promoted_to_fp16 = const()[name = string("const_76_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1024, 1, 32, 32]> lower_cast_fp16 = mul(x = var_6077_cast_fp16, y = const_76_promoted_to_fp16)[name = string("lower_cast_fp16")];
+            bool top_interleave_0 = const()[name = string("top_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> top_cast_fp16 = concat(axis = var_5753, interleave = top_interleave_0, values = (var_6070_cast_fp16, var_392_to_fp16))[name = string("top_cast_fp16")];
+            bool bottom_interleave_0 = const()[name = string("bottom_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 32, 64]> bottom_cast_fp16 = concat(axis = var_5753, interleave = bottom_interleave_0, values = (lower_cast_fp16, var_6073_cast_fp16))[name = string("bottom_cast_fp16")];
+            bool inv_93_interleave_0 = const()[name = string("inv_93_interleave_0"), val = bool(false)];
+            tensor<fp16, [1024, 1, 64, 64]> inv_93_cast_fp16 = concat(axis = var_5724, interleave = inv_93_interleave_0, values = (top_cast_fp16, bottom_cast_fp16))[name = string("inv_93_cast_fp16")];
+            tensor<int32, [4]> var_6086 = const()[name = string("op_6086"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<fp16, [32, 32, 64, 64]> inv_cast_fp16 = reshape(shape = var_6086, x = inv_93_cast_fp16)[name = string("inv_cast_fp16")];
+            bool new_v_transpose_x_0 = const()[name = string("new_v_transpose_x_0"), val = bool(false)];
+            bool new_v_transpose_y_0 = const()[name = string("new_v_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> new_v_cast_fp16 = matmul(transpose_x = new_v_transpose_x_0, transpose_y = new_v_transpose_y_0, x = inv_cast_fp16, y = v_beta_cast_fp16)[name = string("new_v_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_6089_cast_fp16 = exp(x = cum_cast_fp16)[name = string("op_6089_cast_fp16")];
+            tensor<int32, [1]> var_6090_axes_0 = const()[name = string("op_6090_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_6090_cast_fp16 = expand_dims(axes = var_6090_axes_0, x = var_6089_cast_fp16)[name = string("op_6090_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> var_6091_cast_fp16 = mul(x = k_beta_cast_fp16, y = var_6090_cast_fp16)[name = string("op_6091_cast_fp16")];
+            bool k_cumdecay_transpose_x_0 = const()[name = string("k_cumdecay_transpose_x_0"), val = bool(false)];
+            bool k_cumdecay_transpose_y_0 = const()[name = string("k_cumdecay_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 32, 64, 128]> k_cumdecay_cast_fp16 = matmul(transpose_x = k_cumdecay_transpose_x_0, transpose_y = k_cumdecay_transpose_y_0, x = inv_cast_fp16, y = var_6091_cast_fp16)[name = string("k_cumdecay_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> q_dec_cast_fp16 = mul(x = q_63_cast_fp16, y = var_6090_cast_fp16)[name = string("q_dec_cast_fp16")];
+            tensor<fp16, [32, 32, 64]> var_6096_cast_fp16 = exp(x = to_end_cast_fp16)[name = string("op_6096_cast_fp16")];
+            tensor<int32, [1]> var_6097_axes_0 = const()[name = string("op_6097_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 64, 1]> var_6097_cast_fp16 = expand_dims(axes = var_6097_axes_0, x = var_6096_cast_fp16)[name = string("op_6097_cast_fp16")];
+            tensor<fp16, [32, 32, 64, 128]> k_dec_cast_fp16 = mul(x = k_53_cast_fp16, y = var_6097_cast_fp16)[name = string("k_dec_cast_fp16")];
+            tensor<int32, [3]> var_6099_begin_0 = const()[name = string("op_6099_begin_0"), val = tensor<int32, [3]>([0, 0, -1])];
+            tensor<int32, [3]> var_6099_end_0 = const()[name = string("op_6099_end_0"), val = tensor<int32, [3]>([32, 32, 64])];
+            tensor<bool, [3]> var_6099_end_mask_0 = const()[name = string("op_6099_end_mask_0"), val = tensor<bool, [3]>([true, true, true])];
+            tensor<fp16, [32, 32, 1]> var_6099_cast_fp16 = slice_by_index(begin = var_6099_begin_0, end = var_6099_end_0, end_mask = var_6099_end_mask_0, x = cum_cast_fp16)[name = string("op_6099_cast_fp16")];
+            tensor<fp16, [32, 32, 1]> var_6100_cast_fp16 = exp(x = var_6099_cast_fp16)[name = string("op_6100_cast_fp16")];
+            tensor<int32, [1]> chunk_decay_axes_0 = const()[name = string("chunk_decay_axes_0"), val = tensor<int32, [1]>([3])];
+            tensor<fp16, [32, 32, 1, 1]> chunk_decay_cast_fp16 = expand_dims(axes = chunk_decay_axes_0, x = var_6100_cast_fp16)[name = string("chunk_decay_cast_fp16")];
+            tensor<int32, [4]> var_6105_begin_0 = const()[name = string("op_6105_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6105_end_0 = const()[name = string("op_6105_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_6105_end_mask_0 = const()[name = string("op_6105_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6105_squeeze_mask_0 = const()[name = string("op_6105_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6105_cast_fp16 = slice_by_index(begin = var_6105_begin_0, end = var_6105_end_0, end_mask = var_6105_end_mask_0, squeeze_mask = var_6105_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6105_cast_fp16")];
+            tensor<int32, [4]> var_6107_begin_0 = const()[name = string("op_6107_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6107_end_0 = const()[name = string("op_6107_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_6107_end_mask_0 = const()[name = string("op_6107_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6107_squeeze_mask_0 = const()[name = string("op_6107_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6107_cast_fp16 = slice_by_index(begin = var_6107_begin_0, end = var_6107_end_0, end_mask = var_6107_end_mask_0, squeeze_mask = var_6107_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6107_cast_fp16")];
+            bool var_6108_transpose_x_0 = const()[name = string("op_6108_transpose_x_0"), val = bool(false)];
+            bool var_6108_transpose_y_0 = const()[name = string("op_6108_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6108_cast_fp16 = matmul(transpose_x = var_6108_transpose_x_0, transpose_y = var_6108_transpose_y_0, x = var_6107_cast_fp16, y = state_1_to_fp16)[name = string("op_6108_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_321_cast_fp16 = sub(x = var_6105_cast_fp16, y = var_6108_cast_fp16)[name = string("v_new_321_cast_fp16")];
+            tensor<int32, [4]> var_6111_begin_0 = const()[name = string("op_6111_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6111_end_0 = const()[name = string("op_6111_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_6111_end_mask_0 = const()[name = string("op_6111_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6111_squeeze_mask_0 = const()[name = string("op_6111_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6111_cast_fp16 = slice_by_index(begin = var_6111_begin_0, end = var_6111_end_0, end_mask = var_6111_end_mask_0, squeeze_mask = var_6111_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6111_cast_fp16")];
+            bool var_6112_transpose_x_0 = const()[name = string("op_6112_transpose_x_0"), val = bool(false)];
+            bool var_6112_transpose_y_0 = const()[name = string("op_6112_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6112_cast_fp16 = matmul(transpose_x = var_6112_transpose_x_0, transpose_y = var_6112_transpose_y_0, x = var_6111_cast_fp16, y = state_1_to_fp16)[name = string("op_6112_cast_fp16")];
+            tensor<int32, [4]> var_6114_begin_0 = const()[name = string("op_6114_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6114_end_0 = const()[name = string("op_6114_end_0"), val = tensor<int32, [4]>([32, 1, 64, 64])];
+            tensor<bool, [4]> var_6114_end_mask_0 = const()[name = string("op_6114_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6114_squeeze_mask_0 = const()[name = string("op_6114_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6114_cast_fp16 = slice_by_index(begin = var_6114_begin_0, end = var_6114_end_0, end_mask = var_6114_end_mask_0, squeeze_mask = var_6114_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6114_cast_fp16")];
+            bool var_6115_transpose_x_0 = const()[name = string("op_6115_transpose_x_0"), val = bool(false)];
+            bool var_6115_transpose_y_0 = const()[name = string("op_6115_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6115_cast_fp16 = matmul(transpose_x = var_6115_transpose_x_0, transpose_y = var_6115_transpose_y_0, x = var_6114_cast_fp16, y = v_new_321_cast_fp16)[name = string("op_6115_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6116_cast_fp16 = add(x = var_6112_cast_fp16, y = var_6115_cast_fp16)[name = string("op_6116_cast_fp16")];
+            tensor<int32, [4]> var_6118_begin_0 = const()[name = string("op_6118_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6118_end_0 = const()[name = string("op_6118_end_0"), val = tensor<int32, [4]>([32, 1, 1, 1])];
+            tensor<bool, [4]> var_6118_end_mask_0 = const()[name = string("op_6118_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6118_squeeze_mask_0 = const()[name = string("op_6118_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6118_cast_fp16 = slice_by_index(begin = var_6118_begin_0, end = var_6118_end_0, end_mask = var_6118_end_mask_0, squeeze_mask = var_6118_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6118_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6119_cast_fp16 = mul(x = state_1_to_fp16, y = var_6118_cast_fp16)[name = string("op_6119_cast_fp16")];
+            tensor<int32, [4]> var_6121_begin_0 = const()[name = string("op_6121_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6121_end_0 = const()[name = string("op_6121_end_0"), val = tensor<int32, [4]>([32, 1, 64, 128])];
+            tensor<bool, [4]> var_6121_end_mask_0 = const()[name = string("op_6121_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6121_squeeze_mask_0 = const()[name = string("op_6121_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6121_cast_fp16 = slice_by_index(begin = var_6121_begin_0, end = var_6121_end_0, end_mask = var_6121_end_mask_0, squeeze_mask = var_6121_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6121_cast_fp16")];
+            bool var_6123_transpose_x_1 = const()[name = string("op_6123_transpose_x_1"), val = bool(true)];
+            bool var_6123_transpose_y_1 = const()[name = string("op_6123_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6123_cast_fp16 = matmul(transpose_x = var_6123_transpose_x_1, transpose_y = var_6123_transpose_y_1, x = var_6121_cast_fp16, y = v_new_321_cast_fp16)[name = string("op_6123_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_323_cast_fp16 = add(x = var_6119_cast_fp16, y = var_6123_cast_fp16)[name = string("state_323_cast_fp16")];
+            tensor<int32, [4]> var_6126_begin_0 = const()[name = string("op_6126_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_6126_end_0 = const()[name = string("op_6126_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_6126_end_mask_0 = const()[name = string("op_6126_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6126_squeeze_mask_0 = const()[name = string("op_6126_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6126_cast_fp16 = slice_by_index(begin = var_6126_begin_0, end = var_6126_end_0, end_mask = var_6126_end_mask_0, squeeze_mask = var_6126_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6126_cast_fp16")];
+            tensor<int32, [4]> var_6128_begin_0 = const()[name = string("op_6128_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_6128_end_0 = const()[name = string("op_6128_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_6128_end_mask_0 = const()[name = string("op_6128_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6128_squeeze_mask_0 = const()[name = string("op_6128_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6128_cast_fp16 = slice_by_index(begin = var_6128_begin_0, end = var_6128_end_0, end_mask = var_6128_end_mask_0, squeeze_mask = var_6128_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6128_cast_fp16")];
+            bool var_6129_transpose_x_0 = const()[name = string("op_6129_transpose_x_0"), val = bool(false)];
+            bool var_6129_transpose_y_0 = const()[name = string("op_6129_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6129_cast_fp16 = matmul(transpose_x = var_6129_transpose_x_0, transpose_y = var_6129_transpose_y_0, x = var_6128_cast_fp16, y = state_323_cast_fp16)[name = string("op_6129_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_323_cast_fp16 = sub(x = var_6126_cast_fp16, y = var_6129_cast_fp16)[name = string("v_new_323_cast_fp16")];
+            tensor<int32, [4]> var_6132_begin_0 = const()[name = string("op_6132_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_6132_end_0 = const()[name = string("op_6132_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_6132_end_mask_0 = const()[name = string("op_6132_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6132_squeeze_mask_0 = const()[name = string("op_6132_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6132_cast_fp16 = slice_by_index(begin = var_6132_begin_0, end = var_6132_end_0, end_mask = var_6132_end_mask_0, squeeze_mask = var_6132_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6132_cast_fp16")];
+            bool var_6133_transpose_x_0 = const()[name = string("op_6133_transpose_x_0"), val = bool(false)];
+            bool var_6133_transpose_y_0 = const()[name = string("op_6133_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6133_cast_fp16 = matmul(transpose_x = var_6133_transpose_x_0, transpose_y = var_6133_transpose_y_0, x = var_6132_cast_fp16, y = state_323_cast_fp16)[name = string("op_6133_cast_fp16")];
+            tensor<int32, [4]> var_6135_begin_0 = const()[name = string("op_6135_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_6135_end_0 = const()[name = string("op_6135_end_0"), val = tensor<int32, [4]>([32, 2, 64, 64])];
+            tensor<bool, [4]> var_6135_end_mask_0 = const()[name = string("op_6135_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6135_squeeze_mask_0 = const()[name = string("op_6135_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6135_cast_fp16 = slice_by_index(begin = var_6135_begin_0, end = var_6135_end_0, end_mask = var_6135_end_mask_0, squeeze_mask = var_6135_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6135_cast_fp16")];
+            bool var_6136_transpose_x_0 = const()[name = string("op_6136_transpose_x_0"), val = bool(false)];
+            bool var_6136_transpose_y_0 = const()[name = string("op_6136_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6136_cast_fp16 = matmul(transpose_x = var_6136_transpose_x_0, transpose_y = var_6136_transpose_y_0, x = var_6135_cast_fp16, y = v_new_323_cast_fp16)[name = string("op_6136_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6137_cast_fp16 = add(x = var_6133_cast_fp16, y = var_6136_cast_fp16)[name = string("op_6137_cast_fp16")];
+            tensor<int32, [4]> var_6139_begin_0 = const()[name = string("op_6139_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_6139_end_0 = const()[name = string("op_6139_end_0"), val = tensor<int32, [4]>([32, 2, 1, 1])];
+            tensor<bool, [4]> var_6139_end_mask_0 = const()[name = string("op_6139_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6139_squeeze_mask_0 = const()[name = string("op_6139_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6139_cast_fp16 = slice_by_index(begin = var_6139_begin_0, end = var_6139_end_0, end_mask = var_6139_end_mask_0, squeeze_mask = var_6139_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6139_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6140_cast_fp16 = mul(x = state_323_cast_fp16, y = var_6139_cast_fp16)[name = string("op_6140_cast_fp16")];
+            tensor<int32, [4]> var_6142_begin_0 = const()[name = string("op_6142_begin_0"), val = tensor<int32, [4]>([0, 1, 0, 0])];
+            tensor<int32, [4]> var_6142_end_0 = const()[name = string("op_6142_end_0"), val = tensor<int32, [4]>([32, 2, 64, 128])];
+            tensor<bool, [4]> var_6142_end_mask_0 = const()[name = string("op_6142_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6142_squeeze_mask_0 = const()[name = string("op_6142_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6142_cast_fp16 = slice_by_index(begin = var_6142_begin_0, end = var_6142_end_0, end_mask = var_6142_end_mask_0, squeeze_mask = var_6142_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6142_cast_fp16")];
+            bool var_6144_transpose_x_1 = const()[name = string("op_6144_transpose_x_1"), val = bool(true)];
+            bool var_6144_transpose_y_1 = const()[name = string("op_6144_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6144_cast_fp16 = matmul(transpose_x = var_6144_transpose_x_1, transpose_y = var_6144_transpose_y_1, x = var_6142_cast_fp16, y = v_new_323_cast_fp16)[name = string("op_6144_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_325_cast_fp16 = add(x = var_6140_cast_fp16, y = var_6144_cast_fp16)[name = string("state_325_cast_fp16")];
+            tensor<int32, [4]> var_6147_begin_0 = const()[name = string("op_6147_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_6147_end_0 = const()[name = string("op_6147_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_6147_end_mask_0 = const()[name = string("op_6147_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6147_squeeze_mask_0 = const()[name = string("op_6147_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6147_cast_fp16 = slice_by_index(begin = var_6147_begin_0, end = var_6147_end_0, end_mask = var_6147_end_mask_0, squeeze_mask = var_6147_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6147_cast_fp16")];
+            tensor<int32, [4]> var_6149_begin_0 = const()[name = string("op_6149_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_6149_end_0 = const()[name = string("op_6149_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_6149_end_mask_0 = const()[name = string("op_6149_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6149_squeeze_mask_0 = const()[name = string("op_6149_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6149_cast_fp16 = slice_by_index(begin = var_6149_begin_0, end = var_6149_end_0, end_mask = var_6149_end_mask_0, squeeze_mask = var_6149_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6149_cast_fp16")];
+            bool var_6150_transpose_x_0 = const()[name = string("op_6150_transpose_x_0"), val = bool(false)];
+            bool var_6150_transpose_y_0 = const()[name = string("op_6150_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6150_cast_fp16 = matmul(transpose_x = var_6150_transpose_x_0, transpose_y = var_6150_transpose_y_0, x = var_6149_cast_fp16, y = state_325_cast_fp16)[name = string("op_6150_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_325_cast_fp16 = sub(x = var_6147_cast_fp16, y = var_6150_cast_fp16)[name = string("v_new_325_cast_fp16")];
+            tensor<int32, [4]> var_6153_begin_0 = const()[name = string("op_6153_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_6153_end_0 = const()[name = string("op_6153_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_6153_end_mask_0 = const()[name = string("op_6153_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6153_squeeze_mask_0 = const()[name = string("op_6153_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6153_cast_fp16 = slice_by_index(begin = var_6153_begin_0, end = var_6153_end_0, end_mask = var_6153_end_mask_0, squeeze_mask = var_6153_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6153_cast_fp16")];
+            bool var_6154_transpose_x_0 = const()[name = string("op_6154_transpose_x_0"), val = bool(false)];
+            bool var_6154_transpose_y_0 = const()[name = string("op_6154_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6154_cast_fp16 = matmul(transpose_x = var_6154_transpose_x_0, transpose_y = var_6154_transpose_y_0, x = var_6153_cast_fp16, y = state_325_cast_fp16)[name = string("op_6154_cast_fp16")];
+            tensor<int32, [4]> var_6156_begin_0 = const()[name = string("op_6156_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_6156_end_0 = const()[name = string("op_6156_end_0"), val = tensor<int32, [4]>([32, 3, 64, 64])];
+            tensor<bool, [4]> var_6156_end_mask_0 = const()[name = string("op_6156_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6156_squeeze_mask_0 = const()[name = string("op_6156_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6156_cast_fp16 = slice_by_index(begin = var_6156_begin_0, end = var_6156_end_0, end_mask = var_6156_end_mask_0, squeeze_mask = var_6156_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6156_cast_fp16")];
+            bool var_6157_transpose_x_0 = const()[name = string("op_6157_transpose_x_0"), val = bool(false)];
+            bool var_6157_transpose_y_0 = const()[name = string("op_6157_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6157_cast_fp16 = matmul(transpose_x = var_6157_transpose_x_0, transpose_y = var_6157_transpose_y_0, x = var_6156_cast_fp16, y = v_new_325_cast_fp16)[name = string("op_6157_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6158_cast_fp16 = add(x = var_6154_cast_fp16, y = var_6157_cast_fp16)[name = string("op_6158_cast_fp16")];
+            tensor<int32, [4]> var_6160_begin_0 = const()[name = string("op_6160_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_6160_end_0 = const()[name = string("op_6160_end_0"), val = tensor<int32, [4]>([32, 3, 1, 1])];
+            tensor<bool, [4]> var_6160_end_mask_0 = const()[name = string("op_6160_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6160_squeeze_mask_0 = const()[name = string("op_6160_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6160_cast_fp16 = slice_by_index(begin = var_6160_begin_0, end = var_6160_end_0, end_mask = var_6160_end_mask_0, squeeze_mask = var_6160_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6160_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6161_cast_fp16 = mul(x = state_325_cast_fp16, y = var_6160_cast_fp16)[name = string("op_6161_cast_fp16")];
+            tensor<int32, [4]> var_6163_begin_0 = const()[name = string("op_6163_begin_0"), val = tensor<int32, [4]>([0, 2, 0, 0])];
+            tensor<int32, [4]> var_6163_end_0 = const()[name = string("op_6163_end_0"), val = tensor<int32, [4]>([32, 3, 64, 128])];
+            tensor<bool, [4]> var_6163_end_mask_0 = const()[name = string("op_6163_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6163_squeeze_mask_0 = const()[name = string("op_6163_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6163_cast_fp16 = slice_by_index(begin = var_6163_begin_0, end = var_6163_end_0, end_mask = var_6163_end_mask_0, squeeze_mask = var_6163_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6163_cast_fp16")];
+            bool var_6165_transpose_x_1 = const()[name = string("op_6165_transpose_x_1"), val = bool(true)];
+            bool var_6165_transpose_y_1 = const()[name = string("op_6165_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6165_cast_fp16 = matmul(transpose_x = var_6165_transpose_x_1, transpose_y = var_6165_transpose_y_1, x = var_6163_cast_fp16, y = v_new_325_cast_fp16)[name = string("op_6165_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_327_cast_fp16 = add(x = var_6161_cast_fp16, y = var_6165_cast_fp16)[name = string("state_327_cast_fp16")];
+            tensor<int32, [4]> var_6168_begin_0 = const()[name = string("op_6168_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_6168_end_0 = const()[name = string("op_6168_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_6168_end_mask_0 = const()[name = string("op_6168_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6168_squeeze_mask_0 = const()[name = string("op_6168_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6168_cast_fp16 = slice_by_index(begin = var_6168_begin_0, end = var_6168_end_0, end_mask = var_6168_end_mask_0, squeeze_mask = var_6168_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6168_cast_fp16")];
+            tensor<int32, [4]> var_6170_begin_0 = const()[name = string("op_6170_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_6170_end_0 = const()[name = string("op_6170_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_6170_end_mask_0 = const()[name = string("op_6170_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6170_squeeze_mask_0 = const()[name = string("op_6170_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6170_cast_fp16 = slice_by_index(begin = var_6170_begin_0, end = var_6170_end_0, end_mask = var_6170_end_mask_0, squeeze_mask = var_6170_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6170_cast_fp16")];
+            bool var_6171_transpose_x_0 = const()[name = string("op_6171_transpose_x_0"), val = bool(false)];
+            bool var_6171_transpose_y_0 = const()[name = string("op_6171_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6171_cast_fp16 = matmul(transpose_x = var_6171_transpose_x_0, transpose_y = var_6171_transpose_y_0, x = var_6170_cast_fp16, y = state_327_cast_fp16)[name = string("op_6171_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_327_cast_fp16 = sub(x = var_6168_cast_fp16, y = var_6171_cast_fp16)[name = string("v_new_327_cast_fp16")];
+            tensor<int32, [4]> var_6174_begin_0 = const()[name = string("op_6174_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_6174_end_0 = const()[name = string("op_6174_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_6174_end_mask_0 = const()[name = string("op_6174_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6174_squeeze_mask_0 = const()[name = string("op_6174_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6174_cast_fp16 = slice_by_index(begin = var_6174_begin_0, end = var_6174_end_0, end_mask = var_6174_end_mask_0, squeeze_mask = var_6174_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6174_cast_fp16")];
+            bool var_6175_transpose_x_0 = const()[name = string("op_6175_transpose_x_0"), val = bool(false)];
+            bool var_6175_transpose_y_0 = const()[name = string("op_6175_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6175_cast_fp16 = matmul(transpose_x = var_6175_transpose_x_0, transpose_y = var_6175_transpose_y_0, x = var_6174_cast_fp16, y = state_327_cast_fp16)[name = string("op_6175_cast_fp16")];
+            tensor<int32, [4]> var_6177_begin_0 = const()[name = string("op_6177_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_6177_end_0 = const()[name = string("op_6177_end_0"), val = tensor<int32, [4]>([32, 4, 64, 64])];
+            tensor<bool, [4]> var_6177_end_mask_0 = const()[name = string("op_6177_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6177_squeeze_mask_0 = const()[name = string("op_6177_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6177_cast_fp16 = slice_by_index(begin = var_6177_begin_0, end = var_6177_end_0, end_mask = var_6177_end_mask_0, squeeze_mask = var_6177_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6177_cast_fp16")];
+            bool var_6178_transpose_x_0 = const()[name = string("op_6178_transpose_x_0"), val = bool(false)];
+            bool var_6178_transpose_y_0 = const()[name = string("op_6178_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6178_cast_fp16 = matmul(transpose_x = var_6178_transpose_x_0, transpose_y = var_6178_transpose_y_0, x = var_6177_cast_fp16, y = v_new_327_cast_fp16)[name = string("op_6178_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6179_cast_fp16 = add(x = var_6175_cast_fp16, y = var_6178_cast_fp16)[name = string("op_6179_cast_fp16")];
+            tensor<int32, [4]> var_6181_begin_0 = const()[name = string("op_6181_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_6181_end_0 = const()[name = string("op_6181_end_0"), val = tensor<int32, [4]>([32, 4, 1, 1])];
+            tensor<bool, [4]> var_6181_end_mask_0 = const()[name = string("op_6181_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6181_squeeze_mask_0 = const()[name = string("op_6181_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6181_cast_fp16 = slice_by_index(begin = var_6181_begin_0, end = var_6181_end_0, end_mask = var_6181_end_mask_0, squeeze_mask = var_6181_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6181_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6182_cast_fp16 = mul(x = state_327_cast_fp16, y = var_6181_cast_fp16)[name = string("op_6182_cast_fp16")];
+            tensor<int32, [4]> var_6184_begin_0 = const()[name = string("op_6184_begin_0"), val = tensor<int32, [4]>([0, 3, 0, 0])];
+            tensor<int32, [4]> var_6184_end_0 = const()[name = string("op_6184_end_0"), val = tensor<int32, [4]>([32, 4, 64, 128])];
+            tensor<bool, [4]> var_6184_end_mask_0 = const()[name = string("op_6184_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6184_squeeze_mask_0 = const()[name = string("op_6184_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6184_cast_fp16 = slice_by_index(begin = var_6184_begin_0, end = var_6184_end_0, end_mask = var_6184_end_mask_0, squeeze_mask = var_6184_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6184_cast_fp16")];
+            bool var_6186_transpose_x_1 = const()[name = string("op_6186_transpose_x_1"), val = bool(true)];
+            bool var_6186_transpose_y_1 = const()[name = string("op_6186_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6186_cast_fp16 = matmul(transpose_x = var_6186_transpose_x_1, transpose_y = var_6186_transpose_y_1, x = var_6184_cast_fp16, y = v_new_327_cast_fp16)[name = string("op_6186_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_329_cast_fp16 = add(x = var_6182_cast_fp16, y = var_6186_cast_fp16)[name = string("state_329_cast_fp16")];
+            tensor<int32, [4]> var_6189_begin_0 = const()[name = string("op_6189_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_6189_end_0 = const()[name = string("op_6189_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_6189_end_mask_0 = const()[name = string("op_6189_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6189_squeeze_mask_0 = const()[name = string("op_6189_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6189_cast_fp16 = slice_by_index(begin = var_6189_begin_0, end = var_6189_end_0, end_mask = var_6189_end_mask_0, squeeze_mask = var_6189_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6189_cast_fp16")];
+            tensor<int32, [4]> var_6191_begin_0 = const()[name = string("op_6191_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_6191_end_0 = const()[name = string("op_6191_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_6191_end_mask_0 = const()[name = string("op_6191_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6191_squeeze_mask_0 = const()[name = string("op_6191_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6191_cast_fp16 = slice_by_index(begin = var_6191_begin_0, end = var_6191_end_0, end_mask = var_6191_end_mask_0, squeeze_mask = var_6191_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6191_cast_fp16")];
+            bool var_6192_transpose_x_0 = const()[name = string("op_6192_transpose_x_0"), val = bool(false)];
+            bool var_6192_transpose_y_0 = const()[name = string("op_6192_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6192_cast_fp16 = matmul(transpose_x = var_6192_transpose_x_0, transpose_y = var_6192_transpose_y_0, x = var_6191_cast_fp16, y = state_329_cast_fp16)[name = string("op_6192_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_329_cast_fp16 = sub(x = var_6189_cast_fp16, y = var_6192_cast_fp16)[name = string("v_new_329_cast_fp16")];
+            tensor<int32, [4]> var_6195_begin_0 = const()[name = string("op_6195_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_6195_end_0 = const()[name = string("op_6195_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_6195_end_mask_0 = const()[name = string("op_6195_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6195_squeeze_mask_0 = const()[name = string("op_6195_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6195_cast_fp16 = slice_by_index(begin = var_6195_begin_0, end = var_6195_end_0, end_mask = var_6195_end_mask_0, squeeze_mask = var_6195_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6195_cast_fp16")];
+            bool var_6196_transpose_x_0 = const()[name = string("op_6196_transpose_x_0"), val = bool(false)];
+            bool var_6196_transpose_y_0 = const()[name = string("op_6196_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6196_cast_fp16 = matmul(transpose_x = var_6196_transpose_x_0, transpose_y = var_6196_transpose_y_0, x = var_6195_cast_fp16, y = state_329_cast_fp16)[name = string("op_6196_cast_fp16")];
+            tensor<int32, [4]> var_6198_begin_0 = const()[name = string("op_6198_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_6198_end_0 = const()[name = string("op_6198_end_0"), val = tensor<int32, [4]>([32, 5, 64, 64])];
+            tensor<bool, [4]> var_6198_end_mask_0 = const()[name = string("op_6198_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6198_squeeze_mask_0 = const()[name = string("op_6198_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6198_cast_fp16 = slice_by_index(begin = var_6198_begin_0, end = var_6198_end_0, end_mask = var_6198_end_mask_0, squeeze_mask = var_6198_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6198_cast_fp16")];
+            bool var_6199_transpose_x_0 = const()[name = string("op_6199_transpose_x_0"), val = bool(false)];
+            bool var_6199_transpose_y_0 = const()[name = string("op_6199_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6199_cast_fp16 = matmul(transpose_x = var_6199_transpose_x_0, transpose_y = var_6199_transpose_y_0, x = var_6198_cast_fp16, y = v_new_329_cast_fp16)[name = string("op_6199_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6200_cast_fp16 = add(x = var_6196_cast_fp16, y = var_6199_cast_fp16)[name = string("op_6200_cast_fp16")];
+            tensor<int32, [4]> var_6202_begin_0 = const()[name = string("op_6202_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_6202_end_0 = const()[name = string("op_6202_end_0"), val = tensor<int32, [4]>([32, 5, 1, 1])];
+            tensor<bool, [4]> var_6202_end_mask_0 = const()[name = string("op_6202_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6202_squeeze_mask_0 = const()[name = string("op_6202_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6202_cast_fp16 = slice_by_index(begin = var_6202_begin_0, end = var_6202_end_0, end_mask = var_6202_end_mask_0, squeeze_mask = var_6202_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6202_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6203_cast_fp16 = mul(x = state_329_cast_fp16, y = var_6202_cast_fp16)[name = string("op_6203_cast_fp16")];
+            tensor<int32, [4]> var_6205_begin_0 = const()[name = string("op_6205_begin_0"), val = tensor<int32, [4]>([0, 4, 0, 0])];
+            tensor<int32, [4]> var_6205_end_0 = const()[name = string("op_6205_end_0"), val = tensor<int32, [4]>([32, 5, 64, 128])];
+            tensor<bool, [4]> var_6205_end_mask_0 = const()[name = string("op_6205_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6205_squeeze_mask_0 = const()[name = string("op_6205_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6205_cast_fp16 = slice_by_index(begin = var_6205_begin_0, end = var_6205_end_0, end_mask = var_6205_end_mask_0, squeeze_mask = var_6205_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6205_cast_fp16")];
+            bool var_6207_transpose_x_1 = const()[name = string("op_6207_transpose_x_1"), val = bool(true)];
+            bool var_6207_transpose_y_1 = const()[name = string("op_6207_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6207_cast_fp16 = matmul(transpose_x = var_6207_transpose_x_1, transpose_y = var_6207_transpose_y_1, x = var_6205_cast_fp16, y = v_new_329_cast_fp16)[name = string("op_6207_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_331_cast_fp16 = add(x = var_6203_cast_fp16, y = var_6207_cast_fp16)[name = string("state_331_cast_fp16")];
+            tensor<int32, [4]> var_6210_begin_0 = const()[name = string("op_6210_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_6210_end_0 = const()[name = string("op_6210_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_6210_end_mask_0 = const()[name = string("op_6210_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6210_squeeze_mask_0 = const()[name = string("op_6210_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6210_cast_fp16 = slice_by_index(begin = var_6210_begin_0, end = var_6210_end_0, end_mask = var_6210_end_mask_0, squeeze_mask = var_6210_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6210_cast_fp16")];
+            tensor<int32, [4]> var_6212_begin_0 = const()[name = string("op_6212_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_6212_end_0 = const()[name = string("op_6212_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_6212_end_mask_0 = const()[name = string("op_6212_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6212_squeeze_mask_0 = const()[name = string("op_6212_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6212_cast_fp16 = slice_by_index(begin = var_6212_begin_0, end = var_6212_end_0, end_mask = var_6212_end_mask_0, squeeze_mask = var_6212_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6212_cast_fp16")];
+            bool var_6213_transpose_x_0 = const()[name = string("op_6213_transpose_x_0"), val = bool(false)];
+            bool var_6213_transpose_y_0 = const()[name = string("op_6213_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6213_cast_fp16 = matmul(transpose_x = var_6213_transpose_x_0, transpose_y = var_6213_transpose_y_0, x = var_6212_cast_fp16, y = state_331_cast_fp16)[name = string("op_6213_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_331_cast_fp16 = sub(x = var_6210_cast_fp16, y = var_6213_cast_fp16)[name = string("v_new_331_cast_fp16")];
+            tensor<int32, [4]> var_6216_begin_0 = const()[name = string("op_6216_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_6216_end_0 = const()[name = string("op_6216_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_6216_end_mask_0 = const()[name = string("op_6216_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6216_squeeze_mask_0 = const()[name = string("op_6216_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6216_cast_fp16 = slice_by_index(begin = var_6216_begin_0, end = var_6216_end_0, end_mask = var_6216_end_mask_0, squeeze_mask = var_6216_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6216_cast_fp16")];
+            bool var_6217_transpose_x_0 = const()[name = string("op_6217_transpose_x_0"), val = bool(false)];
+            bool var_6217_transpose_y_0 = const()[name = string("op_6217_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6217_cast_fp16 = matmul(transpose_x = var_6217_transpose_x_0, transpose_y = var_6217_transpose_y_0, x = var_6216_cast_fp16, y = state_331_cast_fp16)[name = string("op_6217_cast_fp16")];
+            tensor<int32, [4]> var_6219_begin_0 = const()[name = string("op_6219_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_6219_end_0 = const()[name = string("op_6219_end_0"), val = tensor<int32, [4]>([32, 6, 64, 64])];
+            tensor<bool, [4]> var_6219_end_mask_0 = const()[name = string("op_6219_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6219_squeeze_mask_0 = const()[name = string("op_6219_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6219_cast_fp16 = slice_by_index(begin = var_6219_begin_0, end = var_6219_end_0, end_mask = var_6219_end_mask_0, squeeze_mask = var_6219_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6219_cast_fp16")];
+            bool var_6220_transpose_x_0 = const()[name = string("op_6220_transpose_x_0"), val = bool(false)];
+            bool var_6220_transpose_y_0 = const()[name = string("op_6220_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6220_cast_fp16 = matmul(transpose_x = var_6220_transpose_x_0, transpose_y = var_6220_transpose_y_0, x = var_6219_cast_fp16, y = v_new_331_cast_fp16)[name = string("op_6220_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6221_cast_fp16 = add(x = var_6217_cast_fp16, y = var_6220_cast_fp16)[name = string("op_6221_cast_fp16")];
+            tensor<int32, [4]> var_6223_begin_0 = const()[name = string("op_6223_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_6223_end_0 = const()[name = string("op_6223_end_0"), val = tensor<int32, [4]>([32, 6, 1, 1])];
+            tensor<bool, [4]> var_6223_end_mask_0 = const()[name = string("op_6223_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6223_squeeze_mask_0 = const()[name = string("op_6223_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6223_cast_fp16 = slice_by_index(begin = var_6223_begin_0, end = var_6223_end_0, end_mask = var_6223_end_mask_0, squeeze_mask = var_6223_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6223_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6224_cast_fp16 = mul(x = state_331_cast_fp16, y = var_6223_cast_fp16)[name = string("op_6224_cast_fp16")];
+            tensor<int32, [4]> var_6226_begin_0 = const()[name = string("op_6226_begin_0"), val = tensor<int32, [4]>([0, 5, 0, 0])];
+            tensor<int32, [4]> var_6226_end_0 = const()[name = string("op_6226_end_0"), val = tensor<int32, [4]>([32, 6, 64, 128])];
+            tensor<bool, [4]> var_6226_end_mask_0 = const()[name = string("op_6226_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6226_squeeze_mask_0 = const()[name = string("op_6226_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6226_cast_fp16 = slice_by_index(begin = var_6226_begin_0, end = var_6226_end_0, end_mask = var_6226_end_mask_0, squeeze_mask = var_6226_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6226_cast_fp16")];
+            bool var_6228_transpose_x_1 = const()[name = string("op_6228_transpose_x_1"), val = bool(true)];
+            bool var_6228_transpose_y_1 = const()[name = string("op_6228_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6228_cast_fp16 = matmul(transpose_x = var_6228_transpose_x_1, transpose_y = var_6228_transpose_y_1, x = var_6226_cast_fp16, y = v_new_331_cast_fp16)[name = string("op_6228_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_333_cast_fp16 = add(x = var_6224_cast_fp16, y = var_6228_cast_fp16)[name = string("state_333_cast_fp16")];
+            tensor<int32, [4]> var_6231_begin_0 = const()[name = string("op_6231_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_6231_end_0 = const()[name = string("op_6231_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_6231_end_mask_0 = const()[name = string("op_6231_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6231_squeeze_mask_0 = const()[name = string("op_6231_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6231_cast_fp16 = slice_by_index(begin = var_6231_begin_0, end = var_6231_end_0, end_mask = var_6231_end_mask_0, squeeze_mask = var_6231_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6231_cast_fp16")];
+            tensor<int32, [4]> var_6233_begin_0 = const()[name = string("op_6233_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_6233_end_0 = const()[name = string("op_6233_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_6233_end_mask_0 = const()[name = string("op_6233_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6233_squeeze_mask_0 = const()[name = string("op_6233_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6233_cast_fp16 = slice_by_index(begin = var_6233_begin_0, end = var_6233_end_0, end_mask = var_6233_end_mask_0, squeeze_mask = var_6233_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6233_cast_fp16")];
+            bool var_6234_transpose_x_0 = const()[name = string("op_6234_transpose_x_0"), val = bool(false)];
+            bool var_6234_transpose_y_0 = const()[name = string("op_6234_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6234_cast_fp16 = matmul(transpose_x = var_6234_transpose_x_0, transpose_y = var_6234_transpose_y_0, x = var_6233_cast_fp16, y = state_333_cast_fp16)[name = string("op_6234_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_333_cast_fp16 = sub(x = var_6231_cast_fp16, y = var_6234_cast_fp16)[name = string("v_new_333_cast_fp16")];
+            tensor<int32, [4]> var_6237_begin_0 = const()[name = string("op_6237_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_6237_end_0 = const()[name = string("op_6237_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_6237_end_mask_0 = const()[name = string("op_6237_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6237_squeeze_mask_0 = const()[name = string("op_6237_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6237_cast_fp16 = slice_by_index(begin = var_6237_begin_0, end = var_6237_end_0, end_mask = var_6237_end_mask_0, squeeze_mask = var_6237_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6237_cast_fp16")];
+            bool var_6238_transpose_x_0 = const()[name = string("op_6238_transpose_x_0"), val = bool(false)];
+            bool var_6238_transpose_y_0 = const()[name = string("op_6238_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6238_cast_fp16 = matmul(transpose_x = var_6238_transpose_x_0, transpose_y = var_6238_transpose_y_0, x = var_6237_cast_fp16, y = state_333_cast_fp16)[name = string("op_6238_cast_fp16")];
+            tensor<int32, [4]> var_6240_begin_0 = const()[name = string("op_6240_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_6240_end_0 = const()[name = string("op_6240_end_0"), val = tensor<int32, [4]>([32, 7, 64, 64])];
+            tensor<bool, [4]> var_6240_end_mask_0 = const()[name = string("op_6240_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6240_squeeze_mask_0 = const()[name = string("op_6240_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6240_cast_fp16 = slice_by_index(begin = var_6240_begin_0, end = var_6240_end_0, end_mask = var_6240_end_mask_0, squeeze_mask = var_6240_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6240_cast_fp16")];
+            bool var_6241_transpose_x_0 = const()[name = string("op_6241_transpose_x_0"), val = bool(false)];
+            bool var_6241_transpose_y_0 = const()[name = string("op_6241_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6241_cast_fp16 = matmul(transpose_x = var_6241_transpose_x_0, transpose_y = var_6241_transpose_y_0, x = var_6240_cast_fp16, y = v_new_333_cast_fp16)[name = string("op_6241_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6242_cast_fp16 = add(x = var_6238_cast_fp16, y = var_6241_cast_fp16)[name = string("op_6242_cast_fp16")];
+            tensor<int32, [4]> var_6244_begin_0 = const()[name = string("op_6244_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_6244_end_0 = const()[name = string("op_6244_end_0"), val = tensor<int32, [4]>([32, 7, 1, 1])];
+            tensor<bool, [4]> var_6244_end_mask_0 = const()[name = string("op_6244_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6244_squeeze_mask_0 = const()[name = string("op_6244_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6244_cast_fp16 = slice_by_index(begin = var_6244_begin_0, end = var_6244_end_0, end_mask = var_6244_end_mask_0, squeeze_mask = var_6244_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6244_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6245_cast_fp16 = mul(x = state_333_cast_fp16, y = var_6244_cast_fp16)[name = string("op_6245_cast_fp16")];
+            tensor<int32, [4]> var_6247_begin_0 = const()[name = string("op_6247_begin_0"), val = tensor<int32, [4]>([0, 6, 0, 0])];
+            tensor<int32, [4]> var_6247_end_0 = const()[name = string("op_6247_end_0"), val = tensor<int32, [4]>([32, 7, 64, 128])];
+            tensor<bool, [4]> var_6247_end_mask_0 = const()[name = string("op_6247_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6247_squeeze_mask_0 = const()[name = string("op_6247_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6247_cast_fp16 = slice_by_index(begin = var_6247_begin_0, end = var_6247_end_0, end_mask = var_6247_end_mask_0, squeeze_mask = var_6247_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6247_cast_fp16")];
+            bool var_6249_transpose_x_1 = const()[name = string("op_6249_transpose_x_1"), val = bool(true)];
+            bool var_6249_transpose_y_1 = const()[name = string("op_6249_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6249_cast_fp16 = matmul(transpose_x = var_6249_transpose_x_1, transpose_y = var_6249_transpose_y_1, x = var_6247_cast_fp16, y = v_new_333_cast_fp16)[name = string("op_6249_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_335_cast_fp16 = add(x = var_6245_cast_fp16, y = var_6249_cast_fp16)[name = string("state_335_cast_fp16")];
+            tensor<int32, [4]> var_6252_begin_0 = const()[name = string("op_6252_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_6252_end_0 = const()[name = string("op_6252_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_6252_end_mask_0 = const()[name = string("op_6252_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6252_squeeze_mask_0 = const()[name = string("op_6252_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6252_cast_fp16 = slice_by_index(begin = var_6252_begin_0, end = var_6252_end_0, end_mask = var_6252_end_mask_0, squeeze_mask = var_6252_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6252_cast_fp16")];
+            tensor<int32, [4]> var_6254_begin_0 = const()[name = string("op_6254_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_6254_end_0 = const()[name = string("op_6254_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_6254_end_mask_0 = const()[name = string("op_6254_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6254_squeeze_mask_0 = const()[name = string("op_6254_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6254_cast_fp16 = slice_by_index(begin = var_6254_begin_0, end = var_6254_end_0, end_mask = var_6254_end_mask_0, squeeze_mask = var_6254_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6254_cast_fp16")];
+            bool var_6255_transpose_x_0 = const()[name = string("op_6255_transpose_x_0"), val = bool(false)];
+            bool var_6255_transpose_y_0 = const()[name = string("op_6255_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6255_cast_fp16 = matmul(transpose_x = var_6255_transpose_x_0, transpose_y = var_6255_transpose_y_0, x = var_6254_cast_fp16, y = state_335_cast_fp16)[name = string("op_6255_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_335_cast_fp16 = sub(x = var_6252_cast_fp16, y = var_6255_cast_fp16)[name = string("v_new_335_cast_fp16")];
+            tensor<int32, [4]> var_6258_begin_0 = const()[name = string("op_6258_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_6258_end_0 = const()[name = string("op_6258_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_6258_end_mask_0 = const()[name = string("op_6258_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6258_squeeze_mask_0 = const()[name = string("op_6258_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6258_cast_fp16 = slice_by_index(begin = var_6258_begin_0, end = var_6258_end_0, end_mask = var_6258_end_mask_0, squeeze_mask = var_6258_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6258_cast_fp16")];
+            bool var_6259_transpose_x_0 = const()[name = string("op_6259_transpose_x_0"), val = bool(false)];
+            bool var_6259_transpose_y_0 = const()[name = string("op_6259_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6259_cast_fp16 = matmul(transpose_x = var_6259_transpose_x_0, transpose_y = var_6259_transpose_y_0, x = var_6258_cast_fp16, y = state_335_cast_fp16)[name = string("op_6259_cast_fp16")];
+            tensor<int32, [4]> var_6261_begin_0 = const()[name = string("op_6261_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_6261_end_0 = const()[name = string("op_6261_end_0"), val = tensor<int32, [4]>([32, 8, 64, 64])];
+            tensor<bool, [4]> var_6261_end_mask_0 = const()[name = string("op_6261_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6261_squeeze_mask_0 = const()[name = string("op_6261_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6261_cast_fp16 = slice_by_index(begin = var_6261_begin_0, end = var_6261_end_0, end_mask = var_6261_end_mask_0, squeeze_mask = var_6261_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6261_cast_fp16")];
+            bool var_6262_transpose_x_0 = const()[name = string("op_6262_transpose_x_0"), val = bool(false)];
+            bool var_6262_transpose_y_0 = const()[name = string("op_6262_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6262_cast_fp16 = matmul(transpose_x = var_6262_transpose_x_0, transpose_y = var_6262_transpose_y_0, x = var_6261_cast_fp16, y = v_new_335_cast_fp16)[name = string("op_6262_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6263_cast_fp16 = add(x = var_6259_cast_fp16, y = var_6262_cast_fp16)[name = string("op_6263_cast_fp16")];
+            tensor<int32, [4]> var_6265_begin_0 = const()[name = string("op_6265_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_6265_end_0 = const()[name = string("op_6265_end_0"), val = tensor<int32, [4]>([32, 8, 1, 1])];
+            tensor<bool, [4]> var_6265_end_mask_0 = const()[name = string("op_6265_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6265_squeeze_mask_0 = const()[name = string("op_6265_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6265_cast_fp16 = slice_by_index(begin = var_6265_begin_0, end = var_6265_end_0, end_mask = var_6265_end_mask_0, squeeze_mask = var_6265_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6265_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6266_cast_fp16 = mul(x = state_335_cast_fp16, y = var_6265_cast_fp16)[name = string("op_6266_cast_fp16")];
+            tensor<int32, [4]> var_6268_begin_0 = const()[name = string("op_6268_begin_0"), val = tensor<int32, [4]>([0, 7, 0, 0])];
+            tensor<int32, [4]> var_6268_end_0 = const()[name = string("op_6268_end_0"), val = tensor<int32, [4]>([32, 8, 64, 128])];
+            tensor<bool, [4]> var_6268_end_mask_0 = const()[name = string("op_6268_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6268_squeeze_mask_0 = const()[name = string("op_6268_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6268_cast_fp16 = slice_by_index(begin = var_6268_begin_0, end = var_6268_end_0, end_mask = var_6268_end_mask_0, squeeze_mask = var_6268_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6268_cast_fp16")];
+            bool var_6270_transpose_x_1 = const()[name = string("op_6270_transpose_x_1"), val = bool(true)];
+            bool var_6270_transpose_y_1 = const()[name = string("op_6270_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6270_cast_fp16 = matmul(transpose_x = var_6270_transpose_x_1, transpose_y = var_6270_transpose_y_1, x = var_6268_cast_fp16, y = v_new_335_cast_fp16)[name = string("op_6270_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_337_cast_fp16 = add(x = var_6266_cast_fp16, y = var_6270_cast_fp16)[name = string("state_337_cast_fp16")];
+            tensor<int32, [4]> var_6273_begin_0 = const()[name = string("op_6273_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_6273_end_0 = const()[name = string("op_6273_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_6273_end_mask_0 = const()[name = string("op_6273_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6273_squeeze_mask_0 = const()[name = string("op_6273_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6273_cast_fp16 = slice_by_index(begin = var_6273_begin_0, end = var_6273_end_0, end_mask = var_6273_end_mask_0, squeeze_mask = var_6273_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6273_cast_fp16")];
+            tensor<int32, [4]> var_6275_begin_0 = const()[name = string("op_6275_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_6275_end_0 = const()[name = string("op_6275_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_6275_end_mask_0 = const()[name = string("op_6275_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6275_squeeze_mask_0 = const()[name = string("op_6275_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6275_cast_fp16 = slice_by_index(begin = var_6275_begin_0, end = var_6275_end_0, end_mask = var_6275_end_mask_0, squeeze_mask = var_6275_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6275_cast_fp16")];
+            bool var_6276_transpose_x_0 = const()[name = string("op_6276_transpose_x_0"), val = bool(false)];
+            bool var_6276_transpose_y_0 = const()[name = string("op_6276_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6276_cast_fp16 = matmul(transpose_x = var_6276_transpose_x_0, transpose_y = var_6276_transpose_y_0, x = var_6275_cast_fp16, y = state_337_cast_fp16)[name = string("op_6276_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_337_cast_fp16 = sub(x = var_6273_cast_fp16, y = var_6276_cast_fp16)[name = string("v_new_337_cast_fp16")];
+            tensor<int32, [4]> var_6279_begin_0 = const()[name = string("op_6279_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_6279_end_0 = const()[name = string("op_6279_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_6279_end_mask_0 = const()[name = string("op_6279_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6279_squeeze_mask_0 = const()[name = string("op_6279_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6279_cast_fp16 = slice_by_index(begin = var_6279_begin_0, end = var_6279_end_0, end_mask = var_6279_end_mask_0, squeeze_mask = var_6279_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6279_cast_fp16")];
+            bool var_6280_transpose_x_0 = const()[name = string("op_6280_transpose_x_0"), val = bool(false)];
+            bool var_6280_transpose_y_0 = const()[name = string("op_6280_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6280_cast_fp16 = matmul(transpose_x = var_6280_transpose_x_0, transpose_y = var_6280_transpose_y_0, x = var_6279_cast_fp16, y = state_337_cast_fp16)[name = string("op_6280_cast_fp16")];
+            tensor<int32, [4]> var_6282_begin_0 = const()[name = string("op_6282_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_6282_end_0 = const()[name = string("op_6282_end_0"), val = tensor<int32, [4]>([32, 9, 64, 64])];
+            tensor<bool, [4]> var_6282_end_mask_0 = const()[name = string("op_6282_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6282_squeeze_mask_0 = const()[name = string("op_6282_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6282_cast_fp16 = slice_by_index(begin = var_6282_begin_0, end = var_6282_end_0, end_mask = var_6282_end_mask_0, squeeze_mask = var_6282_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6282_cast_fp16")];
+            bool var_6283_transpose_x_0 = const()[name = string("op_6283_transpose_x_0"), val = bool(false)];
+            bool var_6283_transpose_y_0 = const()[name = string("op_6283_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6283_cast_fp16 = matmul(transpose_x = var_6283_transpose_x_0, transpose_y = var_6283_transpose_y_0, x = var_6282_cast_fp16, y = v_new_337_cast_fp16)[name = string("op_6283_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6284_cast_fp16 = add(x = var_6280_cast_fp16, y = var_6283_cast_fp16)[name = string("op_6284_cast_fp16")];
+            tensor<int32, [4]> var_6286_begin_0 = const()[name = string("op_6286_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_6286_end_0 = const()[name = string("op_6286_end_0"), val = tensor<int32, [4]>([32, 9, 1, 1])];
+            tensor<bool, [4]> var_6286_end_mask_0 = const()[name = string("op_6286_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6286_squeeze_mask_0 = const()[name = string("op_6286_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6286_cast_fp16 = slice_by_index(begin = var_6286_begin_0, end = var_6286_end_0, end_mask = var_6286_end_mask_0, squeeze_mask = var_6286_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6286_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6287_cast_fp16 = mul(x = state_337_cast_fp16, y = var_6286_cast_fp16)[name = string("op_6287_cast_fp16")];
+            tensor<int32, [4]> var_6289_begin_0 = const()[name = string("op_6289_begin_0"), val = tensor<int32, [4]>([0, 8, 0, 0])];
+            tensor<int32, [4]> var_6289_end_0 = const()[name = string("op_6289_end_0"), val = tensor<int32, [4]>([32, 9, 64, 128])];
+            tensor<bool, [4]> var_6289_end_mask_0 = const()[name = string("op_6289_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6289_squeeze_mask_0 = const()[name = string("op_6289_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6289_cast_fp16 = slice_by_index(begin = var_6289_begin_0, end = var_6289_end_0, end_mask = var_6289_end_mask_0, squeeze_mask = var_6289_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6289_cast_fp16")];
+            bool var_6291_transpose_x_1 = const()[name = string("op_6291_transpose_x_1"), val = bool(true)];
+            bool var_6291_transpose_y_1 = const()[name = string("op_6291_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6291_cast_fp16 = matmul(transpose_x = var_6291_transpose_x_1, transpose_y = var_6291_transpose_y_1, x = var_6289_cast_fp16, y = v_new_337_cast_fp16)[name = string("op_6291_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_339_cast_fp16 = add(x = var_6287_cast_fp16, y = var_6291_cast_fp16)[name = string("state_339_cast_fp16")];
+            tensor<int32, [4]> var_6294_begin_0 = const()[name = string("op_6294_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_6294_end_0 = const()[name = string("op_6294_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_6294_end_mask_0 = const()[name = string("op_6294_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6294_squeeze_mask_0 = const()[name = string("op_6294_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6294_cast_fp16 = slice_by_index(begin = var_6294_begin_0, end = var_6294_end_0, end_mask = var_6294_end_mask_0, squeeze_mask = var_6294_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6294_cast_fp16")];
+            tensor<int32, [4]> var_6296_begin_0 = const()[name = string("op_6296_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_6296_end_0 = const()[name = string("op_6296_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_6296_end_mask_0 = const()[name = string("op_6296_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6296_squeeze_mask_0 = const()[name = string("op_6296_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6296_cast_fp16 = slice_by_index(begin = var_6296_begin_0, end = var_6296_end_0, end_mask = var_6296_end_mask_0, squeeze_mask = var_6296_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6296_cast_fp16")];
+            bool var_6297_transpose_x_0 = const()[name = string("op_6297_transpose_x_0"), val = bool(false)];
+            bool var_6297_transpose_y_0 = const()[name = string("op_6297_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6297_cast_fp16 = matmul(transpose_x = var_6297_transpose_x_0, transpose_y = var_6297_transpose_y_0, x = var_6296_cast_fp16, y = state_339_cast_fp16)[name = string("op_6297_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_339_cast_fp16 = sub(x = var_6294_cast_fp16, y = var_6297_cast_fp16)[name = string("v_new_339_cast_fp16")];
+            tensor<int32, [4]> var_6300_begin_0 = const()[name = string("op_6300_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_6300_end_0 = const()[name = string("op_6300_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_6300_end_mask_0 = const()[name = string("op_6300_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6300_squeeze_mask_0 = const()[name = string("op_6300_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6300_cast_fp16 = slice_by_index(begin = var_6300_begin_0, end = var_6300_end_0, end_mask = var_6300_end_mask_0, squeeze_mask = var_6300_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6300_cast_fp16")];
+            bool var_6301_transpose_x_0 = const()[name = string("op_6301_transpose_x_0"), val = bool(false)];
+            bool var_6301_transpose_y_0 = const()[name = string("op_6301_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6301_cast_fp16 = matmul(transpose_x = var_6301_transpose_x_0, transpose_y = var_6301_transpose_y_0, x = var_6300_cast_fp16, y = state_339_cast_fp16)[name = string("op_6301_cast_fp16")];
+            tensor<int32, [4]> var_6303_begin_0 = const()[name = string("op_6303_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_6303_end_0 = const()[name = string("op_6303_end_0"), val = tensor<int32, [4]>([32, 10, 64, 64])];
+            tensor<bool, [4]> var_6303_end_mask_0 = const()[name = string("op_6303_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6303_squeeze_mask_0 = const()[name = string("op_6303_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6303_cast_fp16 = slice_by_index(begin = var_6303_begin_0, end = var_6303_end_0, end_mask = var_6303_end_mask_0, squeeze_mask = var_6303_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6303_cast_fp16")];
+            bool var_6304_transpose_x_0 = const()[name = string("op_6304_transpose_x_0"), val = bool(false)];
+            bool var_6304_transpose_y_0 = const()[name = string("op_6304_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6304_cast_fp16 = matmul(transpose_x = var_6304_transpose_x_0, transpose_y = var_6304_transpose_y_0, x = var_6303_cast_fp16, y = v_new_339_cast_fp16)[name = string("op_6304_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6305_cast_fp16 = add(x = var_6301_cast_fp16, y = var_6304_cast_fp16)[name = string("op_6305_cast_fp16")];
+            tensor<int32, [4]> var_6307_begin_0 = const()[name = string("op_6307_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_6307_end_0 = const()[name = string("op_6307_end_0"), val = tensor<int32, [4]>([32, 10, 1, 1])];
+            tensor<bool, [4]> var_6307_end_mask_0 = const()[name = string("op_6307_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6307_squeeze_mask_0 = const()[name = string("op_6307_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6307_cast_fp16 = slice_by_index(begin = var_6307_begin_0, end = var_6307_end_0, end_mask = var_6307_end_mask_0, squeeze_mask = var_6307_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6307_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6308_cast_fp16 = mul(x = state_339_cast_fp16, y = var_6307_cast_fp16)[name = string("op_6308_cast_fp16")];
+            tensor<int32, [4]> var_6310_begin_0 = const()[name = string("op_6310_begin_0"), val = tensor<int32, [4]>([0, 9, 0, 0])];
+            tensor<int32, [4]> var_6310_end_0 = const()[name = string("op_6310_end_0"), val = tensor<int32, [4]>([32, 10, 64, 128])];
+            tensor<bool, [4]> var_6310_end_mask_0 = const()[name = string("op_6310_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6310_squeeze_mask_0 = const()[name = string("op_6310_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6310_cast_fp16 = slice_by_index(begin = var_6310_begin_0, end = var_6310_end_0, end_mask = var_6310_end_mask_0, squeeze_mask = var_6310_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6310_cast_fp16")];
+            bool var_6312_transpose_x_1 = const()[name = string("op_6312_transpose_x_1"), val = bool(true)];
+            bool var_6312_transpose_y_1 = const()[name = string("op_6312_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6312_cast_fp16 = matmul(transpose_x = var_6312_transpose_x_1, transpose_y = var_6312_transpose_y_1, x = var_6310_cast_fp16, y = v_new_339_cast_fp16)[name = string("op_6312_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_341_cast_fp16 = add(x = var_6308_cast_fp16, y = var_6312_cast_fp16)[name = string("state_341_cast_fp16")];
+            tensor<int32, [4]> var_6315_begin_0 = const()[name = string("op_6315_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_6315_end_0 = const()[name = string("op_6315_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_6315_end_mask_0 = const()[name = string("op_6315_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6315_squeeze_mask_0 = const()[name = string("op_6315_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6315_cast_fp16 = slice_by_index(begin = var_6315_begin_0, end = var_6315_end_0, end_mask = var_6315_end_mask_0, squeeze_mask = var_6315_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6315_cast_fp16")];
+            tensor<int32, [4]> var_6317_begin_0 = const()[name = string("op_6317_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_6317_end_0 = const()[name = string("op_6317_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_6317_end_mask_0 = const()[name = string("op_6317_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6317_squeeze_mask_0 = const()[name = string("op_6317_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6317_cast_fp16 = slice_by_index(begin = var_6317_begin_0, end = var_6317_end_0, end_mask = var_6317_end_mask_0, squeeze_mask = var_6317_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6317_cast_fp16")];
+            bool var_6318_transpose_x_0 = const()[name = string("op_6318_transpose_x_0"), val = bool(false)];
+            bool var_6318_transpose_y_0 = const()[name = string("op_6318_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6318_cast_fp16 = matmul(transpose_x = var_6318_transpose_x_0, transpose_y = var_6318_transpose_y_0, x = var_6317_cast_fp16, y = state_341_cast_fp16)[name = string("op_6318_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_341_cast_fp16 = sub(x = var_6315_cast_fp16, y = var_6318_cast_fp16)[name = string("v_new_341_cast_fp16")];
+            tensor<int32, [4]> var_6321_begin_0 = const()[name = string("op_6321_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_6321_end_0 = const()[name = string("op_6321_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_6321_end_mask_0 = const()[name = string("op_6321_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6321_squeeze_mask_0 = const()[name = string("op_6321_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6321_cast_fp16 = slice_by_index(begin = var_6321_begin_0, end = var_6321_end_0, end_mask = var_6321_end_mask_0, squeeze_mask = var_6321_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6321_cast_fp16")];
+            bool var_6322_transpose_x_0 = const()[name = string("op_6322_transpose_x_0"), val = bool(false)];
+            bool var_6322_transpose_y_0 = const()[name = string("op_6322_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6322_cast_fp16 = matmul(transpose_x = var_6322_transpose_x_0, transpose_y = var_6322_transpose_y_0, x = var_6321_cast_fp16, y = state_341_cast_fp16)[name = string("op_6322_cast_fp16")];
+            tensor<int32, [4]> var_6324_begin_0 = const()[name = string("op_6324_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_6324_end_0 = const()[name = string("op_6324_end_0"), val = tensor<int32, [4]>([32, 11, 64, 64])];
+            tensor<bool, [4]> var_6324_end_mask_0 = const()[name = string("op_6324_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6324_squeeze_mask_0 = const()[name = string("op_6324_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6324_cast_fp16 = slice_by_index(begin = var_6324_begin_0, end = var_6324_end_0, end_mask = var_6324_end_mask_0, squeeze_mask = var_6324_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6324_cast_fp16")];
+            bool var_6325_transpose_x_0 = const()[name = string("op_6325_transpose_x_0"), val = bool(false)];
+            bool var_6325_transpose_y_0 = const()[name = string("op_6325_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6325_cast_fp16 = matmul(transpose_x = var_6325_transpose_x_0, transpose_y = var_6325_transpose_y_0, x = var_6324_cast_fp16, y = v_new_341_cast_fp16)[name = string("op_6325_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6326_cast_fp16 = add(x = var_6322_cast_fp16, y = var_6325_cast_fp16)[name = string("op_6326_cast_fp16")];
+            tensor<int32, [4]> var_6328_begin_0 = const()[name = string("op_6328_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_6328_end_0 = const()[name = string("op_6328_end_0"), val = tensor<int32, [4]>([32, 11, 1, 1])];
+            tensor<bool, [4]> var_6328_end_mask_0 = const()[name = string("op_6328_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6328_squeeze_mask_0 = const()[name = string("op_6328_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6328_cast_fp16 = slice_by_index(begin = var_6328_begin_0, end = var_6328_end_0, end_mask = var_6328_end_mask_0, squeeze_mask = var_6328_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6328_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6329_cast_fp16 = mul(x = state_341_cast_fp16, y = var_6328_cast_fp16)[name = string("op_6329_cast_fp16")];
+            tensor<int32, [4]> var_6331_begin_0 = const()[name = string("op_6331_begin_0"), val = tensor<int32, [4]>([0, 10, 0, 0])];
+            tensor<int32, [4]> var_6331_end_0 = const()[name = string("op_6331_end_0"), val = tensor<int32, [4]>([32, 11, 64, 128])];
+            tensor<bool, [4]> var_6331_end_mask_0 = const()[name = string("op_6331_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6331_squeeze_mask_0 = const()[name = string("op_6331_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6331_cast_fp16 = slice_by_index(begin = var_6331_begin_0, end = var_6331_end_0, end_mask = var_6331_end_mask_0, squeeze_mask = var_6331_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6331_cast_fp16")];
+            bool var_6333_transpose_x_1 = const()[name = string("op_6333_transpose_x_1"), val = bool(true)];
+            bool var_6333_transpose_y_1 = const()[name = string("op_6333_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6333_cast_fp16 = matmul(transpose_x = var_6333_transpose_x_1, transpose_y = var_6333_transpose_y_1, x = var_6331_cast_fp16, y = v_new_341_cast_fp16)[name = string("op_6333_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_343_cast_fp16 = add(x = var_6329_cast_fp16, y = var_6333_cast_fp16)[name = string("state_343_cast_fp16")];
+            tensor<int32, [4]> var_6336_begin_0 = const()[name = string("op_6336_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_6336_end_0 = const()[name = string("op_6336_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_6336_end_mask_0 = const()[name = string("op_6336_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6336_squeeze_mask_0 = const()[name = string("op_6336_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6336_cast_fp16 = slice_by_index(begin = var_6336_begin_0, end = var_6336_end_0, end_mask = var_6336_end_mask_0, squeeze_mask = var_6336_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6336_cast_fp16")];
+            tensor<int32, [4]> var_6338_begin_0 = const()[name = string("op_6338_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_6338_end_0 = const()[name = string("op_6338_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_6338_end_mask_0 = const()[name = string("op_6338_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6338_squeeze_mask_0 = const()[name = string("op_6338_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6338_cast_fp16 = slice_by_index(begin = var_6338_begin_0, end = var_6338_end_0, end_mask = var_6338_end_mask_0, squeeze_mask = var_6338_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6338_cast_fp16")];
+            bool var_6339_transpose_x_0 = const()[name = string("op_6339_transpose_x_0"), val = bool(false)];
+            bool var_6339_transpose_y_0 = const()[name = string("op_6339_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6339_cast_fp16 = matmul(transpose_x = var_6339_transpose_x_0, transpose_y = var_6339_transpose_y_0, x = var_6338_cast_fp16, y = state_343_cast_fp16)[name = string("op_6339_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_343_cast_fp16 = sub(x = var_6336_cast_fp16, y = var_6339_cast_fp16)[name = string("v_new_343_cast_fp16")];
+            tensor<int32, [4]> var_6342_begin_0 = const()[name = string("op_6342_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_6342_end_0 = const()[name = string("op_6342_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_6342_end_mask_0 = const()[name = string("op_6342_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6342_squeeze_mask_0 = const()[name = string("op_6342_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6342_cast_fp16 = slice_by_index(begin = var_6342_begin_0, end = var_6342_end_0, end_mask = var_6342_end_mask_0, squeeze_mask = var_6342_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6342_cast_fp16")];
+            bool var_6343_transpose_x_0 = const()[name = string("op_6343_transpose_x_0"), val = bool(false)];
+            bool var_6343_transpose_y_0 = const()[name = string("op_6343_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6343_cast_fp16 = matmul(transpose_x = var_6343_transpose_x_0, transpose_y = var_6343_transpose_y_0, x = var_6342_cast_fp16, y = state_343_cast_fp16)[name = string("op_6343_cast_fp16")];
+            tensor<int32, [4]> var_6345_begin_0 = const()[name = string("op_6345_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_6345_end_0 = const()[name = string("op_6345_end_0"), val = tensor<int32, [4]>([32, 12, 64, 64])];
+            tensor<bool, [4]> var_6345_end_mask_0 = const()[name = string("op_6345_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6345_squeeze_mask_0 = const()[name = string("op_6345_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6345_cast_fp16 = slice_by_index(begin = var_6345_begin_0, end = var_6345_end_0, end_mask = var_6345_end_mask_0, squeeze_mask = var_6345_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6345_cast_fp16")];
+            bool var_6346_transpose_x_0 = const()[name = string("op_6346_transpose_x_0"), val = bool(false)];
+            bool var_6346_transpose_y_0 = const()[name = string("op_6346_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6346_cast_fp16 = matmul(transpose_x = var_6346_transpose_x_0, transpose_y = var_6346_transpose_y_0, x = var_6345_cast_fp16, y = v_new_343_cast_fp16)[name = string("op_6346_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6347_cast_fp16 = add(x = var_6343_cast_fp16, y = var_6346_cast_fp16)[name = string("op_6347_cast_fp16")];
+            tensor<int32, [4]> var_6349_begin_0 = const()[name = string("op_6349_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_6349_end_0 = const()[name = string("op_6349_end_0"), val = tensor<int32, [4]>([32, 12, 1, 1])];
+            tensor<bool, [4]> var_6349_end_mask_0 = const()[name = string("op_6349_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6349_squeeze_mask_0 = const()[name = string("op_6349_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6349_cast_fp16 = slice_by_index(begin = var_6349_begin_0, end = var_6349_end_0, end_mask = var_6349_end_mask_0, squeeze_mask = var_6349_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6349_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6350_cast_fp16 = mul(x = state_343_cast_fp16, y = var_6349_cast_fp16)[name = string("op_6350_cast_fp16")];
+            tensor<int32, [4]> var_6352_begin_0 = const()[name = string("op_6352_begin_0"), val = tensor<int32, [4]>([0, 11, 0, 0])];
+            tensor<int32, [4]> var_6352_end_0 = const()[name = string("op_6352_end_0"), val = tensor<int32, [4]>([32, 12, 64, 128])];
+            tensor<bool, [4]> var_6352_end_mask_0 = const()[name = string("op_6352_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6352_squeeze_mask_0 = const()[name = string("op_6352_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6352_cast_fp16 = slice_by_index(begin = var_6352_begin_0, end = var_6352_end_0, end_mask = var_6352_end_mask_0, squeeze_mask = var_6352_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6352_cast_fp16")];
+            bool var_6354_transpose_x_1 = const()[name = string("op_6354_transpose_x_1"), val = bool(true)];
+            bool var_6354_transpose_y_1 = const()[name = string("op_6354_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6354_cast_fp16 = matmul(transpose_x = var_6354_transpose_x_1, transpose_y = var_6354_transpose_y_1, x = var_6352_cast_fp16, y = v_new_343_cast_fp16)[name = string("op_6354_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_345_cast_fp16 = add(x = var_6350_cast_fp16, y = var_6354_cast_fp16)[name = string("state_345_cast_fp16")];
+            tensor<int32, [4]> var_6357_begin_0 = const()[name = string("op_6357_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_6357_end_0 = const()[name = string("op_6357_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_6357_end_mask_0 = const()[name = string("op_6357_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6357_squeeze_mask_0 = const()[name = string("op_6357_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6357_cast_fp16 = slice_by_index(begin = var_6357_begin_0, end = var_6357_end_0, end_mask = var_6357_end_mask_0, squeeze_mask = var_6357_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6357_cast_fp16")];
+            tensor<int32, [4]> var_6359_begin_0 = const()[name = string("op_6359_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_6359_end_0 = const()[name = string("op_6359_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_6359_end_mask_0 = const()[name = string("op_6359_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6359_squeeze_mask_0 = const()[name = string("op_6359_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6359_cast_fp16 = slice_by_index(begin = var_6359_begin_0, end = var_6359_end_0, end_mask = var_6359_end_mask_0, squeeze_mask = var_6359_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6359_cast_fp16")];
+            bool var_6360_transpose_x_0 = const()[name = string("op_6360_transpose_x_0"), val = bool(false)];
+            bool var_6360_transpose_y_0 = const()[name = string("op_6360_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6360_cast_fp16 = matmul(transpose_x = var_6360_transpose_x_0, transpose_y = var_6360_transpose_y_0, x = var_6359_cast_fp16, y = state_345_cast_fp16)[name = string("op_6360_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_345_cast_fp16 = sub(x = var_6357_cast_fp16, y = var_6360_cast_fp16)[name = string("v_new_345_cast_fp16")];
+            tensor<int32, [4]> var_6363_begin_0 = const()[name = string("op_6363_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_6363_end_0 = const()[name = string("op_6363_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_6363_end_mask_0 = const()[name = string("op_6363_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6363_squeeze_mask_0 = const()[name = string("op_6363_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6363_cast_fp16 = slice_by_index(begin = var_6363_begin_0, end = var_6363_end_0, end_mask = var_6363_end_mask_0, squeeze_mask = var_6363_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6363_cast_fp16")];
+            bool var_6364_transpose_x_0 = const()[name = string("op_6364_transpose_x_0"), val = bool(false)];
+            bool var_6364_transpose_y_0 = const()[name = string("op_6364_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6364_cast_fp16 = matmul(transpose_x = var_6364_transpose_x_0, transpose_y = var_6364_transpose_y_0, x = var_6363_cast_fp16, y = state_345_cast_fp16)[name = string("op_6364_cast_fp16")];
+            tensor<int32, [4]> var_6366_begin_0 = const()[name = string("op_6366_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_6366_end_0 = const()[name = string("op_6366_end_0"), val = tensor<int32, [4]>([32, 13, 64, 64])];
+            tensor<bool, [4]> var_6366_end_mask_0 = const()[name = string("op_6366_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6366_squeeze_mask_0 = const()[name = string("op_6366_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6366_cast_fp16 = slice_by_index(begin = var_6366_begin_0, end = var_6366_end_0, end_mask = var_6366_end_mask_0, squeeze_mask = var_6366_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6366_cast_fp16")];
+            bool var_6367_transpose_x_0 = const()[name = string("op_6367_transpose_x_0"), val = bool(false)];
+            bool var_6367_transpose_y_0 = const()[name = string("op_6367_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6367_cast_fp16 = matmul(transpose_x = var_6367_transpose_x_0, transpose_y = var_6367_transpose_y_0, x = var_6366_cast_fp16, y = v_new_345_cast_fp16)[name = string("op_6367_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6368_cast_fp16 = add(x = var_6364_cast_fp16, y = var_6367_cast_fp16)[name = string("op_6368_cast_fp16")];
+            tensor<int32, [4]> var_6370_begin_0 = const()[name = string("op_6370_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_6370_end_0 = const()[name = string("op_6370_end_0"), val = tensor<int32, [4]>([32, 13, 1, 1])];
+            tensor<bool, [4]> var_6370_end_mask_0 = const()[name = string("op_6370_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6370_squeeze_mask_0 = const()[name = string("op_6370_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6370_cast_fp16 = slice_by_index(begin = var_6370_begin_0, end = var_6370_end_0, end_mask = var_6370_end_mask_0, squeeze_mask = var_6370_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6370_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6371_cast_fp16 = mul(x = state_345_cast_fp16, y = var_6370_cast_fp16)[name = string("op_6371_cast_fp16")];
+            tensor<int32, [4]> var_6373_begin_0 = const()[name = string("op_6373_begin_0"), val = tensor<int32, [4]>([0, 12, 0, 0])];
+            tensor<int32, [4]> var_6373_end_0 = const()[name = string("op_6373_end_0"), val = tensor<int32, [4]>([32, 13, 64, 128])];
+            tensor<bool, [4]> var_6373_end_mask_0 = const()[name = string("op_6373_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6373_squeeze_mask_0 = const()[name = string("op_6373_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6373_cast_fp16 = slice_by_index(begin = var_6373_begin_0, end = var_6373_end_0, end_mask = var_6373_end_mask_0, squeeze_mask = var_6373_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6373_cast_fp16")];
+            bool var_6375_transpose_x_1 = const()[name = string("op_6375_transpose_x_1"), val = bool(true)];
+            bool var_6375_transpose_y_1 = const()[name = string("op_6375_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6375_cast_fp16 = matmul(transpose_x = var_6375_transpose_x_1, transpose_y = var_6375_transpose_y_1, x = var_6373_cast_fp16, y = v_new_345_cast_fp16)[name = string("op_6375_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_347_cast_fp16 = add(x = var_6371_cast_fp16, y = var_6375_cast_fp16)[name = string("state_347_cast_fp16")];
+            tensor<int32, [4]> var_6378_begin_0 = const()[name = string("op_6378_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_6378_end_0 = const()[name = string("op_6378_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_6378_end_mask_0 = const()[name = string("op_6378_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6378_squeeze_mask_0 = const()[name = string("op_6378_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6378_cast_fp16 = slice_by_index(begin = var_6378_begin_0, end = var_6378_end_0, end_mask = var_6378_end_mask_0, squeeze_mask = var_6378_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6378_cast_fp16")];
+            tensor<int32, [4]> var_6380_begin_0 = const()[name = string("op_6380_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_6380_end_0 = const()[name = string("op_6380_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_6380_end_mask_0 = const()[name = string("op_6380_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6380_squeeze_mask_0 = const()[name = string("op_6380_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6380_cast_fp16 = slice_by_index(begin = var_6380_begin_0, end = var_6380_end_0, end_mask = var_6380_end_mask_0, squeeze_mask = var_6380_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6380_cast_fp16")];
+            bool var_6381_transpose_x_0 = const()[name = string("op_6381_transpose_x_0"), val = bool(false)];
+            bool var_6381_transpose_y_0 = const()[name = string("op_6381_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6381_cast_fp16 = matmul(transpose_x = var_6381_transpose_x_0, transpose_y = var_6381_transpose_y_0, x = var_6380_cast_fp16, y = state_347_cast_fp16)[name = string("op_6381_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_347_cast_fp16 = sub(x = var_6378_cast_fp16, y = var_6381_cast_fp16)[name = string("v_new_347_cast_fp16")];
+            tensor<int32, [4]> var_6384_begin_0 = const()[name = string("op_6384_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_6384_end_0 = const()[name = string("op_6384_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_6384_end_mask_0 = const()[name = string("op_6384_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6384_squeeze_mask_0 = const()[name = string("op_6384_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6384_cast_fp16 = slice_by_index(begin = var_6384_begin_0, end = var_6384_end_0, end_mask = var_6384_end_mask_0, squeeze_mask = var_6384_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6384_cast_fp16")];
+            bool var_6385_transpose_x_0 = const()[name = string("op_6385_transpose_x_0"), val = bool(false)];
+            bool var_6385_transpose_y_0 = const()[name = string("op_6385_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6385_cast_fp16 = matmul(transpose_x = var_6385_transpose_x_0, transpose_y = var_6385_transpose_y_0, x = var_6384_cast_fp16, y = state_347_cast_fp16)[name = string("op_6385_cast_fp16")];
+            tensor<int32, [4]> var_6387_begin_0 = const()[name = string("op_6387_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_6387_end_0 = const()[name = string("op_6387_end_0"), val = tensor<int32, [4]>([32, 14, 64, 64])];
+            tensor<bool, [4]> var_6387_end_mask_0 = const()[name = string("op_6387_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6387_squeeze_mask_0 = const()[name = string("op_6387_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6387_cast_fp16 = slice_by_index(begin = var_6387_begin_0, end = var_6387_end_0, end_mask = var_6387_end_mask_0, squeeze_mask = var_6387_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6387_cast_fp16")];
+            bool var_6388_transpose_x_0 = const()[name = string("op_6388_transpose_x_0"), val = bool(false)];
+            bool var_6388_transpose_y_0 = const()[name = string("op_6388_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6388_cast_fp16 = matmul(transpose_x = var_6388_transpose_x_0, transpose_y = var_6388_transpose_y_0, x = var_6387_cast_fp16, y = v_new_347_cast_fp16)[name = string("op_6388_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6389_cast_fp16 = add(x = var_6385_cast_fp16, y = var_6388_cast_fp16)[name = string("op_6389_cast_fp16")];
+            tensor<int32, [4]> var_6391_begin_0 = const()[name = string("op_6391_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_6391_end_0 = const()[name = string("op_6391_end_0"), val = tensor<int32, [4]>([32, 14, 1, 1])];
+            tensor<bool, [4]> var_6391_end_mask_0 = const()[name = string("op_6391_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6391_squeeze_mask_0 = const()[name = string("op_6391_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6391_cast_fp16 = slice_by_index(begin = var_6391_begin_0, end = var_6391_end_0, end_mask = var_6391_end_mask_0, squeeze_mask = var_6391_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6391_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6392_cast_fp16 = mul(x = state_347_cast_fp16, y = var_6391_cast_fp16)[name = string("op_6392_cast_fp16")];
+            tensor<int32, [4]> var_6394_begin_0 = const()[name = string("op_6394_begin_0"), val = tensor<int32, [4]>([0, 13, 0, 0])];
+            tensor<int32, [4]> var_6394_end_0 = const()[name = string("op_6394_end_0"), val = tensor<int32, [4]>([32, 14, 64, 128])];
+            tensor<bool, [4]> var_6394_end_mask_0 = const()[name = string("op_6394_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6394_squeeze_mask_0 = const()[name = string("op_6394_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6394_cast_fp16 = slice_by_index(begin = var_6394_begin_0, end = var_6394_end_0, end_mask = var_6394_end_mask_0, squeeze_mask = var_6394_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6394_cast_fp16")];
+            bool var_6396_transpose_x_1 = const()[name = string("op_6396_transpose_x_1"), val = bool(true)];
+            bool var_6396_transpose_y_1 = const()[name = string("op_6396_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6396_cast_fp16 = matmul(transpose_x = var_6396_transpose_x_1, transpose_y = var_6396_transpose_y_1, x = var_6394_cast_fp16, y = v_new_347_cast_fp16)[name = string("op_6396_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_349_cast_fp16 = add(x = var_6392_cast_fp16, y = var_6396_cast_fp16)[name = string("state_349_cast_fp16")];
+            tensor<int32, [4]> var_6399_begin_0 = const()[name = string("op_6399_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_6399_end_0 = const()[name = string("op_6399_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_6399_end_mask_0 = const()[name = string("op_6399_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6399_squeeze_mask_0 = const()[name = string("op_6399_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6399_cast_fp16 = slice_by_index(begin = var_6399_begin_0, end = var_6399_end_0, end_mask = var_6399_end_mask_0, squeeze_mask = var_6399_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6399_cast_fp16")];
+            tensor<int32, [4]> var_6401_begin_0 = const()[name = string("op_6401_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_6401_end_0 = const()[name = string("op_6401_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_6401_end_mask_0 = const()[name = string("op_6401_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6401_squeeze_mask_0 = const()[name = string("op_6401_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6401_cast_fp16 = slice_by_index(begin = var_6401_begin_0, end = var_6401_end_0, end_mask = var_6401_end_mask_0, squeeze_mask = var_6401_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6401_cast_fp16")];
+            bool var_6402_transpose_x_0 = const()[name = string("op_6402_transpose_x_0"), val = bool(false)];
+            bool var_6402_transpose_y_0 = const()[name = string("op_6402_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6402_cast_fp16 = matmul(transpose_x = var_6402_transpose_x_0, transpose_y = var_6402_transpose_y_0, x = var_6401_cast_fp16, y = state_349_cast_fp16)[name = string("op_6402_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_349_cast_fp16 = sub(x = var_6399_cast_fp16, y = var_6402_cast_fp16)[name = string("v_new_349_cast_fp16")];
+            tensor<int32, [4]> var_6405_begin_0 = const()[name = string("op_6405_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_6405_end_0 = const()[name = string("op_6405_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_6405_end_mask_0 = const()[name = string("op_6405_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6405_squeeze_mask_0 = const()[name = string("op_6405_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6405_cast_fp16 = slice_by_index(begin = var_6405_begin_0, end = var_6405_end_0, end_mask = var_6405_end_mask_0, squeeze_mask = var_6405_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6405_cast_fp16")];
+            bool var_6406_transpose_x_0 = const()[name = string("op_6406_transpose_x_0"), val = bool(false)];
+            bool var_6406_transpose_y_0 = const()[name = string("op_6406_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6406_cast_fp16 = matmul(transpose_x = var_6406_transpose_x_0, transpose_y = var_6406_transpose_y_0, x = var_6405_cast_fp16, y = state_349_cast_fp16)[name = string("op_6406_cast_fp16")];
+            tensor<int32, [4]> var_6408_begin_0 = const()[name = string("op_6408_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_6408_end_0 = const()[name = string("op_6408_end_0"), val = tensor<int32, [4]>([32, 15, 64, 64])];
+            tensor<bool, [4]> var_6408_end_mask_0 = const()[name = string("op_6408_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6408_squeeze_mask_0 = const()[name = string("op_6408_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6408_cast_fp16 = slice_by_index(begin = var_6408_begin_0, end = var_6408_end_0, end_mask = var_6408_end_mask_0, squeeze_mask = var_6408_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6408_cast_fp16")];
+            bool var_6409_transpose_x_0 = const()[name = string("op_6409_transpose_x_0"), val = bool(false)];
+            bool var_6409_transpose_y_0 = const()[name = string("op_6409_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6409_cast_fp16 = matmul(transpose_x = var_6409_transpose_x_0, transpose_y = var_6409_transpose_y_0, x = var_6408_cast_fp16, y = v_new_349_cast_fp16)[name = string("op_6409_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6410_cast_fp16 = add(x = var_6406_cast_fp16, y = var_6409_cast_fp16)[name = string("op_6410_cast_fp16")];
+            tensor<int32, [4]> var_6412_begin_0 = const()[name = string("op_6412_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_6412_end_0 = const()[name = string("op_6412_end_0"), val = tensor<int32, [4]>([32, 15, 1, 1])];
+            tensor<bool, [4]> var_6412_end_mask_0 = const()[name = string("op_6412_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6412_squeeze_mask_0 = const()[name = string("op_6412_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6412_cast_fp16 = slice_by_index(begin = var_6412_begin_0, end = var_6412_end_0, end_mask = var_6412_end_mask_0, squeeze_mask = var_6412_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6412_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6413_cast_fp16 = mul(x = state_349_cast_fp16, y = var_6412_cast_fp16)[name = string("op_6413_cast_fp16")];
+            tensor<int32, [4]> var_6415_begin_0 = const()[name = string("op_6415_begin_0"), val = tensor<int32, [4]>([0, 14, 0, 0])];
+            tensor<int32, [4]> var_6415_end_0 = const()[name = string("op_6415_end_0"), val = tensor<int32, [4]>([32, 15, 64, 128])];
+            tensor<bool, [4]> var_6415_end_mask_0 = const()[name = string("op_6415_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6415_squeeze_mask_0 = const()[name = string("op_6415_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6415_cast_fp16 = slice_by_index(begin = var_6415_begin_0, end = var_6415_end_0, end_mask = var_6415_end_mask_0, squeeze_mask = var_6415_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6415_cast_fp16")];
+            bool var_6417_transpose_x_1 = const()[name = string("op_6417_transpose_x_1"), val = bool(true)];
+            bool var_6417_transpose_y_1 = const()[name = string("op_6417_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6417_cast_fp16 = matmul(transpose_x = var_6417_transpose_x_1, transpose_y = var_6417_transpose_y_1, x = var_6415_cast_fp16, y = v_new_349_cast_fp16)[name = string("op_6417_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_351_cast_fp16 = add(x = var_6413_cast_fp16, y = var_6417_cast_fp16)[name = string("state_351_cast_fp16")];
+            tensor<int32, [4]> var_6420_begin_0 = const()[name = string("op_6420_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_6420_end_0 = const()[name = string("op_6420_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_6420_end_mask_0 = const()[name = string("op_6420_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6420_squeeze_mask_0 = const()[name = string("op_6420_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6420_cast_fp16 = slice_by_index(begin = var_6420_begin_0, end = var_6420_end_0, end_mask = var_6420_end_mask_0, squeeze_mask = var_6420_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6420_cast_fp16")];
+            tensor<int32, [4]> var_6422_begin_0 = const()[name = string("op_6422_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_6422_end_0 = const()[name = string("op_6422_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_6422_end_mask_0 = const()[name = string("op_6422_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6422_squeeze_mask_0 = const()[name = string("op_6422_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6422_cast_fp16 = slice_by_index(begin = var_6422_begin_0, end = var_6422_end_0, end_mask = var_6422_end_mask_0, squeeze_mask = var_6422_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6422_cast_fp16")];
+            bool var_6423_transpose_x_0 = const()[name = string("op_6423_transpose_x_0"), val = bool(false)];
+            bool var_6423_transpose_y_0 = const()[name = string("op_6423_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6423_cast_fp16 = matmul(transpose_x = var_6423_transpose_x_0, transpose_y = var_6423_transpose_y_0, x = var_6422_cast_fp16, y = state_351_cast_fp16)[name = string("op_6423_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_351_cast_fp16 = sub(x = var_6420_cast_fp16, y = var_6423_cast_fp16)[name = string("v_new_351_cast_fp16")];
+            tensor<int32, [4]> var_6426_begin_0 = const()[name = string("op_6426_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_6426_end_0 = const()[name = string("op_6426_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_6426_end_mask_0 = const()[name = string("op_6426_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6426_squeeze_mask_0 = const()[name = string("op_6426_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6426_cast_fp16 = slice_by_index(begin = var_6426_begin_0, end = var_6426_end_0, end_mask = var_6426_end_mask_0, squeeze_mask = var_6426_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6426_cast_fp16")];
+            bool var_6427_transpose_x_0 = const()[name = string("op_6427_transpose_x_0"), val = bool(false)];
+            bool var_6427_transpose_y_0 = const()[name = string("op_6427_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6427_cast_fp16 = matmul(transpose_x = var_6427_transpose_x_0, transpose_y = var_6427_transpose_y_0, x = var_6426_cast_fp16, y = state_351_cast_fp16)[name = string("op_6427_cast_fp16")];
+            tensor<int32, [4]> var_6429_begin_0 = const()[name = string("op_6429_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_6429_end_0 = const()[name = string("op_6429_end_0"), val = tensor<int32, [4]>([32, 16, 64, 64])];
+            tensor<bool, [4]> var_6429_end_mask_0 = const()[name = string("op_6429_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6429_squeeze_mask_0 = const()[name = string("op_6429_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6429_cast_fp16 = slice_by_index(begin = var_6429_begin_0, end = var_6429_end_0, end_mask = var_6429_end_mask_0, squeeze_mask = var_6429_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6429_cast_fp16")];
+            bool var_6430_transpose_x_0 = const()[name = string("op_6430_transpose_x_0"), val = bool(false)];
+            bool var_6430_transpose_y_0 = const()[name = string("op_6430_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6430_cast_fp16 = matmul(transpose_x = var_6430_transpose_x_0, transpose_y = var_6430_transpose_y_0, x = var_6429_cast_fp16, y = v_new_351_cast_fp16)[name = string("op_6430_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6431_cast_fp16 = add(x = var_6427_cast_fp16, y = var_6430_cast_fp16)[name = string("op_6431_cast_fp16")];
+            tensor<int32, [4]> var_6433_begin_0 = const()[name = string("op_6433_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_6433_end_0 = const()[name = string("op_6433_end_0"), val = tensor<int32, [4]>([32, 16, 1, 1])];
+            tensor<bool, [4]> var_6433_end_mask_0 = const()[name = string("op_6433_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6433_squeeze_mask_0 = const()[name = string("op_6433_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6433_cast_fp16 = slice_by_index(begin = var_6433_begin_0, end = var_6433_end_0, end_mask = var_6433_end_mask_0, squeeze_mask = var_6433_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6433_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6434_cast_fp16 = mul(x = state_351_cast_fp16, y = var_6433_cast_fp16)[name = string("op_6434_cast_fp16")];
+            tensor<int32, [4]> var_6436_begin_0 = const()[name = string("op_6436_begin_0"), val = tensor<int32, [4]>([0, 15, 0, 0])];
+            tensor<int32, [4]> var_6436_end_0 = const()[name = string("op_6436_end_0"), val = tensor<int32, [4]>([32, 16, 64, 128])];
+            tensor<bool, [4]> var_6436_end_mask_0 = const()[name = string("op_6436_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6436_squeeze_mask_0 = const()[name = string("op_6436_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6436_cast_fp16 = slice_by_index(begin = var_6436_begin_0, end = var_6436_end_0, end_mask = var_6436_end_mask_0, squeeze_mask = var_6436_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6436_cast_fp16")];
+            bool var_6438_transpose_x_1 = const()[name = string("op_6438_transpose_x_1"), val = bool(true)];
+            bool var_6438_transpose_y_1 = const()[name = string("op_6438_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6438_cast_fp16 = matmul(transpose_x = var_6438_transpose_x_1, transpose_y = var_6438_transpose_y_1, x = var_6436_cast_fp16, y = v_new_351_cast_fp16)[name = string("op_6438_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_353_cast_fp16 = add(x = var_6434_cast_fp16, y = var_6438_cast_fp16)[name = string("state_353_cast_fp16")];
+            tensor<int32, [4]> var_6441_begin_0 = const()[name = string("op_6441_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_6441_end_0 = const()[name = string("op_6441_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_6441_end_mask_0 = const()[name = string("op_6441_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6441_squeeze_mask_0 = const()[name = string("op_6441_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6441_cast_fp16 = slice_by_index(begin = var_6441_begin_0, end = var_6441_end_0, end_mask = var_6441_end_mask_0, squeeze_mask = var_6441_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6441_cast_fp16")];
+            tensor<int32, [4]> var_6443_begin_0 = const()[name = string("op_6443_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_6443_end_0 = const()[name = string("op_6443_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_6443_end_mask_0 = const()[name = string("op_6443_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6443_squeeze_mask_0 = const()[name = string("op_6443_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6443_cast_fp16 = slice_by_index(begin = var_6443_begin_0, end = var_6443_end_0, end_mask = var_6443_end_mask_0, squeeze_mask = var_6443_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6443_cast_fp16")];
+            bool var_6444_transpose_x_0 = const()[name = string("op_6444_transpose_x_0"), val = bool(false)];
+            bool var_6444_transpose_y_0 = const()[name = string("op_6444_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6444_cast_fp16 = matmul(transpose_x = var_6444_transpose_x_0, transpose_y = var_6444_transpose_y_0, x = var_6443_cast_fp16, y = state_353_cast_fp16)[name = string("op_6444_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_353_cast_fp16 = sub(x = var_6441_cast_fp16, y = var_6444_cast_fp16)[name = string("v_new_353_cast_fp16")];
+            tensor<int32, [4]> var_6447_begin_0 = const()[name = string("op_6447_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_6447_end_0 = const()[name = string("op_6447_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_6447_end_mask_0 = const()[name = string("op_6447_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6447_squeeze_mask_0 = const()[name = string("op_6447_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6447_cast_fp16 = slice_by_index(begin = var_6447_begin_0, end = var_6447_end_0, end_mask = var_6447_end_mask_0, squeeze_mask = var_6447_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6447_cast_fp16")];
+            bool var_6448_transpose_x_0 = const()[name = string("op_6448_transpose_x_0"), val = bool(false)];
+            bool var_6448_transpose_y_0 = const()[name = string("op_6448_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6448_cast_fp16 = matmul(transpose_x = var_6448_transpose_x_0, transpose_y = var_6448_transpose_y_0, x = var_6447_cast_fp16, y = state_353_cast_fp16)[name = string("op_6448_cast_fp16")];
+            tensor<int32, [4]> var_6450_begin_0 = const()[name = string("op_6450_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_6450_end_0 = const()[name = string("op_6450_end_0"), val = tensor<int32, [4]>([32, 17, 64, 64])];
+            tensor<bool, [4]> var_6450_end_mask_0 = const()[name = string("op_6450_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6450_squeeze_mask_0 = const()[name = string("op_6450_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6450_cast_fp16 = slice_by_index(begin = var_6450_begin_0, end = var_6450_end_0, end_mask = var_6450_end_mask_0, squeeze_mask = var_6450_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6450_cast_fp16")];
+            bool var_6451_transpose_x_0 = const()[name = string("op_6451_transpose_x_0"), val = bool(false)];
+            bool var_6451_transpose_y_0 = const()[name = string("op_6451_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6451_cast_fp16 = matmul(transpose_x = var_6451_transpose_x_0, transpose_y = var_6451_transpose_y_0, x = var_6450_cast_fp16, y = v_new_353_cast_fp16)[name = string("op_6451_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6452_cast_fp16 = add(x = var_6448_cast_fp16, y = var_6451_cast_fp16)[name = string("op_6452_cast_fp16")];
+            tensor<int32, [4]> var_6454_begin_0 = const()[name = string("op_6454_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_6454_end_0 = const()[name = string("op_6454_end_0"), val = tensor<int32, [4]>([32, 17, 1, 1])];
+            tensor<bool, [4]> var_6454_end_mask_0 = const()[name = string("op_6454_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6454_squeeze_mask_0 = const()[name = string("op_6454_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6454_cast_fp16 = slice_by_index(begin = var_6454_begin_0, end = var_6454_end_0, end_mask = var_6454_end_mask_0, squeeze_mask = var_6454_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6454_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6455_cast_fp16 = mul(x = state_353_cast_fp16, y = var_6454_cast_fp16)[name = string("op_6455_cast_fp16")];
+            tensor<int32, [4]> var_6457_begin_0 = const()[name = string("op_6457_begin_0"), val = tensor<int32, [4]>([0, 16, 0, 0])];
+            tensor<int32, [4]> var_6457_end_0 = const()[name = string("op_6457_end_0"), val = tensor<int32, [4]>([32, 17, 64, 128])];
+            tensor<bool, [4]> var_6457_end_mask_0 = const()[name = string("op_6457_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6457_squeeze_mask_0 = const()[name = string("op_6457_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6457_cast_fp16 = slice_by_index(begin = var_6457_begin_0, end = var_6457_end_0, end_mask = var_6457_end_mask_0, squeeze_mask = var_6457_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6457_cast_fp16")];
+            bool var_6459_transpose_x_1 = const()[name = string("op_6459_transpose_x_1"), val = bool(true)];
+            bool var_6459_transpose_y_1 = const()[name = string("op_6459_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6459_cast_fp16 = matmul(transpose_x = var_6459_transpose_x_1, transpose_y = var_6459_transpose_y_1, x = var_6457_cast_fp16, y = v_new_353_cast_fp16)[name = string("op_6459_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_355_cast_fp16 = add(x = var_6455_cast_fp16, y = var_6459_cast_fp16)[name = string("state_355_cast_fp16")];
+            tensor<int32, [4]> var_6462_begin_0 = const()[name = string("op_6462_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_6462_end_0 = const()[name = string("op_6462_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_6462_end_mask_0 = const()[name = string("op_6462_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6462_squeeze_mask_0 = const()[name = string("op_6462_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6462_cast_fp16 = slice_by_index(begin = var_6462_begin_0, end = var_6462_end_0, end_mask = var_6462_end_mask_0, squeeze_mask = var_6462_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6462_cast_fp16")];
+            tensor<int32, [4]> var_6464_begin_0 = const()[name = string("op_6464_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_6464_end_0 = const()[name = string("op_6464_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_6464_end_mask_0 = const()[name = string("op_6464_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6464_squeeze_mask_0 = const()[name = string("op_6464_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6464_cast_fp16 = slice_by_index(begin = var_6464_begin_0, end = var_6464_end_0, end_mask = var_6464_end_mask_0, squeeze_mask = var_6464_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6464_cast_fp16")];
+            bool var_6465_transpose_x_0 = const()[name = string("op_6465_transpose_x_0"), val = bool(false)];
+            bool var_6465_transpose_y_0 = const()[name = string("op_6465_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6465_cast_fp16 = matmul(transpose_x = var_6465_transpose_x_0, transpose_y = var_6465_transpose_y_0, x = var_6464_cast_fp16, y = state_355_cast_fp16)[name = string("op_6465_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_355_cast_fp16 = sub(x = var_6462_cast_fp16, y = var_6465_cast_fp16)[name = string("v_new_355_cast_fp16")];
+            tensor<int32, [4]> var_6468_begin_0 = const()[name = string("op_6468_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_6468_end_0 = const()[name = string("op_6468_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_6468_end_mask_0 = const()[name = string("op_6468_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6468_squeeze_mask_0 = const()[name = string("op_6468_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6468_cast_fp16 = slice_by_index(begin = var_6468_begin_0, end = var_6468_end_0, end_mask = var_6468_end_mask_0, squeeze_mask = var_6468_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6468_cast_fp16")];
+            bool var_6469_transpose_x_0 = const()[name = string("op_6469_transpose_x_0"), val = bool(false)];
+            bool var_6469_transpose_y_0 = const()[name = string("op_6469_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6469_cast_fp16 = matmul(transpose_x = var_6469_transpose_x_0, transpose_y = var_6469_transpose_y_0, x = var_6468_cast_fp16, y = state_355_cast_fp16)[name = string("op_6469_cast_fp16")];
+            tensor<int32, [4]> var_6471_begin_0 = const()[name = string("op_6471_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_6471_end_0 = const()[name = string("op_6471_end_0"), val = tensor<int32, [4]>([32, 18, 64, 64])];
+            tensor<bool, [4]> var_6471_end_mask_0 = const()[name = string("op_6471_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6471_squeeze_mask_0 = const()[name = string("op_6471_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6471_cast_fp16 = slice_by_index(begin = var_6471_begin_0, end = var_6471_end_0, end_mask = var_6471_end_mask_0, squeeze_mask = var_6471_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6471_cast_fp16")];
+            bool var_6472_transpose_x_0 = const()[name = string("op_6472_transpose_x_0"), val = bool(false)];
+            bool var_6472_transpose_y_0 = const()[name = string("op_6472_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6472_cast_fp16 = matmul(transpose_x = var_6472_transpose_x_0, transpose_y = var_6472_transpose_y_0, x = var_6471_cast_fp16, y = v_new_355_cast_fp16)[name = string("op_6472_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6473_cast_fp16 = add(x = var_6469_cast_fp16, y = var_6472_cast_fp16)[name = string("op_6473_cast_fp16")];
+            tensor<int32, [4]> var_6475_begin_0 = const()[name = string("op_6475_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_6475_end_0 = const()[name = string("op_6475_end_0"), val = tensor<int32, [4]>([32, 18, 1, 1])];
+            tensor<bool, [4]> var_6475_end_mask_0 = const()[name = string("op_6475_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6475_squeeze_mask_0 = const()[name = string("op_6475_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6475_cast_fp16 = slice_by_index(begin = var_6475_begin_0, end = var_6475_end_0, end_mask = var_6475_end_mask_0, squeeze_mask = var_6475_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6475_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6476_cast_fp16 = mul(x = state_355_cast_fp16, y = var_6475_cast_fp16)[name = string("op_6476_cast_fp16")];
+            tensor<int32, [4]> var_6478_begin_0 = const()[name = string("op_6478_begin_0"), val = tensor<int32, [4]>([0, 17, 0, 0])];
+            tensor<int32, [4]> var_6478_end_0 = const()[name = string("op_6478_end_0"), val = tensor<int32, [4]>([32, 18, 64, 128])];
+            tensor<bool, [4]> var_6478_end_mask_0 = const()[name = string("op_6478_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6478_squeeze_mask_0 = const()[name = string("op_6478_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6478_cast_fp16 = slice_by_index(begin = var_6478_begin_0, end = var_6478_end_0, end_mask = var_6478_end_mask_0, squeeze_mask = var_6478_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6478_cast_fp16")];
+            bool var_6480_transpose_x_1 = const()[name = string("op_6480_transpose_x_1"), val = bool(true)];
+            bool var_6480_transpose_y_1 = const()[name = string("op_6480_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6480_cast_fp16 = matmul(transpose_x = var_6480_transpose_x_1, transpose_y = var_6480_transpose_y_1, x = var_6478_cast_fp16, y = v_new_355_cast_fp16)[name = string("op_6480_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_357_cast_fp16 = add(x = var_6476_cast_fp16, y = var_6480_cast_fp16)[name = string("state_357_cast_fp16")];
+            tensor<int32, [4]> var_6483_begin_0 = const()[name = string("op_6483_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_6483_end_0 = const()[name = string("op_6483_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_6483_end_mask_0 = const()[name = string("op_6483_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6483_squeeze_mask_0 = const()[name = string("op_6483_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6483_cast_fp16 = slice_by_index(begin = var_6483_begin_0, end = var_6483_end_0, end_mask = var_6483_end_mask_0, squeeze_mask = var_6483_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6483_cast_fp16")];
+            tensor<int32, [4]> var_6485_begin_0 = const()[name = string("op_6485_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_6485_end_0 = const()[name = string("op_6485_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_6485_end_mask_0 = const()[name = string("op_6485_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6485_squeeze_mask_0 = const()[name = string("op_6485_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6485_cast_fp16 = slice_by_index(begin = var_6485_begin_0, end = var_6485_end_0, end_mask = var_6485_end_mask_0, squeeze_mask = var_6485_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6485_cast_fp16")];
+            bool var_6486_transpose_x_0 = const()[name = string("op_6486_transpose_x_0"), val = bool(false)];
+            bool var_6486_transpose_y_0 = const()[name = string("op_6486_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6486_cast_fp16 = matmul(transpose_x = var_6486_transpose_x_0, transpose_y = var_6486_transpose_y_0, x = var_6485_cast_fp16, y = state_357_cast_fp16)[name = string("op_6486_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_357_cast_fp16 = sub(x = var_6483_cast_fp16, y = var_6486_cast_fp16)[name = string("v_new_357_cast_fp16")];
+            tensor<int32, [4]> var_6489_begin_0 = const()[name = string("op_6489_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_6489_end_0 = const()[name = string("op_6489_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_6489_end_mask_0 = const()[name = string("op_6489_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6489_squeeze_mask_0 = const()[name = string("op_6489_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6489_cast_fp16 = slice_by_index(begin = var_6489_begin_0, end = var_6489_end_0, end_mask = var_6489_end_mask_0, squeeze_mask = var_6489_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6489_cast_fp16")];
+            bool var_6490_transpose_x_0 = const()[name = string("op_6490_transpose_x_0"), val = bool(false)];
+            bool var_6490_transpose_y_0 = const()[name = string("op_6490_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6490_cast_fp16 = matmul(transpose_x = var_6490_transpose_x_0, transpose_y = var_6490_transpose_y_0, x = var_6489_cast_fp16, y = state_357_cast_fp16)[name = string("op_6490_cast_fp16")];
+            tensor<int32, [4]> var_6492_begin_0 = const()[name = string("op_6492_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_6492_end_0 = const()[name = string("op_6492_end_0"), val = tensor<int32, [4]>([32, 19, 64, 64])];
+            tensor<bool, [4]> var_6492_end_mask_0 = const()[name = string("op_6492_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6492_squeeze_mask_0 = const()[name = string("op_6492_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6492_cast_fp16 = slice_by_index(begin = var_6492_begin_0, end = var_6492_end_0, end_mask = var_6492_end_mask_0, squeeze_mask = var_6492_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6492_cast_fp16")];
+            bool var_6493_transpose_x_0 = const()[name = string("op_6493_transpose_x_0"), val = bool(false)];
+            bool var_6493_transpose_y_0 = const()[name = string("op_6493_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6493_cast_fp16 = matmul(transpose_x = var_6493_transpose_x_0, transpose_y = var_6493_transpose_y_0, x = var_6492_cast_fp16, y = v_new_357_cast_fp16)[name = string("op_6493_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6494_cast_fp16 = add(x = var_6490_cast_fp16, y = var_6493_cast_fp16)[name = string("op_6494_cast_fp16")];
+            tensor<int32, [4]> var_6496_begin_0 = const()[name = string("op_6496_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_6496_end_0 = const()[name = string("op_6496_end_0"), val = tensor<int32, [4]>([32, 19, 1, 1])];
+            tensor<bool, [4]> var_6496_end_mask_0 = const()[name = string("op_6496_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6496_squeeze_mask_0 = const()[name = string("op_6496_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6496_cast_fp16 = slice_by_index(begin = var_6496_begin_0, end = var_6496_end_0, end_mask = var_6496_end_mask_0, squeeze_mask = var_6496_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6496_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6497_cast_fp16 = mul(x = state_357_cast_fp16, y = var_6496_cast_fp16)[name = string("op_6497_cast_fp16")];
+            tensor<int32, [4]> var_6499_begin_0 = const()[name = string("op_6499_begin_0"), val = tensor<int32, [4]>([0, 18, 0, 0])];
+            tensor<int32, [4]> var_6499_end_0 = const()[name = string("op_6499_end_0"), val = tensor<int32, [4]>([32, 19, 64, 128])];
+            tensor<bool, [4]> var_6499_end_mask_0 = const()[name = string("op_6499_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6499_squeeze_mask_0 = const()[name = string("op_6499_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6499_cast_fp16 = slice_by_index(begin = var_6499_begin_0, end = var_6499_end_0, end_mask = var_6499_end_mask_0, squeeze_mask = var_6499_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6499_cast_fp16")];
+            bool var_6501_transpose_x_1 = const()[name = string("op_6501_transpose_x_1"), val = bool(true)];
+            bool var_6501_transpose_y_1 = const()[name = string("op_6501_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6501_cast_fp16 = matmul(transpose_x = var_6501_transpose_x_1, transpose_y = var_6501_transpose_y_1, x = var_6499_cast_fp16, y = v_new_357_cast_fp16)[name = string("op_6501_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_359_cast_fp16 = add(x = var_6497_cast_fp16, y = var_6501_cast_fp16)[name = string("state_359_cast_fp16")];
+            tensor<int32, [4]> var_6504_begin_0 = const()[name = string("op_6504_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_6504_end_0 = const()[name = string("op_6504_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_6504_end_mask_0 = const()[name = string("op_6504_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6504_squeeze_mask_0 = const()[name = string("op_6504_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6504_cast_fp16 = slice_by_index(begin = var_6504_begin_0, end = var_6504_end_0, end_mask = var_6504_end_mask_0, squeeze_mask = var_6504_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6504_cast_fp16")];
+            tensor<int32, [4]> var_6506_begin_0 = const()[name = string("op_6506_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_6506_end_0 = const()[name = string("op_6506_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_6506_end_mask_0 = const()[name = string("op_6506_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6506_squeeze_mask_0 = const()[name = string("op_6506_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6506_cast_fp16 = slice_by_index(begin = var_6506_begin_0, end = var_6506_end_0, end_mask = var_6506_end_mask_0, squeeze_mask = var_6506_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6506_cast_fp16")];
+            bool var_6507_transpose_x_0 = const()[name = string("op_6507_transpose_x_0"), val = bool(false)];
+            bool var_6507_transpose_y_0 = const()[name = string("op_6507_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6507_cast_fp16 = matmul(transpose_x = var_6507_transpose_x_0, transpose_y = var_6507_transpose_y_0, x = var_6506_cast_fp16, y = state_359_cast_fp16)[name = string("op_6507_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_359_cast_fp16 = sub(x = var_6504_cast_fp16, y = var_6507_cast_fp16)[name = string("v_new_359_cast_fp16")];
+            tensor<int32, [4]> var_6510_begin_0 = const()[name = string("op_6510_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_6510_end_0 = const()[name = string("op_6510_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_6510_end_mask_0 = const()[name = string("op_6510_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6510_squeeze_mask_0 = const()[name = string("op_6510_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6510_cast_fp16 = slice_by_index(begin = var_6510_begin_0, end = var_6510_end_0, end_mask = var_6510_end_mask_0, squeeze_mask = var_6510_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6510_cast_fp16")];
+            bool var_6511_transpose_x_0 = const()[name = string("op_6511_transpose_x_0"), val = bool(false)];
+            bool var_6511_transpose_y_0 = const()[name = string("op_6511_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6511_cast_fp16 = matmul(transpose_x = var_6511_transpose_x_0, transpose_y = var_6511_transpose_y_0, x = var_6510_cast_fp16, y = state_359_cast_fp16)[name = string("op_6511_cast_fp16")];
+            tensor<int32, [4]> var_6513_begin_0 = const()[name = string("op_6513_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_6513_end_0 = const()[name = string("op_6513_end_0"), val = tensor<int32, [4]>([32, 20, 64, 64])];
+            tensor<bool, [4]> var_6513_end_mask_0 = const()[name = string("op_6513_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6513_squeeze_mask_0 = const()[name = string("op_6513_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6513_cast_fp16 = slice_by_index(begin = var_6513_begin_0, end = var_6513_end_0, end_mask = var_6513_end_mask_0, squeeze_mask = var_6513_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6513_cast_fp16")];
+            bool var_6514_transpose_x_0 = const()[name = string("op_6514_transpose_x_0"), val = bool(false)];
+            bool var_6514_transpose_y_0 = const()[name = string("op_6514_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6514_cast_fp16 = matmul(transpose_x = var_6514_transpose_x_0, transpose_y = var_6514_transpose_y_0, x = var_6513_cast_fp16, y = v_new_359_cast_fp16)[name = string("op_6514_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6515_cast_fp16 = add(x = var_6511_cast_fp16, y = var_6514_cast_fp16)[name = string("op_6515_cast_fp16")];
+            tensor<int32, [4]> var_6517_begin_0 = const()[name = string("op_6517_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_6517_end_0 = const()[name = string("op_6517_end_0"), val = tensor<int32, [4]>([32, 20, 1, 1])];
+            tensor<bool, [4]> var_6517_end_mask_0 = const()[name = string("op_6517_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6517_squeeze_mask_0 = const()[name = string("op_6517_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6517_cast_fp16 = slice_by_index(begin = var_6517_begin_0, end = var_6517_end_0, end_mask = var_6517_end_mask_0, squeeze_mask = var_6517_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6517_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6518_cast_fp16 = mul(x = state_359_cast_fp16, y = var_6517_cast_fp16)[name = string("op_6518_cast_fp16")];
+            tensor<int32, [4]> var_6520_begin_0 = const()[name = string("op_6520_begin_0"), val = tensor<int32, [4]>([0, 19, 0, 0])];
+            tensor<int32, [4]> var_6520_end_0 = const()[name = string("op_6520_end_0"), val = tensor<int32, [4]>([32, 20, 64, 128])];
+            tensor<bool, [4]> var_6520_end_mask_0 = const()[name = string("op_6520_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6520_squeeze_mask_0 = const()[name = string("op_6520_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6520_cast_fp16 = slice_by_index(begin = var_6520_begin_0, end = var_6520_end_0, end_mask = var_6520_end_mask_0, squeeze_mask = var_6520_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6520_cast_fp16")];
+            bool var_6522_transpose_x_1 = const()[name = string("op_6522_transpose_x_1"), val = bool(true)];
+            bool var_6522_transpose_y_1 = const()[name = string("op_6522_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6522_cast_fp16 = matmul(transpose_x = var_6522_transpose_x_1, transpose_y = var_6522_transpose_y_1, x = var_6520_cast_fp16, y = v_new_359_cast_fp16)[name = string("op_6522_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_361_cast_fp16 = add(x = var_6518_cast_fp16, y = var_6522_cast_fp16)[name = string("state_361_cast_fp16")];
+            tensor<int32, [4]> var_6525_begin_0 = const()[name = string("op_6525_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_6525_end_0 = const()[name = string("op_6525_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_6525_end_mask_0 = const()[name = string("op_6525_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6525_squeeze_mask_0 = const()[name = string("op_6525_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6525_cast_fp16 = slice_by_index(begin = var_6525_begin_0, end = var_6525_end_0, end_mask = var_6525_end_mask_0, squeeze_mask = var_6525_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6525_cast_fp16")];
+            tensor<int32, [4]> var_6527_begin_0 = const()[name = string("op_6527_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_6527_end_0 = const()[name = string("op_6527_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_6527_end_mask_0 = const()[name = string("op_6527_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6527_squeeze_mask_0 = const()[name = string("op_6527_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6527_cast_fp16 = slice_by_index(begin = var_6527_begin_0, end = var_6527_end_0, end_mask = var_6527_end_mask_0, squeeze_mask = var_6527_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6527_cast_fp16")];
+            bool var_6528_transpose_x_0 = const()[name = string("op_6528_transpose_x_0"), val = bool(false)];
+            bool var_6528_transpose_y_0 = const()[name = string("op_6528_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6528_cast_fp16 = matmul(transpose_x = var_6528_transpose_x_0, transpose_y = var_6528_transpose_y_0, x = var_6527_cast_fp16, y = state_361_cast_fp16)[name = string("op_6528_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_361_cast_fp16 = sub(x = var_6525_cast_fp16, y = var_6528_cast_fp16)[name = string("v_new_361_cast_fp16")];
+            tensor<int32, [4]> var_6531_begin_0 = const()[name = string("op_6531_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_6531_end_0 = const()[name = string("op_6531_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_6531_end_mask_0 = const()[name = string("op_6531_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6531_squeeze_mask_0 = const()[name = string("op_6531_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6531_cast_fp16 = slice_by_index(begin = var_6531_begin_0, end = var_6531_end_0, end_mask = var_6531_end_mask_0, squeeze_mask = var_6531_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6531_cast_fp16")];
+            bool var_6532_transpose_x_0 = const()[name = string("op_6532_transpose_x_0"), val = bool(false)];
+            bool var_6532_transpose_y_0 = const()[name = string("op_6532_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6532_cast_fp16 = matmul(transpose_x = var_6532_transpose_x_0, transpose_y = var_6532_transpose_y_0, x = var_6531_cast_fp16, y = state_361_cast_fp16)[name = string("op_6532_cast_fp16")];
+            tensor<int32, [4]> var_6534_begin_0 = const()[name = string("op_6534_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_6534_end_0 = const()[name = string("op_6534_end_0"), val = tensor<int32, [4]>([32, 21, 64, 64])];
+            tensor<bool, [4]> var_6534_end_mask_0 = const()[name = string("op_6534_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6534_squeeze_mask_0 = const()[name = string("op_6534_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6534_cast_fp16 = slice_by_index(begin = var_6534_begin_0, end = var_6534_end_0, end_mask = var_6534_end_mask_0, squeeze_mask = var_6534_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6534_cast_fp16")];
+            bool var_6535_transpose_x_0 = const()[name = string("op_6535_transpose_x_0"), val = bool(false)];
+            bool var_6535_transpose_y_0 = const()[name = string("op_6535_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6535_cast_fp16 = matmul(transpose_x = var_6535_transpose_x_0, transpose_y = var_6535_transpose_y_0, x = var_6534_cast_fp16, y = v_new_361_cast_fp16)[name = string("op_6535_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6536_cast_fp16 = add(x = var_6532_cast_fp16, y = var_6535_cast_fp16)[name = string("op_6536_cast_fp16")];
+            tensor<int32, [4]> var_6538_begin_0 = const()[name = string("op_6538_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_6538_end_0 = const()[name = string("op_6538_end_0"), val = tensor<int32, [4]>([32, 21, 1, 1])];
+            tensor<bool, [4]> var_6538_end_mask_0 = const()[name = string("op_6538_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6538_squeeze_mask_0 = const()[name = string("op_6538_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6538_cast_fp16 = slice_by_index(begin = var_6538_begin_0, end = var_6538_end_0, end_mask = var_6538_end_mask_0, squeeze_mask = var_6538_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6538_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6539_cast_fp16 = mul(x = state_361_cast_fp16, y = var_6538_cast_fp16)[name = string("op_6539_cast_fp16")];
+            tensor<int32, [4]> var_6541_begin_0 = const()[name = string("op_6541_begin_0"), val = tensor<int32, [4]>([0, 20, 0, 0])];
+            tensor<int32, [4]> var_6541_end_0 = const()[name = string("op_6541_end_0"), val = tensor<int32, [4]>([32, 21, 64, 128])];
+            tensor<bool, [4]> var_6541_end_mask_0 = const()[name = string("op_6541_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6541_squeeze_mask_0 = const()[name = string("op_6541_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6541_cast_fp16 = slice_by_index(begin = var_6541_begin_0, end = var_6541_end_0, end_mask = var_6541_end_mask_0, squeeze_mask = var_6541_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6541_cast_fp16")];
+            bool var_6543_transpose_x_1 = const()[name = string("op_6543_transpose_x_1"), val = bool(true)];
+            bool var_6543_transpose_y_1 = const()[name = string("op_6543_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6543_cast_fp16 = matmul(transpose_x = var_6543_transpose_x_1, transpose_y = var_6543_transpose_y_1, x = var_6541_cast_fp16, y = v_new_361_cast_fp16)[name = string("op_6543_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_363_cast_fp16 = add(x = var_6539_cast_fp16, y = var_6543_cast_fp16)[name = string("state_363_cast_fp16")];
+            tensor<int32, [4]> var_6546_begin_0 = const()[name = string("op_6546_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_6546_end_0 = const()[name = string("op_6546_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_6546_end_mask_0 = const()[name = string("op_6546_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6546_squeeze_mask_0 = const()[name = string("op_6546_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6546_cast_fp16 = slice_by_index(begin = var_6546_begin_0, end = var_6546_end_0, end_mask = var_6546_end_mask_0, squeeze_mask = var_6546_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6546_cast_fp16")];
+            tensor<int32, [4]> var_6548_begin_0 = const()[name = string("op_6548_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_6548_end_0 = const()[name = string("op_6548_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_6548_end_mask_0 = const()[name = string("op_6548_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6548_squeeze_mask_0 = const()[name = string("op_6548_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6548_cast_fp16 = slice_by_index(begin = var_6548_begin_0, end = var_6548_end_0, end_mask = var_6548_end_mask_0, squeeze_mask = var_6548_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6548_cast_fp16")];
+            bool var_6549_transpose_x_0 = const()[name = string("op_6549_transpose_x_0"), val = bool(false)];
+            bool var_6549_transpose_y_0 = const()[name = string("op_6549_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6549_cast_fp16 = matmul(transpose_x = var_6549_transpose_x_0, transpose_y = var_6549_transpose_y_0, x = var_6548_cast_fp16, y = state_363_cast_fp16)[name = string("op_6549_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_363_cast_fp16 = sub(x = var_6546_cast_fp16, y = var_6549_cast_fp16)[name = string("v_new_363_cast_fp16")];
+            tensor<int32, [4]> var_6552_begin_0 = const()[name = string("op_6552_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_6552_end_0 = const()[name = string("op_6552_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_6552_end_mask_0 = const()[name = string("op_6552_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6552_squeeze_mask_0 = const()[name = string("op_6552_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6552_cast_fp16 = slice_by_index(begin = var_6552_begin_0, end = var_6552_end_0, end_mask = var_6552_end_mask_0, squeeze_mask = var_6552_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6552_cast_fp16")];
+            bool var_6553_transpose_x_0 = const()[name = string("op_6553_transpose_x_0"), val = bool(false)];
+            bool var_6553_transpose_y_0 = const()[name = string("op_6553_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6553_cast_fp16 = matmul(transpose_x = var_6553_transpose_x_0, transpose_y = var_6553_transpose_y_0, x = var_6552_cast_fp16, y = state_363_cast_fp16)[name = string("op_6553_cast_fp16")];
+            tensor<int32, [4]> var_6555_begin_0 = const()[name = string("op_6555_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_6555_end_0 = const()[name = string("op_6555_end_0"), val = tensor<int32, [4]>([32, 22, 64, 64])];
+            tensor<bool, [4]> var_6555_end_mask_0 = const()[name = string("op_6555_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6555_squeeze_mask_0 = const()[name = string("op_6555_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6555_cast_fp16 = slice_by_index(begin = var_6555_begin_0, end = var_6555_end_0, end_mask = var_6555_end_mask_0, squeeze_mask = var_6555_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6555_cast_fp16")];
+            bool var_6556_transpose_x_0 = const()[name = string("op_6556_transpose_x_0"), val = bool(false)];
+            bool var_6556_transpose_y_0 = const()[name = string("op_6556_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6556_cast_fp16 = matmul(transpose_x = var_6556_transpose_x_0, transpose_y = var_6556_transpose_y_0, x = var_6555_cast_fp16, y = v_new_363_cast_fp16)[name = string("op_6556_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6557_cast_fp16 = add(x = var_6553_cast_fp16, y = var_6556_cast_fp16)[name = string("op_6557_cast_fp16")];
+            tensor<int32, [4]> var_6559_begin_0 = const()[name = string("op_6559_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_6559_end_0 = const()[name = string("op_6559_end_0"), val = tensor<int32, [4]>([32, 22, 1, 1])];
+            tensor<bool, [4]> var_6559_end_mask_0 = const()[name = string("op_6559_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6559_squeeze_mask_0 = const()[name = string("op_6559_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6559_cast_fp16 = slice_by_index(begin = var_6559_begin_0, end = var_6559_end_0, end_mask = var_6559_end_mask_0, squeeze_mask = var_6559_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6559_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6560_cast_fp16 = mul(x = state_363_cast_fp16, y = var_6559_cast_fp16)[name = string("op_6560_cast_fp16")];
+            tensor<int32, [4]> var_6562_begin_0 = const()[name = string("op_6562_begin_0"), val = tensor<int32, [4]>([0, 21, 0, 0])];
+            tensor<int32, [4]> var_6562_end_0 = const()[name = string("op_6562_end_0"), val = tensor<int32, [4]>([32, 22, 64, 128])];
+            tensor<bool, [4]> var_6562_end_mask_0 = const()[name = string("op_6562_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6562_squeeze_mask_0 = const()[name = string("op_6562_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6562_cast_fp16 = slice_by_index(begin = var_6562_begin_0, end = var_6562_end_0, end_mask = var_6562_end_mask_0, squeeze_mask = var_6562_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6562_cast_fp16")];
+            bool var_6564_transpose_x_1 = const()[name = string("op_6564_transpose_x_1"), val = bool(true)];
+            bool var_6564_transpose_y_1 = const()[name = string("op_6564_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6564_cast_fp16 = matmul(transpose_x = var_6564_transpose_x_1, transpose_y = var_6564_transpose_y_1, x = var_6562_cast_fp16, y = v_new_363_cast_fp16)[name = string("op_6564_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_365_cast_fp16 = add(x = var_6560_cast_fp16, y = var_6564_cast_fp16)[name = string("state_365_cast_fp16")];
+            tensor<int32, [4]> var_6567_begin_0 = const()[name = string("op_6567_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_6567_end_0 = const()[name = string("op_6567_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_6567_end_mask_0 = const()[name = string("op_6567_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6567_squeeze_mask_0 = const()[name = string("op_6567_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6567_cast_fp16 = slice_by_index(begin = var_6567_begin_0, end = var_6567_end_0, end_mask = var_6567_end_mask_0, squeeze_mask = var_6567_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6567_cast_fp16")];
+            tensor<int32, [4]> var_6569_begin_0 = const()[name = string("op_6569_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_6569_end_0 = const()[name = string("op_6569_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_6569_end_mask_0 = const()[name = string("op_6569_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6569_squeeze_mask_0 = const()[name = string("op_6569_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6569_cast_fp16 = slice_by_index(begin = var_6569_begin_0, end = var_6569_end_0, end_mask = var_6569_end_mask_0, squeeze_mask = var_6569_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6569_cast_fp16")];
+            bool var_6570_transpose_x_0 = const()[name = string("op_6570_transpose_x_0"), val = bool(false)];
+            bool var_6570_transpose_y_0 = const()[name = string("op_6570_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6570_cast_fp16 = matmul(transpose_x = var_6570_transpose_x_0, transpose_y = var_6570_transpose_y_0, x = var_6569_cast_fp16, y = state_365_cast_fp16)[name = string("op_6570_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_365_cast_fp16 = sub(x = var_6567_cast_fp16, y = var_6570_cast_fp16)[name = string("v_new_365_cast_fp16")];
+            tensor<int32, [4]> var_6573_begin_0 = const()[name = string("op_6573_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_6573_end_0 = const()[name = string("op_6573_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_6573_end_mask_0 = const()[name = string("op_6573_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6573_squeeze_mask_0 = const()[name = string("op_6573_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6573_cast_fp16 = slice_by_index(begin = var_6573_begin_0, end = var_6573_end_0, end_mask = var_6573_end_mask_0, squeeze_mask = var_6573_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6573_cast_fp16")];
+            bool var_6574_transpose_x_0 = const()[name = string("op_6574_transpose_x_0"), val = bool(false)];
+            bool var_6574_transpose_y_0 = const()[name = string("op_6574_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6574_cast_fp16 = matmul(transpose_x = var_6574_transpose_x_0, transpose_y = var_6574_transpose_y_0, x = var_6573_cast_fp16, y = state_365_cast_fp16)[name = string("op_6574_cast_fp16")];
+            tensor<int32, [4]> var_6576_begin_0 = const()[name = string("op_6576_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_6576_end_0 = const()[name = string("op_6576_end_0"), val = tensor<int32, [4]>([32, 23, 64, 64])];
+            tensor<bool, [4]> var_6576_end_mask_0 = const()[name = string("op_6576_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6576_squeeze_mask_0 = const()[name = string("op_6576_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6576_cast_fp16 = slice_by_index(begin = var_6576_begin_0, end = var_6576_end_0, end_mask = var_6576_end_mask_0, squeeze_mask = var_6576_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6576_cast_fp16")];
+            bool var_6577_transpose_x_0 = const()[name = string("op_6577_transpose_x_0"), val = bool(false)];
+            bool var_6577_transpose_y_0 = const()[name = string("op_6577_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6577_cast_fp16 = matmul(transpose_x = var_6577_transpose_x_0, transpose_y = var_6577_transpose_y_0, x = var_6576_cast_fp16, y = v_new_365_cast_fp16)[name = string("op_6577_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6578_cast_fp16 = add(x = var_6574_cast_fp16, y = var_6577_cast_fp16)[name = string("op_6578_cast_fp16")];
+            tensor<int32, [4]> var_6580_begin_0 = const()[name = string("op_6580_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_6580_end_0 = const()[name = string("op_6580_end_0"), val = tensor<int32, [4]>([32, 23, 1, 1])];
+            tensor<bool, [4]> var_6580_end_mask_0 = const()[name = string("op_6580_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6580_squeeze_mask_0 = const()[name = string("op_6580_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6580_cast_fp16 = slice_by_index(begin = var_6580_begin_0, end = var_6580_end_0, end_mask = var_6580_end_mask_0, squeeze_mask = var_6580_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6580_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6581_cast_fp16 = mul(x = state_365_cast_fp16, y = var_6580_cast_fp16)[name = string("op_6581_cast_fp16")];
+            tensor<int32, [4]> var_6583_begin_0 = const()[name = string("op_6583_begin_0"), val = tensor<int32, [4]>([0, 22, 0, 0])];
+            tensor<int32, [4]> var_6583_end_0 = const()[name = string("op_6583_end_0"), val = tensor<int32, [4]>([32, 23, 64, 128])];
+            tensor<bool, [4]> var_6583_end_mask_0 = const()[name = string("op_6583_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6583_squeeze_mask_0 = const()[name = string("op_6583_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6583_cast_fp16 = slice_by_index(begin = var_6583_begin_0, end = var_6583_end_0, end_mask = var_6583_end_mask_0, squeeze_mask = var_6583_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6583_cast_fp16")];
+            bool var_6585_transpose_x_1 = const()[name = string("op_6585_transpose_x_1"), val = bool(true)];
+            bool var_6585_transpose_y_1 = const()[name = string("op_6585_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6585_cast_fp16 = matmul(transpose_x = var_6585_transpose_x_1, transpose_y = var_6585_transpose_y_1, x = var_6583_cast_fp16, y = v_new_365_cast_fp16)[name = string("op_6585_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_367_cast_fp16 = add(x = var_6581_cast_fp16, y = var_6585_cast_fp16)[name = string("state_367_cast_fp16")];
+            tensor<int32, [4]> var_6588_begin_0 = const()[name = string("op_6588_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_6588_end_0 = const()[name = string("op_6588_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_6588_end_mask_0 = const()[name = string("op_6588_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6588_squeeze_mask_0 = const()[name = string("op_6588_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6588_cast_fp16 = slice_by_index(begin = var_6588_begin_0, end = var_6588_end_0, end_mask = var_6588_end_mask_0, squeeze_mask = var_6588_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6588_cast_fp16")];
+            tensor<int32, [4]> var_6590_begin_0 = const()[name = string("op_6590_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_6590_end_0 = const()[name = string("op_6590_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_6590_end_mask_0 = const()[name = string("op_6590_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6590_squeeze_mask_0 = const()[name = string("op_6590_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6590_cast_fp16 = slice_by_index(begin = var_6590_begin_0, end = var_6590_end_0, end_mask = var_6590_end_mask_0, squeeze_mask = var_6590_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6590_cast_fp16")];
+            bool var_6591_transpose_x_0 = const()[name = string("op_6591_transpose_x_0"), val = bool(false)];
+            bool var_6591_transpose_y_0 = const()[name = string("op_6591_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6591_cast_fp16 = matmul(transpose_x = var_6591_transpose_x_0, transpose_y = var_6591_transpose_y_0, x = var_6590_cast_fp16, y = state_367_cast_fp16)[name = string("op_6591_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_367_cast_fp16 = sub(x = var_6588_cast_fp16, y = var_6591_cast_fp16)[name = string("v_new_367_cast_fp16")];
+            tensor<int32, [4]> var_6594_begin_0 = const()[name = string("op_6594_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_6594_end_0 = const()[name = string("op_6594_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_6594_end_mask_0 = const()[name = string("op_6594_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6594_squeeze_mask_0 = const()[name = string("op_6594_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6594_cast_fp16 = slice_by_index(begin = var_6594_begin_0, end = var_6594_end_0, end_mask = var_6594_end_mask_0, squeeze_mask = var_6594_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6594_cast_fp16")];
+            bool var_6595_transpose_x_0 = const()[name = string("op_6595_transpose_x_0"), val = bool(false)];
+            bool var_6595_transpose_y_0 = const()[name = string("op_6595_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6595_cast_fp16 = matmul(transpose_x = var_6595_transpose_x_0, transpose_y = var_6595_transpose_y_0, x = var_6594_cast_fp16, y = state_367_cast_fp16)[name = string("op_6595_cast_fp16")];
+            tensor<int32, [4]> var_6597_begin_0 = const()[name = string("op_6597_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_6597_end_0 = const()[name = string("op_6597_end_0"), val = tensor<int32, [4]>([32, 24, 64, 64])];
+            tensor<bool, [4]> var_6597_end_mask_0 = const()[name = string("op_6597_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6597_squeeze_mask_0 = const()[name = string("op_6597_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6597_cast_fp16 = slice_by_index(begin = var_6597_begin_0, end = var_6597_end_0, end_mask = var_6597_end_mask_0, squeeze_mask = var_6597_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6597_cast_fp16")];
+            bool var_6598_transpose_x_0 = const()[name = string("op_6598_transpose_x_0"), val = bool(false)];
+            bool var_6598_transpose_y_0 = const()[name = string("op_6598_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6598_cast_fp16 = matmul(transpose_x = var_6598_transpose_x_0, transpose_y = var_6598_transpose_y_0, x = var_6597_cast_fp16, y = v_new_367_cast_fp16)[name = string("op_6598_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6599_cast_fp16 = add(x = var_6595_cast_fp16, y = var_6598_cast_fp16)[name = string("op_6599_cast_fp16")];
+            tensor<int32, [4]> var_6601_begin_0 = const()[name = string("op_6601_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_6601_end_0 = const()[name = string("op_6601_end_0"), val = tensor<int32, [4]>([32, 24, 1, 1])];
+            tensor<bool, [4]> var_6601_end_mask_0 = const()[name = string("op_6601_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6601_squeeze_mask_0 = const()[name = string("op_6601_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6601_cast_fp16 = slice_by_index(begin = var_6601_begin_0, end = var_6601_end_0, end_mask = var_6601_end_mask_0, squeeze_mask = var_6601_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6601_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6602_cast_fp16 = mul(x = state_367_cast_fp16, y = var_6601_cast_fp16)[name = string("op_6602_cast_fp16")];
+            tensor<int32, [4]> var_6604_begin_0 = const()[name = string("op_6604_begin_0"), val = tensor<int32, [4]>([0, 23, 0, 0])];
+            tensor<int32, [4]> var_6604_end_0 = const()[name = string("op_6604_end_0"), val = tensor<int32, [4]>([32, 24, 64, 128])];
+            tensor<bool, [4]> var_6604_end_mask_0 = const()[name = string("op_6604_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6604_squeeze_mask_0 = const()[name = string("op_6604_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6604_cast_fp16 = slice_by_index(begin = var_6604_begin_0, end = var_6604_end_0, end_mask = var_6604_end_mask_0, squeeze_mask = var_6604_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6604_cast_fp16")];
+            bool var_6606_transpose_x_1 = const()[name = string("op_6606_transpose_x_1"), val = bool(true)];
+            bool var_6606_transpose_y_1 = const()[name = string("op_6606_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6606_cast_fp16 = matmul(transpose_x = var_6606_transpose_x_1, transpose_y = var_6606_transpose_y_1, x = var_6604_cast_fp16, y = v_new_367_cast_fp16)[name = string("op_6606_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_369_cast_fp16 = add(x = var_6602_cast_fp16, y = var_6606_cast_fp16)[name = string("state_369_cast_fp16")];
+            tensor<int32, [4]> var_6609_begin_0 = const()[name = string("op_6609_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_6609_end_0 = const()[name = string("op_6609_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_6609_end_mask_0 = const()[name = string("op_6609_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6609_squeeze_mask_0 = const()[name = string("op_6609_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6609_cast_fp16 = slice_by_index(begin = var_6609_begin_0, end = var_6609_end_0, end_mask = var_6609_end_mask_0, squeeze_mask = var_6609_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6609_cast_fp16")];
+            tensor<int32, [4]> var_6611_begin_0 = const()[name = string("op_6611_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_6611_end_0 = const()[name = string("op_6611_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_6611_end_mask_0 = const()[name = string("op_6611_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6611_squeeze_mask_0 = const()[name = string("op_6611_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6611_cast_fp16 = slice_by_index(begin = var_6611_begin_0, end = var_6611_end_0, end_mask = var_6611_end_mask_0, squeeze_mask = var_6611_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6611_cast_fp16")];
+            bool var_6612_transpose_x_0 = const()[name = string("op_6612_transpose_x_0"), val = bool(false)];
+            bool var_6612_transpose_y_0 = const()[name = string("op_6612_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6612_cast_fp16 = matmul(transpose_x = var_6612_transpose_x_0, transpose_y = var_6612_transpose_y_0, x = var_6611_cast_fp16, y = state_369_cast_fp16)[name = string("op_6612_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_369_cast_fp16 = sub(x = var_6609_cast_fp16, y = var_6612_cast_fp16)[name = string("v_new_369_cast_fp16")];
+            tensor<int32, [4]> var_6615_begin_0 = const()[name = string("op_6615_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_6615_end_0 = const()[name = string("op_6615_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_6615_end_mask_0 = const()[name = string("op_6615_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6615_squeeze_mask_0 = const()[name = string("op_6615_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6615_cast_fp16 = slice_by_index(begin = var_6615_begin_0, end = var_6615_end_0, end_mask = var_6615_end_mask_0, squeeze_mask = var_6615_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6615_cast_fp16")];
+            bool var_6616_transpose_x_0 = const()[name = string("op_6616_transpose_x_0"), val = bool(false)];
+            bool var_6616_transpose_y_0 = const()[name = string("op_6616_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6616_cast_fp16 = matmul(transpose_x = var_6616_transpose_x_0, transpose_y = var_6616_transpose_y_0, x = var_6615_cast_fp16, y = state_369_cast_fp16)[name = string("op_6616_cast_fp16")];
+            tensor<int32, [4]> var_6618_begin_0 = const()[name = string("op_6618_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_6618_end_0 = const()[name = string("op_6618_end_0"), val = tensor<int32, [4]>([32, 25, 64, 64])];
+            tensor<bool, [4]> var_6618_end_mask_0 = const()[name = string("op_6618_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6618_squeeze_mask_0 = const()[name = string("op_6618_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6618_cast_fp16 = slice_by_index(begin = var_6618_begin_0, end = var_6618_end_0, end_mask = var_6618_end_mask_0, squeeze_mask = var_6618_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6618_cast_fp16")];
+            bool var_6619_transpose_x_0 = const()[name = string("op_6619_transpose_x_0"), val = bool(false)];
+            bool var_6619_transpose_y_0 = const()[name = string("op_6619_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6619_cast_fp16 = matmul(transpose_x = var_6619_transpose_x_0, transpose_y = var_6619_transpose_y_0, x = var_6618_cast_fp16, y = v_new_369_cast_fp16)[name = string("op_6619_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6620_cast_fp16 = add(x = var_6616_cast_fp16, y = var_6619_cast_fp16)[name = string("op_6620_cast_fp16")];
+            tensor<int32, [4]> var_6622_begin_0 = const()[name = string("op_6622_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_6622_end_0 = const()[name = string("op_6622_end_0"), val = tensor<int32, [4]>([32, 25, 1, 1])];
+            tensor<bool, [4]> var_6622_end_mask_0 = const()[name = string("op_6622_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6622_squeeze_mask_0 = const()[name = string("op_6622_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6622_cast_fp16 = slice_by_index(begin = var_6622_begin_0, end = var_6622_end_0, end_mask = var_6622_end_mask_0, squeeze_mask = var_6622_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6622_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6623_cast_fp16 = mul(x = state_369_cast_fp16, y = var_6622_cast_fp16)[name = string("op_6623_cast_fp16")];
+            tensor<int32, [4]> var_6625_begin_0 = const()[name = string("op_6625_begin_0"), val = tensor<int32, [4]>([0, 24, 0, 0])];
+            tensor<int32, [4]> var_6625_end_0 = const()[name = string("op_6625_end_0"), val = tensor<int32, [4]>([32, 25, 64, 128])];
+            tensor<bool, [4]> var_6625_end_mask_0 = const()[name = string("op_6625_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6625_squeeze_mask_0 = const()[name = string("op_6625_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6625_cast_fp16 = slice_by_index(begin = var_6625_begin_0, end = var_6625_end_0, end_mask = var_6625_end_mask_0, squeeze_mask = var_6625_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6625_cast_fp16")];
+            bool var_6627_transpose_x_1 = const()[name = string("op_6627_transpose_x_1"), val = bool(true)];
+            bool var_6627_transpose_y_1 = const()[name = string("op_6627_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6627_cast_fp16 = matmul(transpose_x = var_6627_transpose_x_1, transpose_y = var_6627_transpose_y_1, x = var_6625_cast_fp16, y = v_new_369_cast_fp16)[name = string("op_6627_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_371_cast_fp16 = add(x = var_6623_cast_fp16, y = var_6627_cast_fp16)[name = string("state_371_cast_fp16")];
+            tensor<int32, [4]> var_6630_begin_0 = const()[name = string("op_6630_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_6630_end_0 = const()[name = string("op_6630_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_6630_end_mask_0 = const()[name = string("op_6630_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6630_squeeze_mask_0 = const()[name = string("op_6630_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6630_cast_fp16 = slice_by_index(begin = var_6630_begin_0, end = var_6630_end_0, end_mask = var_6630_end_mask_0, squeeze_mask = var_6630_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6630_cast_fp16")];
+            tensor<int32, [4]> var_6632_begin_0 = const()[name = string("op_6632_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_6632_end_0 = const()[name = string("op_6632_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_6632_end_mask_0 = const()[name = string("op_6632_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6632_squeeze_mask_0 = const()[name = string("op_6632_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6632_cast_fp16 = slice_by_index(begin = var_6632_begin_0, end = var_6632_end_0, end_mask = var_6632_end_mask_0, squeeze_mask = var_6632_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6632_cast_fp16")];
+            bool var_6633_transpose_x_0 = const()[name = string("op_6633_transpose_x_0"), val = bool(false)];
+            bool var_6633_transpose_y_0 = const()[name = string("op_6633_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6633_cast_fp16 = matmul(transpose_x = var_6633_transpose_x_0, transpose_y = var_6633_transpose_y_0, x = var_6632_cast_fp16, y = state_371_cast_fp16)[name = string("op_6633_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_371_cast_fp16 = sub(x = var_6630_cast_fp16, y = var_6633_cast_fp16)[name = string("v_new_371_cast_fp16")];
+            tensor<int32, [4]> var_6636_begin_0 = const()[name = string("op_6636_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_6636_end_0 = const()[name = string("op_6636_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_6636_end_mask_0 = const()[name = string("op_6636_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6636_squeeze_mask_0 = const()[name = string("op_6636_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6636_cast_fp16 = slice_by_index(begin = var_6636_begin_0, end = var_6636_end_0, end_mask = var_6636_end_mask_0, squeeze_mask = var_6636_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6636_cast_fp16")];
+            bool var_6637_transpose_x_0 = const()[name = string("op_6637_transpose_x_0"), val = bool(false)];
+            bool var_6637_transpose_y_0 = const()[name = string("op_6637_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6637_cast_fp16 = matmul(transpose_x = var_6637_transpose_x_0, transpose_y = var_6637_transpose_y_0, x = var_6636_cast_fp16, y = state_371_cast_fp16)[name = string("op_6637_cast_fp16")];
+            tensor<int32, [4]> var_6639_begin_0 = const()[name = string("op_6639_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_6639_end_0 = const()[name = string("op_6639_end_0"), val = tensor<int32, [4]>([32, 26, 64, 64])];
+            tensor<bool, [4]> var_6639_end_mask_0 = const()[name = string("op_6639_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6639_squeeze_mask_0 = const()[name = string("op_6639_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6639_cast_fp16 = slice_by_index(begin = var_6639_begin_0, end = var_6639_end_0, end_mask = var_6639_end_mask_0, squeeze_mask = var_6639_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6639_cast_fp16")];
+            bool var_6640_transpose_x_0 = const()[name = string("op_6640_transpose_x_0"), val = bool(false)];
+            bool var_6640_transpose_y_0 = const()[name = string("op_6640_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6640_cast_fp16 = matmul(transpose_x = var_6640_transpose_x_0, transpose_y = var_6640_transpose_y_0, x = var_6639_cast_fp16, y = v_new_371_cast_fp16)[name = string("op_6640_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6641_cast_fp16 = add(x = var_6637_cast_fp16, y = var_6640_cast_fp16)[name = string("op_6641_cast_fp16")];
+            tensor<int32, [4]> var_6643_begin_0 = const()[name = string("op_6643_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_6643_end_0 = const()[name = string("op_6643_end_0"), val = tensor<int32, [4]>([32, 26, 1, 1])];
+            tensor<bool, [4]> var_6643_end_mask_0 = const()[name = string("op_6643_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6643_squeeze_mask_0 = const()[name = string("op_6643_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6643_cast_fp16 = slice_by_index(begin = var_6643_begin_0, end = var_6643_end_0, end_mask = var_6643_end_mask_0, squeeze_mask = var_6643_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6643_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6644_cast_fp16 = mul(x = state_371_cast_fp16, y = var_6643_cast_fp16)[name = string("op_6644_cast_fp16")];
+            tensor<int32, [4]> var_6646_begin_0 = const()[name = string("op_6646_begin_0"), val = tensor<int32, [4]>([0, 25, 0, 0])];
+            tensor<int32, [4]> var_6646_end_0 = const()[name = string("op_6646_end_0"), val = tensor<int32, [4]>([32, 26, 64, 128])];
+            tensor<bool, [4]> var_6646_end_mask_0 = const()[name = string("op_6646_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6646_squeeze_mask_0 = const()[name = string("op_6646_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6646_cast_fp16 = slice_by_index(begin = var_6646_begin_0, end = var_6646_end_0, end_mask = var_6646_end_mask_0, squeeze_mask = var_6646_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6646_cast_fp16")];
+            bool var_6648_transpose_x_1 = const()[name = string("op_6648_transpose_x_1"), val = bool(true)];
+            bool var_6648_transpose_y_1 = const()[name = string("op_6648_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6648_cast_fp16 = matmul(transpose_x = var_6648_transpose_x_1, transpose_y = var_6648_transpose_y_1, x = var_6646_cast_fp16, y = v_new_371_cast_fp16)[name = string("op_6648_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_373_cast_fp16 = add(x = var_6644_cast_fp16, y = var_6648_cast_fp16)[name = string("state_373_cast_fp16")];
+            tensor<int32, [4]> var_6651_begin_0 = const()[name = string("op_6651_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_6651_end_0 = const()[name = string("op_6651_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_6651_end_mask_0 = const()[name = string("op_6651_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6651_squeeze_mask_0 = const()[name = string("op_6651_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6651_cast_fp16 = slice_by_index(begin = var_6651_begin_0, end = var_6651_end_0, end_mask = var_6651_end_mask_0, squeeze_mask = var_6651_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6651_cast_fp16")];
+            tensor<int32, [4]> var_6653_begin_0 = const()[name = string("op_6653_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_6653_end_0 = const()[name = string("op_6653_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_6653_end_mask_0 = const()[name = string("op_6653_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6653_squeeze_mask_0 = const()[name = string("op_6653_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6653_cast_fp16 = slice_by_index(begin = var_6653_begin_0, end = var_6653_end_0, end_mask = var_6653_end_mask_0, squeeze_mask = var_6653_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6653_cast_fp16")];
+            bool var_6654_transpose_x_0 = const()[name = string("op_6654_transpose_x_0"), val = bool(false)];
+            bool var_6654_transpose_y_0 = const()[name = string("op_6654_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6654_cast_fp16 = matmul(transpose_x = var_6654_transpose_x_0, transpose_y = var_6654_transpose_y_0, x = var_6653_cast_fp16, y = state_373_cast_fp16)[name = string("op_6654_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_373_cast_fp16 = sub(x = var_6651_cast_fp16, y = var_6654_cast_fp16)[name = string("v_new_373_cast_fp16")];
+            tensor<int32, [4]> var_6657_begin_0 = const()[name = string("op_6657_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_6657_end_0 = const()[name = string("op_6657_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_6657_end_mask_0 = const()[name = string("op_6657_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6657_squeeze_mask_0 = const()[name = string("op_6657_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6657_cast_fp16 = slice_by_index(begin = var_6657_begin_0, end = var_6657_end_0, end_mask = var_6657_end_mask_0, squeeze_mask = var_6657_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6657_cast_fp16")];
+            bool var_6658_transpose_x_0 = const()[name = string("op_6658_transpose_x_0"), val = bool(false)];
+            bool var_6658_transpose_y_0 = const()[name = string("op_6658_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6658_cast_fp16 = matmul(transpose_x = var_6658_transpose_x_0, transpose_y = var_6658_transpose_y_0, x = var_6657_cast_fp16, y = state_373_cast_fp16)[name = string("op_6658_cast_fp16")];
+            tensor<int32, [4]> var_6660_begin_0 = const()[name = string("op_6660_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_6660_end_0 = const()[name = string("op_6660_end_0"), val = tensor<int32, [4]>([32, 27, 64, 64])];
+            tensor<bool, [4]> var_6660_end_mask_0 = const()[name = string("op_6660_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6660_squeeze_mask_0 = const()[name = string("op_6660_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6660_cast_fp16 = slice_by_index(begin = var_6660_begin_0, end = var_6660_end_0, end_mask = var_6660_end_mask_0, squeeze_mask = var_6660_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6660_cast_fp16")];
+            bool var_6661_transpose_x_0 = const()[name = string("op_6661_transpose_x_0"), val = bool(false)];
+            bool var_6661_transpose_y_0 = const()[name = string("op_6661_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6661_cast_fp16 = matmul(transpose_x = var_6661_transpose_x_0, transpose_y = var_6661_transpose_y_0, x = var_6660_cast_fp16, y = v_new_373_cast_fp16)[name = string("op_6661_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6662_cast_fp16 = add(x = var_6658_cast_fp16, y = var_6661_cast_fp16)[name = string("op_6662_cast_fp16")];
+            tensor<int32, [4]> var_6664_begin_0 = const()[name = string("op_6664_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_6664_end_0 = const()[name = string("op_6664_end_0"), val = tensor<int32, [4]>([32, 27, 1, 1])];
+            tensor<bool, [4]> var_6664_end_mask_0 = const()[name = string("op_6664_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6664_squeeze_mask_0 = const()[name = string("op_6664_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6664_cast_fp16 = slice_by_index(begin = var_6664_begin_0, end = var_6664_end_0, end_mask = var_6664_end_mask_0, squeeze_mask = var_6664_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6664_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6665_cast_fp16 = mul(x = state_373_cast_fp16, y = var_6664_cast_fp16)[name = string("op_6665_cast_fp16")];
+            tensor<int32, [4]> var_6667_begin_0 = const()[name = string("op_6667_begin_0"), val = tensor<int32, [4]>([0, 26, 0, 0])];
+            tensor<int32, [4]> var_6667_end_0 = const()[name = string("op_6667_end_0"), val = tensor<int32, [4]>([32, 27, 64, 128])];
+            tensor<bool, [4]> var_6667_end_mask_0 = const()[name = string("op_6667_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6667_squeeze_mask_0 = const()[name = string("op_6667_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6667_cast_fp16 = slice_by_index(begin = var_6667_begin_0, end = var_6667_end_0, end_mask = var_6667_end_mask_0, squeeze_mask = var_6667_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6667_cast_fp16")];
+            bool var_6669_transpose_x_1 = const()[name = string("op_6669_transpose_x_1"), val = bool(true)];
+            bool var_6669_transpose_y_1 = const()[name = string("op_6669_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6669_cast_fp16 = matmul(transpose_x = var_6669_transpose_x_1, transpose_y = var_6669_transpose_y_1, x = var_6667_cast_fp16, y = v_new_373_cast_fp16)[name = string("op_6669_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_375_cast_fp16 = add(x = var_6665_cast_fp16, y = var_6669_cast_fp16)[name = string("state_375_cast_fp16")];
+            tensor<int32, [4]> var_6672_begin_0 = const()[name = string("op_6672_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_6672_end_0 = const()[name = string("op_6672_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_6672_end_mask_0 = const()[name = string("op_6672_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6672_squeeze_mask_0 = const()[name = string("op_6672_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6672_cast_fp16 = slice_by_index(begin = var_6672_begin_0, end = var_6672_end_0, end_mask = var_6672_end_mask_0, squeeze_mask = var_6672_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6672_cast_fp16")];
+            tensor<int32, [4]> var_6674_begin_0 = const()[name = string("op_6674_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_6674_end_0 = const()[name = string("op_6674_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_6674_end_mask_0 = const()[name = string("op_6674_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6674_squeeze_mask_0 = const()[name = string("op_6674_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6674_cast_fp16 = slice_by_index(begin = var_6674_begin_0, end = var_6674_end_0, end_mask = var_6674_end_mask_0, squeeze_mask = var_6674_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6674_cast_fp16")];
+            bool var_6675_transpose_x_0 = const()[name = string("op_6675_transpose_x_0"), val = bool(false)];
+            bool var_6675_transpose_y_0 = const()[name = string("op_6675_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6675_cast_fp16 = matmul(transpose_x = var_6675_transpose_x_0, transpose_y = var_6675_transpose_y_0, x = var_6674_cast_fp16, y = state_375_cast_fp16)[name = string("op_6675_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_375_cast_fp16 = sub(x = var_6672_cast_fp16, y = var_6675_cast_fp16)[name = string("v_new_375_cast_fp16")];
+            tensor<int32, [4]> var_6678_begin_0 = const()[name = string("op_6678_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_6678_end_0 = const()[name = string("op_6678_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_6678_end_mask_0 = const()[name = string("op_6678_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6678_squeeze_mask_0 = const()[name = string("op_6678_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6678_cast_fp16 = slice_by_index(begin = var_6678_begin_0, end = var_6678_end_0, end_mask = var_6678_end_mask_0, squeeze_mask = var_6678_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6678_cast_fp16")];
+            bool var_6679_transpose_x_0 = const()[name = string("op_6679_transpose_x_0"), val = bool(false)];
+            bool var_6679_transpose_y_0 = const()[name = string("op_6679_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6679_cast_fp16 = matmul(transpose_x = var_6679_transpose_x_0, transpose_y = var_6679_transpose_y_0, x = var_6678_cast_fp16, y = state_375_cast_fp16)[name = string("op_6679_cast_fp16")];
+            tensor<int32, [4]> var_6681_begin_0 = const()[name = string("op_6681_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_6681_end_0 = const()[name = string("op_6681_end_0"), val = tensor<int32, [4]>([32, 28, 64, 64])];
+            tensor<bool, [4]> var_6681_end_mask_0 = const()[name = string("op_6681_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6681_squeeze_mask_0 = const()[name = string("op_6681_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6681_cast_fp16 = slice_by_index(begin = var_6681_begin_0, end = var_6681_end_0, end_mask = var_6681_end_mask_0, squeeze_mask = var_6681_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6681_cast_fp16")];
+            bool var_6682_transpose_x_0 = const()[name = string("op_6682_transpose_x_0"), val = bool(false)];
+            bool var_6682_transpose_y_0 = const()[name = string("op_6682_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6682_cast_fp16 = matmul(transpose_x = var_6682_transpose_x_0, transpose_y = var_6682_transpose_y_0, x = var_6681_cast_fp16, y = v_new_375_cast_fp16)[name = string("op_6682_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6683_cast_fp16 = add(x = var_6679_cast_fp16, y = var_6682_cast_fp16)[name = string("op_6683_cast_fp16")];
+            tensor<int32, [4]> var_6685_begin_0 = const()[name = string("op_6685_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_6685_end_0 = const()[name = string("op_6685_end_0"), val = tensor<int32, [4]>([32, 28, 1, 1])];
+            tensor<bool, [4]> var_6685_end_mask_0 = const()[name = string("op_6685_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6685_squeeze_mask_0 = const()[name = string("op_6685_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6685_cast_fp16 = slice_by_index(begin = var_6685_begin_0, end = var_6685_end_0, end_mask = var_6685_end_mask_0, squeeze_mask = var_6685_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6685_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6686_cast_fp16 = mul(x = state_375_cast_fp16, y = var_6685_cast_fp16)[name = string("op_6686_cast_fp16")];
+            tensor<int32, [4]> var_6688_begin_0 = const()[name = string("op_6688_begin_0"), val = tensor<int32, [4]>([0, 27, 0, 0])];
+            tensor<int32, [4]> var_6688_end_0 = const()[name = string("op_6688_end_0"), val = tensor<int32, [4]>([32, 28, 64, 128])];
+            tensor<bool, [4]> var_6688_end_mask_0 = const()[name = string("op_6688_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6688_squeeze_mask_0 = const()[name = string("op_6688_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6688_cast_fp16 = slice_by_index(begin = var_6688_begin_0, end = var_6688_end_0, end_mask = var_6688_end_mask_0, squeeze_mask = var_6688_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6688_cast_fp16")];
+            bool var_6690_transpose_x_1 = const()[name = string("op_6690_transpose_x_1"), val = bool(true)];
+            bool var_6690_transpose_y_1 = const()[name = string("op_6690_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6690_cast_fp16 = matmul(transpose_x = var_6690_transpose_x_1, transpose_y = var_6690_transpose_y_1, x = var_6688_cast_fp16, y = v_new_375_cast_fp16)[name = string("op_6690_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_377_cast_fp16 = add(x = var_6686_cast_fp16, y = var_6690_cast_fp16)[name = string("state_377_cast_fp16")];
+            tensor<int32, [4]> var_6693_begin_0 = const()[name = string("op_6693_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_6693_end_0 = const()[name = string("op_6693_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_6693_end_mask_0 = const()[name = string("op_6693_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6693_squeeze_mask_0 = const()[name = string("op_6693_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6693_cast_fp16 = slice_by_index(begin = var_6693_begin_0, end = var_6693_end_0, end_mask = var_6693_end_mask_0, squeeze_mask = var_6693_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6693_cast_fp16")];
+            tensor<int32, [4]> var_6695_begin_0 = const()[name = string("op_6695_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_6695_end_0 = const()[name = string("op_6695_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_6695_end_mask_0 = const()[name = string("op_6695_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6695_squeeze_mask_0 = const()[name = string("op_6695_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6695_cast_fp16 = slice_by_index(begin = var_6695_begin_0, end = var_6695_end_0, end_mask = var_6695_end_mask_0, squeeze_mask = var_6695_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6695_cast_fp16")];
+            bool var_6696_transpose_x_0 = const()[name = string("op_6696_transpose_x_0"), val = bool(false)];
+            bool var_6696_transpose_y_0 = const()[name = string("op_6696_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6696_cast_fp16 = matmul(transpose_x = var_6696_transpose_x_0, transpose_y = var_6696_transpose_y_0, x = var_6695_cast_fp16, y = state_377_cast_fp16)[name = string("op_6696_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_377_cast_fp16 = sub(x = var_6693_cast_fp16, y = var_6696_cast_fp16)[name = string("v_new_377_cast_fp16")];
+            tensor<int32, [4]> var_6699_begin_0 = const()[name = string("op_6699_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_6699_end_0 = const()[name = string("op_6699_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_6699_end_mask_0 = const()[name = string("op_6699_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6699_squeeze_mask_0 = const()[name = string("op_6699_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6699_cast_fp16 = slice_by_index(begin = var_6699_begin_0, end = var_6699_end_0, end_mask = var_6699_end_mask_0, squeeze_mask = var_6699_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6699_cast_fp16")];
+            bool var_6700_transpose_x_0 = const()[name = string("op_6700_transpose_x_0"), val = bool(false)];
+            bool var_6700_transpose_y_0 = const()[name = string("op_6700_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6700_cast_fp16 = matmul(transpose_x = var_6700_transpose_x_0, transpose_y = var_6700_transpose_y_0, x = var_6699_cast_fp16, y = state_377_cast_fp16)[name = string("op_6700_cast_fp16")];
+            tensor<int32, [4]> var_6702_begin_0 = const()[name = string("op_6702_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_6702_end_0 = const()[name = string("op_6702_end_0"), val = tensor<int32, [4]>([32, 29, 64, 64])];
+            tensor<bool, [4]> var_6702_end_mask_0 = const()[name = string("op_6702_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6702_squeeze_mask_0 = const()[name = string("op_6702_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6702_cast_fp16 = slice_by_index(begin = var_6702_begin_0, end = var_6702_end_0, end_mask = var_6702_end_mask_0, squeeze_mask = var_6702_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6702_cast_fp16")];
+            bool var_6703_transpose_x_0 = const()[name = string("op_6703_transpose_x_0"), val = bool(false)];
+            bool var_6703_transpose_y_0 = const()[name = string("op_6703_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6703_cast_fp16 = matmul(transpose_x = var_6703_transpose_x_0, transpose_y = var_6703_transpose_y_0, x = var_6702_cast_fp16, y = v_new_377_cast_fp16)[name = string("op_6703_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6704_cast_fp16 = add(x = var_6700_cast_fp16, y = var_6703_cast_fp16)[name = string("op_6704_cast_fp16")];
+            tensor<int32, [4]> var_6706_begin_0 = const()[name = string("op_6706_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_6706_end_0 = const()[name = string("op_6706_end_0"), val = tensor<int32, [4]>([32, 29, 1, 1])];
+            tensor<bool, [4]> var_6706_end_mask_0 = const()[name = string("op_6706_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6706_squeeze_mask_0 = const()[name = string("op_6706_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6706_cast_fp16 = slice_by_index(begin = var_6706_begin_0, end = var_6706_end_0, end_mask = var_6706_end_mask_0, squeeze_mask = var_6706_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6706_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6707_cast_fp16 = mul(x = state_377_cast_fp16, y = var_6706_cast_fp16)[name = string("op_6707_cast_fp16")];
+            tensor<int32, [4]> var_6709_begin_0 = const()[name = string("op_6709_begin_0"), val = tensor<int32, [4]>([0, 28, 0, 0])];
+            tensor<int32, [4]> var_6709_end_0 = const()[name = string("op_6709_end_0"), val = tensor<int32, [4]>([32, 29, 64, 128])];
+            tensor<bool, [4]> var_6709_end_mask_0 = const()[name = string("op_6709_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6709_squeeze_mask_0 = const()[name = string("op_6709_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6709_cast_fp16 = slice_by_index(begin = var_6709_begin_0, end = var_6709_end_0, end_mask = var_6709_end_mask_0, squeeze_mask = var_6709_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6709_cast_fp16")];
+            bool var_6711_transpose_x_1 = const()[name = string("op_6711_transpose_x_1"), val = bool(true)];
+            bool var_6711_transpose_y_1 = const()[name = string("op_6711_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6711_cast_fp16 = matmul(transpose_x = var_6711_transpose_x_1, transpose_y = var_6711_transpose_y_1, x = var_6709_cast_fp16, y = v_new_377_cast_fp16)[name = string("op_6711_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_379_cast_fp16 = add(x = var_6707_cast_fp16, y = var_6711_cast_fp16)[name = string("state_379_cast_fp16")];
+            tensor<int32, [4]> var_6714_begin_0 = const()[name = string("op_6714_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_6714_end_0 = const()[name = string("op_6714_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_6714_end_mask_0 = const()[name = string("op_6714_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6714_squeeze_mask_0 = const()[name = string("op_6714_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6714_cast_fp16 = slice_by_index(begin = var_6714_begin_0, end = var_6714_end_0, end_mask = var_6714_end_mask_0, squeeze_mask = var_6714_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6714_cast_fp16")];
+            tensor<int32, [4]> var_6716_begin_0 = const()[name = string("op_6716_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_6716_end_0 = const()[name = string("op_6716_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_6716_end_mask_0 = const()[name = string("op_6716_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6716_squeeze_mask_0 = const()[name = string("op_6716_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6716_cast_fp16 = slice_by_index(begin = var_6716_begin_0, end = var_6716_end_0, end_mask = var_6716_end_mask_0, squeeze_mask = var_6716_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6716_cast_fp16")];
+            bool var_6717_transpose_x_0 = const()[name = string("op_6717_transpose_x_0"), val = bool(false)];
+            bool var_6717_transpose_y_0 = const()[name = string("op_6717_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6717_cast_fp16 = matmul(transpose_x = var_6717_transpose_x_0, transpose_y = var_6717_transpose_y_0, x = var_6716_cast_fp16, y = state_379_cast_fp16)[name = string("op_6717_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_379_cast_fp16 = sub(x = var_6714_cast_fp16, y = var_6717_cast_fp16)[name = string("v_new_379_cast_fp16")];
+            tensor<int32, [4]> var_6720_begin_0 = const()[name = string("op_6720_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_6720_end_0 = const()[name = string("op_6720_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_6720_end_mask_0 = const()[name = string("op_6720_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6720_squeeze_mask_0 = const()[name = string("op_6720_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6720_cast_fp16 = slice_by_index(begin = var_6720_begin_0, end = var_6720_end_0, end_mask = var_6720_end_mask_0, squeeze_mask = var_6720_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6720_cast_fp16")];
+            bool var_6721_transpose_x_0 = const()[name = string("op_6721_transpose_x_0"), val = bool(false)];
+            bool var_6721_transpose_y_0 = const()[name = string("op_6721_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6721_cast_fp16 = matmul(transpose_x = var_6721_transpose_x_0, transpose_y = var_6721_transpose_y_0, x = var_6720_cast_fp16, y = state_379_cast_fp16)[name = string("op_6721_cast_fp16")];
+            tensor<int32, [4]> var_6723_begin_0 = const()[name = string("op_6723_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_6723_end_0 = const()[name = string("op_6723_end_0"), val = tensor<int32, [4]>([32, 30, 64, 64])];
+            tensor<bool, [4]> var_6723_end_mask_0 = const()[name = string("op_6723_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6723_squeeze_mask_0 = const()[name = string("op_6723_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6723_cast_fp16 = slice_by_index(begin = var_6723_begin_0, end = var_6723_end_0, end_mask = var_6723_end_mask_0, squeeze_mask = var_6723_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6723_cast_fp16")];
+            bool var_6724_transpose_x_0 = const()[name = string("op_6724_transpose_x_0"), val = bool(false)];
+            bool var_6724_transpose_y_0 = const()[name = string("op_6724_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6724_cast_fp16 = matmul(transpose_x = var_6724_transpose_x_0, transpose_y = var_6724_transpose_y_0, x = var_6723_cast_fp16, y = v_new_379_cast_fp16)[name = string("op_6724_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6725_cast_fp16 = add(x = var_6721_cast_fp16, y = var_6724_cast_fp16)[name = string("op_6725_cast_fp16")];
+            tensor<int32, [4]> var_6727_begin_0 = const()[name = string("op_6727_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_6727_end_0 = const()[name = string("op_6727_end_0"), val = tensor<int32, [4]>([32, 30, 1, 1])];
+            tensor<bool, [4]> var_6727_end_mask_0 = const()[name = string("op_6727_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6727_squeeze_mask_0 = const()[name = string("op_6727_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6727_cast_fp16 = slice_by_index(begin = var_6727_begin_0, end = var_6727_end_0, end_mask = var_6727_end_mask_0, squeeze_mask = var_6727_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6727_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6728_cast_fp16 = mul(x = state_379_cast_fp16, y = var_6727_cast_fp16)[name = string("op_6728_cast_fp16")];
+            tensor<int32, [4]> var_6730_begin_0 = const()[name = string("op_6730_begin_0"), val = tensor<int32, [4]>([0, 29, 0, 0])];
+            tensor<int32, [4]> var_6730_end_0 = const()[name = string("op_6730_end_0"), val = tensor<int32, [4]>([32, 30, 64, 128])];
+            tensor<bool, [4]> var_6730_end_mask_0 = const()[name = string("op_6730_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6730_squeeze_mask_0 = const()[name = string("op_6730_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6730_cast_fp16 = slice_by_index(begin = var_6730_begin_0, end = var_6730_end_0, end_mask = var_6730_end_mask_0, squeeze_mask = var_6730_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6730_cast_fp16")];
+            bool var_6732_transpose_x_1 = const()[name = string("op_6732_transpose_x_1"), val = bool(true)];
+            bool var_6732_transpose_y_1 = const()[name = string("op_6732_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6732_cast_fp16 = matmul(transpose_x = var_6732_transpose_x_1, transpose_y = var_6732_transpose_y_1, x = var_6730_cast_fp16, y = v_new_379_cast_fp16)[name = string("op_6732_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_381_cast_fp16 = add(x = var_6728_cast_fp16, y = var_6732_cast_fp16)[name = string("state_381_cast_fp16")];
+            tensor<int32, [4]> var_6735_begin_0 = const()[name = string("op_6735_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_6735_end_0 = const()[name = string("op_6735_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_6735_end_mask_0 = const()[name = string("op_6735_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6735_squeeze_mask_0 = const()[name = string("op_6735_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6735_cast_fp16 = slice_by_index(begin = var_6735_begin_0, end = var_6735_end_0, end_mask = var_6735_end_mask_0, squeeze_mask = var_6735_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6735_cast_fp16")];
+            tensor<int32, [4]> var_6737_begin_0 = const()[name = string("op_6737_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_6737_end_0 = const()[name = string("op_6737_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_6737_end_mask_0 = const()[name = string("op_6737_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6737_squeeze_mask_0 = const()[name = string("op_6737_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6737_cast_fp16 = slice_by_index(begin = var_6737_begin_0, end = var_6737_end_0, end_mask = var_6737_end_mask_0, squeeze_mask = var_6737_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6737_cast_fp16")];
+            bool var_6738_transpose_x_0 = const()[name = string("op_6738_transpose_x_0"), val = bool(false)];
+            bool var_6738_transpose_y_0 = const()[name = string("op_6738_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6738_cast_fp16 = matmul(transpose_x = var_6738_transpose_x_0, transpose_y = var_6738_transpose_y_0, x = var_6737_cast_fp16, y = state_381_cast_fp16)[name = string("op_6738_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_381_cast_fp16 = sub(x = var_6735_cast_fp16, y = var_6738_cast_fp16)[name = string("v_new_381_cast_fp16")];
+            tensor<int32, [4]> var_6741_begin_0 = const()[name = string("op_6741_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_6741_end_0 = const()[name = string("op_6741_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_6741_end_mask_0 = const()[name = string("op_6741_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6741_squeeze_mask_0 = const()[name = string("op_6741_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6741_cast_fp16 = slice_by_index(begin = var_6741_begin_0, end = var_6741_end_0, end_mask = var_6741_end_mask_0, squeeze_mask = var_6741_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6741_cast_fp16")];
+            bool var_6742_transpose_x_0 = const()[name = string("op_6742_transpose_x_0"), val = bool(false)];
+            bool var_6742_transpose_y_0 = const()[name = string("op_6742_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6742_cast_fp16 = matmul(transpose_x = var_6742_transpose_x_0, transpose_y = var_6742_transpose_y_0, x = var_6741_cast_fp16, y = state_381_cast_fp16)[name = string("op_6742_cast_fp16")];
+            tensor<int32, [4]> var_6744_begin_0 = const()[name = string("op_6744_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_6744_end_0 = const()[name = string("op_6744_end_0"), val = tensor<int32, [4]>([32, 31, 64, 64])];
+            tensor<bool, [4]> var_6744_end_mask_0 = const()[name = string("op_6744_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6744_squeeze_mask_0 = const()[name = string("op_6744_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6744_cast_fp16 = slice_by_index(begin = var_6744_begin_0, end = var_6744_end_0, end_mask = var_6744_end_mask_0, squeeze_mask = var_6744_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6744_cast_fp16")];
+            bool var_6745_transpose_x_0 = const()[name = string("op_6745_transpose_x_0"), val = bool(false)];
+            bool var_6745_transpose_y_0 = const()[name = string("op_6745_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6745_cast_fp16 = matmul(transpose_x = var_6745_transpose_x_0, transpose_y = var_6745_transpose_y_0, x = var_6744_cast_fp16, y = v_new_381_cast_fp16)[name = string("op_6745_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6746_cast_fp16 = add(x = var_6742_cast_fp16, y = var_6745_cast_fp16)[name = string("op_6746_cast_fp16")];
+            tensor<int32, [4]> var_6748_begin_0 = const()[name = string("op_6748_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_6748_end_0 = const()[name = string("op_6748_end_0"), val = tensor<int32, [4]>([32, 31, 1, 1])];
+            tensor<bool, [4]> var_6748_end_mask_0 = const()[name = string("op_6748_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6748_squeeze_mask_0 = const()[name = string("op_6748_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 1, 1]> var_6748_cast_fp16 = slice_by_index(begin = var_6748_begin_0, end = var_6748_end_0, end_mask = var_6748_end_mask_0, squeeze_mask = var_6748_squeeze_mask_0, x = chunk_decay_cast_fp16)[name = string("op_6748_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> var_6749_cast_fp16 = mul(x = state_381_cast_fp16, y = var_6748_cast_fp16)[name = string("op_6749_cast_fp16")];
+            tensor<int32, [4]> var_6751_begin_0 = const()[name = string("op_6751_begin_0"), val = tensor<int32, [4]>([0, 30, 0, 0])];
+            tensor<int32, [4]> var_6751_end_0 = const()[name = string("op_6751_end_0"), val = tensor<int32, [4]>([32, 31, 64, 128])];
+            tensor<bool, [4]> var_6751_end_mask_0 = const()[name = string("op_6751_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6751_squeeze_mask_0 = const()[name = string("op_6751_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6751_cast_fp16 = slice_by_index(begin = var_6751_begin_0, end = var_6751_end_0, end_mask = var_6751_end_mask_0, squeeze_mask = var_6751_squeeze_mask_0, x = k_dec_cast_fp16)[name = string("op_6751_cast_fp16")];
+            bool var_6753_transpose_x_1 = const()[name = string("op_6753_transpose_x_1"), val = bool(true)];
+            bool var_6753_transpose_y_1 = const()[name = string("op_6753_transpose_y_1"), val = bool(false)];
+            tensor<fp16, [32, 128, 128]> var_6753_cast_fp16 = matmul(transpose_x = var_6753_transpose_x_1, transpose_y = var_6753_transpose_y_1, x = var_6751_cast_fp16, y = v_new_381_cast_fp16)[name = string("op_6753_cast_fp16")];
+            tensor<fp16, [32, 128, 128]> state_cast_fp16 = add(x = var_6749_cast_fp16, y = var_6753_cast_fp16)[name = string("state_cast_fp16")];
+            tensor<int32, [4]> var_6756_begin_0 = const()[name = string("op_6756_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_6756_end_0 = const()[name = string("op_6756_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_6756_end_mask_0 = const()[name = string("op_6756_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6756_squeeze_mask_0 = const()[name = string("op_6756_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6756_cast_fp16 = slice_by_index(begin = var_6756_begin_0, end = var_6756_end_0, end_mask = var_6756_end_mask_0, squeeze_mask = var_6756_squeeze_mask_0, x = new_v_cast_fp16)[name = string("op_6756_cast_fp16")];
+            tensor<int32, [4]> var_6758_begin_0 = const()[name = string("op_6758_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_6758_end_0 = const()[name = string("op_6758_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_6758_end_mask_0 = const()[name = string("op_6758_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6758_squeeze_mask_0 = const()[name = string("op_6758_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6758_cast_fp16 = slice_by_index(begin = var_6758_begin_0, end = var_6758_end_0, end_mask = var_6758_end_mask_0, squeeze_mask = var_6758_squeeze_mask_0, x = k_cumdecay_cast_fp16)[name = string("op_6758_cast_fp16")];
+            bool var_6759_transpose_x_0 = const()[name = string("op_6759_transpose_x_0"), val = bool(false)];
+            bool var_6759_transpose_y_0 = const()[name = string("op_6759_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6759_cast_fp16 = matmul(transpose_x = var_6759_transpose_x_0, transpose_y = var_6759_transpose_y_0, x = var_6758_cast_fp16, y = state_cast_fp16)[name = string("op_6759_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> v_new_cast_fp16 = sub(x = var_6756_cast_fp16, y = var_6759_cast_fp16)[name = string("v_new_cast_fp16")];
+            tensor<int32, [4]> var_6762_begin_0 = const()[name = string("op_6762_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_6762_end_0 = const()[name = string("op_6762_end_0"), val = tensor<int32, [4]>([32, 32, 64, 128])];
+            tensor<bool, [4]> var_6762_end_mask_0 = const()[name = string("op_6762_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6762_squeeze_mask_0 = const()[name = string("op_6762_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 128]> var_6762_cast_fp16 = slice_by_index(begin = var_6762_begin_0, end = var_6762_end_0, end_mask = var_6762_end_mask_0, squeeze_mask = var_6762_squeeze_mask_0, x = q_dec_cast_fp16)[name = string("op_6762_cast_fp16")];
+            bool var_6763_transpose_x_0 = const()[name = string("op_6763_transpose_x_0"), val = bool(false)];
+            bool var_6763_transpose_y_0 = const()[name = string("op_6763_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6763_cast_fp16 = matmul(transpose_x = var_6763_transpose_x_0, transpose_y = var_6763_transpose_y_0, x = var_6762_cast_fp16, y = state_cast_fp16)[name = string("op_6763_cast_fp16")];
+            tensor<int32, [4]> var_6765_begin_0 = const()[name = string("op_6765_begin_0"), val = tensor<int32, [4]>([0, 31, 0, 0])];
+            tensor<int32, [4]> var_6765_end_0 = const()[name = string("op_6765_end_0"), val = tensor<int32, [4]>([32, 32, 64, 64])];
+            tensor<bool, [4]> var_6765_end_mask_0 = const()[name = string("op_6765_end_mask_0"), val = tensor<bool, [4]>([true, false, true, true])];
+            tensor<bool, [4]> var_6765_squeeze_mask_0 = const()[name = string("op_6765_squeeze_mask_0"), val = tensor<bool, [4]>([false, true, false, false])];
+            tensor<fp16, [32, 64, 64]> var_6765_cast_fp16 = slice_by_index(begin = var_6765_begin_0, end = var_6765_end_0, end_mask = var_6765_end_mask_0, squeeze_mask = var_6765_squeeze_mask_0, x = attn_intra_cast_fp16)[name = string("op_6765_cast_fp16")];
+            bool var_6766_transpose_x_0 = const()[name = string("op_6766_transpose_x_0"), val = bool(false)];
+            bool var_6766_transpose_y_0 = const()[name = string("op_6766_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [32, 64, 128]> var_6766_cast_fp16 = matmul(transpose_x = var_6766_transpose_x_0, transpose_y = var_6766_transpose_y_0, x = var_6765_cast_fp16, y = v_new_cast_fp16)[name = string("op_6766_cast_fp16")];
+            tensor<fp16, [32, 64, 128]> var_6767_cast_fp16 = add(x = var_6763_cast_fp16, y = var_6766_cast_fp16)[name = string("op_6767_cast_fp16")];
+            int32 var_6769_axis_0 = const()[name = string("op_6769_axis_0"), val = int32(1)];
+            tensor<fp16, [32, 32, 64, 128]> var_6769_cast_fp16 = stack(axis = var_6769_axis_0, values = (var_6116_cast_fp16, var_6137_cast_fp16, var_6158_cast_fp16, var_6179_cast_fp16, var_6200_cast_fp16, var_6221_cast_fp16, var_6242_cast_fp16, var_6263_cast_fp16, var_6284_cast_fp16, var_6305_cast_fp16, var_6326_cast_fp16, var_6347_cast_fp16, var_6368_cast_fp16, var_6389_cast_fp16, var_6410_cast_fp16, var_6431_cast_fp16, var_6452_cast_fp16, var_6473_cast_fp16, var_6494_cast_fp16, var_6515_cast_fp16, var_6536_cast_fp16, var_6557_cast_fp16, var_6578_cast_fp16, var_6599_cast_fp16, var_6620_cast_fp16, var_6641_cast_fp16, var_6662_cast_fp16, var_6683_cast_fp16, var_6704_cast_fp16, var_6725_cast_fp16, var_6746_cast_fp16, var_6767_cast_fp16))[name = string("op_6769_cast_fp16")];
+            tensor<int32, [3]> var_6770 = const()[name = string("op_6770"), val = tensor<int32, [3]>([32, 2048, 128])];
+            tensor<fp16, [32, 2048, 128]> var_6771_cast_fp16 = reshape(shape = var_6770, x = var_6769_cast_fp16)[name = string("op_6771_cast_fp16")];
+            tensor<int32, [3]> var_6772 = const()[name = string("op_6772"), val = tensor<int32, [3]>([1, 0, 2])];
+            tensor<int32, [4]> input_95_begin_0 = const()[name = string("input_95_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> input_95_end_0 = const()[name = string("input_95_end_0"), val = tensor<int32, [4]>([1, 2048, 32, 128])];
+            tensor<bool, [4]> input_95_end_mask_0 = const()[name = string("input_95_end_mask_0"), val = tensor<bool, [4]>([false, true, true, true])];
+            tensor<bool, [4]> input_95_squeeze_mask_0 = const()[name = string("input_95_squeeze_mask_0"), val = tensor<bool, [4]>([true, false, false, false])];
+            tensor<fp16, [2048, 32, 128]> input_95_cast_fp16 = slice_by_index(begin = input_95_begin_0, end = input_95_end_0, end_mask = input_95_end_mask_0, squeeze_mask = input_95_squeeze_mask_0, x = z_cast_fp16)[name = string("input_95_cast_fp16")];
+            fp16 var_5752_promoted_1_to_fp16 = const()[name = string("op_5752_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [2048, 32, 128]> x_169_cast_fp16 = transpose(perm = var_6772, x = var_6771_cast_fp16)[name = string("transpose_4")];
+            tensor<fp16, [2048, 32, 128]> var_6776_cast_fp16 = pow(x = x_169_cast_fp16, y = var_5752_promoted_1_to_fp16)[name = string("op_6776_cast_fp16")];
+            tensor<int32, [1]> var_6778_axes_0 = const()[name = string("op_6778_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6778_keep_dims_0 = const()[name = string("op_6778_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [2048, 32, 1]> var_6778_cast_fp16 = reduce_mean(axes = var_6778_axes_0, keep_dims = var_6778_keep_dims_0, x = var_6776_cast_fp16)[name = string("op_6778_cast_fp16")];
+            fp16 var_6779_to_fp16 = const()[name = string("op_6779_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [2048, 32, 1]> var_6780_cast_fp16 = add(x = var_6778_cast_fp16, y = var_6779_to_fp16)[name = string("op_6780_cast_fp16")];
+            fp32 var_6781_epsilon_0 = const()[name = string("op_6781_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [2048, 32, 1]> var_6781_cast_fp16 = rsqrt(epsilon = var_6781_epsilon_0, x = var_6780_cast_fp16)[name = string("op_6781_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> x_171_cast_fp16 = mul(x = x_169_cast_fp16, y = var_6781_cast_fp16)[name = string("x_171_cast_fp16")];
+            tensor<fp16, [128]> layers_6_linear_attn_norm_weight_to_fp16 = const()[name = string("layers_6_linear_attn_norm_weight_to_fp16"), val = tensor<fp16, [128]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421837376)))];
+            tensor<fp16, [2048, 32, 128]> var_6783_cast_fp16 = mul(x = layers_6_linear_attn_norm_weight_to_fp16, y = x_171_cast_fp16)[name = string("op_6783_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> var_6784_cast_fp16 = silu(x = input_95_cast_fp16)[name = string("op_6784_cast_fp16")];
+            tensor<fp16, [2048, 32, 128]> core_cast_fp16 = mul(x = var_6783_cast_fp16, y = var_6784_cast_fp16)[name = string("core_cast_fp16")];
+            tensor<int32, [3]> var_6786 = const()[name = string("op_6786"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> input_97_cast_fp16 = reshape(shape = var_6786, x = core_cast_fp16)[name = string("input_97_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_6_linear_attn_out_proj_weight_to_fp16 = const()[name = string("layers_6_linear_attn_out_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1421837696)))];
+            tensor<fp16, [1, 2048, 2560]> linear_51_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_6_linear_attn_out_proj_weight_to_fp16, x = input_97_cast_fp16)[name = string("linear_51_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_173_cast_fp16 = add(x = x_151_cast_fp16, y = linear_51_cast_fp16)[name = string("x_173_cast_fp16")];
+            fp16 var_5752_promoted_2_to_fp16 = const()[name = string("op_5752_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_6792_cast_fp16 = pow(x = x_173_cast_fp16, y = var_5752_promoted_2_to_fp16)[name = string("op_6792_cast_fp16")];
+            tensor<int32, [1]> var_6794_axes_0 = const()[name = string("op_6794_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6794_keep_dims_0 = const()[name = string("op_6794_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_6794_cast_fp16 = reduce_mean(axes = var_6794_axes_0, keep_dims = var_6794_keep_dims_0, x = var_6792_cast_fp16)[name = string("op_6794_cast_fp16")];
+            fp16 var_6795_to_fp16 = const()[name = string("op_6795_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_6796_cast_fp16 = add(x = var_6794_cast_fp16, y = var_6795_to_fp16)[name = string("op_6796_cast_fp16")];
+            fp32 var_6797_epsilon_0 = const()[name = string("op_6797_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_6797_cast_fp16 = rsqrt(epsilon = var_6797_epsilon_0, x = var_6796_cast_fp16)[name = string("op_6797_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_175_cast_fp16 = mul(x = x_173_cast_fp16, y = var_6797_cast_fp16)[name = string("x_175_cast_fp16")];
+            tensor<fp16, [2560]> var_6800_to_fp16 = const()[name = string("op_6800_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1442809280)))];
+            tensor<fp16, [1, 2048, 2560]> input_99_cast_fp16 = mul(x = x_175_cast_fp16, y = var_6800_to_fp16)[name = string("input_99_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_6_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_6_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1442814464)))];
+            tensor<fp16, [1, 2048, 9216]> linear_52_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_6_mlp_gate_proj_weight_to_fp16, x = input_99_cast_fp16)[name = string("linear_52_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_6807_cast_fp16 = silu(x = linear_52_cast_fp16)[name = string("op_6807_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_6_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_6_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1490000448)))];
+            tensor<fp16, [1, 2048, 9216]> linear_53_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_6_mlp_up_proj_weight_to_fp16, x = input_99_cast_fp16)[name = string("linear_53_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_103_cast_fp16 = mul(x = var_6807_cast_fp16, y = linear_53_cast_fp16)[name = string("input_103_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_6_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_6_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1537186432)))];
+            tensor<fp16, [1, 2048, 2560]> linear_54_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_6_mlp_down_proj_weight_to_fp16, x = input_103_cast_fp16)[name = string("linear_54_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_177_cast_fp16 = add(x = x_173_cast_fp16, y = linear_54_cast_fp16)[name = string("x_177_cast_fp16")];
+            int32 var_6827 = const()[name = string("op_6827"), val = int32(-1)];
+            fp16 var_6826_promoted_to_fp16 = const()[name = string("op_6826_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_6836_cast_fp16 = pow(x = x_177_cast_fp16, y = var_6826_promoted_to_fp16)[name = string("op_6836_cast_fp16")];
+            tensor<int32, [1]> var_6838_axes_0 = const()[name = string("op_6838_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6838_keep_dims_0 = const()[name = string("op_6838_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_6838_cast_fp16 = reduce_mean(axes = var_6838_axes_0, keep_dims = var_6838_keep_dims_0, x = var_6836_cast_fp16)[name = string("op_6838_cast_fp16")];
+            fp16 var_6839_to_fp16 = const()[name = string("op_6839_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_6840_cast_fp16 = add(x = var_6838_cast_fp16, y = var_6839_to_fp16)[name = string("op_6840_cast_fp16")];
+            fp32 var_6841_epsilon_0 = const()[name = string("op_6841_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_6841_cast_fp16 = rsqrt(epsilon = var_6841_epsilon_0, x = var_6840_cast_fp16)[name = string("op_6841_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_179_cast_fp16 = mul(x = x_177_cast_fp16, y = var_6841_cast_fp16)[name = string("x_179_cast_fp16")];
+            tensor<fp16, [2560]> var_6844_to_fp16 = const()[name = string("op_6844_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1584372416)))];
+            tensor<fp16, [1, 2048, 2560]> x_181_cast_fp16 = mul(x = x_179_cast_fp16, y = var_6844_to_fp16)[name = string("x_181_cast_fp16")];
+            tensor<fp16, [8192, 2560]> layers_7_self_attn_q_proj_weight_to_fp16 = const()[name = string("layers_7_self_attn_q_proj_weight_to_fp16"), val = tensor<fp16, [8192, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1584377600)))];
+            tensor<fp16, [1, 2048, 8192]> linear_55_cast_fp16 = linear(bias = linear_0_bias_0_to_fp16, weight = layers_7_self_attn_q_proj_weight_to_fp16, x = x_181_cast_fp16)[name = string("linear_55_cast_fp16")];
+            tensor<int32, [4]> var_6855 = const()[name = string("op_6855"), val = tensor<int32, [4]>([1, 2048, 16, 512])];
+            tensor<fp16, [1, 2048, 16, 512]> qg_cast_fp16 = reshape(shape = var_6855, x = linear_55_cast_fp16)[name = string("qg_cast_fp16")];
+            tensor<int32, [4]> x_183_begin_0 = const()[name = string("x_183_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> x_183_end_0 = const()[name = string("x_183_end_0"), val = tensor<int32, [4]>([1, 2048, 16, 256])];
+            tensor<bool, [4]> x_183_end_mask_0 = const()[name = string("x_183_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 2048, 16, 256]> x_183_cast_fp16 = slice_by_index(begin = x_183_begin_0, end = x_183_end_0, end_mask = x_183_end_mask_0, x = qg_cast_fp16)[name = string("x_183_cast_fp16")];
+            tensor<int32, [4]> gate_5_begin_0 = const()[name = string("gate_5_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 256])];
+            tensor<int32, [4]> gate_5_end_0 = const()[name = string("gate_5_end_0"), val = tensor<int32, [4]>([1, 2048, 16, 512])];
+            tensor<bool, [4]> gate_5_end_mask_0 = const()[name = string("gate_5_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 2048, 16, 256]> gate_5_cast_fp16 = slice_by_index(begin = gate_5_begin_0, end = gate_5_end_0, end_mask = gate_5_end_mask_0, x = qg_cast_fp16)[name = string("gate_5_cast_fp16")];
+            tensor<int32, [3]> var_6859 = const()[name = string("op_6859"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 4096]> gate_cast_fp16 = reshape(shape = var_6859, x = gate_5_cast_fp16)[name = string("gate_cast_fp16")];
+            fp16 var_6826_promoted_1_to_fp16 = const()[name = string("op_6826_promoted_1_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 16, 256]> var_6862_cast_fp16 = pow(x = x_183_cast_fp16, y = var_6826_promoted_1_to_fp16)[name = string("op_6862_cast_fp16")];
+            tensor<int32, [1]> var_6864_axes_0 = const()[name = string("op_6864_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6864_keep_dims_0 = const()[name = string("op_6864_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 16, 1]> var_6864_cast_fp16 = reduce_mean(axes = var_6864_axes_0, keep_dims = var_6864_keep_dims_0, x = var_6862_cast_fp16)[name = string("op_6864_cast_fp16")];
+            fp16 var_6865_to_fp16 = const()[name = string("op_6865_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 16, 1]> var_6866_cast_fp16 = add(x = var_6864_cast_fp16, y = var_6865_to_fp16)[name = string("op_6866_cast_fp16")];
+            fp32 var_6867_epsilon_0 = const()[name = string("op_6867_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 16, 1]> var_6867_cast_fp16 = rsqrt(epsilon = var_6867_epsilon_0, x = var_6866_cast_fp16)[name = string("op_6867_cast_fp16")];
+            tensor<fp16, [1, 2048, 16, 256]> x_185_cast_fp16 = mul(x = x_183_cast_fp16, y = var_6867_cast_fp16)[name = string("x_185_cast_fp16")];
+            tensor<fp16, [256]> var_6870_to_fp16 = const()[name = string("op_6870_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1626320704)))];
+            tensor<fp16, [1, 2048, 16, 256]> var_6871_cast_fp16 = mul(x = x_185_cast_fp16, y = var_6870_to_fp16)[name = string("op_6871_cast_fp16")];
+            tensor<int32, [4]> q_65_perm_0 = const()[name = string("q_65_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1024, 2560]> layers_7_self_attn_k_proj_weight_to_fp16 = const()[name = string("layers_7_self_attn_k_proj_weight_to_fp16"), val = tensor<fp16, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1626321280)))];
+            tensor<fp16, [1, 2048, 1024]> linear_56_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_7_self_attn_k_proj_weight_to_fp16, x = x_181_cast_fp16)[name = string("linear_56_cast_fp16")];
+            tensor<int32, [4]> var_6875 = const()[name = string("op_6875"), val = tensor<int32, [4]>([1, 2048, 4, 256])];
+            tensor<fp16, [1, 2048, 4, 256]> x_187_cast_fp16 = reshape(shape = var_6875, x = linear_56_cast_fp16)[name = string("x_187_cast_fp16")];
+            fp16 var_6826_promoted_2_to_fp16 = const()[name = string("op_6826_promoted_2_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 4, 256]> var_6878_cast_fp16 = pow(x = x_187_cast_fp16, y = var_6826_promoted_2_to_fp16)[name = string("op_6878_cast_fp16")];
+            tensor<int32, [1]> var_6880_axes_0 = const()[name = string("op_6880_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6880_keep_dims_0 = const()[name = string("op_6880_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 4, 1]> var_6880_cast_fp16 = reduce_mean(axes = var_6880_axes_0, keep_dims = var_6880_keep_dims_0, x = var_6878_cast_fp16)[name = string("op_6880_cast_fp16")];
+            fp16 var_6881_to_fp16 = const()[name = string("op_6881_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 4, 1]> var_6882_cast_fp16 = add(x = var_6880_cast_fp16, y = var_6881_to_fp16)[name = string("op_6882_cast_fp16")];
+            fp32 var_6883_epsilon_0 = const()[name = string("op_6883_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 4, 1]> var_6883_cast_fp16 = rsqrt(epsilon = var_6883_epsilon_0, x = var_6882_cast_fp16)[name = string("op_6883_cast_fp16")];
+            tensor<fp16, [1, 2048, 4, 256]> x_189_cast_fp16 = mul(x = x_187_cast_fp16, y = var_6883_cast_fp16)[name = string("x_189_cast_fp16")];
+            tensor<fp16, [256]> var_6886_to_fp16 = const()[name = string("op_6886_to_fp16"), val = tensor<fp16, [256]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1631564224)))];
+            tensor<fp16, [1, 2048, 4, 256]> var_6887_cast_fp16 = mul(x = x_189_cast_fp16, y = var_6886_to_fp16)[name = string("op_6887_cast_fp16")];
+            tensor<int32, [4]> k_55_perm_0 = const()[name = string("k_55_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<fp16, [1024, 2560]> layers_7_self_attn_v_proj_weight_to_fp16 = const()[name = string("layers_7_self_attn_v_proj_weight_to_fp16"), val = tensor<fp16, [1024, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1631564800)))];
+            tensor<fp16, [1, 2048, 1024]> linear_57_cast_fp16 = linear(bias = linear_25_bias_0_to_fp16, weight = layers_7_self_attn_v_proj_weight_to_fp16, x = x_181_cast_fp16)[name = string("linear_57_cast_fp16")];
+            tensor<int32, [4]> var_6891 = const()[name = string("op_6891"), val = tensor<int32, [4]>([1, 2048, 4, 256])];
+            tensor<fp16, [1, 2048, 4, 256]> var_6892_cast_fp16 = reshape(shape = var_6891, x = linear_57_cast_fp16)[name = string("op_6892_cast_fp16")];
+            tensor<int32, [4]> v_29_perm_0 = const()[name = string("v_29_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [4]> var_6898_begin_0 = const()[name = string("op_6898_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6898_end_0 = const()[name = string("op_6898_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 64])];
+            tensor<bool, [4]> var_6898_end_mask_0 = const()[name = string("op_6898_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 2048, 256]> q_65_cast_fp16 = transpose(perm = q_65_perm_0, x = var_6871_cast_fp16)[name = string("transpose_3")];
+            tensor<fp16, [1, 16, 2048, 64]> var_6898_cast_fp16 = slice_by_index(begin = var_6898_begin_0, end = var_6898_end_0, end_mask = var_6898_end_mask_0, x = q_65_cast_fp16)[name = string("op_6898_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 64]> var_6899_cast_fp16 = mul(x = var_6898_cast_fp16, y = c_1_cast_fp16)[name = string("op_6899_cast_fp16")];
+            tensor<int32, [4]> var_6906_begin_0 = const()[name = string("op_6906_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_6906_end_0 = const()[name = string("op_6906_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 64])];
+            tensor<bool, [4]> var_6906_end_mask_0 = const()[name = string("op_6906_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 2048, 32]> var_6906_cast_fp16 = slice_by_index(begin = var_6906_begin_0, end = var_6906_end_0, end_mask = var_6906_end_mask_0, x = var_6898_cast_fp16)[name = string("op_6906_cast_fp16")];
+            fp16 const_79_promoted_to_fp16 = const()[name = string("const_79_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 16, 2048, 32]> var_6907_cast_fp16 = mul(x = var_6906_cast_fp16, y = const_79_promoted_to_fp16)[name = string("op_6907_cast_fp16")];
+            tensor<int32, [4]> var_6908_begin_0 = const()[name = string("op_6908_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6908_end_0 = const()[name = string("op_6908_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 32])];
+            tensor<bool, [4]> var_6908_end_mask_0 = const()[name = string("op_6908_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 16, 2048, 32]> var_6908_cast_fp16 = slice_by_index(begin = var_6908_begin_0, end = var_6908_end_0, end_mask = var_6908_end_mask_0, x = var_6898_cast_fp16)[name = string("op_6908_cast_fp16")];
+            bool var_6910_interleave_0 = const()[name = string("op_6910_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 2048, 64]> var_6910_cast_fp16 = concat(axis = var_6827, interleave = var_6910_interleave_0, values = (var_6907_cast_fp16, var_6908_cast_fp16))[name = string("op_6910_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 64]> var_6911_cast_fp16 = mul(x = var_6910_cast_fp16, y = s_1_cast_fp16)[name = string("op_6911_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 64]> var_6912_cast_fp16 = add(x = var_6899_cast_fp16, y = var_6911_cast_fp16)[name = string("op_6912_cast_fp16")];
+            tensor<int32, [4]> var_6913_begin_0 = const()[name = string("op_6913_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_6913_end_0 = const()[name = string("op_6913_end_0"), val = tensor<int32, [4]>([1, 16, 2048, 256])];
+            tensor<bool, [4]> var_6913_end_mask_0 = const()[name = string("op_6913_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 16, 2048, 192]> var_6913_cast_fp16 = slice_by_index(begin = var_6913_begin_0, end = var_6913_end_0, end_mask = var_6913_end_mask_0, x = q_65_cast_fp16)[name = string("op_6913_cast_fp16")];
+            bool q_interleave_0 = const()[name = string("q_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 2048, 256]> q_cast_fp16 = concat(axis = var_6827, interleave = q_interleave_0, values = (var_6912_cast_fp16, var_6913_cast_fp16))[name = string("q_cast_fp16")];
+            tensor<int32, [4]> var_6916_begin_0 = const()[name = string("op_6916_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6916_end_0 = const()[name = string("op_6916_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 64])];
+            tensor<bool, [4]> var_6916_end_mask_0 = const()[name = string("op_6916_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 2048, 256]> k_55_cast_fp16 = transpose(perm = k_55_perm_0, x = var_6887_cast_fp16)[name = string("transpose_2")];
+            tensor<fp16, [1, 4, 2048, 64]> var_6916_cast_fp16 = slice_by_index(begin = var_6916_begin_0, end = var_6916_end_0, end_mask = var_6916_end_mask_0, x = k_55_cast_fp16)[name = string("op_6916_cast_fp16")];
+            tensor<fp16, [1, 4, 2048, 64]> var_6917_cast_fp16 = mul(x = var_6916_cast_fp16, y = c_1_cast_fp16)[name = string("op_6917_cast_fp16")];
+            tensor<int32, [4]> var_6924_begin_0 = const()[name = string("op_6924_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 32])];
+            tensor<int32, [4]> var_6924_end_0 = const()[name = string("op_6924_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 64])];
+            tensor<bool, [4]> var_6924_end_mask_0 = const()[name = string("op_6924_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 2048, 32]> var_6924_cast_fp16 = slice_by_index(begin = var_6924_begin_0, end = var_6924_end_0, end_mask = var_6924_end_mask_0, x = var_6916_cast_fp16)[name = string("op_6924_cast_fp16")];
+            fp16 const_81_promoted_to_fp16 = const()[name = string("const_81_promoted_to_fp16"), val = fp16(-0x1p+0)];
+            tensor<fp16, [1, 4, 2048, 32]> var_6925_cast_fp16 = mul(x = var_6924_cast_fp16, y = const_81_promoted_to_fp16)[name = string("op_6925_cast_fp16")];
+            tensor<int32, [4]> var_6926_begin_0 = const()[name = string("op_6926_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 0])];
+            tensor<int32, [4]> var_6926_end_0 = const()[name = string("op_6926_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 32])];
+            tensor<bool, [4]> var_6926_end_mask_0 = const()[name = string("op_6926_end_mask_0"), val = tensor<bool, [4]>([true, true, true, false])];
+            tensor<fp16, [1, 4, 2048, 32]> var_6926_cast_fp16 = slice_by_index(begin = var_6926_begin_0, end = var_6926_end_0, end_mask = var_6926_end_mask_0, x = var_6916_cast_fp16)[name = string("op_6926_cast_fp16")];
+            bool var_6928_interleave_0 = const()[name = string("op_6928_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 2048, 64]> var_6928_cast_fp16 = concat(axis = var_6827, interleave = var_6928_interleave_0, values = (var_6925_cast_fp16, var_6926_cast_fp16))[name = string("op_6928_cast_fp16")];
+            tensor<fp16, [1, 4, 2048, 64]> var_6929_cast_fp16 = mul(x = var_6928_cast_fp16, y = s_1_cast_fp16)[name = string("op_6929_cast_fp16")];
+            tensor<fp16, [1, 4, 2048, 64]> var_6930_cast_fp16 = add(x = var_6917_cast_fp16, y = var_6929_cast_fp16)[name = string("op_6930_cast_fp16")];
+            tensor<int32, [4]> var_6931_begin_0 = const()[name = string("op_6931_begin_0"), val = tensor<int32, [4]>([0, 0, 0, 64])];
+            tensor<int32, [4]> var_6931_end_0 = const()[name = string("op_6931_end_0"), val = tensor<int32, [4]>([1, 4, 2048, 256])];
+            tensor<bool, [4]> var_6931_end_mask_0 = const()[name = string("op_6931_end_mask_0"), val = tensor<bool, [4]>([true, true, true, true])];
+            tensor<fp16, [1, 4, 2048, 192]> var_6931_cast_fp16 = slice_by_index(begin = var_6931_begin_0, end = var_6931_end_0, end_mask = var_6931_end_mask_0, x = k_55_cast_fp16)[name = string("op_6931_cast_fp16")];
+            bool k_57_interleave_0 = const()[name = string("k_57_interleave_0"), val = bool(false)];
+            tensor<fp16, [1, 4, 2048, 256]> k_57_cast_fp16 = concat(axis = var_6827, interleave = k_57_interleave_0, values = (var_6930_cast_fp16, var_6931_cast_fp16))[name = string("k_57_cast_fp16")];
+            tensor<int32, [1]> var_6936_axes_0 = const()[name = string("op_6936_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 1, 2048, 256]> var_6936_cast_fp16 = expand_dims(axes = var_6936_axes_0, x = k_57_cast_fp16)[name = string("op_6936_cast_fp16")];
+            tensor<int32, [5]> var_6938_reps_0 = const()[name = string("op_6938_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 2048, 256]> var_6938_cast_fp16 = tile(reps = var_6938_reps_0, x = var_6936_cast_fp16)[name = string("op_6938_cast_fp16")];
+            tensor<int32, [4]> var_6939 = const()[name = string("op_6939"), val = tensor<int32, [4]>([1, 16, 2048, 256])];
+            tensor<fp16, [1, 16, 2048, 256]> k_cast_fp16 = reshape(shape = var_6939, x = var_6938_cast_fp16)[name = string("k_cast_fp16")];
+            tensor<int32, [1]> var_6943_axes_0 = const()[name = string("op_6943_axes_0"), val = tensor<int32, [1]>([2])];
+            tensor<fp16, [1, 4, 2048, 256]> v_29_cast_fp16 = transpose(perm = v_29_perm_0, x = var_6892_cast_fp16)[name = string("transpose_1")];
+            tensor<fp16, [1, 4, 1, 2048, 256]> var_6943_cast_fp16 = expand_dims(axes = var_6943_axes_0, x = v_29_cast_fp16)[name = string("op_6943_cast_fp16")];
+            tensor<int32, [5]> var_6945_reps_0 = const()[name = string("op_6945_reps_0"), val = tensor<int32, [5]>([1, 1, 4, 1, 1])];
+            tensor<fp16, [1, 4, 4, 2048, 256]> var_6945_cast_fp16 = tile(reps = var_6945_reps_0, x = var_6943_cast_fp16)[name = string("op_6945_cast_fp16")];
+            tensor<int32, [4]> var_6946 = const()[name = string("op_6946"), val = tensor<int32, [4]>([1, 16, 2048, 256])];
+            tensor<fp16, [1, 16, 2048, 256]> v_cast_fp16 = reshape(shape = var_6946, x = var_6945_cast_fp16)[name = string("v_cast_fp16")];
+            bool var_6949_transpose_x_1 = const()[name = string("op_6949_transpose_x_1"), val = bool(false)];
+            bool var_6949_transpose_y_1 = const()[name = string("op_6949_transpose_y_1"), val = bool(true)];
+            tensor<fp16, [1, 16, 2048, 2048]> var_6949_cast_fp16 = matmul(transpose_x = var_6949_transpose_x_1, transpose_y = var_6949_transpose_y_1, x = q_cast_fp16, y = k_cast_fp16)[name = string("op_6949_cast_fp16")];
+            fp16 var_6950_to_fp16 = const()[name = string("op_6950_to_fp16"), val = fp16(0x1p-4)];
+            tensor<fp16, [1, 16, 2048, 2048]> var_6951_cast_fp16 = mul(x = var_6949_cast_fp16, y = var_6950_to_fp16)[name = string("op_6951_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 2048]> scores_cast_fp16 = add(x = var_6951_cast_fp16, y = mask_to_fp16)[name = string("scores_cast_fp16")];
+            tensor<fp16, [1, 16, 2048, 2048]> var_6953_cast_fp16 = softmax(axis = var_6827, x = scores_cast_fp16)[name = string("op_6953_cast_fp16")];
+            bool out_5_transpose_x_0 = const()[name = string("out_5_transpose_x_0"), val = bool(false)];
+            bool out_5_transpose_y_0 = const()[name = string("out_5_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1, 16, 2048, 256]> out_5_cast_fp16 = matmul(transpose_x = out_5_transpose_x_0, transpose_y = out_5_transpose_y_0, x = var_6953_cast_fp16, y = v_cast_fp16)[name = string("out_5_cast_fp16")];
+            tensor<int32, [4]> var_6955_perm_0 = const()[name = string("op_6955_perm_0"), val = tensor<int32, [4]>([0, 2, 1, 3])];
+            tensor<int32, [3]> var_6956 = const()[name = string("op_6956"), val = tensor<int32, [3]>([1, 2048, 4096])];
+            tensor<fp16, [1, 2048, 16, 256]> var_6955_cast_fp16 = transpose(perm = var_6955_perm_0, x = out_5_cast_fp16)[name = string("transpose_0")];
+            tensor<fp16, [1, 2048, 4096]> out_cast_fp16 = reshape(shape = var_6956, x = var_6955_cast_fp16)[name = string("out_cast_fp16")];
+            tensor<fp16, [1, 2048, 4096]> var_6958_cast_fp16 = sigmoid(x = gate_cast_fp16)[name = string("op_6958_cast_fp16")];
+            tensor<fp16, [1, 2048, 4096]> input_105_cast_fp16 = mul(x = out_cast_fp16, y = var_6958_cast_fp16)[name = string("input_105_cast_fp16")];
+            tensor<fp16, [2560, 4096]> layers_7_self_attn_o_proj_weight_to_fp16 = const()[name = string("layers_7_self_attn_o_proj_weight_to_fp16"), val = tensor<fp16, [2560, 4096]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1636807744)))];
+            tensor<fp16, [1, 2048, 2560]> linear_58_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_7_self_attn_o_proj_weight_to_fp16, x = input_105_cast_fp16)[name = string("linear_58_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_195_cast_fp16 = add(x = x_177_cast_fp16, y = linear_58_cast_fp16)[name = string("x_195_cast_fp16")];
+            fp16 var_6826_promoted_3_to_fp16 = const()[name = string("op_6826_promoted_3_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2048, 2560]> var_6964_cast_fp16 = pow(x = x_195_cast_fp16, y = var_6826_promoted_3_to_fp16)[name = string("op_6964_cast_fp16")];
+            tensor<int32, [1]> var_6966_axes_0 = const()[name = string("op_6966_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6966_keep_dims_0 = const()[name = string("op_6966_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 2048, 1]> var_6966_cast_fp16 = reduce_mean(axes = var_6966_axes_0, keep_dims = var_6966_keep_dims_0, x = var_6964_cast_fp16)[name = string("op_6966_cast_fp16")];
+            fp16 var_6967_to_fp16 = const()[name = string("op_6967_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 2048, 1]> var_6968_cast_fp16 = add(x = var_6966_cast_fp16, y = var_6967_to_fp16)[name = string("op_6968_cast_fp16")];
+            fp32 var_6969_epsilon_0 = const()[name = string("op_6969_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 2048, 1]> var_6969_cast_fp16 = rsqrt(epsilon = var_6969_epsilon_0, x = var_6968_cast_fp16)[name = string("op_6969_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> x_197_cast_fp16 = mul(x = x_195_cast_fp16, y = var_6969_cast_fp16)[name = string("x_197_cast_fp16")];
+            tensor<fp16, [2560]> var_6972_to_fp16 = const()[name = string("op_6972_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1657779328)))];
+            tensor<fp16, [1, 2048, 2560]> input_107_cast_fp16 = mul(x = x_197_cast_fp16, y = var_6972_to_fp16)[name = string("input_107_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_7_mlp_gate_proj_weight_to_fp16 = const()[name = string("layers_7_mlp_gate_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1657784512)))];
+            tensor<fp16, [1, 2048, 9216]> linear_59_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_7_mlp_gate_proj_weight_to_fp16, x = input_107_cast_fp16)[name = string("linear_59_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> var_6979_cast_fp16 = silu(x = linear_59_cast_fp16)[name = string("op_6979_cast_fp16")];
+            tensor<fp16, [9216, 2560]> layers_7_mlp_up_proj_weight_to_fp16 = const()[name = string("layers_7_mlp_up_proj_weight_to_fp16"), val = tensor<fp16, [9216, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1704970496)))];
+            tensor<fp16, [1, 2048, 9216]> linear_60_cast_fp16 = linear(bias = linear_5_bias_0_to_fp16, weight = layers_7_mlp_up_proj_weight_to_fp16, x = input_107_cast_fp16)[name = string("linear_60_cast_fp16")];
+            tensor<fp16, [1, 2048, 9216]> input_111_cast_fp16 = mul(x = var_6979_cast_fp16, y = linear_60_cast_fp16)[name = string("input_111_cast_fp16")];
+            tensor<fp16, [2560, 9216]> layers_7_mlp_down_proj_weight_to_fp16 = const()[name = string("layers_7_mlp_down_proj_weight_to_fp16"), val = tensor<fp16, [2560, 9216]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1752156480)))];
+            tensor<fp16, [1, 2048, 2560]> linear_61_cast_fp16 = linear(bias = linear_4_bias_0_to_fp16, weight = layers_7_mlp_down_proj_weight_to_fp16, x = input_111_cast_fp16)[name = string("linear_61_cast_fp16")];
+            tensor<fp16, [1, 2048, 2560]> hidden_cast_fp16 = add(x = x_195_cast_fp16, y = linear_61_cast_fp16)[name = string("hidden_cast_fp16")];
+            tensor<int32, [3]> var_6988_begin_0 = const()[name = string("op_6988_begin_0"), val = tensor<int32, [3]>([0, 0, 0])];
+            tensor<int32, [3]> var_6988_end_0 = const()[name = string("op_6988_end_0"), val = tensor<int32, [3]>([1, 2048, 2560])];
+            tensor<bool, [3]> var_6988_end_mask_0 = const()[name = string("op_6988_end_mask_0"), val = tensor<bool, [3]>([false, true, true])];
+            tensor<bool, [3]> var_6988_squeeze_mask_0 = const()[name = string("op_6988_squeeze_mask_0"), val = tensor<bool, [3]>([true, false, false])];
+            tensor<fp16, [2048, 2560]> var_6988_cast_fp16 = slice_by_index(begin = var_6988_begin_0, end = var_6988_end_0, end_mask = var_6988_end_mask_0, squeeze_mask = var_6988_squeeze_mask_0, x = hidden_cast_fp16)[name = string("op_6988_cast_fp16")];
+            bool x_199_transpose_x_0 = const()[name = string("x_199_transpose_x_0"), val = bool(false)];
+            bool x_199_transpose_y_0 = const()[name = string("x_199_transpose_y_0"), val = bool(false)];
+            tensor<fp16, [1, 2560]> x_199_cast_fp16 = matmul(transpose_x = x_199_transpose_x_0, transpose_y = x_199_transpose_y_0, x = last_onehot, y = var_6988_cast_fp16)[name = string("x_199_cast_fp16")];
+            fp16 var_6994_promoted_to_fp16 = const()[name = string("op_6994_promoted_to_fp16"), val = fp16(0x1p+1)];
+            tensor<fp16, [1, 2560]> var_6996_cast_fp16 = pow(x = x_199_cast_fp16, y = var_6994_promoted_to_fp16)[name = string("op_6996_cast_fp16")];
+            tensor<int32, [1]> var_6998_axes_0 = const()[name = string("op_6998_axes_0"), val = tensor<int32, [1]>([-1])];
+            bool var_6998_keep_dims_0 = const()[name = string("op_6998_keep_dims_0"), val = bool(true)];
+            tensor<fp16, [1, 1]> var_6998_cast_fp16 = reduce_mean(axes = var_6998_axes_0, keep_dims = var_6998_keep_dims_0, x = var_6996_cast_fp16)[name = string("op_6998_cast_fp16")];
+            fp16 var_6999_to_fp16 = const()[name = string("op_6999_to_fp16"), val = fp16(0x1.1p-20)];
+            tensor<fp16, [1, 1]> var_7000_cast_fp16 = add(x = var_6998_cast_fp16, y = var_6999_to_fp16)[name = string("op_7000_cast_fp16")];
+            fp32 var_7001_epsilon_0 = const()[name = string("op_7001_epsilon_0"), val = fp32(0x1.197998p-40)];
+            tensor<fp16, [1, 1]> var_7001_cast_fp16 = rsqrt(epsilon = var_7001_epsilon_0, x = var_7000_cast_fp16)[name = string("op_7001_cast_fp16")];
+            tensor<fp16, [1, 2560]> x_cast_fp16 = mul(x = x_199_cast_fp16, y = var_7001_cast_fp16)[name = string("x_cast_fp16")];
+            tensor<fp16, [2560]> var_7004_to_fp16 = const()[name = string("op_7004_to_fp16"), val = tensor<fp16, [2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1799342464)))];
+            tensor<fp16, [1, 2560]> input_cast_fp16 = mul(x = x_cast_fp16, y = var_7004_to_fp16)[name = string("input_cast_fp16")];
+            tensor<fp16, [26, 2560]> letter_head_weight_to_fp16 = const()[name = string("letter_head_weight_to_fp16"), val = tensor<fp16, [26, 2560]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1799347648)))];
+            tensor<fp16, [26]> linear_62_bias_0_to_fp16 = const()[name = string("linear_62_bias_0_to_fp16"), val = tensor<fp16, [26]>(BLOBFILE(path = string("@model_path/weights/weight.bin"), offset = uint64(1799480832)))];
+            tensor<fp16, [1, 26]> letter_logits = linear(bias = linear_62_bias_0_to_fp16, weight = letter_head_weight_to_fp16, x = input_cast_fp16)[name = string("linear_62_cast_fp16")];
+        } -> (letter_logits);
+}
\ No newline at end of file