diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..d4e802e6f9b578fa46377f4e60144844db7143e6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf filter=lfs diff=lfs merge=lfs -text diff --git a/Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf b/Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf new file mode 100644 index 0000000000000000000000000000000000000000..d7a22b6ddc650445f61d9833e423eddc4fc74c29 --- /dev/null +++ b/Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56a1eca8639ad1161a236f852063844f09d0c53dc49920291040f02f2e9e805e +size 18744342464 diff --git a/SHA256SUMS b/SHA256SUMS new file mode 100644 index 0000000000000000000000000000000000000000..be37d0e128e9f01d9ae4bf03a41d6bf366fc9cc1 --- /dev/null +++ b/SHA256SUMS @@ -0,0 +1,6 @@ +51ad9aa42908c0a765b10ed9fdebe99663fffe3bc0e1a15cbc4e2954effe266a Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-COHERENT.gguf +122e62e39ed7141970ca913af5438db86fcae2bfbfd1ad93455f6a149861a4ce Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-FAST.gguf +56a1eca8639ad1161a236f852063844f09d0c53dc49920291040f02f2e9e805e Nex-N2.5-mini-imatrix-Q4_0-ROCmFP4-STRIX_LEAN.gguf +7e5afffc822c64a7b43a1223d0094fb60895a521f6c89b912409d25091bd17f8 Nex-N2.5-mini.imatrix +9183c7ba8510fb9628edd2265a4cad8c02b3aecec3cbdf65620f47494e75836d chat_template_enable_thinking.jinja +a80a82977e4787a838a612316c79a6533c1b098fcac19372cc78929120162a4e mmproj-Nex-N2.5-mini-BF16.gguf diff --git a/chat_template_enable_thinking.jinja b/chat_template_enable_thinking.jinja new file mode 100644 index 0000000000000000000000000000000000000000..14be67548588eb60c46c0c2464f4227b58858cc6 --- /dev/null +++ b/chat_template_enable_thinking.jinja @@ -0,0 +1,155 @@ +{%- if reasoning_effort is not defined and enable_thinking is defined %}{%- set reasoning_effort = 'high' if enable_thinking else 'none' %}{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if reasoning_effort is not defined or reasoning_effort is none %} + {{- '' }} + {%- elif reasoning_effort == 'none' %} + {{- '\n\n\n\n' }} + {%- elif reasoning_effort == 'high' %} + {{- '\n' }} + {%- else %} + {{- '' }} + {%- endif %} +{%- endif %} diff --git a/recipe/logs/N4_kld_q102.log b/recipe/logs/N4_kld_q102.log new file mode 100644 index 0000000000000000000000000000000000000000..30cb7faf61b53fc7941bcf6a1ea6f32c601b1028 --- /dev/null +++ b/recipe/logs/N4_kld_q102.log @@ -0,0 +1,92 @@ +0.00.046.192 I common_init_result: fitting params to device memory ... +0.00.046.197 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on) +0.00.468.601 W llama_model_loader: direct I/O is enabled, disabling mmap +0.26.097.875 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.26.161.936 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.26.456.383 I +0.26.456.524 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.26.593.623 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1 +0.29.575.306 I kl_divergence: 2.98 seconds per pass - ETA 1.98 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 5.9944 ± 0.4685 0.05151 ± 0.01742 0.11333 ± 0.00681 10.320 ± 0.638 % 87.390 ± 1.038 % + 2 6.9832 ± 0.3895 0.04665 ± 0.01189 0.09838 ± 0.00400 8.897 ± 0.415 % 87.048 ± 0.743 % + 3 7.5417 ± 0.3512 0.07003 ± 0.00990 0.10487 ± 0.00406 9.267 ± 0.393 % 87.195 ± 0.603 % + 4 7.8954 ± 0.3289 0.08016 ± 0.00897 0.11061 ± 0.00503 9.327 ± 0.343 % 86.901 ± 0.527 % + 5 7.6999 ± 0.2880 0.07519 ± 0.00798 0.10848 ± 0.00428 9.187 ± 0.313 % 87.097 ± 0.469 % + 6 6.6265 ± 0.2194 0.07502 ± 0.00740 0.11000 ± 0.00421 9.941 ± 0.317 % 87.455 ± 0.423 % + 7 6.1509 ± 0.1859 0.06918 ± 0.00711 0.11837 ± 0.00459 10.426 ± 0.309 % 87.530 ± 0.390 % + 8 6.0395 ± 0.1692 0.06322 ± 0.00648 0.11517 ± 0.00408 10.254 ± 0.281 % 87.476 ± 0.366 % + 9 6.3422 ± 0.1682 0.05959 ± 0.00609 0.11478 ± 0.00370 10.054 ± 0.261 % 87.216 ± 0.348 % + 10 6.4559 ± 0.1635 0.05892 ± 0.00570 0.11065 ± 0.00337 9.793 ± 0.242 % 87.243 ± 0.330 % + 11 6.5040 ± 0.1565 0.05755 ± 0.00536 0.10715 ± 0.00309 9.593 ± 0.227 % 87.283 ± 0.314 % + 12 6.7574 ± 0.1569 0.05744 ± 0.00504 0.10450 ± 0.00284 9.400 ± 0.214 % 87.211 ± 0.301 % + 13 6.8020 ± 0.1514 0.05713 ± 0.00478 0.10247 ± 0.00264 9.285 ± 0.203 % 87.232 ± 0.289 % + 14 6.8553 ± 0.1467 0.05617 ± 0.00457 0.10045 ± 0.00247 9.123 ± 0.192 % 87.236 ± 0.279 % + 15 6.8957 ± 0.1427 0.05582 ± 0.00438 0.09956 ± 0.00232 9.076 ± 0.184 % 87.247 ± 0.269 % + 16 7.0571 ± 0.1415 0.05224 ± 0.00422 0.09847 ± 0.00219 8.961 ± 0.175 % 87.243 ± 0.261 % + 17 7.0917 ± 0.1375 0.05016 ± 0.00405 0.09704 ± 0.00208 8.871 ± 0.169 % 87.281 ± 0.253 % + 18 7.1772 ± 0.1353 0.04896 ± 0.00392 0.09635 ± 0.00198 8.799 ± 0.162 % 87.243 ± 0.246 % + 19 7.1236 ± 0.1311 0.04836 ± 0.00378 0.09524 ± 0.00190 8.763 ± 0.159 % 87.303 ± 0.239 % + 20 6.8769 ± 0.1227 0.05245 ± 0.00378 0.10018 ± 0.00190 9.053 ± 0.155 % 87.175 ± 0.234 % + 21 6.8931 ± 0.1198 0.05231 ± 0.00368 0.10066 ± 0.00184 9.029 ± 0.150 % 87.120 ± 0.229 % + 22 6.9236 ± 0.1178 0.05472 ± 0.00363 0.10218 ± 0.00181 9.063 ± 0.147 % 87.168 ± 0.223 % + 23 6.9807 ± 0.1162 0.05553 ± 0.00354 0.10187 ± 0.00175 9.045 ± 0.143 % 87.131 ± 0.218 % + 24 6.9769 ± 0.1134 0.05494 ± 0.00347 0.10222 ± 0.00175 9.051 ± 0.143 % 87.093 ± 0.214 % + 25 7.0111 ± 0.1117 0.05513 ± 0.00339 0.10169 ± 0.00169 8.997 ± 0.139 % 87.015 ± 0.210 % + 26 6.9763 ± 0.1088 0.05463 ± 0.00332 0.10165 ± 0.00165 9.014 ± 0.137 % 87.055 ± 0.206 % + 27 7.1528 ± 0.1102 0.05528 ± 0.00326 0.10141 ± 0.00162 8.950 ± 0.134 % 87.079 ± 0.202 % + 28 7.2365 ± 0.1098 0.05405 ± 0.00318 0.10027 ± 0.00157 8.866 ± 0.131 % 87.118 ± 0.198 % + 29 7.2391 ± 0.1079 0.05527 ± 0.00313 0.10044 ± 0.00154 8.907 ± 0.129 % 87.077 ± 0.195 % + 30 7.1873 ± 0.1051 0.05609 ± 0.00307 0.10003 ± 0.00150 8.887 ± 0.126 % 87.061 ± 0.192 % + 31 7.0778 ± 0.1015 0.05524 ± 0.00300 0.09893 ± 0.00145 8.840 ± 0.123 % 87.154 ± 0.188 % + 32 6.9728 ± 0.0983 0.05456 ± 0.00300 0.10164 ± 0.00161 8.995 ± 0.124 % 87.121 ± 0.185 % + 33 6.9098 ± 0.0957 0.05537 ± 0.00295 0.10136 ± 0.00157 8.987 ± 0.122 % 87.153 ± 0.182 % + 34 6.8891 ± 0.0938 0.05453 ± 0.00288 0.10042 ± 0.00153 8.943 ± 0.120 % 87.154 ± 0.179 % + 35 6.9008 ± 0.0926 0.05442 ± 0.00283 0.09974 ± 0.00149 8.883 ± 0.118 % 87.214 ± 0.176 % + 36 6.9190 ± 0.0917 0.05458 ± 0.00279 0.09958 ± 0.00146 8.857 ± 0.116 % 87.197 ± 0.174 % + 37 6.8168 ± 0.0888 0.05349 ± 0.00273 0.09856 ± 0.00142 8.817 ± 0.113 % 87.242 ± 0.171 % + 38 6.7442 ± 0.0864 0.05330 ± 0.00269 0.09804 ± 0.00139 8.824 ± 0.112 % 87.277 ± 0.169 % + 39 6.6583 ± 0.0839 0.05282 ± 0.00265 0.09776 ± 0.00137 8.827 ± 0.110 % 87.247 ± 0.167 % + 40 6.5617 ± 0.0812 0.05204 ± 0.00261 0.09707 ± 0.00134 8.800 ± 0.108 % 87.287 ± 0.165 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.561727 ± 0.081240 +Mean PPL(base) : 6.228979 ± 0.075322 +Cor(ln(PPL(Q)), ln(PPL(base))): 97.76% +Mean ln(PPL(Q)/PPL(base)) : 0.052041 ± 0.002607 +Mean PPL(Q)/PPL(base) : 1.053419 ± 0.002746 +Mean PPL(Q)-PPL(base) : 0.332749 ± 0.017588 + +====== KL divergence statistics ====== +Mean KLD: 0.097071 ± 0.001342 +Maximum KLD: 12.257548 +99.9% KLD: 3.289070 +99.0% KLD: 0.874485 +95.0% KLD: 0.332573 +90.0% KLD: 0.209309 +Median KLD: 0.043524 +10.0% KLD: 0.000605 + 5.0% KLD: 0.000161 + 1.0% KLD: -0.000023 + 0.1% KLD: -0.000269 +Minimum KLD: -0.000603 + +====== Token probability statistics ====== +Mean Δp: -0.366 ± 0.043 % +Maximum Δp: 99.335% +99.9% Δp: 53.261% +99.0% Δp: 23.761% +95.0% Δp: 10.784% +90.0% Δp: 6.407% +75.0% Δp: 1.292% +Median Δp: -0.004% +25.0% Δp: -1.463% +10.0% Δp: -6.895% + 5.0% Δp: -12.449% + 1.0% Δp: -30.951% + 0.1% Δp: -71.092% +Minimum Δp: -99.867% +RMS Δp : 8.800 ± 0.108 % +Same top p: 87.287 ± 0.165 % + diff --git a/recipe/logs/N4_kld_q103.log b/recipe/logs/N4_kld_q103.log new file mode 100644 index 0000000000000000000000000000000000000000..4d51de494489a4ce280a7c0d44bd2d05aa7aad43 --- /dev/null +++ b/recipe/logs/N4_kld_q103.log @@ -0,0 +1,92 @@ +0.00.043.507 I common_init_result: fitting params to device memory ... +0.00.043.512 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on) +0.00.547.415 W llama_model_loader: direct I/O is enabled, disabling mmap +0.28.910.009 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.28.974.230 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.29.190.163 I +0.29.190.272 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.29.311.893 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1 +0.31.982.391 I kl_divergence: 2.67 seconds per pass - ETA 1.77 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 6.3324 ± 0.5080 0.10635 ± 0.02072 0.14199 ± 0.00911 10.822 ± 0.626 % 85.728 ± 1.094 % + 2 7.2445 ± 0.4085 0.08338 ± 0.01320 0.11857 ± 0.00596 9.324 ± 0.416 % 86.510 ± 0.755 % + 3 7.6437 ± 0.3567 0.08347 ± 0.01046 0.11993 ± 0.00567 9.224 ± 0.356 % 85.956 ± 0.627 % + 4 7.8662 ± 0.3254 0.07645 ± 0.00919 0.12254 ± 0.00560 9.291 ± 0.320 % 85.997 ± 0.543 % + 5 7.6438 ± 0.2844 0.06788 ± 0.00816 0.11930 ± 0.00461 9.150 ± 0.277 % 86.041 ± 0.485 % + 6 6.5928 ± 0.2169 0.06992 ± 0.00760 0.12081 ± 0.00444 9.877 ± 0.294 % 86.298 ± 0.439 % + 7 6.0938 ± 0.1829 0.05985 ± 0.00717 0.12740 ± 0.00482 10.188 ± 0.281 % 86.468 ± 0.404 % + 8 6.0037 ± 0.1675 0.05727 ± 0.00661 0.12543 ± 0.00429 10.065 ± 0.258 % 86.522 ± 0.377 % + 9 6.3212 ± 0.1671 0.05627 ± 0.00620 0.12516 ± 0.00387 9.872 ± 0.238 % 86.282 ± 0.359 % + 10 6.4505 ± 0.1629 0.05808 ± 0.00580 0.12152 ± 0.00352 9.739 ± 0.221 % 86.354 ± 0.339 % + 11 6.5075 ± 0.1561 0.05809 ± 0.00543 0.11797 ± 0.00323 9.554 ± 0.208 % 86.404 ± 0.323 % + 12 6.7603 ± 0.1566 0.05787 ± 0.00509 0.11420 ± 0.00297 9.348 ± 0.196 % 86.355 ± 0.310 % + 13 6.7920 ± 0.1508 0.05567 ± 0.00486 0.11244 ± 0.00278 9.245 ± 0.186 % 86.397 ± 0.297 % + 14 6.8545 ± 0.1464 0.05605 ± 0.00465 0.11100 ± 0.00260 9.142 ± 0.177 % 86.385 ± 0.287 % + 15 6.8911 ± 0.1423 0.05516 ± 0.00446 0.11053 ± 0.00247 9.124 ± 0.169 % 86.354 ± 0.277 % + 16 7.0572 ± 0.1413 0.05225 ± 0.00431 0.10982 ± 0.00235 9.046 ± 0.163 % 86.254 ± 0.269 % + 17 7.0949 ± 0.1372 0.05061 ± 0.00415 0.10844 ± 0.00223 8.972 ± 0.157 % 86.263 ± 0.261 % + 18 7.1844 ± 0.1351 0.04995 ± 0.00401 0.10792 ± 0.00213 8.944 ± 0.152 % 86.168 ± 0.254 % + 19 7.1415 ± 0.1312 0.05088 ± 0.00389 0.10652 ± 0.00203 8.872 ± 0.147 % 86.191 ± 0.247 % + 20 6.8778 ± 0.1223 0.05258 ± 0.00387 0.11191 ± 0.00202 9.238 ± 0.145 % 86.144 ± 0.242 % + 21 6.8965 ± 0.1195 0.05280 ± 0.00379 0.11255 ± 0.00196 9.249 ± 0.143 % 86.110 ± 0.236 % + 22 6.9128 ± 0.1172 0.05316 ± 0.00370 0.11332 ± 0.00190 9.279 ± 0.140 % 86.150 ± 0.230 % + 23 6.9751 ± 0.1157 0.05473 ± 0.00362 0.11327 ± 0.00185 9.276 ± 0.138 % 86.124 ± 0.225 % + 24 6.9742 ± 0.1131 0.05456 ± 0.00353 0.11363 ± 0.00182 9.287 ± 0.137 % 86.091 ± 0.221 % + 25 7.0117 ± 0.1115 0.05521 ± 0.00346 0.11333 ± 0.00176 9.261 ± 0.133 % 86.088 ± 0.216 % + 26 6.9819 ± 0.1087 0.05543 ± 0.00339 0.11378 ± 0.00175 9.294 ± 0.131 % 86.127 ± 0.212 % + 27 7.1532 ± 0.1100 0.05533 ± 0.00332 0.11307 ± 0.00170 9.223 ± 0.128 % 86.148 ± 0.208 % + 28 7.2405 ± 0.1097 0.05461 ± 0.00324 0.11201 ± 0.00165 9.147 ± 0.126 % 86.193 ± 0.204 % + 29 7.2447 ± 0.1078 0.05604 ± 0.00319 0.11216 ± 0.00161 9.196 ± 0.125 % 86.133 ± 0.201 % + 30 7.1827 ± 0.1049 0.05545 ± 0.00314 0.11174 ± 0.00157 9.180 ± 0.122 % 86.103 ± 0.197 % + 31 7.0787 ± 0.1014 0.05537 ± 0.00308 0.11105 ± 0.00153 9.164 ± 0.119 % 86.201 ± 0.194 % + 32 6.9666 ± 0.0980 0.05367 ± 0.00307 0.11404 ± 0.00170 9.317 ± 0.121 % 86.150 ± 0.191 % + 33 6.9006 ± 0.0953 0.05404 ± 0.00303 0.11358 ± 0.00167 9.303 ± 0.119 % 86.167 ± 0.188 % + 34 6.8792 ± 0.0934 0.05310 ± 0.00297 0.11250 ± 0.00162 9.246 ± 0.117 % 86.180 ± 0.185 % + 35 6.8869 ± 0.0922 0.05240 ± 0.00291 0.11158 ± 0.00158 9.198 ± 0.116 % 86.242 ± 0.182 % + 36 6.9043 ± 0.0912 0.05245 ± 0.00287 0.11125 ± 0.00154 9.156 ± 0.113 % 86.239 ± 0.180 % + 37 6.8080 ± 0.0885 0.05220 ± 0.00282 0.11015 ± 0.00151 9.118 ± 0.111 % 86.304 ± 0.177 % + 38 6.7280 ± 0.0860 0.05090 ± 0.00278 0.10979 ± 0.00147 9.111 ± 0.109 % 86.343 ± 0.174 % + 39 6.6466 ± 0.0835 0.05105 ± 0.00274 0.10941 ± 0.00145 9.111 ± 0.107 % 86.370 ± 0.172 % + 40 6.5498 ± 0.0809 0.05023 ± 0.00270 0.10878 ± 0.00142 9.108 ± 0.106 % 86.405 ± 0.169 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.549829 ± 0.080919 +Mean PPL(base) : 6.228979 ± 0.075322 +Cor(ln(PPL(Q)), ln(PPL(base))): 97.58% +Mean ln(PPL(Q)/PPL(base)) : 0.050227 ± 0.002704 +Mean PPL(Q)/PPL(base) : 1.051509 ± 0.002843 +Mean PPL(Q)-PPL(base) : 0.320850 ± 0.018076 + +====== KL divergence statistics ====== +Mean KLD: 0.108779 ± 0.001420 +Maximum KLD: 12.086925 +99.9% KLD: 3.352409 +99.0% KLD: 0.956811 +95.0% KLD: 0.370801 +90.0% KLD: 0.235309 +Median KLD: 0.049730 +10.0% KLD: 0.000721 + 5.0% KLD: 0.000197 + 1.0% KLD: -0.000038 + 0.1% KLD: -0.000343 +Minimum KLD: -0.000800 + +====== Token probability statistics ====== +Mean Δp: -0.515 ± 0.045 % +Maximum Δp: 98.721% +99.9% Δp: 52.916% +99.0% Δp: 24.226% +95.0% Δp: 11.222% +90.0% Δp: 6.613% +75.0% Δp: 1.299% +Median Δp: -0.007% +25.0% Δp: -1.668% +10.0% Δp: -7.814% + 5.0% Δp: -13.545% + 1.0% Δp: -32.559% + 0.1% Δp: -70.472% +Minimum Δp: -99.855% +RMS Δp : 9.108 ± 0.106 % +Same top p: 86.405 ± 0.169 % + diff --git a/recipe/logs/N4v_kld_q102.log b/recipe/logs/N4v_kld_q102.log new file mode 100644 index 0000000000000000000000000000000000000000..a2dc9e2a79f1f84fdb22f343f89fc5075e1eb3cc --- /dev/null +++ b/recipe/logs/N4v_kld_q102.log @@ -0,0 +1,93 @@ +0.00.044.057 I common_init_result: fitting params to device memory ... +0.00.044.061 I common_init_result: (for bugs during this step try to reproduce them with -fit off, or provide --verbose logs if the bug only occurs with -fit on) +0.00.458.402 W llama_model_loader: direct I/O is enabled, disabling mmap +0.01.547.350 W read_raw_unsafe: Falling back to buffered IO due to Bad address +0.29.871.824 W llama_context: n_ctx_seq (2048) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.29.921.987 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.30.136.871 I +0.30.137.012 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.30.272.958 I kl_divergence: computing over 40 chunks, n_ctx=2048, batch_size=2048, n_seq=1 +0.33.203.394 I kl_divergence: 2.93 seconds per pass - ETA 1.95 minutes + +chunk PPL ln(PPL(Q)/PPL(base)) KL Divergence Δp RMS Same top p + 1 5.9966 ± 0.4697 0.05187 ± 0.01811 0.11459 ± 0.00733 10.192 ± 0.639 % 86.217 ± 1.078 % + 2 6.9326 ± 0.3865 0.03938 ± 0.01202 0.09983 ± 0.00427 8.906 ± 0.415 % 86.364 ± 0.759 % + 3 7.4953 ± 0.3486 0.06386 ± 0.01034 0.10617 ± 0.00431 9.400 ± 0.398 % 86.771 ± 0.612 % + 4 7.8932 ± 0.3295 0.07988 ± 0.00943 0.11534 ± 0.00606 9.599 ± 0.356 % 86.779 ± 0.530 % + 5 7.6968 ± 0.2885 0.07479 ± 0.00839 0.11294 ± 0.00512 9.430 ± 0.323 % 86.979 ± 0.471 % + 6 6.6156 ± 0.2191 0.07337 ± 0.00761 0.11303 ± 0.00469 10.110 ± 0.316 % 87.341 ± 0.424 % + 7 6.1496 ± 0.1860 0.06895 ± 0.00729 0.12010 ± 0.00470 10.509 ± 0.310 % 87.544 ± 0.390 % + 8 6.0524 ± 0.1698 0.06536 ± 0.00664 0.11729 ± 0.00419 10.338 ± 0.282 % 87.414 ± 0.367 % + 9 6.3606 ± 0.1689 0.06248 ± 0.00621 0.11624 ± 0.00378 10.074 ± 0.260 % 87.053 ± 0.350 % + 10 6.4785 ± 0.1643 0.06242 ± 0.00577 0.11173 ± 0.00343 9.809 ± 0.243 % 87.185 ± 0.330 % + 11 6.5332 ± 0.1574 0.06204 ± 0.00542 0.10800 ± 0.00313 9.617 ± 0.228 % 87.203 ± 0.315 % + 12 6.7836 ± 0.1577 0.06132 ± 0.00509 0.10489 ± 0.00288 9.401 ± 0.215 % 87.154 ± 0.302 % + 13 6.8332 ± 0.1523 0.06171 ± 0.00483 0.10310 ± 0.00268 9.272 ± 0.203 % 87.112 ± 0.291 % + 14 6.8837 ± 0.1476 0.06029 ± 0.00462 0.10108 ± 0.00251 9.107 ± 0.193 % 87.222 ± 0.279 % + 15 6.9222 ± 0.1435 0.05966 ± 0.00443 0.10045 ± 0.00236 9.064 ± 0.184 % 87.129 ± 0.270 % + 16 7.0981 ± 0.1428 0.05802 ± 0.00427 0.09952 ± 0.00223 8.947 ± 0.176 % 87.170 ± 0.261 % + 17 7.1355 ± 0.1388 0.05632 ± 0.00410 0.09812 ± 0.00211 8.873 ± 0.170 % 87.229 ± 0.253 % + 18 7.2233 ± 0.1366 0.05535 ± 0.00396 0.09744 ± 0.00201 8.809 ± 0.164 % 87.151 ± 0.247 % + 19 7.1804 ± 0.1327 0.05631 ± 0.00385 0.09656 ± 0.00193 8.767 ± 0.160 % 87.138 ± 0.240 % + 20 6.9267 ± 0.1241 0.05967 ± 0.00383 0.10094 ± 0.00190 9.066 ± 0.156 % 87.043 ± 0.235 % + 21 6.9448 ± 0.1213 0.05978 ± 0.00374 0.10135 ± 0.00184 9.031 ± 0.151 % 87.032 ± 0.229 % + 22 6.9675 ± 0.1190 0.06104 ± 0.00367 0.10256 ± 0.00183 9.077 ± 0.149 % 87.035 ± 0.224 % + 23 7.0270 ± 0.1174 0.06214 ± 0.00359 0.10236 ± 0.00177 9.057 ± 0.146 % 86.986 ± 0.219 % + 24 7.0192 ± 0.1145 0.06099 ± 0.00349 0.10191 ± 0.00171 9.016 ± 0.143 % 86.983 ± 0.215 % + 25 7.0530 ± 0.1127 0.06108 ± 0.00342 0.10141 ± 0.00166 8.977 ± 0.139 % 86.952 ± 0.211 % + 26 7.0192 ± 0.1099 0.06075 ± 0.00334 0.10143 ± 0.00162 8.999 ± 0.137 % 86.973 ± 0.206 % + 27 7.1950 ± 0.1112 0.06116 ± 0.00329 0.10126 ± 0.00160 8.943 ± 0.134 % 86.988 ± 0.202 % + 28 7.2801 ± 0.1108 0.06006 ± 0.00321 0.10017 ± 0.00155 8.860 ± 0.131 % 86.975 ± 0.199 % + 29 7.2833 ± 0.1089 0.06136 ± 0.00316 0.10033 ± 0.00152 8.904 ± 0.130 % 86.925 ± 0.196 % + 30 7.2262 ± 0.1061 0.06148 ± 0.00310 0.09994 ± 0.00148 8.892 ± 0.127 % 86.937 ± 0.192 % + 31 7.1166 ± 0.1024 0.06072 ± 0.00302 0.09878 ± 0.00144 8.837 ± 0.124 % 87.043 ± 0.189 % + 32 7.0141 ± 0.0992 0.06047 ± 0.00303 0.10174 ± 0.00161 9.036 ± 0.126 % 87.014 ± 0.186 % + 33 6.9480 ± 0.0965 0.06088 ± 0.00298 0.10139 ± 0.00157 9.021 ± 0.124 % 87.046 ± 0.183 % + 34 6.9248 ± 0.0946 0.05971 ± 0.00291 0.10041 ± 0.00152 8.964 ± 0.121 % 87.091 ± 0.180 % + 35 6.9370 ± 0.0934 0.05965 ± 0.00285 0.09981 ± 0.00149 8.914 ± 0.119 % 87.139 ± 0.177 % + 36 6.9569 ± 0.0925 0.06004 ± 0.00282 0.09978 ± 0.00146 8.900 ± 0.118 % 87.159 ± 0.174 % + 37 6.8518 ± 0.0895 0.05860 ± 0.00276 0.09882 ± 0.00142 8.866 ± 0.115 % 87.192 ± 0.172 % + 38 6.7745 ± 0.0870 0.05779 ± 0.00272 0.09818 ± 0.00139 8.852 ± 0.114 % 87.256 ± 0.169 % + 39 6.6883 ± 0.0845 0.05732 ± 0.00267 0.09776 ± 0.00137 8.864 ± 0.112 % 87.232 ± 0.167 % + 40 6.5921 ± 0.0819 0.05666 ± 0.00263 0.09716 ± 0.00134 8.853 ± 0.111 % 87.265 ± 0.165 % + +====== Perplexity statistics ====== +Mean PPL(Q) : 6.592123 ± 0.081874 +Mean PPL(base) : 6.228979 ± 0.075322 +Cor(ln(PPL(Q)), ln(PPL(base))): 97.73% +Mean ln(PPL(Q)/PPL(base)) : 0.056663 ± 0.002630 +Mean PPL(Q)/PPL(base) : 1.058299 ± 0.002783 +Mean PPL(Q)-PPL(base) : 0.363144 ± 0.017958 + +====== KL divergence statistics ====== +Mean KLD: 0.097165 ± 0.001337 +Maximum KLD: 15.969161 +99.9% KLD: 3.145117 +99.0% KLD: 0.858566 +95.0% KLD: 0.329361 +90.0% KLD: 0.207554 +Median KLD: 0.043968 +10.0% KLD: 0.000631 + 5.0% KLD: 0.000159 + 1.0% KLD: -0.000029 + 0.1% KLD: -0.000286 +Minimum KLD: -0.000564 + +====== Token probability statistics ====== +Mean Δp: -0.372 ± 0.044 % +Maximum Δp: 98.904% +99.9% Δp: 55.573% +99.0% Δp: 23.490% +95.0% Δp: 10.880% +90.0% Δp: 6.394% +75.0% Δp: 1.299% +Median Δp: -0.005% +25.0% Δp: -1.458% +10.0% Δp: -7.029% + 5.0% Δp: -12.485% + 1.0% Δp: -31.169% + 0.1% Δp: -73.630% +Minimum Δp: -99.736% +RMS Δp : 8.853 ± 0.111 % +Same top p: 87.265 ± 0.165 % + diff --git a/recipe/logs/N6_bench.log b/recipe/logs/N6_bench.log new file mode 100644 index 0000000000000000000000000000000000000000..2a015406f4a9656648d88606897317b688d0cc3c --- /dev/null +++ b/recipe/logs/N6_bench.log @@ -0,0 +1,46 @@ +[2026-09-16T23:22:44Z] waiting for the quiet-box lock (no HF upload during speed runs) +[2026-09-17T00:32:03Z] quiet-box lock held +[2026-09-17T00:32:03Z] bench n-q106-rocm +{"label": "n-q106-rocm", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "code", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 63.94, "tg_min": 63.56, "tg_max": 63.96, "pp_median": 1158.2, "prompt_n": 7098, "accept": null} +[2026-09-17T00:33:06Z] bench n-q106-vk +{"label": "n-q106-vk", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "code", "load_s": 4.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 68.16, "tg_min": 68.1, "tg_max": 68.21, "pp_median": 1006.0, "prompt_n": 7096, "accept": null} +[2026-09-17T00:33:54Z] bench n-q102-rocm +{"label": "n-q102-rocm", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_COHERENT.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "code", "load_s": 24.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_COHERENT.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 61.74, "tg_min": 61.74, "tg_max": 61.8, "pp_median": 1186.7, "prompt_n": 7100, "accept": null} +[2026-09-17T00:34:58Z] bench n-q102-vk +{"label": "n-q102-vk", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_COHERENT.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "code", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_COHERENT.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 68.0, "tg_min": 67.88, "tg_max": 68.09, "pp_median": 1001.4, "prompt_n": 7098, "accept": null} +[2026-09-17T00:36:05Z] bench n-q103-rocm +{"label": "n-q103-rocm", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_FAST.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "code", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_FAST.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 63.31, "tg_min": 63.14, "tg_max": 63.38, "pp_median": 1156.6, "prompt_n": 7100, "accept": null} +[2026-09-17T00:37:08Z] bench n-q103-vk +{"label": "n-q103-vk", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_FAST.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "code", "load_s": 20.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_FAST.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 68.62, "tg_min": 68.6, "tg_max": 68.67, "pp_median": 998.9, "prompt_n": 7098, "accept": null} +[2026-09-17T00:38:12Z] bench n-q106i-rocm +{"label": "n-q106i-rocm", "model": "Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "code", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out-imat/Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 63.29, "tg_min": 63.19, "tg_max": 63.41, "pp_median": 1145.3, "prompt_n": 7100, "accept": null} +[2026-09-17T00:39:16Z] bench n-q106i-vk +{"label": "n-q106i-vk", "model": "Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "code", "load_s": 20.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out-imat/Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 67.77, "tg_min": 67.74, "tg_max": 67.86, "pp_median": 995.0, "prompt_n": 7098, "accept": null} +[2026-09-17T00:40:20Z] bench n-q102i-rocm +{"label": "n-q102i-rocm", "model": "Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_COHERENT.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "code", "load_s": 24.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out-imat/Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_COHERENT.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 61.56, "tg_min": 61.53, "tg_max": 61.63, "pp_median": 1183.6, "prompt_n": 7098, "accept": null} +[2026-09-17T00:41:25Z] bench n-q102i-vk +{"label": "n-q102i-vk", "model": "Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_COHERENT.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "code", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out-imat/Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_COHERENT.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 67.81, "tg_min": 67.77, "tg_max": 67.85, "pp_median": 998.7, "prompt_n": 7097, "accept": null} +[2026-09-17T00:42:32Z] bench n-q103i-rocm +{"label": "n-q103i-rocm", "model": "Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_FAST.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "code", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out-imat/Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_FAST.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 63.03, "tg_min": 62.55, "tg_max": 63.11, "pp_median": 1154.4, "prompt_n": 7101, "accept": null} +[2026-09-17T00:43:35Z] bench n-q103i-vk +{"label": "n-q103i-vk", "model": "Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_FAST.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "code", "load_s": 20.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out-imat/Nex-N2.5-mini-imatrix-Q4_0_ROCMFP4_FAST.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 68.44, "tg_min": 68.42, "tg_max": 68.56, "pp_median": 993.9, "prompt_n": 7098, "accept": null} +[2026-09-17T00:44:39Z] bench n-q106-rocm-prose +{"label": "n-q106-rocm-prose", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "ROCm0", "ctx": 65536, "workload": "prose", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf -dev ROCm0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 63.06, "tg_min": 62.94, "tg_max": 63.09, "pp_median": 1139.1, "prompt_n": 7454, "accept": null} +[2026-09-17T00:45:44Z] bench n-q106-vk-prose +{"label": "n-q106-vk-prose", "model": "Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "draft": null, "nmax": null, "strict": false, "bin": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin", "dev": "Vulkan0", "ctx": 65536, "workload": "prose", "load_s": 22.0, "cmd": "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin/llama-server -m /mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf -dev Vulkan0 -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -c 65536 -b 2048 -ub 1024 --host 127.0.0.1 --port 18600 --no-webui", "tg_median": 67.34, "tg_min": 67.31, "tg_max": 67.35, "pp_median": 977.7, "prompt_n": 7454, "accept": null} +[2026-09-17T00:46:53Z] cachegate n-c3-q106 +788d", "cold_sha": "a8545d39788d", "instr": "Write a function that returns every regular expression used in this file. ok ok ok", "warm_instr": "Summarise this file. (warm-up 3) ok ok ok ok"} + {"variant": 4, "L": 7039, "warm_prompt_n": 1028, "warm_cache_n": 6011, "cold_prompt_n": 7039, "cold_cache_n": 0, "aligned": true, "warm_draft": [null, null], "cold_draft": [null, null], "identical": true, "first_diff_char": null, "warm_sha": "af97b2dde4e2", "cold_sha": "af97b2dde4e2", "instr": "Write a function that maps each class in this file to its base classes. ok", "warm_instr": "Summarise this file. (warm-up 4) ok ok ok ok"} +{"label": "n-c3-q106", "gate": "cachegate3-aligned", "n": 5, "L": 7039, "expect_cache_n": 6011, "all_reused": true, "all_aligned": true, "identical": 5, "result": "PASS"} +[2026-09-17T00:48:34Z] tools n-tools-q106 +alse enum: unit=fahrenheit + PASS think=False correct-decline: content='391' + PASS think=False multi-turn: final='Tokyo is currently **21°C** with clear skies.' + PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + PASS think=False parallel: calls=['lima', 'oslo'] +{"label": "n-tools-q106", "passed": 6, "total": 14, "detail": {"multi-arg|think=True": false, "nested-object|think=True": false, "enum|think=True": false, "correct-decline|think=True": false, "multi-turn|think=True": false, "streaming|think=True": false, "parallel|think=True": false, "multi-arg|think=False": true, "nested-object|think=False": false, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} +[2026-09-17T00:49:16Z] vision n-vision-q106-faon +{"label": "n-vision-q106-faon", "fa": "on", "mtp": false, "expected": "red,blue,circle,square", "answer": "The image shows two shapes: a red circle on the left and a blue square on the right.", "hits": ["red", "blue", "circle", "square"], "error": null, "server_died": false, "server_log_errors": [], "result": "PASS"} +[2026-09-17T00:49:42Z] vision n-vision-q106-faoff +{"label": "n-vision-q106-faoff", "fa": "off", "mtp": false, "expected": "red,blue,circle,square", "answer": "The image shows two simple shapes:\n\n- A **red circle** on the left.\n- A **blue square** on the right.", "hits": ["red", "blue", "circle", "square"], "error": null, "server_died": false, "server_log_errors": [], "result": "PASS"} +[2026-09-17T00:50:08Z] NEX_BENCH_DONE diff --git a/recipe/logs/N6t_tools_roff.log b/recipe/logs/N6t_tools_roff.log new file mode 100644 index 0000000000000000000000000000000000000000..b98a370e024ff7697e707592d5381f5aec98422d --- /dev/null +++ b/recipe/logs/N6t_tools_roff.log @@ -0,0 +1,63 @@ + PASS think=True multi-arg: args={'city': 'Paris', 'unit': 'celsius'} + FAIL think=True nested-object: exception KeyError('tool_calls') + PASS think=True enum: unit=fahrenheit + PASS think=True correct-decline: content='391' + PASS think=True multi-turn: final='Tokyo is **21°C** and **clear** right now.' + PASS think=True streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + PASS think=True parallel: calls=['lima', 'oslo'] + PASS think=False multi-arg: args={'city': 'Paris', 'unit': 'celsius'} + PASS think=False nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} + PASS think=False enum: unit=fahrenheit + PASS think=False correct-decline: content='391' + PASS think=False multi-turn: final='Tokyo is **21°C** and **clear**.' + PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + PASS think=False parallel: calls=['lima', 'oslo'] +{"label": "n-tools-q106-roff", "passed": 13, "total": 14, "detail": {"multi-arg|think=True": true, "nested-object|think=True": false, "enum|think=True": true, "correct-decline|think=True": true, "multi-turn|think=True": true, "streaming|think=True": true, "parallel|think=True": true, "multi-arg|think=False": true, "nested-object|think=False": true, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} + PASS think=True multi-arg: args={'city': 'Paris', 'unit': 'celsius'} + PASS think=True nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} + PASS think=True enum: unit=fahrenheit + PASS think=True correct-decline: content='391' + PASS think=True multi-turn: final='Tokyo is **21°C** and **clear**.' + PASS think=True streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + FAIL think=True parallel: calls=['oslo'] + PASS think=False multi-arg: args={'city': 'Paris', 'unit': 'celsius'} + PASS think=False nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} + PASS think=False enum: unit=fahrenheit + PASS think=False correct-decline: content='391' + PASS think=False multi-turn: final='Tokyo’s current weather is **21°C and clear**.' + PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + PASS think=False parallel: calls=['lima', 'oslo'] +{"label": "n-tools-q106-roff-r2", "passed": 13, "total": 14, "detail": {"multi-arg|think=True": true, "nested-object|think=True": true, "enum|think=True": true, "correct-decline|think=True": true, "multi-turn|think=True": true, "streaming|think=True": true, "parallel|think=True": false, "multi-arg|think=False": true, "nested-object|think=False": true, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} + PASS think=True multi-arg: args={'city': 'Paris', 'unit': 'celsius'} + PASS think=True nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} + PASS think=True enum: unit=fahrenheit + PASS think=True correct-decline: content='391' + PASS think=True multi-turn: final='Tokyo is currently **21°C** with **clear skies**.' + PASS think=True streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + PASS think=True parallel: calls=['lima', 'oslo'] + PASS think=False multi-arg: args={'city': 'Paris', 'unit': 'celsius'} + PASS think=False nested-object: args={'title': 'Design review', 'when': {'date': '2026-10-02', 'time': '14:00'}, 'attendees': ['ana@x.io', 'bo@x.io']} + PASS think=False enum: unit=fahrenheit + PASS think=False correct-decline: content='391' + PASS think=False multi-turn: final='Tokyo’s weather is **21°C and clear**.' + PASS think=False streaming: stream args={'city': 'Rome', 'unit': 'celsius'} + PASS think=False parallel: calls=['lima', 'oslo'] +{"label": "n-tools-q106-roff-r3", "passed": 14, "total": 14, "detail": {"multi-arg|think=True": true, "nested-object|think=True": true, "enum|think=True": true, "correct-decline|think=True": true, "multi-turn|think=True": true, "streaming|think=True": true, "parallel|think=True": true, "multi-arg|think=False": true, "nested-object|think=False": true, "enum|think=False": true, "correct-decline|think=False": true, "multi-turn|think=False": true, "streaming|think=False": true, "parallel|think=False": true}} +{"label": "n-vision-q106-roff-faon", "fa": "on", "mtp": false, "expected": "red,blue,circle,square", "answer": "The image shows two shapes: a red circle on the left and a blue square on the right.", "hits": ["red", "blue", "circle", "square"], "error": null, "server_died": false, "server_log_errors": [], "result": "PASS"} +probe no-kwargs correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +probe no-kwargs single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +probe no-kwargs multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +probe enable_thinking=false correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +probe enable_thinking=false single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +probe enable_thinking=false multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +probe reasoning_effort=high correct-decline {"content": "We need answer directly. 391.\n\n\n391", "reasoning_len": 0, "tool_calls": [], "leaks": [""]} +probe reasoning_effort=high single-word {"content": "We need need output exactly ready.\n\n\nready", "reasoning_len": 0, "tool_calls": [], "leaks": [""]} +probe reasoning_effort=high multi-arg {"content": "We need need tool. Current weather Paris celsius.\n\n\n", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": [""]} +probe reasoning_effort=medium correct-decline {"content": "\n\n\n\n391", "reasoning_len": 0, "tool_calls": [], "leaks": [""]} +probe reasoning_effort=medium single-word {"content": "\n\n\n\nready", "reasoning_len": 0, "tool_calls": [], "leaks": [""]} +probe reasoning_effort=medium multi-arg {"content": "\n\n\n\n", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": [""]} +probe reasoning_effort=none correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +probe reasoning_effort=none single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +probe reasoning_effort=none multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +NEX_TOOLS_TPL_DONE +rc=0 diff --git a/recipe/logs/N8b_seats.log b/recipe/logs/N8b_seats.log new file mode 100644 index 0000000000000000000000000000000000000000..85d9d3f3fdedbfa8eb540bda4d84bfbbc57354cb --- /dev/null +++ b/recipe/logs/N8b_seats.log @@ -0,0 +1,8 @@ +[2026-09-17T01:06:02Z] plan: [('max1-nex-fast', 'ROCm0', 'fa on', 262144, '31000M', True), ('max1-nex-fast-imat', 'ROCm0', 'fa on', 262144, '31000M', True)] +[2026-09-17T01:06:04Z] max1-nex-fast written (ROCm0, -fa on, ctx 262144, MemoryMax 31000M) -> smoke test +{"unit": "max1-nex-fast", "port": 8097, "load_s": 25, "time": "2026-09-17T01:06:29Z", "direct_reply": "ready", "direct_tg": 61.724584902166534, "default_reply": "", "default_reasoning_len": 5, "default_leak": false, "gateway_model": "nex-n2.5-mini-fast@max1", "gateway_reply": "ready", "result": "FAIL"} +[2026-09-17T01:06:30Z] max1-nex-fast stopped (enabled: disabled) +[2026-09-17T01:06:36Z] max1-nex-fast-imat written (ROCm0, -fa on, ctx 262144, MemoryMax 31000M) -> smoke test +{"unit": "max1-nex-fast-imat", "port": 8098, "load_s": 25, "time": "2026-09-17T01:07:02Z", "direct_reply": "ready", "direct_tg": 47.69058349428906, "default_reply": "", "default_reasoning_len": 5, "default_leak": false, "gateway_model": "nex-n2.5-mini-fast-imatrix@max1", "gateway_reply": "ready", "result": "FAIL"} +[2026-09-17T01:07:03Z] max1-nex-fast-imat stopped (enabled: disabled) +[2026-09-17T01:07:08Z] NEX_SEATS_DONE fail=1 diff --git a/recipe/logs/N8c_seats.log b/recipe/logs/N8c_seats.log new file mode 100644 index 0000000000000000000000000000000000000000..d944c6f5f40adcff90c553aa9fa5544460ba045b --- /dev/null +++ b/recipe/logs/N8c_seats.log @@ -0,0 +1,8 @@ +[2026-09-17T01:19:09Z] plan: [('max1-nex-fast', 'ROCm0', 'fa on', 262144, '31000M', True), ('max1-nex-fast-imat', 'ROCm0', 'fa on', 262144, '31000M', True)] +[2026-09-17T01:19:10Z] max1-nex-fast written (ROCm0, -fa on, ctx 262144, MemoryMax 31000M) -> smoke test +{"unit": "max1-nex-fast", "port": 8097, "load_s": 30, "time": "2026-09-17T01:19:40Z", "direct_reply": "ready", "direct_tg": 42.57493188010899, "default_reply": "ready", "default_reasoning_len": 0, "default_leak": false, "thinking_reply": "", "thinking_reasoning_len": 5, "thinking_leak": false, "gateway_model": "nex-n2.5-mini-fast@max1", "gateway_reply": "ready", "result": "FAIL"} +[2026-09-17T01:19:42Z] max1-nex-fast stopped (enabled: disabled) +[2026-09-17T01:19:48Z] max1-nex-fast-imat written (ROCm0, -fa on, ctx 262144, MemoryMax 31000M) -> smoke test +{"unit": "max1-nex-fast-imat", "port": 8098, "load_s": 25, "time": "2026-09-17T01:20:13Z", "direct_reply": "ready", "direct_tg": 45.33297066956798, "default_reply": "ready", "default_reasoning_len": 0, "default_leak": false, "thinking_reply": "", "thinking_reasoning_len": 5, "thinking_leak": false, "gateway_model": "nex-n2.5-mini-fast-imatrix@max1", "gateway_reply": "ready", "result": "FAIL"} +[2026-09-17T01:20:15Z] max1-nex-fast-imat stopped (enabled: disabled) +[2026-09-17T01:20:20Z] NEX_SEATS_DONE fail=1 diff --git a/recipe/logs/Q1_q106.log b/recipe/logs/Q1_q106.log new file mode 100644 index 0000000000000000000000000000000000000000..f9b2ce047f223f3f347fc8f8786777e3bc6f1064 --- /dev/null +++ b/recipe/logs/Q1_q106.log @@ -0,0 +1,786 @@ +ggml_rocm_init: found 1 ROCm devices (Total VRAM: 131072 MiB): + Device 0: AMD Radeon Graphics, gfx1151 (0x1151), VMM: no, Wave Size: 32, VRAM: 131072 MiB +ggml_vulkan: Found 1 Vulkan devices: +ggml_vulkan: 0 = AMD Radeon Graphics (RADV GFX1151) (radv) | uma: 1 | fp16: 1 | bf16: 0 | warp size: 64 | shared memory: 65536 | int dot: 0 | matrix cores: KHR_coopmat +llama_print_build_info: build = 1 (d3ca537) +llama_print_build_info: built with GNU 13.3.0 for Linux x86_64 +main: quantizing 'gguf/Nex-N2.5-mini-BF16.gguf' to 'out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' as Q4_0_ROCMFP4_STRIX_LEAN using 16 threads +llama_model_loader: loaded meta data with 37 key-value pairs and 733 tensors from gguf/Nex-N2.5-mini-BF16.gguf (version GGUF V3 (latest)) +llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output. +llama_model_loader: - kv 0: general.architecture str = qwen35moe +llama_model_loader: - kv 1: general.type str = model +llama_model_loader: - kv 2: general.name str = Nex-N2.5-mini +llama_model_loader: - kv 3: general.size_label str = 256x2.6B +llama_model_loader: - kv 4: general.license str = apache-2.0 +llama_model_loader: - kv 5: general.tags arr[str,1] = ["text-generation"] +llama_model_loader: - kv 6: qwen35moe.block_count u32 = 40 +llama_model_loader: - kv 7: qwen35moe.context_length u32 = 262144 +llama_model_loader: - kv 8: qwen35moe.embedding_length u32 = 2048 +llama_model_loader: - kv 9: qwen35moe.attention.head_count u32 = 16 +llama_model_loader: - kv 10: qwen35moe.attention.head_count_kv u32 = 2 +llama_model_loader: - kv 11: qwen35moe.rope.dimension_sections arr[i32,4] = [11, 11, 10, 0] +llama_model_loader: - kv 12: qwen35moe.rope.freq_base f32 = 10000000.000000 +llama_model_loader: - kv 13: qwen35moe.attention.layer_norm_rms_epsilon f32 = 0.000001 +llama_model_loader: - kv 14: qwen35moe.expert_count u32 = 256 +llama_model_loader: - kv 15: qwen35moe.expert_used_count u32 = 8 +llama_model_loader: - kv 16: qwen35moe.attention.key_length u32 = 256 +llama_model_loader: - kv 17: qwen35moe.attention.value_length u32 = 256 +llama_model_loader: - kv 18: general.file_type u32 = 32 +llama_model_loader: - kv 19: qwen35moe.expert_feed_forward_length u32 = 512 +llama_model_loader: - kv 20: qwen35moe.expert_shared_feed_forward_length u32 = 512 +llama_model_loader: - kv 21: qwen35moe.ssm.conv_kernel u32 = 4 +llama_model_loader: - kv 22: qwen35moe.ssm.state_size u32 = 128 +llama_model_loader: - kv 23: qwen35moe.ssm.group_count u32 = 16 +llama_model_loader: - kv 24: qwen35moe.ssm.time_step_rank u32 = 32 +llama_model_loader: - kv 25: qwen35moe.ssm.inner_size u32 = 4096 +llama_model_loader: - kv 26: qwen35moe.full_attention_interval u32 = 4 +llama_model_loader: - kv 27: qwen35moe.rope.dimension_count u32 = 64 +llama_model_loader: - kv 28: general.quantization_version u32 = 2 +llama_model_loader: - kv 29: tokenizer.ggml.model str = gpt2 +llama_model_loader: - kv 30: tokenizer.ggml.pre str = qwen35 +llama_model_loader: - kv 31: tokenizer.ggml.tokens arr[str,248320] = ["!", "\"", "#", "$", "%", "&", "'", ... +llama_model_loader: - kv 32: tokenizer.ggml.token_type arr[i32,248320] = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ... +llama_model_loader: - kv 33: tokenizer.ggml.merges arr[str,247587] = ["Ġ Ġ", "ĠĠ ĠĠ", "i n", "Ġ t",... +llama_model_loader: - kv 34: tokenizer.ggml.eos_token_id u32 = 248046 +llama_model_loader: - kv 35: tokenizer.ggml.padding_token_id u32 = 248044 +llama_model_loader: - kv 36: tokenizer.chat_template str = {%- set image_count = namespace(value... +llama_model_loader: - type f32: 301 tensors +llama_model_loader: - type bf16: 432 tensors +[ 1/ 733] output.weight - [ 2048, 248320, 1, 1], type = bf16, converting to q6_K .. size = 970.00 MiB -> 397.85 MiB +[ 2/ 733] output_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 3/ 733] token_embd.weight - [ 2048, 248320, 1, 1], type = bf16, converting to q5_K .. size = 970.00 MiB -> 333.44 MiB +[ 4/ 733] blk.0.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 5/ 733] blk.0.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 6/ 733] blk.0.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 7/ 733] blk.0.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 8/ 733] blk.0.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 9/ 733] blk.0.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 10/ 733] blk.0.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 11/ 733] blk.0.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 12/ 733] blk.0.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 13/ 733] blk.0.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 14/ 733] blk.0.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 15/ 733] blk.0.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 16/ 733] blk.0.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 17/ 733] blk.0.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 18/ 733] blk.0.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 19/ 733] blk.0.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 20/ 733] blk.0.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 21/ 733] blk.0.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 22/ 733] blk.0.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 23/ 733] blk.1.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 24/ 733] blk.1.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 25/ 733] blk.1.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 26/ 733] blk.1.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 27/ 733] blk.1.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 28/ 733] blk.1.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 29/ 733] blk.1.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 30/ 733] blk.1.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 31/ 733] blk.1.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 32/ 733] blk.1.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 33/ 733] blk.1.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 34/ 733] blk.1.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 35/ 733] blk.1.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 36/ 733] blk.1.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 37/ 733] blk.1.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 38/ 733] blk.1.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 39/ 733] blk.1.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 40/ 733] blk.1.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 41/ 733] blk.1.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 42/ 733] blk.2.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 43/ 733] blk.2.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 44/ 733] blk.2.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 45/ 733] blk.2.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 46/ 733] blk.2.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 47/ 733] blk.2.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 48/ 733] blk.2.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 49/ 733] blk.2.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 50/ 733] blk.2.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 51/ 733] blk.2.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 52/ 733] blk.2.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 53/ 733] blk.2.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 54/ 733] blk.2.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 55/ 733] blk.2.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 56/ 733] blk.2.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 57/ 733] blk.2.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 58/ 733] blk.2.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 59/ 733] blk.2.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 60/ 733] blk.2.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 61/ 733] blk.3.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 62/ 733] blk.3.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 63/ 733] blk.3.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 64/ 733] blk.3.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 65/ 733] blk.3.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 66/ 733] blk.3.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 67/ 733] blk.3.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 68/ 733] blk.3.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 69/ 733] blk.3.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 70/ 733] blk.3.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 71/ 733] blk.3.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 72/ 733] blk.3.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 73/ 733] blk.3.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 74/ 733] blk.3.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 75/ 733] blk.3.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 76/ 733] blk.3.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 77/ 733] blk.4.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 78/ 733] blk.4.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 79/ 733] blk.4.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 80/ 733] blk.4.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 81/ 733] blk.4.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 82/ 733] blk.4.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 83/ 733] blk.4.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 84/ 733] blk.4.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 85/ 733] blk.4.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 86/ 733] blk.4.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 87/ 733] blk.4.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 88/ 733] blk.4.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 89/ 733] blk.4.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 90/ 733] blk.4.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 91/ 733] blk.4.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 92/ 733] blk.4.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 93/ 733] blk.4.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 94/ 733] blk.4.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 95/ 733] blk.4.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 96/ 733] blk.5.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 97/ 733] blk.5.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 98/ 733] blk.5.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 99/ 733] blk.5.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 100/ 733] blk.5.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 101/ 733] blk.5.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 102/ 733] blk.5.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 103/ 733] blk.5.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 104/ 733] blk.5.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 105/ 733] blk.5.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 106/ 733] blk.5.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 107/ 733] blk.5.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 108/ 733] blk.5.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 109/ 733] blk.5.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 110/ 733] blk.5.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 111/ 733] blk.5.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 112/ 733] blk.5.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 113/ 733] blk.5.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 114/ 733] blk.5.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 115/ 733] blk.6.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 116/ 733] blk.6.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 117/ 733] blk.6.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 118/ 733] blk.6.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 119/ 733] blk.6.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 120/ 733] blk.6.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 121/ 733] blk.6.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 122/ 733] blk.6.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 123/ 733] blk.6.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 124/ 733] blk.6.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 125/ 733] blk.6.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 126/ 733] blk.6.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 127/ 733] blk.6.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 128/ 733] blk.6.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 129/ 733] blk.6.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 130/ 733] blk.6.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 131/ 733] blk.6.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 132/ 733] blk.6.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 133/ 733] blk.6.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 134/ 733] blk.7.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 135/ 733] blk.7.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 136/ 733] blk.7.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 137/ 733] blk.7.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 138/ 733] blk.7.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 139/ 733] blk.7.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 140/ 733] blk.7.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 141/ 733] blk.7.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 142/ 733] blk.7.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 143/ 733] blk.7.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 144/ 733] blk.7.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 145/ 733] blk.7.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 146/ 733] blk.7.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 147/ 733] blk.7.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 148/ 733] blk.7.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 149/ 733] blk.7.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 150/ 733] blk.8.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 151/ 733] blk.8.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 152/ 733] blk.8.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 153/ 733] blk.8.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 154/ 733] blk.8.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 155/ 733] blk.8.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 156/ 733] blk.8.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 157/ 733] blk.8.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 158/ 733] blk.8.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 159/ 733] blk.8.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 160/ 733] blk.8.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 161/ 733] blk.8.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 162/ 733] blk.8.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 163/ 733] blk.8.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 164/ 733] blk.8.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 165/ 733] blk.8.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 166/ 733] blk.8.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 167/ 733] blk.8.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 168/ 733] blk.8.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 169/ 733] blk.9.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 170/ 733] blk.9.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 171/ 733] blk.9.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 172/ 733] blk.9.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 173/ 733] blk.9.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 174/ 733] blk.9.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 175/ 733] blk.9.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 176/ 733] blk.9.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 177/ 733] blk.9.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 178/ 733] blk.9.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 179/ 733] blk.9.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 180/ 733] blk.9.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 181/ 733] blk.9.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 182/ 733] blk.9.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 183/ 733] blk.9.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 184/ 733] blk.9.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 185/ 733] blk.9.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 186/ 733] blk.9.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 187/ 733] blk.9.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 188/ 733] blk.10.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 189/ 733] blk.10.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 190/ 733] blk.10.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 191/ 733] blk.10.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 192/ 733] blk.10.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 193/ 733] blk.10.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 194/ 733] blk.10.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 195/ 733] blk.10.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 196/ 733] blk.10.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 197/ 733] blk.10.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 198/ 733] blk.10.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 199/ 733] blk.10.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 200/ 733] blk.10.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 201/ 733] blk.10.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 202/ 733] blk.10.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 203/ 733] blk.10.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 204/ 733] blk.10.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 205/ 733] blk.10.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 206/ 733] blk.10.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 207/ 733] blk.11.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 208/ 733] blk.11.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 209/ 733] blk.11.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 210/ 733] blk.11.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 211/ 733] blk.11.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 212/ 733] blk.11.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 213/ 733] blk.11.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 214/ 733] blk.11.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 215/ 733] blk.11.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 216/ 733] blk.11.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 217/ 733] blk.11.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 218/ 733] blk.11.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 219/ 733] blk.11.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 220/ 733] blk.11.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 221/ 733] blk.11.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 222/ 733] blk.11.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 223/ 733] blk.12.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 224/ 733] blk.12.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 225/ 733] blk.12.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 226/ 733] blk.12.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 227/ 733] blk.12.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 228/ 733] blk.12.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 229/ 733] blk.12.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 230/ 733] blk.12.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 231/ 733] blk.12.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 232/ 733] blk.12.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 233/ 733] blk.12.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 234/ 733] blk.12.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 235/ 733] blk.12.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 236/ 733] blk.12.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 237/ 733] blk.12.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 238/ 733] blk.12.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 239/ 733] blk.12.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 240/ 733] blk.12.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 241/ 733] blk.12.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 242/ 733] blk.13.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 243/ 733] blk.13.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 244/ 733] blk.13.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 245/ 733] blk.13.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 246/ 733] blk.13.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 247/ 733] blk.13.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 248/ 733] blk.13.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 249/ 733] blk.13.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 250/ 733] blk.13.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 251/ 733] blk.13.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 252/ 733] blk.13.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 253/ 733] blk.13.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 254/ 733] blk.13.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 255/ 733] blk.13.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 256/ 733] blk.13.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 257/ 733] blk.13.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 258/ 733] blk.13.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 259/ 733] blk.13.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 260/ 733] blk.13.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 261/ 733] blk.14.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 262/ 733] blk.14.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 263/ 733] blk.14.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 264/ 733] blk.14.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 265/ 733] blk.14.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 266/ 733] blk.14.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 267/ 733] blk.14.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 268/ 733] blk.14.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 269/ 733] blk.14.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 270/ 733] blk.14.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 271/ 733] blk.14.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 272/ 733] blk.14.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 273/ 733] blk.14.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 274/ 733] blk.14.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 275/ 733] blk.14.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 276/ 733] blk.14.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 277/ 733] blk.14.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 278/ 733] blk.14.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 279/ 733] blk.14.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 280/ 733] blk.15.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 281/ 733] blk.15.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 282/ 733] blk.15.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 283/ 733] blk.15.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 284/ 733] blk.15.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 285/ 733] blk.15.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 286/ 733] blk.15.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 287/ 733] blk.15.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 288/ 733] blk.15.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 289/ 733] blk.15.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 290/ 733] blk.15.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 291/ 733] blk.15.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 292/ 733] blk.15.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 293/ 733] blk.15.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 294/ 733] blk.15.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 295/ 733] blk.15.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 296/ 733] blk.16.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 297/ 733] blk.16.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 298/ 733] blk.16.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 299/ 733] blk.16.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 300/ 733] blk.16.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 301/ 733] blk.16.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 302/ 733] blk.16.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 303/ 733] blk.16.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 304/ 733] blk.16.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 305/ 733] blk.16.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 306/ 733] blk.16.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 307/ 733] blk.16.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 308/ 733] blk.16.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 309/ 733] blk.16.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 310/ 733] blk.16.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 311/ 733] blk.16.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 312/ 733] blk.16.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 313/ 733] blk.16.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 314/ 733] blk.16.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 315/ 733] blk.17.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 316/ 733] blk.17.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 317/ 733] blk.17.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 318/ 733] blk.17.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 319/ 733] blk.17.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 320/ 733] blk.17.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 321/ 733] blk.17.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 322/ 733] blk.17.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 323/ 733] blk.17.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 324/ 733] blk.17.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 325/ 733] blk.17.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 326/ 733] blk.17.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 327/ 733] blk.17.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 328/ 733] blk.17.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 329/ 733] blk.17.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 330/ 733] blk.17.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 331/ 733] blk.17.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 332/ 733] blk.17.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 333/ 733] blk.17.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 334/ 733] blk.18.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 335/ 733] blk.18.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 336/ 733] blk.18.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 337/ 733] blk.18.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 338/ 733] blk.18.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 339/ 733] blk.18.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 340/ 733] blk.18.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 341/ 733] blk.18.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 342/ 733] blk.18.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 343/ 733] blk.18.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 344/ 733] blk.18.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 345/ 733] blk.18.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 346/ 733] blk.18.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 347/ 733] blk.18.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 348/ 733] blk.18.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 349/ 733] blk.18.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 350/ 733] blk.18.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 351/ 733] blk.18.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 352/ 733] blk.18.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 353/ 733] blk.19.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 354/ 733] blk.19.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 355/ 733] blk.19.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 356/ 733] blk.19.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 357/ 733] blk.19.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 358/ 733] blk.19.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 359/ 733] blk.19.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 360/ 733] blk.19.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 361/ 733] blk.19.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 362/ 733] blk.19.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 363/ 733] blk.19.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 364/ 733] blk.19.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 365/ 733] blk.19.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 366/ 733] blk.19.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 367/ 733] blk.19.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 368/ 733] blk.19.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 369/ 733] blk.20.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 370/ 733] blk.20.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 371/ 733] blk.20.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 372/ 733] blk.20.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 373/ 733] blk.20.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 374/ 733] blk.20.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 375/ 733] blk.20.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 376/ 733] blk.20.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 377/ 733] blk.20.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 378/ 733] blk.20.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 379/ 733] blk.20.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 380/ 733] blk.20.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 381/ 733] blk.20.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 382/ 733] blk.20.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 383/ 733] blk.20.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 384/ 733] blk.20.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 385/ 733] blk.20.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 386/ 733] blk.20.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 387/ 733] blk.20.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 388/ 733] blk.21.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 389/ 733] blk.21.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 390/ 733] blk.21.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 391/ 733] blk.21.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 392/ 733] blk.21.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 393/ 733] blk.21.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 394/ 733] blk.21.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 395/ 733] blk.21.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 396/ 733] blk.21.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 397/ 733] blk.21.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 398/ 733] blk.21.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 399/ 733] blk.21.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 400/ 733] blk.21.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 401/ 733] blk.21.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 402/ 733] blk.21.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 403/ 733] blk.21.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 404/ 733] blk.21.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 405/ 733] blk.21.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 406/ 733] blk.21.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 407/ 733] blk.22.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 408/ 733] blk.22.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 409/ 733] blk.22.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 410/ 733] blk.22.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 411/ 733] blk.22.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 412/ 733] blk.22.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 413/ 733] blk.22.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 414/ 733] blk.22.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 415/ 733] blk.22.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 416/ 733] blk.22.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 417/ 733] blk.22.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 418/ 733] blk.22.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 419/ 733] blk.22.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 420/ 733] blk.22.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 421/ 733] blk.22.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 422/ 733] blk.22.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 423/ 733] blk.22.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 424/ 733] blk.22.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 425/ 733] blk.22.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 426/ 733] blk.23.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 427/ 733] blk.23.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 428/ 733] blk.23.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 429/ 733] blk.23.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 430/ 733] blk.23.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 431/ 733] blk.23.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 432/ 733] blk.23.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 433/ 733] blk.23.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 434/ 733] blk.23.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 435/ 733] blk.23.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 436/ 733] blk.23.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 437/ 733] blk.23.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 438/ 733] blk.23.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 439/ 733] blk.23.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 440/ 733] blk.23.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 441/ 733] blk.23.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 442/ 733] blk.24.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 443/ 733] blk.24.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 444/ 733] blk.24.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 445/ 733] blk.24.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 446/ 733] blk.24.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 447/ 733] blk.24.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 448/ 733] blk.24.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 449/ 733] blk.24.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 450/ 733] blk.24.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 451/ 733] blk.24.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 452/ 733] blk.24.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 453/ 733] blk.24.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 454/ 733] blk.24.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 455/ 733] blk.24.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 456/ 733] blk.24.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 457/ 733] blk.24.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 458/ 733] blk.24.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 459/ 733] blk.24.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 460/ 733] blk.24.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 461/ 733] blk.25.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 462/ 733] blk.25.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 463/ 733] blk.25.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 464/ 733] blk.25.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 465/ 733] blk.25.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 466/ 733] blk.25.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 467/ 733] blk.25.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 468/ 733] blk.25.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 469/ 733] blk.25.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 470/ 733] blk.25.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 471/ 733] blk.25.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 472/ 733] blk.25.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 473/ 733] blk.25.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 474/ 733] blk.25.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 475/ 733] blk.25.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 476/ 733] blk.25.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 477/ 733] blk.25.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 478/ 733] blk.25.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 479/ 733] blk.25.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 480/ 733] blk.26.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 481/ 733] blk.26.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 482/ 733] blk.26.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 483/ 733] blk.26.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 484/ 733] blk.26.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 485/ 733] blk.26.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 486/ 733] blk.26.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 487/ 733] blk.26.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 488/ 733] blk.26.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 489/ 733] blk.26.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 490/ 733] blk.26.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 491/ 733] blk.26.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 492/ 733] blk.26.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 493/ 733] blk.26.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 494/ 733] blk.26.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 495/ 733] blk.26.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 496/ 733] blk.26.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 497/ 733] blk.26.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 498/ 733] blk.26.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 499/ 733] blk.27.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 500/ 733] blk.27.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 501/ 733] blk.27.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 502/ 733] blk.27.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 503/ 733] blk.27.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 504/ 733] blk.27.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 505/ 733] blk.27.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 506/ 733] blk.27.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 507/ 733] blk.27.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 508/ 733] blk.27.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 509/ 733] blk.27.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 510/ 733] blk.27.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 511/ 733] blk.27.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 512/ 733] blk.27.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 513/ 733] blk.27.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 514/ 733] blk.27.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 515/ 733] blk.28.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 516/ 733] blk.28.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 517/ 733] blk.28.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 518/ 733] blk.28.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 519/ 733] blk.28.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 520/ 733] blk.28.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 521/ 733] blk.28.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 522/ 733] blk.28.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 523/ 733] blk.28.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 524/ 733] blk.28.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 525/ 733] blk.28.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 526/ 733] blk.28.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 527/ 733] blk.28.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 528/ 733] blk.28.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 529/ 733] blk.28.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 530/ 733] blk.28.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 531/ 733] blk.28.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 532/ 733] blk.28.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 533/ 733] blk.28.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 534/ 733] blk.29.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 535/ 733] blk.29.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 536/ 733] blk.29.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 537/ 733] blk.29.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 538/ 733] blk.29.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 539/ 733] blk.29.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 540/ 733] blk.29.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 541/ 733] blk.29.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 542/ 733] blk.29.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 543/ 733] blk.29.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 544/ 733] blk.29.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 545/ 733] blk.29.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 546/ 733] blk.29.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 547/ 733] blk.29.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 548/ 733] blk.29.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 549/ 733] blk.29.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 550/ 733] blk.29.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 551/ 733] blk.29.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 552/ 733] blk.29.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 553/ 733] blk.30.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 554/ 733] blk.30.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 555/ 733] blk.30.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 556/ 733] blk.30.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 557/ 733] blk.30.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 558/ 733] blk.30.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 559/ 733] blk.30.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 560/ 733] blk.30.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 561/ 733] blk.30.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 562/ 733] blk.30.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 563/ 733] blk.30.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 564/ 733] blk.30.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 565/ 733] blk.30.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 566/ 733] blk.30.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 567/ 733] blk.30.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 568/ 733] blk.30.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 569/ 733] blk.30.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 570/ 733] blk.30.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 571/ 733] blk.30.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 572/ 733] blk.31.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 573/ 733] blk.31.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 574/ 733] blk.31.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 575/ 733] blk.31.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 576/ 733] blk.31.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 577/ 733] blk.31.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 578/ 733] blk.31.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 579/ 733] blk.31.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 580/ 733] blk.31.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 581/ 733] blk.31.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 582/ 733] blk.31.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 583/ 733] blk.31.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 584/ 733] blk.31.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 585/ 733] blk.31.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 586/ 733] blk.31.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 587/ 733] blk.31.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 588/ 733] blk.32.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 589/ 733] blk.32.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 590/ 733] blk.32.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 591/ 733] blk.32.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 592/ 733] blk.32.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 593/ 733] blk.32.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 594/ 733] blk.32.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 595/ 733] blk.32.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 596/ 733] blk.32.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 597/ 733] blk.32.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 598/ 733] blk.32.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 599/ 733] blk.32.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 600/ 733] blk.32.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 601/ 733] blk.32.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 602/ 733] blk.32.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 603/ 733] blk.32.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 604/ 733] blk.32.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 605/ 733] blk.32.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 606/ 733] blk.32.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 607/ 733] blk.33.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 608/ 733] blk.33.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 609/ 733] blk.33.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 610/ 733] blk.33.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 611/ 733] blk.33.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 612/ 733] blk.33.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 613/ 733] blk.33.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 614/ 733] blk.33.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 615/ 733] blk.33.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 616/ 733] blk.33.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 617/ 733] blk.33.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 618/ 733] blk.33.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 619/ 733] blk.33.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 620/ 733] blk.33.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 621/ 733] blk.33.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 622/ 733] blk.33.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 623/ 733] blk.33.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 624/ 733] blk.33.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 625/ 733] blk.33.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 626/ 733] blk.34.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 627/ 733] blk.34.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 628/ 733] blk.34.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 629/ 733] blk.34.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 630/ 733] blk.34.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 631/ 733] blk.34.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 632/ 733] blk.34.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 633/ 733] blk.34.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 634/ 733] blk.34.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 635/ 733] blk.34.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 636/ 733] blk.34.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 637/ 733] blk.34.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 638/ 733] blk.34.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 639/ 733] blk.34.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 640/ 733] blk.34.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 641/ 733] blk.34.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 642/ 733] blk.34.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 643/ 733] blk.34.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 644/ 733] blk.34.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 645/ 733] blk.35.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 646/ 733] blk.35.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 647/ 733] blk.35.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 648/ 733] blk.35.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 649/ 733] blk.35.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 650/ 733] blk.35.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 651/ 733] blk.35.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 652/ 733] blk.35.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 653/ 733] blk.35.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 654/ 733] blk.35.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 655/ 733] blk.35.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 656/ 733] blk.35.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 657/ 733] blk.35.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 658/ 733] blk.35.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 659/ 733] blk.35.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 660/ 733] blk.35.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 661/ 733] blk.36.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 662/ 733] blk.36.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 663/ 733] blk.36.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 664/ 733] blk.36.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 665/ 733] blk.36.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 666/ 733] blk.36.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 667/ 733] blk.36.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 668/ 733] blk.36.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 669/ 733] blk.36.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 670/ 733] blk.36.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 671/ 733] blk.36.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 672/ 733] blk.36.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 673/ 733] blk.36.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 674/ 733] blk.36.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 675/ 733] blk.36.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 676/ 733] blk.36.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 677/ 733] blk.36.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 678/ 733] blk.36.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 679/ 733] blk.36.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 680/ 733] blk.37.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 681/ 733] blk.37.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 682/ 733] blk.37.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 683/ 733] blk.37.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 684/ 733] blk.37.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 685/ 733] blk.37.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 686/ 733] blk.37.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 687/ 733] blk.37.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 688/ 733] blk.37.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 689/ 733] blk.37.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 690/ 733] blk.37.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 691/ 733] blk.37.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 692/ 733] blk.37.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 693/ 733] blk.37.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 694/ 733] blk.37.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 695/ 733] blk.37.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 696/ 733] blk.37.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 697/ 733] blk.37.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 698/ 733] blk.37.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 699/ 733] blk.38.attn_gate.weight - [ 2048, 4096, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 700/ 733] blk.38.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 701/ 733] blk.38.attn_qkv.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 32.00 MiB -> 9.00 MiB +[ 702/ 733] blk.38.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 703/ 733] blk.38.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 704/ 733] blk.38.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 705/ 733] blk.38.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 706/ 733] blk.38.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 707/ 733] blk.38.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 708/ 733] blk.38.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 709/ 733] blk.38.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 710/ 733] blk.38.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 711/ 733] blk.38.ssm_a - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 712/ 733] blk.38.ssm_alpha.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 713/ 733] blk.38.ssm_beta.weight - [ 2048, 32, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 0.12 MiB -> 0.03 MiB +[ 714/ 733] blk.38.ssm_conv1d.weight - [ 4, 8192, 1, 1], type = f32, size = 0.125 MiB +[ 715/ 733] blk.38.ssm_dt.bias - [ 32, 1, 1, 1], type = f32, size = 0.000 MiB +[ 716/ 733] blk.38.ssm_norm.weight - [ 128, 1, 1, 1], type = f32, size = 0.000 MiB +[ 717/ 733] blk.38.ssm_out.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 718/ 733] blk.39.attn_k.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 719/ 733] blk.39.attn_k_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 720/ 733] blk.39.attn_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 721/ 733] blk.39.attn_output.weight - [ 4096, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 16.00 MiB -> 4.25 MiB +[ 722/ 733] blk.39.attn_q.weight - [ 2048, 8192, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 32.00 MiB -> 8.50 MiB +[ 723/ 733] blk.39.attn_q_norm.weight - [ 256, 1, 1, 1], type = f32, size = 0.001 MiB +[ 724/ 733] blk.39.attn_v.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4 .. size = 2.00 MiB -> 0.56 MiB +[ 725/ 733] blk.39.ffn_down_exps.weight - [ 512, 2048, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 726/ 733] blk.39.ffn_down_shexp.weight - [ 512, 2048, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 727/ 733] blk.39.ffn_gate_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 728/ 733] blk.39.ffn_gate_inp.weight - [ 2048, 256, 1, 1], type = f32, size = 2.000 MiB +[ 729/ 733] blk.39.ffn_gate_inp_shexp.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +[ 730/ 733] blk.39.ffn_gate_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 731/ 733] blk.39.ffn_up_exps.weight - [ 2048, 512, 256, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 512.00 MiB -> 136.00 MiB +[ 732/ 733] blk.39.ffn_up_shexp.weight - [ 2048, 512, 1, 1], type = bf16, converting to q4_0_rocmfp4_fast .. size = 2.00 MiB -> 0.53 MiB +[ 733/ 733] blk.39.post_attention_norm.weight - [ 2048, 1, 1, 1], type = f32, size = 0.008 MiB +llama_model_quantize_impl: model size = 66152.24 MiB (16.01 BPW) +llama_model_quantize_impl: quant size = 17865.52 MiB (4.32 BPW) + +main: quantize time = 394871.16 ms +main: total time = 394871.16 ms diff --git a/recipe/logs/b_n-tools-q106-roff-probe.log b/recipe/logs/b_n-tools-q106-roff-probe.log new file mode 100644 index 0000000000000000000000000000000000000000..d3e5bf2211e6f31e520eefccba7b2bb762323796 --- /dev/null +++ b/recipe/logs/b_n-tools-q106-roff-probe.log @@ -0,0 +1,285 @@ +0.00.108.543 I log_info: verbosity = 3 (adjust with the `-lv N` CLI arg) +0.00.108.553 I device_info: +0.00.108.699 I - ROCm0 : AMD Radeon Graphics (131072 MiB, 123524 MiB free) +0.00.108.869 I - Vulkan0 : AMD Radeon Graphics (RADV GFX1151) (132096 MiB, 131922 MiB free) +0.00.108.878 I - CPU : AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (127438 MiB, 127438 MiB free) +0.00.108.975 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.109.043 I srv init: running without SSL +0.00.109.069 I srv init: using 31 threads for HTTP server +0.00.109.070 I srv init: the WebUI is disabled +0.00.109.152 I srv start: binding port with default address family +0.00.110.377 I srv main: loading model +0.00.110.386 I srv load_model: loading model '/mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' +0.00.167.016 W llama_model_loader: direct I/O is enabled, disabling mmap +0.22.284.156 W llama_context: n_ctx_seq (65536) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.22.593.161 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.22.997.376 I srv load_model: initializing slots, n_slots = 1 +0.23.238.804 W srv load_model: speculative decoding will use checkpoints +0.23.238.818 W common_speculative_init: no implementations specified for speculative decoding +0.23.238.821 I slot load_model: id 0 | task -1 | new slot, n_ctx = 65536 +0.23.239.043 I srv load_model: prompt cache RAM enabled: limit_mib=8192 +0.23.239.047 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.23.239.108 I srv init: idle slots will be saved to prompt cache upon starting a new task +0.23.286.447 I init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + + + + +' +0.23.329.268 I srv init: init: chat template, thinking = 0 +0.23.329.322 I srv main: model loaded +0.23.329.333 I srv main: server is listening on http://127.0.0.1:18652 +0.23.329.348 I srv update_slots: all slots are idle +0.24.570.143 I srv params_from_: Chat format: peg-native +0.24.570.532 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.24.570.537 I srv get_availabl: updating prompt cache +0.24.570.545 I srv load: - looking for better prompt, base f_keep = -1.000, sim = 0.000 +0.24.570.551 I srv update: - cache state: 0 prompts, 0.000 MiB (limits: 8192.000 MiB, 65536 tokens, 8589934592 est) +0.24.570.554 I srv get_availabl: prompt cache update took 0.01 ms +0.24.570.926 I reasoning-budget: activated, budget=2147483647 tokens +0.24.570.930 I reasoning-budget: deactivated (natural end) +0.24.570.950 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.25.270.625 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.25.485.037 I slot print_timing: id 0 | task 0 | +prompt eval time = 747.50 ms / 427 tokens ( 1.75 ms per token, 571.23 tokens per second) + eval time = 166.53 ms / 4 tokens ( 41.63 ms per token, 24.02 tokens per second) + total time = 914.04 ms / 431 tokens +0.25.485.248 I slot release: id 0 | task 0 | stop processing: n_tokens = 430, truncated = 0 +0.25.485.267 I srv update_slots: all slots are idle +0.25.528.499 I srv params_from_: Chat format: peg-native +0.25.530.258 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.942 +0.25.530.747 I reasoning-budget: activated, budget=2147483647 tokens +0.25.530.752 I reasoning-budget: deactivated (natural end) +0.25.530.840 I slot launch_slot_: id 0 | task 6 | processing task, is_child = 0 +0.25.530.860 W slot update_slots: id 0 | task 6 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.25.530.863 I slot update_slots: id 0 | task 6 | Checking checkpoint with [422, 422] against 405... +0.25.530.865 W slot update_slots: id 0 | task 6 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.25.530.870 W slot update_slots: id 0 | task 6 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.26.123.530 I slot create_check: id 0 | task 6 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.26.214.070 I slot print_timing: id 0 | task 6 | +prompt eval time = 644.28 ms / 421 tokens ( 1.53 ms per token, 653.44 tokens per second) + eval time = 38.90 ms / 2 tokens ( 19.45 ms per token, 51.42 tokens per second) + total time = 683.18 ms / 423 tokens +0.26.214.297 I slot release: id 0 | task 6 | stop processing: n_tokens = 422, truncated = 0 +0.26.214.362 I srv update_slots: all slots are idle +0.26.263.300 I srv params_from_: Chat format: peg-native +0.26.263.827 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.960 +0.26.264.204 I reasoning-budget: activated, budget=2147483647 tokens +0.26.264.213 I reasoning-budget: deactivated (natural end) +0.26.264.273 I slot launch_slot_: id 0 | task 10 | processing task, is_child = 0 +0.26.264.297 W slot update_slots: id 0 | task 10 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.26.264.300 I slot update_slots: id 0 | task 10 | Checking checkpoint with [416, 416] against 405... +0.26.264.302 W slot update_slots: id 0 | task 10 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.26.264.309 W slot update_slots: id 0 | task 10 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.26.862.333 I slot create_check: id 0 | task 10 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.27.692.349 I slot print_timing: id 0 | task 10 | +prompt eval time = 646.46 ms / 424 tokens ( 1.52 ms per token, 655.88 tokens per second) + eval time = 781.59 ms / 39 tokens ( 20.04 ms per token, 49.90 tokens per second) + total time = 1428.04 ms / 463 tokens +0.27.692.422 I slot release: id 0 | task 10 | stop processing: n_tokens = 462, truncated = 0 +0.27.692.446 I srv update_slots: all slots are idle +0.27.714.727 I srv params_from_: Chat format: peg-native +0.27.715.273 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.951 (> 0.100 thold), f_keep = 0.879 +0.27.715.596 I reasoning-budget: activated, budget=2147483647 tokens +0.27.715.599 I reasoning-budget: deactivated (natural end) +0.27.715.649 I slot launch_slot_: id 0 | task 51 | processing task, is_child = 0 +0.27.715.665 W slot update_slots: id 0 | task 51 | n_past = 406, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.27.715.666 I slot update_slots: id 0 | task 51 | Checking checkpoint with [419, 419] against 406... +0.27.715.667 W slot update_slots: id 0 | task 51 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.27.715.671 W slot update_slots: id 0 | task 51 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.28.253.015 I slot create_check: id 0 | task 51 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.28.373.714 I slot print_timing: id 0 | task 51 | +prompt eval time = 579.69 ms / 427 tokens ( 1.36 ms per token, 736.60 tokens per second) + eval time = 78.34 ms / 4 tokens ( 19.59 ms per token, 51.06 tokens per second) + total time = 658.04 ms / 431 tokens +0.28.373.794 I slot release: id 0 | task 51 | stop processing: n_tokens = 430, truncated = 0 +0.28.373.824 I srv update_slots: all slots are idle +0.28.388.433 I srv params_from_: Chat format: peg-native +0.28.388.929 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.942 +0.28.389.500 I reasoning-budget: activated, budget=2147483647 tokens +0.28.389.507 I reasoning-budget: deactivated (natural end) +0.28.389.598 I slot launch_slot_: id 0 | task 57 | processing task, is_child = 0 +0.28.389.627 W slot update_slots: id 0 | task 57 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.28.389.631 I slot update_slots: id 0 | task 57 | Checking checkpoint with [422, 422] against 405... +0.28.389.633 W slot update_slots: id 0 | task 57 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.28.389.641 W slot update_slots: id 0 | task 57 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.28.926.082 I slot create_check: id 0 | task 57 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.28.986.227 I slot print_timing: id 0 | task 57 | +prompt eval time = 569.23 ms / 421 tokens ( 1.35 ms per token, 739.59 tokens per second) + eval time = 27.35 ms / 2 tokens ( 13.67 ms per token, 73.13 tokens per second) + total time = 596.58 ms / 423 tokens +0.28.986.366 I slot release: id 0 | task 57 | stop processing: n_tokens = 422, truncated = 0 +0.28.986.404 I srv update_slots: all slots are idle +0.29.001.891 I srv params_from_: Chat format: peg-native +0.29.002.333 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.960 +0.29.002.794 I reasoning-budget: activated, budget=2147483647 tokens +0.29.002.796 I reasoning-budget: deactivated (natural end) +0.29.002.839 I slot launch_slot_: id 0 | task 61 | processing task, is_child = 0 +0.29.002.854 W slot update_slots: id 0 | task 61 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.29.002.855 I slot update_slots: id 0 | task 61 | Checking checkpoint with [416, 416] against 405... +0.29.002.857 W slot update_slots: id 0 | task 61 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.29.002.861 W slot update_slots: id 0 | task 61 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.29.671.242 I slot create_check: id 0 | task 61 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.30.481.274 I slot print_timing: id 0 | task 61 | +prompt eval time = 711.37 ms / 424 tokens ( 1.68 ms per token, 596.03 tokens per second) + eval time = 767.02 ms / 39 tokens ( 19.67 ms per token, 50.85 tokens per second) + total time = 1478.38 ms / 463 tokens +0.30.481.478 I slot release: id 0 | task 61 | stop processing: n_tokens = 462, truncated = 0 +0.30.481.535 I srv update_slots: all slots are idle +0.30.525.398 I srv params_from_: Chat format: peg-native +0.30.527.228 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.879 +0.30.527.832 I reasoning-budget: activated, budget=2147483647 tokens +0.30.527.839 I reasoning-budget: deactivated (natural end) +0.30.527.933 I slot launch_slot_: id 0 | task 102 | processing task, is_child = 0 +0.30.527.959 W slot update_slots: id 0 | task 102 | n_past = 406, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.30.527.963 I slot update_slots: id 0 | task 102 | Checking checkpoint with [419, 419] against 406... +0.30.527.965 W slot update_slots: id 0 | task 102 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.30.527.974 W slot update_slots: id 0 | task 102 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.31.124.220 I slot create_check: id 0 | task 102 | created context checkpoint 1 of 32 (pos_min = 420, pos_max = 420, n_tokens = 421, size = 62.813 MiB) +0.31.601.068 I slot print_timing: id 0 | task 102 | +prompt eval time = 634.63 ms / 425 tokens ( 1.49 ms per token, 669.68 tokens per second) + eval time = 438.43 ms / 17 tokens ( 25.79 ms per token, 38.77 tokens per second) + total time = 1073.07 ms / 442 tokens +0.31.601.327 I slot release: id 0 | task 102 | stop processing: n_tokens = 441, truncated = 0 +0.31.601.391 I srv update_slots: all slots are idle +0.31.656.355 I srv params_from_: Chat format: peg-native +0.31.658.634 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.967 (> 0.100 thold), f_keep = 0.918 +0.31.659.308 I reasoning-budget: activated, budget=2147483647 tokens +0.31.659.314 I reasoning-budget: deactivated (natural end) +0.31.659.409 I slot launch_slot_: id 0 | task 121 | processing task, is_child = 0 +0.31.659.433 W slot update_slots: id 0 | task 121 | n_past = 405, slot.prompt.tokens.size() = 441, seq_id = 0, pos_min = 440, n_swa = 0 +0.31.659.436 I slot update_slots: id 0 | task 121 | Checking checkpoint with [420, 420] against 405... +0.31.659.438 W slot update_slots: id 0 | task 121 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.31.659.444 W slot update_slots: id 0 | task 121 | erased invalidated context checkpoint (pos_min = 420, pos_max = 420, n_tokens = 421, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.257.720 I slot create_check: id 0 | task 121 | created context checkpoint 1 of 32 (pos_min = 414, pos_max = 414, n_tokens = 415, size = 62.813 MiB) +0.32.614.361 I slot print_timing: id 0 | task 121 | +prompt eval time = 643.10 ms / 419 tokens ( 1.53 ms per token, 651.54 tokens per second) + eval time = 311.81 ms / 12 tokens ( 25.98 ms per token, 38.49 tokens per second) + total time = 954.90 ms / 431 tokens +0.32.614.571 I slot release: id 0 | task 121 | stop processing: n_tokens = 430, truncated = 0 +0.32.614.632 I srv update_slots: all slots are idle +0.32.639.909 I srv params_from_: Chat format: peg-native +0.32.640.349 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.960 (> 0.100 thold), f_keep = 0.942 +0.32.640.975 I reasoning-budget: activated, budget=2147483647 tokens +0.32.640.982 I reasoning-budget: deactivated (natural end) +0.32.641.069 I slot launch_slot_: id 0 | task 135 | processing task, is_child = 0 +0.32.641.093 W slot update_slots: id 0 | task 135 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.32.641.097 I slot update_slots: id 0 | task 135 | Checking checkpoint with [414, 414] against 405... +0.32.641.099 W slot update_slots: id 0 | task 135 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.32.641.107 W slot update_slots: id 0 | task 135 | erased invalidated context checkpoint (pos_min = 414, pos_max = 414, n_tokens = 415, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.33.188.406 I slot create_check: id 0 | task 135 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.34.408.355 I slot print_timing: id 0 | task 135 | +prompt eval time = 604.54 ms / 422 tokens ( 1.43 ms per token, 698.05 tokens per second) + eval time = 1162.69 ms / 53 tokens ( 21.94 ms per token, 45.58 tokens per second) + total time = 1767.23 ms / 475 tokens +0.34.408.563 I slot release: id 0 | task 135 | stop processing: n_tokens = 474, truncated = 0 +0.34.408.627 I srv update_slots: all slots are idle +0.34.455.891 I srv params_from_: Chat format: peg-native +0.34.457.714 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.958 (> 0.100 thold), f_keep = 0.857 +0.34.458.254 I reasoning-budget: activated, budget=2147483647 tokens +0.34.458.261 I reasoning-budget: deactivated (natural end) +0.34.458.355 I slot launch_slot_: id 0 | task 190 | processing task, is_child = 0 +0.34.458.379 W slot update_slots: id 0 | task 190 | n_past = 406, slot.prompt.tokens.size() = 474, seq_id = 0, pos_min = 473, n_swa = 0 +0.34.458.382 I slot update_slots: id 0 | task 190 | Checking checkpoint with [417, 417] against 406... +0.34.458.384 W slot update_slots: id 0 | task 190 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.34.458.391 W slot update_slots: id 0 | task 190 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.35.097.680 I slot create_check: id 0 | task 190 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.35.342.520 I slot print_timing: id 0 | task 190 | +prompt eval time = 694.85 ms / 424 tokens ( 1.64 ms per token, 610.21 tokens per second) + eval time = 189.28 ms / 7 tokens ( 27.04 ms per token, 36.98 tokens per second) + total time = 884.13 ms / 431 tokens +0.35.342.606 I slot release: id 0 | task 190 | stop processing: n_tokens = 430, truncated = 0 +0.35.342.636 I srv update_slots: all slots are idle +0.35.396.955 I srv params_from_: Chat format: peg-native +0.35.399.122 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.969 (> 0.100 thold), f_keep = 0.942 +0.35.399.482 I reasoning-budget: activated, budget=2147483647 tokens +0.35.399.485 I reasoning-budget: deactivated (natural end) +0.35.399.534 I slot launch_slot_: id 0 | task 199 | processing task, is_child = 0 +0.35.399.550 W slot update_slots: id 0 | task 199 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.35.399.552 I slot update_slots: id 0 | task 199 | Checking checkpoint with [419, 419] against 405... +0.35.399.553 W slot update_slots: id 0 | task 199 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.35.399.556 W slot update_slots: id 0 | task 199 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.35.975.338 I slot create_check: id 0 | task 199 | created context checkpoint 1 of 32 (pos_min = 413, pos_max = 413, n_tokens = 414, size = 62.813 MiB) +0.36.211.213 I slot print_timing: id 0 | task 199 | +prompt eval time = 638.31 ms / 418 tokens ( 1.53 ms per token, 654.86 tokens per second) + eval time = 173.34 ms / 5 tokens ( 34.67 ms per token, 28.84 tokens per second) + total time = 811.65 ms / 423 tokens +0.36.211.296 I slot release: id 0 | task 199 | stop processing: n_tokens = 422, truncated = 0 +0.36.211.327 I srv update_slots: all slots are idle +0.36.257.343 I srv params_from_: Chat format: peg-native +0.36.259.204 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.960 +0.36.259.770 I reasoning-budget: activated, budget=2147483647 tokens +0.36.259.773 I reasoning-budget: deactivated (natural end) +0.36.259.860 I slot launch_slot_: id 0 | task 206 | processing task, is_child = 0 +0.36.259.881 W slot update_slots: id 0 | task 206 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.36.259.884 I slot update_slots: id 0 | task 206 | Checking checkpoint with [413, 413] against 405... +0.36.259.886 W slot update_slots: id 0 | task 206 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.36.259.891 W slot update_slots: id 0 | task 206 | erased invalidated context checkpoint (pos_min = 413, pos_max = 413, n_tokens = 414, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.36.853.203 I slot create_check: id 0 | task 206 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.37.881.460 I slot print_timing: id 0 | task 206 | +prompt eval time = 638.50 ms / 421 tokens ( 1.52 ms per token, 659.35 tokens per second) + eval time = 983.06 ms / 42 tokens ( 23.41 ms per token, 42.72 tokens per second) + total time = 1621.57 ms / 463 tokens +0.37.881.529 I slot release: id 0 | task 206 | stop processing: n_tokens = 462, truncated = 0 +0.37.881.557 I srv update_slots: all slots are idle +0.37.893.541 I srv params_from_: Chat format: peg-native +0.37.894.001 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.951 (> 0.100 thold), f_keep = 0.879 +0.37.894.205 I reasoning-budget: activated, budget=2147483647 tokens +0.37.894.207 I reasoning-budget: deactivated (natural end) +0.37.894.246 I slot launch_slot_: id 0 | task 250 | processing task, is_child = 0 +0.37.894.256 W slot update_slots: id 0 | task 250 | n_past = 406, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.37.894.257 I slot update_slots: id 0 | task 250 | Checking checkpoint with [416, 416] against 406... +0.37.894.258 W slot update_slots: id 0 | task 250 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.37.894.260 W slot update_slots: id 0 | task 250 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.38.426.246 I slot create_check: id 0 | task 250 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.38.626.110 I slot print_timing: id 0 | task 250 | +prompt eval time = 594.65 ms / 427 tokens ( 1.39 ms per token, 718.07 tokens per second) + eval time = 137.18 ms / 4 tokens ( 34.29 ms per token, 29.16 tokens per second) + total time = 731.83 ms / 431 tokens +0.38.626.211 I slot release: id 0 | task 250 | stop processing: n_tokens = 430, truncated = 0 +0.38.626.244 I srv update_slots: all slots are idle +0.38.640.241 I srv params_from_: Chat format: peg-native +0.38.640.929 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.942 +0.38.641.272 I reasoning-budget: activated, budget=2147483647 tokens +0.38.641.274 I reasoning-budget: deactivated (natural end) +0.38.641.328 I slot launch_slot_: id 0 | task 256 | processing task, is_child = 0 +0.38.641.341 W slot update_slots: id 0 | task 256 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.38.641.343 I slot update_slots: id 0 | task 256 | Checking checkpoint with [422, 422] against 405... +0.38.641.344 W slot update_slots: id 0 | task 256 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.38.641.348 W slot update_slots: id 0 | task 256 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.39.216.267 I slot create_check: id 0 | task 256 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.39.278.250 I slot print_timing: id 0 | task 256 | +prompt eval time = 607.81 ms / 421 tokens ( 1.44 ms per token, 692.65 tokens per second) + eval time = 29.08 ms / 2 tokens ( 14.54 ms per token, 68.78 tokens per second) + total time = 636.89 ms / 423 tokens +0.39.278.403 I slot release: id 0 | task 256 | stop processing: n_tokens = 422, truncated = 0 +0.39.278.445 I srv update_slots: all slots are idle +0.39.315.466 I srv params_from_: Chat format: peg-native +0.39.315.935 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.960 +0.39.316.462 I reasoning-budget: activated, budget=2147483647 tokens +0.39.316.465 I reasoning-budget: deactivated (natural end) +0.39.316.537 I slot launch_slot_: id 0 | task 260 | processing task, is_child = 0 +0.39.316.555 W slot update_slots: id 0 | task 260 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.39.316.557 I slot update_slots: id 0 | task 260 | Checking checkpoint with [416, 416] against 405... +0.39.316.559 W slot update_slots: id 0 | task 260 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.39.316.565 W slot update_slots: id 0 | task 260 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.39.992.109 I slot create_check: id 0 | task 260 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.40.804.656 I slot print_timing: id 0 | task 260 | +prompt eval time = 726.64 ms / 424 tokens ( 1.71 ms per token, 583.51 tokens per second) + eval time = 761.44 ms / 39 tokens ( 19.52 ms per token, 51.22 tokens per second) + total time = 1488.09 ms / 463 tokens +0.40.804.764 I slot release: id 0 | task 260 | stop processing: n_tokens = 462, truncated = 0 +0.40.804.795 I srv update_slots: all slots are idle +0.40.806.202 I srv operator(): operator(): cleaning up before exit... diff --git a/recipe/logs/b_n-tools-q106-roff-r2.log b/recipe/logs/b_n-tools-q106-roff-r2.log new file mode 100644 index 0000000000000000000000000000000000000000..476330733b94f68bfedd9f85a9683ea45987b6f1 --- /dev/null +++ b/recipe/logs/b_n-tools-q106-roff-r2.log @@ -0,0 +1,301 @@ +0.00.115.039 I log_info: verbosity = 3 (adjust with the `-lv N` CLI arg) +0.00.115.043 I device_info: +0.00.115.119 I - ROCm0 : AMD Radeon Graphics (131072 MiB, 123866 MiB free) +0.00.115.243 I - Vulkan0 : AMD Radeon Graphics (RADV GFX1151) (132096 MiB, 131922 MiB free) +0.00.115.248 I - CPU : AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (127438 MiB, 127438 MiB free) +0.00.115.311 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.115.325 I srv init: running without SSL +0.00.115.360 I srv init: using 31 threads for HTTP server +0.00.115.361 I srv init: the WebUI is disabled +0.00.115.422 I srv start: binding port with default address family +0.00.116.607 I srv main: loading model +0.00.116.616 I srv load_model: loading model '/mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' +0.00.167.815 W llama_model_loader: direct I/O is enabled, disabling mmap +0.22.638.462 W llama_context: n_ctx_seq (65536) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.22.931.507 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.23.329.767 I srv load_model: initializing slots, n_slots = 1 +0.23.568.311 W srv load_model: speculative decoding will use checkpoints +0.23.568.330 W common_speculative_init: no implementations specified for speculative decoding +0.23.568.334 I slot load_model: id 0 | task -1 | new slot, n_ctx = 65536 +0.23.568.522 I srv load_model: prompt cache RAM enabled: limit_mib=8192 +0.23.568.527 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.23.568.581 I srv init: idle slots will be saved to prompt cache upon starting a new task +0.23.617.611 I init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + + + + +' +0.23.662.046 I srv init: init: chat template, thinking = 0 +0.23.662.133 I srv main: model loaded +0.23.662.139 I srv main: server is listening on http://127.0.0.1:18600 +0.23.662.179 I srv update_slots: all slots are idle +0.24.822.880 I srv params_from_: Chat format: peg-native +0.24.825.218 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.24.825.223 I srv get_availabl: updating prompt cache +0.24.825.232 I srv load: - looking for better prompt, base f_keep = -1.000, sim = 0.000 +0.24.825.240 I srv update: - cache state: 0 prompts, 0.000 MiB (limits: 8192.000 MiB, 65536 tokens, 8589934592 est) +0.24.825.243 I srv get_availabl: prompt cache update took 0.02 ms +0.24.825.968 I reasoning-budget: activated, budget=2147483647 tokens +0.24.825.999 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.25.491.127 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.25.806.479 I reasoning-budget: deactivated (natural end) +0.26.592.455 I slot print_timing: id 0 | task 0 | +prompt eval time = 699.89 ms / 422 tokens ( 1.66 ms per token, 602.95 tokens per second) + eval time = 1066.52 ms / 52 tokens ( 20.51 ms per token, 48.76 tokens per second) + total time = 1766.41 ms / 474 tokens +0.26.592.534 I slot release: id 0 | task 0 | stop processing: n_tokens = 473, truncated = 0 +0.26.592.543 I srv update_slots: all slots are idle +0.26.605.042 I srv params_from_: Chat format: peg-native +0.26.605.398 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.906 (> 0.100 thold), f_keep = 0.856 +0.26.605.730 I reasoning-budget: activated, budget=2147483647 tokens +0.26.605.800 I slot launch_slot_: id 0 | task 54 | processing task, is_child = 0 +0.26.605.816 W slot update_slots: id 0 | task 54 | n_past = 405, slot.prompt.tokens.size() = 473, seq_id = 0, pos_min = 472, n_swa = 0 +0.26.605.818 I slot update_slots: id 0 | task 54 | Checking checkpoint with [417, 417] against 405... +0.26.605.820 W slot update_slots: id 0 | task 54 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.26.605.826 W slot update_slots: id 0 | task 54 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.27.159.769 I slot create_check: id 0 | task 54 | created context checkpoint 1 of 32 (pos_min = 442, pos_max = 442, n_tokens = 443, size = 62.813 MiB) +0.27.788.657 I reasoning-budget: deactivated (natural end) +0.29.292.130 I slot print_timing: id 0 | task 54 | n_decoded = 100, tg = 47.73 t/s +0.29.582.323 I slot print_timing: id 0 | task 54 | +prompt eval time = 591.18 ms / 447 tokens ( 1.32 ms per token, 756.11 tokens per second) + eval time = 2385.31 ms / 115 tokens ( 20.74 ms per token, 48.21 tokens per second) + total time = 2976.49 ms / 562 tokens +0.29.582.414 I slot release: id 0 | task 54 | stop processing: n_tokens = 561, truncated = 0 +0.29.582.457 I srv update_slots: all slots are idle +0.29.597.465 I srv params_from_: Chat format: peg-native +0.29.597.831 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.953 (> 0.100 thold), f_keep = 0.722 +0.29.598.059 I reasoning-budget: activated, budget=2147483647 tokens +0.29.598.094 I slot launch_slot_: id 0 | task 171 | processing task, is_child = 0 +0.29.598.104 W slot update_slots: id 0 | task 171 | n_past = 405, slot.prompt.tokens.size() = 561, seq_id = 0, pos_min = 560, n_swa = 0 +0.29.598.104 I slot update_slots: id 0 | task 171 | Checking checkpoint with [442, 442] against 405... +0.29.598.105 W slot update_slots: id 0 | task 171 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.29.598.107 W slot update_slots: id 0 | task 171 | erased invalidated context checkpoint (pos_min = 442, pos_max = 442, n_tokens = 443, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.30.126.859 I slot create_check: id 0 | task 171 | created context checkpoint 1 of 32 (pos_min = 420, pos_max = 420, n_tokens = 421, size = 62.813 MiB) +0.30.663.476 I reasoning-budget: deactivated (natural end) +0.31.461.262 I slot print_timing: id 0 | task 171 | +prompt eval time = 582.92 ms / 425 tokens ( 1.37 ms per token, 729.09 tokens per second) + eval time = 1280.22 ms / 63 tokens ( 20.32 ms per token, 49.21 tokens per second) + total time = 1863.14 ms / 488 tokens +0.31.461.343 I slot release: id 0 | task 171 | stop processing: n_tokens = 487, truncated = 0 +0.31.461.374 I srv update_slots: all slots are idle +0.31.475.859 I srv params_from_: Chat format: peg-native +0.31.476.263 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.953 (> 0.100 thold), f_keep = 0.832 +0.31.476.455 I reasoning-budget: activated, budget=2147483647 tokens +0.31.476.489 I slot launch_slot_: id 0 | task 236 | processing task, is_child = 0 +0.31.476.500 W slot update_slots: id 0 | task 236 | n_past = 405, slot.prompt.tokens.size() = 487, seq_id = 0, pos_min = 486, n_swa = 0 +0.31.476.501 I slot update_slots: id 0 | task 236 | Checking checkpoint with [420, 420] against 405... +0.31.476.502 W slot update_slots: id 0 | task 236 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.31.476.505 W slot update_slots: id 0 | task 236 | erased invalidated context checkpoint (pos_min = 420, pos_max = 420, n_tokens = 421, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.014.645 I slot create_check: id 0 | task 236 | created context checkpoint 1 of 32 (pos_min = 420, pos_max = 420, n_tokens = 421, size = 62.813 MiB) +0.32.309.451 I reasoning-budget: deactivated (natural end) +0.32.409.234 I slot print_timing: id 0 | task 236 | +prompt eval time = 590.97 ms / 425 tokens ( 1.39 ms per token, 719.16 tokens per second) + eval time = 341.75 ms / 17 tokens ( 20.10 ms per token, 49.74 tokens per second) + total time = 932.72 ms / 442 tokens +0.32.409.323 I slot release: id 0 | task 236 | stop processing: n_tokens = 441, truncated = 0 +0.32.409.352 I srv update_slots: all slots are idle +0.32.432.559 I srv params_from_: Chat format: peg-native +0.32.432.999 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.921 +0.32.433.226 I reasoning-budget: activated, budget=2147483647 tokens +0.32.433.266 I slot launch_slot_: id 0 | task 255 | processing task, is_child = 0 +0.32.433.278 W slot update_slots: id 0 | task 255 | n_past = 406, slot.prompt.tokens.size() = 441, seq_id = 0, pos_min = 440, n_swa = 0 +0.32.433.279 I slot update_slots: id 0 | task 255 | Checking checkpoint with [420, 420] against 406... +0.32.433.280 W slot update_slots: id 0 | task 255 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.32.433.284 W slot update_slots: id 0 | task 255 | erased invalidated context checkpoint (pos_min = 420, pos_max = 420, n_tokens = 421, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.967.808 I slot create_check: id 0 | task 255 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.33.683.392 I reasoning-budget: deactivated (natural end) +0.34.620.880 I slot print_timing: id 0 | task 255 | +prompt eval time = 589.92 ms / 422 tokens ( 1.40 ms per token, 715.35 tokens per second) + eval time = 1597.65 ms / 69 tokens ( 23.15 ms per token, 43.19 tokens per second) + total time = 2187.57 ms / 491 tokens +0.34.621.091 I slot release: id 0 | task 255 | stop processing: n_tokens = 490, truncated = 0 +0.34.621.151 I srv update_slots: all slots are idle +0.34.685.806 I srv params_from_: Chat format: peg-native +0.34.687.624 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.854 (> 0.100 thold), f_keep = 0.859 +0.34.688.188 I reasoning-budget: activated, budget=2147483647 tokens +0.34.688.287 I slot launch_slot_: id 0 | task 326 | processing task, is_child = 0 +0.34.688.310 W slot update_slots: id 0 | task 326 | n_past = 421, slot.prompt.tokens.size() = 490, seq_id = 0, pos_min = 489, n_swa = 0 +0.34.688.313 I slot update_slots: id 0 | task 326 | Checking checkpoint with [417, 417] against 421... +0.34.696.358 W slot update_slots: id 0 | task 326 | restored context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_past = 418, size = 62.813 MiB) +0.34.932.382 I slot create_check: id 0 | task 326 | created context checkpoint 2 of 32 (pos_min = 488, pos_max = 488, n_tokens = 489, size = 62.813 MiB) +0.35.494.281 I reasoning-budget: deactivated (natural end) +0.35.771.860 I slot print_timing: id 0 | task 326 | +prompt eval time = 289.45 ms / 75 tokens ( 3.86 ms per token, 259.11 tokens per second) + eval time = 794.07 ms / 36 tokens ( 22.06 ms per token, 45.34 tokens per second) + total time = 1083.52 ms / 111 tokens +0.35.772.048 I slot release: id 0 | task 326 | stop processing: n_tokens = 528, truncated = 0 +0.35.772.102 I srv update_slots: all slots are idle +0.35.790.981 I srv params_from_: Chat format: peg-native +0.35.791.515 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.972 (> 0.100 thold), f_keep = 0.777 +0.35.792.014 I reasoning-budget: activated, budget=2147483647 tokens +0.35.792.106 I slot launch_slot_: id 0 | task 364 | processing task, is_child = 0 +0.35.792.127 W slot update_slots: id 0 | task 364 | n_past = 410, slot.prompt.tokens.size() = 528, seq_id = 0, pos_min = 527, n_swa = 0 +0.35.792.130 I slot update_slots: id 0 | task 364 | Checking checkpoint with [488, 488] against 410... +0.35.792.132 I slot update_slots: id 0 | task 364 | Checking checkpoint with [417, 417] against 410... +0.35.792.133 W slot update_slots: id 0 | task 364 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.35.792.140 W slot update_slots: id 0 | task 364 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.35.793.559 W slot update_slots: id 0 | task 364 | erased invalidated context checkpoint (pos_min = 488, pos_max = 488, n_tokens = 489, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.36.350.507 I slot create_check: id 0 | task 364 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.36.640.348 I reasoning-budget: deactivated (natural end) +0.37.440.104 I slot print_timing: id 0 | task 364 | +prompt eval time = 591.78 ms / 422 tokens ( 1.40 ms per token, 713.10 tokens per second) + eval time = 1056.18 ms / 52 tokens ( 20.31 ms per token, 49.23 tokens per second) + total time = 1647.97 ms / 474 tokens +0.37.440.166 I slot release: id 0 | task 364 | stop processing: n_tokens = 473, truncated = 0 +0.37.440.190 I srv update_slots: all slots are idle +0.37.456.395 I srv params_from_: Chat format: peg-native +0.37.456.822 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.935 (> 0.100 thold), f_keep = 0.856 +0.37.457.040 I reasoning-budget: activated, budget=2147483647 tokens +0.37.457.081 I slot launch_slot_: id 0 | task 418 | processing task, is_child = 0 +0.37.457.090 W slot update_slots: id 0 | task 418 | n_past = 405, slot.prompt.tokens.size() = 473, seq_id = 0, pos_min = 472, n_swa = 0 +0.37.457.092 I slot update_slots: id 0 | task 418 | Checking checkpoint with [417, 417] against 405... +0.37.457.092 W slot update_slots: id 0 | task 418 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.37.457.095 W slot update_slots: id 0 | task 418 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.37.989.158 I slot create_check: id 0 | task 418 | created context checkpoint 1 of 32 (pos_min = 428, pos_max = 428, n_tokens = 429, size = 62.813 MiB) +0.39.035.888 I reasoning-budget: deactivated (natural end) +0.39.898.949 I slot print_timing: id 0 | task 418 | +prompt eval time = 571.43 ms / 433 tokens ( 1.32 ms per token, 757.75 tokens per second) + eval time = 1870.42 ms / 92 tokens ( 20.33 ms per token, 49.19 tokens per second) + total time = 2441.84 ms / 525 tokens +0.39.899.036 I slot release: id 0 | task 418 | stop processing: n_tokens = 524, truncated = 0 +0.39.899.069 I srv update_slots: all slots are idle +0.39.931.009 I srv params_from_: Chat format: peg-native +0.39.931.485 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.773 +0.39.932.043 I reasoning-budget: activated, budget=2147483647 tokens +0.39.932.046 I reasoning-budget: deactivated (natural end) +0.39.932.130 I slot launch_slot_: id 0 | task 512 | processing task, is_child = 0 +0.39.932.153 W slot update_slots: id 0 | task 512 | n_past = 405, slot.prompt.tokens.size() = 524, seq_id = 0, pos_min = 523, n_swa = 0 +0.39.932.156 I slot update_slots: id 0 | task 512 | Checking checkpoint with [428, 428] against 405... +0.39.932.157 W slot update_slots: id 0 | task 512 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.39.932.162 W slot update_slots: id 0 | task 512 | erased invalidated context checkpoint (pos_min = 428, pos_max = 428, n_tokens = 429, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.40.516.543 I slot create_check: id 0 | task 512 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.41.428.023 I slot print_timing: id 0 | task 512 | +prompt eval time = 630.82 ms / 424 tokens ( 1.49 ms per token, 672.15 tokens per second) + eval time = 865.02 ms / 39 tokens ( 22.18 ms per token, 45.09 tokens per second) + total time = 1495.84 ms / 463 tokens +0.41.428.279 I slot release: id 0 | task 512 | stop processing: n_tokens = 462, truncated = 0 +0.41.428.346 I srv update_slots: all slots are idle +0.41.471.490 I srv params_from_: Chat format: peg-native +0.41.472.003 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.902 (> 0.100 thold), f_keep = 0.877 +0.41.472.302 I reasoning-budget: activated, budget=2147483647 tokens +0.41.472.304 I reasoning-budget: deactivated (natural end) +0.41.472.355 I slot launch_slot_: id 0 | task 553 | processing task, is_child = 0 +0.41.472.368 W slot update_slots: id 0 | task 553 | n_past = 405, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.41.472.369 I slot update_slots: id 0 | task 553 | Checking checkpoint with [419, 419] against 405... +0.41.472.370 W slot update_slots: id 0 | task 553 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.41.472.373 W slot update_slots: id 0 | task 553 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.42.065.024 I slot create_check: id 0 | task 553 | created context checkpoint 1 of 32 (pos_min = 444, pos_max = 444, n_tokens = 445, size = 62.813 MiB) +0.44.104.338 I slot print_timing: id 0 | task 553 | +prompt eval time = 633.39 ms / 449 tokens ( 1.41 ms per token, 708.88 tokens per second) + eval time = 1998.54 ms / 86 tokens ( 23.24 ms per token, 43.03 tokens per second) + total time = 2631.93 ms / 535 tokens +0.44.104.526 I slot release: id 0 | task 553 | stop processing: n_tokens = 534, truncated = 0 +0.44.104.644 I srv update_slots: all slots are idle +0.44.130.603 I srv params_from_: Chat format: peg-native +0.44.130.995 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.948 (> 0.100 thold), f_keep = 0.758 +0.44.131.303 I reasoning-budget: activated, budget=2147483647 tokens +0.44.131.305 I reasoning-budget: deactivated (natural end) +0.44.131.351 I slot launch_slot_: id 0 | task 641 | processing task, is_child = 0 +0.44.131.364 W slot update_slots: id 0 | task 641 | n_past = 405, slot.prompt.tokens.size() = 534, seq_id = 0, pos_min = 533, n_swa = 0 +0.44.131.365 I slot update_slots: id 0 | task 641 | Checking checkpoint with [444, 444] against 405... +0.44.131.366 W slot update_slots: id 0 | task 641 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.44.131.369 W slot update_slots: id 0 | task 641 | erased invalidated context checkpoint (pos_min = 444, pos_max = 444, n_tokens = 445, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.44.685.484 I slot create_check: id 0 | task 641 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.45.671.889 I slot print_timing: id 0 | task 641 | +prompt eval time = 602.32 ms / 427 tokens ( 1.41 ms per token, 708.93 tokens per second) + eval time = 938.19 ms / 39 tokens ( 24.06 ms per token, 41.57 tokens per second) + total time = 1540.51 ms / 466 tokens +0.45.671.971 I slot release: id 0 | task 641 | stop processing: n_tokens = 465, truncated = 0 +0.45.671.999 I srv update_slots: all slots are idle +0.45.686.558 I srv params_from_: Chat format: peg-native +0.45.686.878 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.948 (> 0.100 thold), f_keep = 0.871 +0.45.687.125 I reasoning-budget: activated, budget=2147483647 tokens +0.45.687.126 I reasoning-budget: deactivated (natural end) +0.45.687.183 I slot launch_slot_: id 0 | task 682 | processing task, is_child = 0 +0.45.687.196 W slot update_slots: id 0 | task 682 | n_past = 405, slot.prompt.tokens.size() = 465, seq_id = 0, pos_min = 464, n_swa = 0 +0.45.687.197 I slot update_slots: id 0 | task 682 | Checking checkpoint with [422, 422] against 405... +0.45.687.199 W slot update_slots: id 0 | task 682 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.45.687.202 W slot update_slots: id 0 | task 682 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.46.216.440 I slot create_check: id 0 | task 682 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.46.339.458 I slot print_timing: id 0 | task 682 | +prompt eval time = 571.41 ms / 427 tokens ( 1.34 ms per token, 747.28 tokens per second) + eval time = 80.84 ms / 4 tokens ( 20.21 ms per token, 49.48 tokens per second) + total time = 652.25 ms / 431 tokens +0.46.339.548 I slot release: id 0 | task 682 | stop processing: n_tokens = 430, truncated = 0 +0.46.339.578 I srv update_slots: all slots are idle +0.46.360.275 I srv params_from_: Chat format: peg-native +0.46.360.733 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.958 (> 0.100 thold), f_keep = 0.944 +0.46.361.269 I reasoning-budget: activated, budget=2147483647 tokens +0.46.361.274 I reasoning-budget: deactivated (natural end) +0.46.361.357 I slot launch_slot_: id 0 | task 688 | processing task, is_child = 0 +0.46.361.380 W slot update_slots: id 0 | task 688 | n_past = 406, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.46.361.383 I slot update_slots: id 0 | task 688 | Checking checkpoint with [422, 422] against 406... +0.46.361.385 W slot update_slots: id 0 | task 688 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.46.361.392 W slot update_slots: id 0 | task 688 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.46.930.125 I slot create_check: id 0 | task 688 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.47.764.185 I slot print_timing: id 0 | task 688 | +prompt eval time = 611.34 ms / 424 tokens ( 1.44 ms per token, 693.56 tokens per second) + eval time = 791.43 ms / 40 tokens ( 19.79 ms per token, 50.54 tokens per second) + total time = 1402.77 ms / 464 tokens +0.47.764.366 I slot release: id 0 | task 688 | stop processing: n_tokens = 463, truncated = 0 +0.47.764.400 I srv update_slots: all slots are idle +0.47.803.235 I srv params_from_: Chat format: peg-native +0.47.803.673 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.935 (> 0.100 thold), f_keep = 1.000 +0.47.804.329 I reasoning-budget: activated, budget=2147483647 tokens +0.47.804.335 I reasoning-budget: deactivated (natural end) +0.47.804.467 I slot launch_slot_: id 0 | task 730 | processing task, is_child = 0 +0.48.002.540 I slot create_check: id 0 | task 730 | created context checkpoint 2 of 32 (pos_min = 490, pos_max = 490, n_tokens = 491, size = 62.813 MiB) +0.48.372.317 I slot print_timing: id 0 | task 730 | +prompt eval time = 256.59 ms / 32 tokens ( 8.02 ms per token, 124.71 tokens per second) + eval time = 311.22 ms / 14 tokens ( 22.23 ms per token, 44.98 tokens per second) + total time = 567.81 ms / 46 tokens +0.48.372.410 I slot release: id 0 | task 730 | stop processing: n_tokens = 508, truncated = 0 +0.48.372.439 I srv update_slots: all slots are idle +0.48.412.286 I srv params_from_: Chat format: peg-native +0.48.412.805 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.967 (> 0.100 thold), f_keep = 0.807 +0.48.413.102 I reasoning-budget: activated, budget=2147483647 tokens +0.48.413.104 I reasoning-budget: deactivated (natural end) +0.48.413.157 I slot launch_slot_: id 0 | task 746 | processing task, is_child = 0 +0.48.413.170 W slot update_slots: id 0 | task 746 | n_past = 410, slot.prompt.tokens.size() = 508, seq_id = 0, pos_min = 507, n_swa = 0 +0.48.413.172 I slot update_slots: id 0 | task 746 | Checking checkpoint with [490, 490] against 410... +0.48.413.172 I slot update_slots: id 0 | task 746 | Checking checkpoint with [419, 419] against 410... +0.48.413.173 W slot update_slots: id 0 | task 746 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.48.413.176 W slot update_slots: id 0 | task 746 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.48.414.558 W slot update_slots: id 0 | task 746 | erased invalidated context checkpoint (pos_min = 490, pos_max = 490, n_tokens = 491, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.48.958.571 I slot create_check: id 0 | task 746 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.49.822.447 I slot print_timing: id 0 | task 746 | +prompt eval time = 585.25 ms / 424 tokens ( 1.38 ms per token, 724.47 tokens per second) + eval time = 824.00 ms / 40 tokens ( 20.60 ms per token, 48.54 tokens per second) + total time = 1409.26 ms / 464 tokens +0.49.822.516 I slot release: id 0 | task 746 | stop processing: n_tokens = 463, truncated = 0 +0.49.822.541 I srv update_slots: all slots are idle +0.49.837.403 I srv params_from_: Chat format: peg-native +0.49.837.865 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.931 (> 0.100 thold), f_keep = 0.875 +0.49.838.376 I reasoning-budget: activated, budget=2147483647 tokens +0.49.838.380 I reasoning-budget: deactivated (natural end) +0.49.838.465 I slot launch_slot_: id 0 | task 788 | processing task, is_child = 0 +0.49.838.487 W slot update_slots: id 0 | task 788 | n_past = 405, slot.prompt.tokens.size() = 463, seq_id = 0, pos_min = 462, n_swa = 0 +0.49.838.490 I slot update_slots: id 0 | task 788 | Checking checkpoint with [419, 419] against 405... +0.49.838.491 W slot update_slots: id 0 | task 788 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.49.838.496 W slot update_slots: id 0 | task 788 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.50.402.186 I slot create_check: id 0 | task 788 | created context checkpoint 1 of 32 (pos_min = 430, pos_max = 430, n_tokens = 431, size = 62.813 MiB) +0.52.152.175 I slot print_timing: id 0 | task 788 | +prompt eval time = 613.67 ms / 435 tokens ( 1.41 ms per token, 708.85 tokens per second) + eval time = 1699.98 ms / 80 tokens ( 21.25 ms per token, 47.06 tokens per second) + total time = 2313.65 ms / 515 tokens +0.52.152.608 I slot release: id 0 | task 788 | stop processing: n_tokens = 514, truncated = 0 +0.52.152.669 I srv update_slots: all slots are idle +0.52.154.330 I srv operator(): operator(): cleaning up before exit... diff --git a/recipe/logs/b_n-tools-q106-tpl-probe.log b/recipe/logs/b_n-tools-q106-tpl-probe.log new file mode 100644 index 0000000000000000000000000000000000000000..a5c128db41eb68531735abb1cceb6b80623387f3 --- /dev/null +++ b/recipe/logs/b_n-tools-q106-tpl-probe.log @@ -0,0 +1,136 @@ +0.00.135.941 I log_info: verbosity = 3 (adjust with the `-lv N` CLI arg) +0.00.135.945 I device_info: +0.00.136.023 I - ROCm0 : AMD Radeon Graphics (131072 MiB, 122347 MiB free) +0.00.136.132 I - Vulkan0 : AMD Radeon Graphics (RADV GFX1151) (132096 MiB, 131922 MiB free) +0.00.136.137 I - CPU : AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (127438 MiB, 127438 MiB free) +0.00.136.208 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.136.238 I srv init: running without SSL +0.00.136.260 I srv init: using 31 threads for HTTP server +0.00.136.261 I srv init: the WebUI is disabled +0.00.136.327 I srv start: binding port with default address family +0.00.137.600 I srv main: loading model +0.00.137.603 I srv load_model: loading model '/mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' +0.00.256.612 W llama_model_loader: direct I/O is enabled, disabling mmap +0.24.306.998 W llama_context: n_ctx_seq (65536) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.24.593.444 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.24.922.506 I srv load_model: initializing slots, n_slots = 1 +0.25.146.190 W srv load_model: speculative decoding will use checkpoints +0.25.146.203 W common_speculative_init: no implementations specified for speculative decoding +0.25.146.206 I slot load_model: id 0 | task -1 | new slot, n_ctx = 65536 +0.25.146.286 I srv load_model: prompt cache RAM enabled: limit_mib=8192 +0.25.146.287 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.25.146.331 I srv init: idle slots will be saved to prompt cache upon starting a new task +0.25.160.750 I init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + +' +0.25.170.675 I srv init: init: chat template, thinking = 1 +0.25.170.721 I srv main: model loaded +0.25.170.724 I srv main: server is listening on http://127.0.0.1:18652 +0.25.170.745 I srv update_slots: all slots are idle +0.26.899.613 I srv params_from_: Chat format: peg-native +0.26.900.013 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.26.900.015 I srv get_availabl: updating prompt cache +0.26.900.021 I srv load: - looking for better prompt, base f_keep = -1.000, sim = 0.000 +0.26.900.027 I srv update: - cache state: 0 prompts, 0.000 MiB (limits: 8192.000 MiB, 65536 tokens, 8589934592 est) +0.26.900.029 I srv get_availabl: prompt cache update took 0.01 ms +0.26.900.306 I reasoning-budget: activated, budget=2147483647 tokens +0.26.900.326 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.27.527.394 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 420, pos_max = 420, n_tokens = 421, size = 62.813 MiB) +0.27.799.897 I reasoning-budget: deactivated (natural end) +0.27.893.084 I slot print_timing: id 0 | task 0 | +prompt eval time = 660.83 ms / 425 tokens ( 1.55 ms per token, 643.13 tokens per second) + eval time = 331.90 ms / 17 tokens ( 19.52 ms per token, 51.22 tokens per second) + total time = 992.73 ms / 442 tokens +0.27.893.160 I slot release: id 0 | task 0 | stop processing: n_tokens = 441, truncated = 0 +0.27.893.171 I srv update_slots: all slots are idle +0.27.910.042 I srv params_from_: Chat format: peg-native +0.27.910.501 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.921 +0.27.910.926 I reasoning-budget: activated, budget=2147483647 tokens +0.27.911.011 I slot launch_slot_: id 0 | task 19 | processing task, is_child = 0 +0.27.911.034 W slot update_slots: id 0 | task 19 | n_past = 406, slot.prompt.tokens.size() = 441, seq_id = 0, pos_min = 440, n_swa = 0 +0.27.911.036 I slot update_slots: id 0 | task 19 | Checking checkpoint with [420, 420] against 406... +0.27.911.038 W slot update_slots: id 0 | task 19 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.27.911.042 W slot update_slots: id 0 | task 19 | erased invalidated context checkpoint (pos_min = 420, pos_max = 420, n_tokens = 421, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.28.555.134 I slot create_check: id 0 | task 19 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.28.851.175 I reasoning-budget: deactivated (natural end) +0.29.588.543 I slot print_timing: id 0 | task 19 | +prompt eval time = 695.53 ms / 422 tokens ( 1.65 ms per token, 606.73 tokens per second) + eval time = 981.97 ms / 53 tokens ( 18.53 ms per token, 53.97 tokens per second) + total time = 1677.50 ms / 475 tokens +0.29.588.615 I slot release: id 0 | task 19 | stop processing: n_tokens = 474, truncated = 0 +0.29.588.642 I srv update_slots: all slots are idle +0.29.622.104 I srv params_from_: Chat format: peg-native +0.29.622.588 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.958 (> 0.100 thold), f_keep = 0.857 +0.29.623.144 I reasoning-budget: activated, budget=2147483647 tokens +0.29.623.228 I slot launch_slot_: id 0 | task 74 | processing task, is_child = 0 +0.29.623.250 W slot update_slots: id 0 | task 74 | n_past = 406, slot.prompt.tokens.size() = 474, seq_id = 0, pos_min = 473, n_swa = 0 +0.29.623.253 I slot update_slots: id 0 | task 74 | Checking checkpoint with [417, 417] against 406... +0.29.623.255 W slot update_slots: id 0 | task 74 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.29.623.260 W slot update_slots: id 0 | task 74 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.30.170.371 I slot create_check: id 0 | task 74 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.30.268.328 I reasoning-budget: deactivated (natural end) +0.30.408.646 I slot print_timing: id 0 | task 74 | +prompt eval time = 594.93 ms / 424 tokens ( 1.40 ms per token, 712.68 tokens per second) + eval time = 190.44 ms / 7 tokens ( 27.21 ms per token, 36.76 tokens per second) + total time = 785.37 ms / 431 tokens +0.30.408.832 I slot release: id 0 | task 74 | stop processing: n_tokens = 430, truncated = 0 +0.30.408.910 I srv update_slots: all slots are idle +0.30.423.352 I srv params_from_: Chat format: peg-native +0.30.423.958 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.964 (> 0.100 thold), f_keep = 0.944 +0.30.424.428 I reasoning-budget: activated, budget=2147483647 tokens +0.30.424.483 I slot launch_slot_: id 0 | task 83 | processing task, is_child = 0 +0.30.424.498 W slot update_slots: id 0 | task 83 | n_past = 406, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.30.424.499 I slot update_slots: id 0 | task 83 | Checking checkpoint with [419, 419] against 406... +0.30.424.500 W slot update_slots: id 0 | task 83 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.30.424.505 W slot update_slots: id 0 | task 83 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.30.909.972 I slot create_check: id 0 | task 83 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.30.972.923 I reasoning-budget: deactivated (natural end) +0.31.775.290 I slot print_timing: id 0 | task 83 | +prompt eval time = 521.51 ms / 421 tokens ( 1.24 ms per token, 807.27 tokens per second) + eval time = 829.27 ms / 42 tokens ( 19.74 ms per token, 50.65 tokens per second) + total time = 1350.78 ms / 463 tokens +0.31.775.369 I slot release: id 0 | task 83 | stop processing: n_tokens = 462, truncated = 0 +0.31.775.395 I srv update_slots: all slots are idle +0.31.814.609 I srv params_from_: Chat format: peg-native +0.31.815.041 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.951 (> 0.100 thold), f_keep = 0.879 +0.31.815.337 I reasoning-budget: activated, budget=2147483647 tokens +0.31.815.385 I slot launch_slot_: id 0 | task 127 | processing task, is_child = 0 +0.31.815.397 W slot update_slots: id 0 | task 127 | n_past = 406, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.31.815.398 I slot update_slots: id 0 | task 127 | Checking checkpoint with [416, 416] against 406... +0.31.815.399 W slot update_slots: id 0 | task 127 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.31.815.402 W slot update_slots: id 0 | task 127 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.362.167 I slot create_check: id 0 | task 127 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.32.500.485 I slot print_timing: id 0 | task 127 | +prompt eval time = 585.39 ms / 427 tokens ( 1.37 ms per token, 729.43 tokens per second) + eval time = 99.68 ms / 4 tokens ( 24.92 ms per token, 40.13 tokens per second) + total time = 685.07 ms / 431 tokens +0.32.500.582 I slot release: id 0 | task 127 | stop processing: n_tokens = 430, truncated = 0 +0.32.500.610 I srv update_slots: all slots are idle +0.32.535.431 I srv params_from_: Chat format: peg-native +0.32.535.929 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.958 (> 0.100 thold), f_keep = 0.944 +0.32.536.504 I reasoning-budget: activated, budget=2147483647 tokens +0.32.536.587 I slot launch_slot_: id 0 | task 133 | processing task, is_child = 0 +0.32.536.611 W slot update_slots: id 0 | task 133 | n_past = 406, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.32.536.614 I slot update_slots: id 0 | task 133 | Checking checkpoint with [422, 422] against 406... +0.32.536.616 W slot update_slots: id 0 | task 133 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.32.536.622 W slot update_slots: id 0 | task 133 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.33.112.942 I slot create_check: id 0 | task 133 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.34.067.962 I slot print_timing: id 0 | task 133 | +prompt eval time = 615.83 ms / 424 tokens ( 1.45 ms per token, 688.51 tokens per second) + eval time = 915.49 ms / 39 tokens ( 23.47 ms per token, 42.60 tokens per second) + total time = 1531.32 ms / 463 tokens +0.34.068.202 I slot release: id 0 | task 133 | stop processing: n_tokens = 462, truncated = 0 +0.34.068.265 I srv update_slots: all slots are idle +0.34.070.064 I srv operator(): operator(): cleaning up before exit... diff --git a/recipe/logs/b_n-tools-q106-tpl.log b/recipe/logs/b_n-tools-q106-tpl.log new file mode 100644 index 0000000000000000000000000000000000000000..c04e68742d67a48950563b064edcf298eb8ee8c2 --- /dev/null +++ b/recipe/logs/b_n-tools-q106-tpl.log @@ -0,0 +1,299 @@ +0.00.148.040 I log_info: verbosity = 3 (adjust with the `-lv N` CLI arg) +0.00.148.048 I device_info: +0.00.148.185 I - ROCm0 : AMD Radeon Graphics (131072 MiB, 122384 MiB free) +0.00.148.371 I - Vulkan0 : AMD Radeon Graphics (RADV GFX1151) (132096 MiB, 131922 MiB free) +0.00.148.380 I - CPU : AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (127438 MiB, 127438 MiB free) +0.00.148.475 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.148.535 I srv init: running without SSL +0.00.148.567 I srv init: using 31 threads for HTTP server +0.00.148.569 I srv init: the WebUI is disabled +0.00.148.676 I srv start: binding port with default address family +0.00.149.917 I srv main: loading model +0.00.149.927 I srv load_model: loading model '/mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' +0.00.318.164 W llama_model_loader: direct I/O is enabled, disabling mmap +0.24.651.499 W llama_context: n_ctx_seq (65536) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.24.848.592 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.25.156.208 I srv load_model: initializing slots, n_slots = 1 +0.25.370.167 W srv load_model: speculative decoding will use checkpoints +0.25.370.182 W common_speculative_init: no implementations specified for speculative decoding +0.25.370.188 I slot load_model: id 0 | task -1 | new slot, n_ctx = 65536 +0.25.370.351 I srv load_model: prompt cache RAM enabled: limit_mib=8192 +0.25.370.398 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.25.370.453 I srv init: idle slots will be saved to prompt cache upon starting a new task +0.25.419.174 I init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + +' +0.25.441.368 I srv init: init: chat template, thinking = 1 +0.25.441.453 I srv main: model loaded +0.25.441.460 I srv main: server is listening on http://127.0.0.1:18600 +0.25.441.468 I srv update_slots: all slots are idle +0.26.789.568 I srv params_from_: Chat format: peg-native +0.26.789.991 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.26.789.993 I srv get_availabl: updating prompt cache +0.26.790.000 I srv load: - looking for better prompt, base f_keep = -1.000, sim = 0.000 +0.26.790.006 I srv update: - cache state: 0 prompts, 0.000 MiB (limits: 8192.000 MiB, 65536 tokens, 8589934592 est) +0.26.790.010 I srv get_availabl: prompt cache update took 0.01 ms +0.26.790.277 I reasoning-budget: activated, budget=2147483647 tokens +0.26.790.294 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.27.476.977 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.27.881.416 I reasoning-budget: deactivated (natural end) +0.28.806.104 I slot print_timing: id 0 | task 0 | +prompt eval time = 751.68 ms / 422 tokens ( 1.78 ms per token, 561.41 tokens per second) + eval time = 1264.10 ms / 52 tokens ( 24.31 ms per token, 41.14 tokens per second) + total time = 2015.78 ms / 474 tokens +0.28.806.213 I slot release: id 0 | task 0 | stop processing: n_tokens = 473, truncated = 0 +0.28.806.225 I srv update_slots: all slots are idle +0.28.824.839 I srv params_from_: Chat format: peg-native +0.28.825.237 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.906 (> 0.100 thold), f_keep = 0.856 +0.28.825.422 I reasoning-budget: activated, budget=2147483647 tokens +0.28.825.471 I slot launch_slot_: id 0 | task 54 | processing task, is_child = 0 +0.28.825.483 W slot update_slots: id 0 | task 54 | n_past = 405, slot.prompt.tokens.size() = 473, seq_id = 0, pos_min = 472, n_swa = 0 +0.28.825.485 I slot update_slots: id 0 | task 54 | Checking checkpoint with [417, 417] against 405... +0.28.825.486 W slot update_slots: id 0 | task 54 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.28.825.489 W slot update_slots: id 0 | task 54 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.29.247.541 I slot create_check: id 0 | task 54 | created context checkpoint 1 of 32 (pos_min = 442, pos_max = 442, n_tokens = 443, size = 62.813 MiB) +0.29.847.933 I reasoning-budget: deactivated (natural end) +0.31.561.799 I slot print_timing: id 0 | task 54 | n_decoded = 100, tg = 43.85 t/s +0.31.768.383 I slot print_timing: id 0 | task 54 | +prompt eval time = 455.75 ms / 447 tokens ( 1.02 ms per token, 980.80 tokens per second) + eval time = 2487.14 ms / 110 tokens ( 22.61 ms per token, 44.23 tokens per second) + total time = 2942.89 ms / 557 tokens +0.31.768.453 I slot release: id 0 | task 54 | stop processing: n_tokens = 556, truncated = 0 +0.31.768.488 I srv update_slots: all slots are idle +0.31.780.986 I srv params_from_: Chat format: peg-native +0.31.781.515 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.953 (> 0.100 thold), f_keep = 0.728 +0.31.781.866 I reasoning-budget: activated, budget=2147483647 tokens +0.31.781.944 I slot launch_slot_: id 0 | task 166 | processing task, is_child = 0 +0.31.781.959 W slot update_slots: id 0 | task 166 | n_past = 405, slot.prompt.tokens.size() = 556, seq_id = 0, pos_min = 555, n_swa = 0 +0.31.781.960 I slot update_slots: id 0 | task 166 | Checking checkpoint with [442, 442] against 405... +0.31.781.961 W slot update_slots: id 0 | task 166 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.31.781.965 W slot update_slots: id 0 | task 166 | erased invalidated context checkpoint (pos_min = 442, pos_max = 442, n_tokens = 443, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.284.242 I slot create_check: id 0 | task 166 | created context checkpoint 1 of 32 (pos_min = 420, pos_max = 420, n_tokens = 421, size = 62.813 MiB) +0.32.618.089 I reasoning-budget: deactivated (natural end) +0.33.467.501 I slot print_timing: id 0 | task 166 | +prompt eval time = 568.15 ms / 425 tokens ( 1.34 ms per token, 748.05 tokens per second) + eval time = 1117.36 ms / 52 tokens ( 21.49 ms per token, 46.54 tokens per second) + total time = 1685.50 ms / 477 tokens +0.33.467.767 I slot release: id 0 | task 166 | stop processing: n_tokens = 476, truncated = 0 +0.33.467.836 I srv update_slots: all slots are idle +0.33.510.830 I srv params_from_: Chat format: peg-native +0.33.511.252 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.953 (> 0.100 thold), f_keep = 0.851 +0.33.511.444 I reasoning-budget: activated, budget=2147483647 tokens +0.33.511.478 I slot launch_slot_: id 0 | task 220 | processing task, is_child = 0 +0.33.511.488 W slot update_slots: id 0 | task 220 | n_past = 405, slot.prompt.tokens.size() = 476, seq_id = 0, pos_min = 475, n_swa = 0 +0.33.511.489 I slot update_slots: id 0 | task 220 | Checking checkpoint with [420, 420] against 405... +0.33.511.490 W slot update_slots: id 0 | task 220 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.33.511.494 W slot update_slots: id 0 | task 220 | erased invalidated context checkpoint (pos_min = 420, pos_max = 420, n_tokens = 421, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.34.023.181 I slot create_check: id 0 | task 220 | created context checkpoint 1 of 32 (pos_min = 420, pos_max = 420, n_tokens = 421, size = 62.813 MiB) +0.34.447.908 I reasoning-budget: deactivated (natural end) +0.34.540.055 I slot print_timing: id 0 | task 220 | +prompt eval time = 555.59 ms / 425 tokens ( 1.31 ms per token, 764.95 tokens per second) + eval time = 472.93 ms / 23 tokens ( 20.56 ms per token, 48.63 tokens per second) + total time = 1028.52 ms / 448 tokens +0.34.540.230 I slot release: id 0 | task 220 | stop processing: n_tokens = 447, truncated = 0 +0.34.540.267 I srv update_slots: all slots are idle +0.34.592.769 I srv params_from_: Chat format: peg-native +0.34.593.328 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.908 +0.34.593.568 I reasoning-budget: activated, budget=2147483647 tokens +0.34.593.618 I slot launch_slot_: id 0 | task 245 | processing task, is_child = 0 +0.34.593.629 W slot update_slots: id 0 | task 245 | n_past = 406, slot.prompt.tokens.size() = 447, seq_id = 0, pos_min = 446, n_swa = 0 +0.34.593.631 I slot update_slots: id 0 | task 245 | Checking checkpoint with [420, 420] against 406... +0.34.593.632 W slot update_slots: id 0 | task 245 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.34.593.635 W slot update_slots: id 0 | task 245 | erased invalidated context checkpoint (pos_min = 420, pos_max = 420, n_tokens = 421, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.35.150.983 I slot create_check: id 0 | task 245 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.35.748.562 I reasoning-budget: deactivated (natural end) +0.36.564.011 I slot print_timing: id 0 | task 245 | +prompt eval time = 621.88 ms / 422 tokens ( 1.47 ms per token, 678.58 tokens per second) + eval time = 1348.48 ms / 64 tokens ( 21.07 ms per token, 47.46 tokens per second) + total time = 1970.37 ms / 486 tokens +0.36.564.097 I slot release: id 0 | task 245 | stop processing: n_tokens = 485, truncated = 0 +0.36.564.126 I srv update_slots: all slots are idle +0.36.618.709 I srv params_from_: Chat format: peg-native +0.36.619.282 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.854 (> 0.100 thold), f_keep = 0.868 +0.36.619.528 I reasoning-budget: activated, budget=2147483647 tokens +0.36.619.573 I slot launch_slot_: id 0 | task 311 | processing task, is_child = 0 +0.36.619.585 W slot update_slots: id 0 | task 311 | n_past = 421, slot.prompt.tokens.size() = 485, seq_id = 0, pos_min = 484, n_swa = 0 +0.36.619.587 I slot update_slots: id 0 | task 311 | Checking checkpoint with [417, 417] against 421... +0.36.623.567 W slot update_slots: id 0 | task 311 | restored context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_past = 418, size = 62.813 MiB) +0.36.859.461 I slot create_check: id 0 | task 311 | created context checkpoint 2 of 32 (pos_min = 488, pos_max = 488, n_tokens = 489, size = 62.813 MiB) +0.37.463.364 I reasoning-budget: deactivated (natural end) +0.37.731.529 I slot print_timing: id 0 | task 311 | +prompt eval time = 287.07 ms / 75 tokens ( 3.83 ms per token, 261.26 tokens per second) + eval time = 824.84 ms / 39 tokens ( 21.15 ms per token, 47.28 tokens per second) + total time = 1111.91 ms / 114 tokens +0.37.731.631 I slot release: id 0 | task 311 | stop processing: n_tokens = 531, truncated = 0 +0.37.731.658 I srv update_slots: all slots are idle +0.37.774.243 I srv params_from_: Chat format: peg-native +0.37.774.624 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.972 (> 0.100 thold), f_keep = 0.772 +0.37.774.821 I reasoning-budget: activated, budget=2147483647 tokens +0.37.774.854 I slot launch_slot_: id 0 | task 352 | processing task, is_child = 0 +0.37.774.864 W slot update_slots: id 0 | task 352 | n_past = 410, slot.prompt.tokens.size() = 531, seq_id = 0, pos_min = 530, n_swa = 0 +0.37.774.865 I slot update_slots: id 0 | task 352 | Checking checkpoint with [488, 488] against 410... +0.37.774.866 I slot update_slots: id 0 | task 352 | Checking checkpoint with [417, 417] against 410... +0.37.774.867 W slot update_slots: id 0 | task 352 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.37.774.870 W slot update_slots: id 0 | task 352 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.37.775.796 W slot update_slots: id 0 | task 352 | erased invalidated context checkpoint (pos_min = 488, pos_max = 488, n_tokens = 489, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.38.322.268 I slot create_check: id 0 | task 352 | created context checkpoint 1 of 32 (pos_min = 417, pos_max = 417, n_tokens = 418, size = 62.813 MiB) +0.38.647.904 I reasoning-budget: deactivated (natural end) +0.39.470.609 I slot print_timing: id 0 | task 352 | +prompt eval time = 598.33 ms / 422 tokens ( 1.42 ms per token, 705.29 tokens per second) + eval time = 1097.37 ms / 54 tokens ( 20.32 ms per token, 49.21 tokens per second) + total time = 1695.70 ms / 476 tokens +0.39.470.815 I slot release: id 0 | task 352 | stop processing: n_tokens = 475, truncated = 0 +0.39.470.868 I srv update_slots: all slots are idle +0.39.526.260 I srv params_from_: Chat format: peg-native +0.39.528.818 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.935 (> 0.100 thold), f_keep = 0.853 +0.39.529.421 I reasoning-budget: activated, budget=2147483647 tokens +0.39.529.515 I slot launch_slot_: id 0 | task 408 | processing task, is_child = 0 +0.39.529.535 W slot update_slots: id 0 | task 408 | n_past = 405, slot.prompt.tokens.size() = 475, seq_id = 0, pos_min = 474, n_swa = 0 +0.39.529.537 I slot update_slots: id 0 | task 408 | Checking checkpoint with [417, 417] against 405... +0.39.529.539 W slot update_slots: id 0 | task 408 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.39.529.547 W slot update_slots: id 0 | task 408 | erased invalidated context checkpoint (pos_min = 417, pos_max = 417, n_tokens = 418, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.40.127.515 I slot create_check: id 0 | task 408 | created context checkpoint 1 of 32 (pos_min = 428, pos_max = 428, n_tokens = 429, size = 62.813 MiB) +0.40.699.453 I reasoning-budget: deactivated (natural end) +0.42.177.715 I slot print_timing: id 0 | task 408 | n_decoded = 100, tg = 49.81 t/s +0.42.310.602 I slot print_timing: id 0 | task 408 | +prompt eval time = 640.38 ms / 433 tokens ( 1.48 ms per token, 676.16 tokens per second) + eval time = 2140.67 ms / 107 tokens ( 20.01 ms per token, 49.98 tokens per second) + total time = 2781.05 ms / 540 tokens +0.42.310.711 I slot release: id 0 | task 408 | stop processing: n_tokens = 539, truncated = 0 +0.42.310.745 I srv update_slots: all slots are idle +0.42.345.777 I srv params_from_: Chat format: peg-native +0.42.346.201 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.751 +0.42.346.446 I reasoning-budget: activated, budget=2147483647 tokens +0.42.346.448 I reasoning-budget: deactivated (natural end) +0.42.346.491 I slot launch_slot_: id 0 | task 517 | processing task, is_child = 0 +0.42.346.503 W slot update_slots: id 0 | task 517 | n_past = 405, slot.prompt.tokens.size() = 539, seq_id = 0, pos_min = 538, n_swa = 0 +0.42.346.504 I slot update_slots: id 0 | task 517 | Checking checkpoint with [428, 428] against 405... +0.42.346.506 W slot update_slots: id 0 | task 517 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.42.346.509 W slot update_slots: id 0 | task 517 | erased invalidated context checkpoint (pos_min = 428, pos_max = 428, n_tokens = 429, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.42.884.527 I slot create_check: id 0 | task 517 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.43.790.791 I slot print_timing: id 0 | task 517 | +prompt eval time = 576.47 ms / 424 tokens ( 1.36 ms per token, 735.51 tokens per second) + eval time = 867.80 ms / 39 tokens ( 22.25 ms per token, 44.94 tokens per second) + total time = 1444.27 ms / 463 tokens +0.43.790.894 I slot release: id 0 | task 517 | stop processing: n_tokens = 462, truncated = 0 +0.43.790.929 I srv update_slots: all slots are idle +0.43.803.670 I srv params_from_: Chat format: peg-native +0.43.804.041 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.902 (> 0.100 thold), f_keep = 0.877 +0.43.804.257 I reasoning-budget: activated, budget=2147483647 tokens +0.43.804.260 I reasoning-budget: deactivated (natural end) +0.43.804.297 I slot launch_slot_: id 0 | task 558 | processing task, is_child = 0 +0.43.804.307 W slot update_slots: id 0 | task 558 | n_past = 405, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.43.804.307 I slot update_slots: id 0 | task 558 | Checking checkpoint with [419, 419] against 405... +0.43.804.308 W slot update_slots: id 0 | task 558 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.43.804.311 W slot update_slots: id 0 | task 558 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.44.348.687 I slot create_check: id 0 | task 558 | created context checkpoint 1 of 32 (pos_min = 444, pos_max = 444, n_tokens = 445, size = 62.813 MiB) +0.46.245.975 I slot print_timing: id 0 | task 558 | +prompt eval time = 596.97 ms / 449 tokens ( 1.33 ms per token, 752.13 tokens per second) + eval time = 1844.68 ms / 86 tokens ( 21.45 ms per token, 46.62 tokens per second) + total time = 2441.65 ms / 535 tokens +0.46.246.065 I slot release: id 0 | task 558 | stop processing: n_tokens = 534, truncated = 0 +0.46.246.093 I srv update_slots: all slots are idle +0.46.272.193 I srv params_from_: Chat format: peg-native +0.46.272.663 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.948 (> 0.100 thold), f_keep = 0.758 +0.46.273.230 I reasoning-budget: activated, budget=2147483647 tokens +0.46.273.238 I reasoning-budget: deactivated (natural end) +0.46.273.319 I slot launch_slot_: id 0 | task 646 | processing task, is_child = 0 +0.46.273.341 W slot update_slots: id 0 | task 646 | n_past = 405, slot.prompt.tokens.size() = 534, seq_id = 0, pos_min = 533, n_swa = 0 +0.46.273.345 I slot update_slots: id 0 | task 646 | Checking checkpoint with [444, 444] against 405... +0.46.273.347 W slot update_slots: id 0 | task 646 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.46.273.352 W slot update_slots: id 0 | task 646 | erased invalidated context checkpoint (pos_min = 444, pos_max = 444, n_tokens = 445, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.46.832.222 I slot create_check: id 0 | task 646 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.47.645.063 I slot print_timing: id 0 | task 646 | +prompt eval time = 593.12 ms / 427 tokens ( 1.39 ms per token, 719.92 tokens per second) + eval time = 778.59 ms / 39 tokens ( 19.96 ms per token, 50.09 tokens per second) + total time = 1371.71 ms / 466 tokens +0.47.645.151 I slot release: id 0 | task 646 | stop processing: n_tokens = 465, truncated = 0 +0.47.645.177 I srv update_slots: all slots are idle +0.47.664.123 I srv params_from_: Chat format: peg-native +0.47.664.583 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.948 (> 0.100 thold), f_keep = 0.871 +0.47.665.092 I reasoning-budget: activated, budget=2147483647 tokens +0.47.665.095 I reasoning-budget: deactivated (natural end) +0.47.665.179 I slot launch_slot_: id 0 | task 687 | processing task, is_child = 0 +0.47.665.201 W slot update_slots: id 0 | task 687 | n_past = 405, slot.prompt.tokens.size() = 465, seq_id = 0, pos_min = 464, n_swa = 0 +0.47.665.204 I slot update_slots: id 0 | task 687 | Checking checkpoint with [422, 422] against 405... +0.47.665.206 W slot update_slots: id 0 | task 687 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.47.665.212 W slot update_slots: id 0 | task 687 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.48.219.459 I slot create_check: id 0 | task 687 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.48.341.963 I slot print_timing: id 0 | task 687 | +prompt eval time = 588.36 ms / 427 tokens ( 1.38 ms per token, 725.75 tokens per second) + eval time = 88.39 ms / 4 tokens ( 22.10 ms per token, 45.25 tokens per second) + total time = 676.75 ms / 431 tokens +0.48.342.059 I slot release: id 0 | task 687 | stop processing: n_tokens = 430, truncated = 0 +0.48.342.091 I srv update_slots: all slots are idle +0.48.357.137 I srv params_from_: Chat format: peg-native +0.48.357.617 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.958 (> 0.100 thold), f_keep = 0.944 +0.48.358.008 I reasoning-budget: activated, budget=2147483647 tokens +0.48.358.014 I reasoning-budget: deactivated (natural end) +0.48.358.075 I slot launch_slot_: id 0 | task 693 | processing task, is_child = 0 +0.48.358.093 W slot update_slots: id 0 | task 693 | n_past = 406, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.48.358.095 I slot update_slots: id 0 | task 693 | Checking checkpoint with [422, 422] against 406... +0.48.358.096 W slot update_slots: id 0 | task 693 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.48.358.101 W slot update_slots: id 0 | task 693 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.49.035.367 I slot create_check: id 0 | task 693 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.49.869.527 I slot print_timing: id 0 | task 693 | +prompt eval time = 708.91 ms / 424 tokens ( 1.67 ms per token, 598.10 tokens per second) + eval time = 802.48 ms / 40 tokens ( 20.06 ms per token, 49.85 tokens per second) + total time = 1511.39 ms / 464 tokens +0.49.869.785 I slot release: id 0 | task 693 | stop processing: n_tokens = 463, truncated = 0 +0.49.869.856 I srv update_slots: all slots are idle +0.49.916.557 I srv params_from_: Chat format: peg-native +0.49.916.971 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.935 (> 0.100 thold), f_keep = 1.000 +0.49.917.213 I reasoning-budget: activated, budget=2147483647 tokens +0.49.917.215 I reasoning-budget: deactivated (natural end) +0.49.917.257 I slot launch_slot_: id 0 | task 735 | processing task, is_child = 0 +0.50.095.825 I slot create_check: id 0 | task 735 | created context checkpoint 2 of 32 (pos_min = 490, pos_max = 490, n_tokens = 491, size = 62.813 MiB) +0.50.567.914 I slot print_timing: id 0 | task 735 | +prompt eval time = 242.43 ms / 32 tokens ( 7.58 ms per token, 132.00 tokens per second) + eval time = 408.20 ms / 15 tokens ( 27.21 ms per token, 36.75 tokens per second) + total time = 650.63 ms / 47 tokens +0.50.567.999 I slot release: id 0 | task 735 | stop processing: n_tokens = 509, truncated = 0 +0.50.568.024 I srv update_slots: all slots are idle +0.50.583.295 I srv params_from_: Chat format: peg-native +0.50.584.124 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.967 (> 0.100 thold), f_keep = 0.806 +0.50.584.432 I reasoning-budget: activated, budget=2147483647 tokens +0.50.584.436 I reasoning-budget: deactivated (natural end) +0.50.584.488 I slot launch_slot_: id 0 | task 752 | processing task, is_child = 0 +0.50.584.503 W slot update_slots: id 0 | task 752 | n_past = 410, slot.prompt.tokens.size() = 509, seq_id = 0, pos_min = 508, n_swa = 0 +0.50.584.505 I slot update_slots: id 0 | task 752 | Checking checkpoint with [490, 490] against 410... +0.50.584.506 I slot update_slots: id 0 | task 752 | Checking checkpoint with [419, 419] against 410... +0.50.584.507 W slot update_slots: id 0 | task 752 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.50.584.510 W slot update_slots: id 0 | task 752 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.50.585.691 W slot update_slots: id 0 | task 752 | erased invalidated context checkpoint (pos_min = 490, pos_max = 490, n_tokens = 491, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.51.126.714 I slot create_check: id 0 | task 752 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.52.045.755 I slot print_timing: id 0 | task 752 | +prompt eval time = 592.34 ms / 424 tokens ( 1.40 ms per token, 715.81 tokens per second) + eval time = 868.87 ms / 40 tokens ( 21.72 ms per token, 46.04 tokens per second) + total time = 1461.21 ms / 464 tokens +0.52.045.943 I slot release: id 0 | task 752 | stop processing: n_tokens = 463, truncated = 0 +0.52.045.996 I srv update_slots: all slots are idle +0.52.104.739 I srv params_from_: Chat format: peg-native +0.52.107.431 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.931 (> 0.100 thold), f_keep = 0.875 +0.52.108.068 I reasoning-budget: activated, budget=2147483647 tokens +0.52.108.073 I reasoning-budget: deactivated (natural end) +0.52.108.179 I slot launch_slot_: id 0 | task 794 | processing task, is_child = 0 +0.52.108.202 W slot update_slots: id 0 | task 794 | n_past = 405, slot.prompt.tokens.size() = 463, seq_id = 0, pos_min = 462, n_swa = 0 +0.52.108.206 I slot update_slots: id 0 | task 794 | Checking checkpoint with [419, 419] against 405... +0.52.108.208 W slot update_slots: id 0 | task 794 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.52.108.217 W slot update_slots: id 0 | task 794 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.52.715.448 I slot create_check: id 0 | task 794 | created context checkpoint 1 of 32 (pos_min = 430, pos_max = 430, n_tokens = 431, size = 62.813 MiB) +0.54.554.592 I slot print_timing: id 0 | task 794 | +prompt eval time = 642.08 ms / 435 tokens ( 1.48 ms per token, 677.48 tokens per second) + eval time = 1804.29 ms / 80 tokens ( 22.55 ms per token, 44.34 tokens per second) + total time = 2446.37 ms / 515 tokens +0.54.554.809 I slot release: id 0 | task 794 | stop processing: n_tokens = 514, truncated = 0 +0.54.554.839 I srv update_slots: all slots are idle +0.54.555.755 I srv operator(): operator(): cleaning up before exit... diff --git a/recipe/logs/b_n-vision-q106-c1-faon.log b/recipe/logs/b_n-vision-q106-c1-faon.log new file mode 100644 index 0000000000000000000000000000000000000000..6a1876d5af86d331f21d6711b5385152fa163372 --- /dev/null +++ b/recipe/logs/b_n-vision-q106-c1-faon.log @@ -0,0 +1,71 @@ +0.00.059.279 W Setting 'enable_thinking' via --chat-template-kwargs is deprecated. Use --reasoning on / --reasoning off instead. +0.00.069.767 I log_info: verbosity = 3 (adjust with the `-lv N` CLI arg) +0.00.069.776 I device_info: +0.00.069.939 I - ROCm0 : AMD Radeon Graphics (131072 MiB, 123862 MiB free) +0.00.070.129 I - Vulkan0 : AMD Radeon Graphics (RADV GFX1151) (132096 MiB, 131922 MiB free) +0.00.070.138 I - CPU : AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (127438 MiB, 127438 MiB free) +0.00.070.226 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.070.307 I srv init: running without SSL +0.00.070.398 I srv init: using 31 threads for HTTP server +0.00.070.401 I srv init: the WebUI is disabled +0.00.070.610 I srv start: binding port with default address family +0.00.071.886 I srv main: loading model +0.00.071.903 I srv load_model: loading model '/mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' +0.00.118.721 W llama_model_loader: direct I/O is enabled, disabling mmap +0.21.623.100 W llama_context: n_ctx_seq (65536) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.21.824.309 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.22.126.084 W load_hparams: Qwen-VL models require at minimum 1024 image tokens to function correctly on grounding tasks +0.22.126.088 W load_hparams: if you encounter problems with accuracy, try adding --image-min-tokens 1024 +0.22.126.088 W load_hparams: more info: https://github.com/ggml-org/llama.cpp/issues/16842 + +0.22.393.368 I srv load_model: loaded multimodal model, '/mnt/models/nex-n2.5-mini/out/mmproj-Nex-N2.5-mini-BF16.gguf' +0.22.393.383 I srv load_model: initializing slots, n_slots = 1 +0.22.705.455 W srv load_model: speculative decoding will use checkpoints +0.22.705.465 W common_speculative_init: no implementations specified for speculative decoding +0.22.705.468 I slot load_model: id 0 | task -1 | new slot, n_ctx = 65536 +0.22.705.551 I srv load_model: prompt cache RAM enabled: limit_mib=8192 +0.22.705.569 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.22.705.622 I srv init: idle slots will be saved to prompt cache upon starting a new task +0.22.759.943 I init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant + + + + +' +0.22.797.944 I srv init: init: chat template, thinking = 1 +0.22.798.029 I srv main: model loaded +0.22.798.044 I srv main: server is listening on http://127.0.0.1:18600 +0.22.798.080 I srv update_slots: all slots are idle +0.23.997.761 I srv params_from_: Chat format: peg-native +0.23.998.009 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.23.998.016 I srv get_availabl: updating prompt cache +0.23.998.027 I srv load: - looking for better prompt, base f_keep = -1.000, sim = 0.000 +0.23.998.035 I srv update: - cache state: 0 prompts, 0.000 MiB (limits: 8192.000 MiB, 65536 tokens, 8589934592 est) +0.23.998.038 I srv get_availabl: prompt cache update took 0.02 ms +0.23.998.225 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.24.059.202 I srv process_chun: processing image... +0.24.395.132 W find_slot: non-consecutive token position 4 after 3 for sequence 0 with 196 new tokens +0.24.395.722 W find_slot: non-consecutive token position 4 after 3 for sequence 0 with 196 new tokens +0.24.862.757 I srv process_chun: image processed in 803 ms +0.24.863.042 W find_slot: non-consecutive token position 34 after 4 for sequence 0 with 17 new tokens +0.24.863.090 W find_slot: non-consecutive token position 34 after 4 for sequence 0 with 17 new tokens +0.24.963.440 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 34, pos_max = 34, n_tokens = 217, size = 62.813 MiB) +0.25.458.909 I slot print_timing: id 0 | task 0 | +prompt eval time = 1019.46 ms / 221 tokens ( 4.61 ms per token, 216.78 tokens per second) + eval time = 441.18 ms / 21 tokens ( 21.01 ms per token, 47.60 tokens per second) + total time = 1460.64 ms / 242 tokens +0.25.458.950 I slot release: id 0 | task 0 | stop processing: n_tokens = 241, truncated = 0 +0.25.458.959 I srv update_slots: all slots are idle +0.26.459.931 I srv operator(): operator(): cleaning up before exit... diff --git a/recipe/logs/probe_reasoning.log b/recipe/logs/probe_reasoning.log new file mode 100644 index 0000000000000000000000000000000000000000..f5c3c71cef7bf98f9a6d94bb22b2fcc1296544b7 --- /dev/null +++ b/recipe/logs/probe_reasoning.log @@ -0,0 +1,16 @@ +default correct-decline {"content": "We need answer directly. 391.\n\n\n391", "reasoning": "", "tool_calls": [], "leaks": [""]} +default multi-arg {"content": "We need need tool. Current weather Paris celsius.\n\n\n", "reasoning": "", "tool_calls": ["get_weather"], "leaks": [""]} +default parallel {"content": "We need need call each city. Parallel okay. Must use commentary tool. Then summarize.\n\n\n", "reasoning": "", "tool_calls": ["get_weather", "get_weather"], "leaks": [""]} +fmt-deepseek correct-decline {"content": "We need answer directly. 391.\n\n\n391", "reasoning": "", "tool_calls": [], "leaks": [""]} +fmt-deepseek multi-arg {"content": "We need need tool. Current weather Paris celsius.\n\n\n", "reasoning": "", "tool_calls": ["get_weather"], "leaks": [""]} +fmt-deepseek parallel {"content": "We need need call each city. Parallel okay. Must use commentary tool. Then summarize.\n\n\n", "reasoning": "", "tool_calls": ["get_weather", "get_weather"], "leaks": [""]} +srv-kwargs-high correct-decline {"content": "We need answer directly. 391.\n\n\n391", "reasoning": "", "tool_calls": [], "leaks": [""]} +srv-kwargs-high multi-arg {"content": "We need need tool. Current weather Paris celsius.\n\n\n", "reasoning": "", "tool_calls": ["get_weather"], "leaks": [""]} +srv-kwargs-high parallel {"content": "We need need call each city. Parallel okay. Must use commentary tool. Then summarize.\n\n\n", "reasoning": "", "tool_calls": ["get_weather", "get_weather"], "leaks": [""]} +reasoning-on correct-decline {"content": "We need answer directly. 391.\n\n\n391", "reasoning": "", "tool_calls": [], "leaks": [""]} +reasoning-on multi-arg {"content": "We need need tool. Current weather Paris celsius.\n\n\n", "reasoning": "", "tool_calls": ["get_weather"], "leaks": [""]} +reasoning-on parallel {"content": "We need need call each city. Parallel okay. Must use commentary tool. Then summarize.\n\n\n", "reasoning": "", "tool_calls": ["get_weather", "get_weather"], "leaks": [""]} +tpl-enable-thinking correct-decline {"content": "391", "reasoning": "We need answer directly. 391.\n", "tool_calls": [], "leaks": []} +tpl-enable-thinking multi-arg {"content": "", "reasoning": "We need need tool. Current weather Paris celsius.\n", "tool_calls": ["get_weather"], "leaks": []} +tpl-enable-thinking parallel {"content": "", "reasoning": "We need need call each city. Parallel okay. Must use commentary tool. Then summarize.\n", "tool_calls": ["get_weather", "get_weather"], "leaks": []} +NEX_REASONING_PROBE_DONE /mnt/models/nex-n2.5-mini/results/nex_reasoning_probe.json diff --git a/recipe/logs/probe_seat_default.log b/recipe/logs/probe_seat_default.log new file mode 100644 index 0000000000000000000000000000000000000000..bf97c4f77d3182238c414d6d92a461a74b9bbe83 --- /dev/null +++ b/recipe/logs/probe_seat_default.log @@ -0,0 +1,19 @@ +C1 no-kwargs correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C1 no-kwargs single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C1 no-kwargs multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +C1 enable_thinking=true correct-decline {"content": "391", "reasoning_len": 30, "tool_calls": [], "leaks": []} +C1 enable_thinking=true single-word {"content": "ready", "reasoning_len": 35, "tool_calls": [], "leaks": []} +C1 enable_thinking=true multi-arg {"content": "", "reasoning_len": 50, "tool_calls": ["get_weather"], "leaks": []} +C1 enable_thinking=false correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C1 enable_thinking=false single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C1 enable_thinking=false multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +C2 no-kwargs correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C2 no-kwargs single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C2 no-kwargs multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +C2 enable_thinking=true correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C2 enable_thinking=true single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C2 enable_thinking=true multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +C2 enable_thinking=false correct-decline {"content": "391", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C2 enable_thinking=false single-word {"content": "ready", "reasoning_len": 0, "tool_calls": [], "leaks": []} +C2 enable_thinking=false multi-arg {"content": "", "reasoning_len": 0, "tool_calls": ["get_weather"], "leaks": []} +NEX_SEAT_DEFAULT_PROBE_DONE diff --git a/recipe/logs/probe_seat_default_C2.log b/recipe/logs/probe_seat_default_C2.log new file mode 100644 index 0000000000000000000000000000000000000000..3607d380bb2ec1749cc8571546f64bb9a825d70f --- /dev/null +++ b/recipe/logs/probe_seat_default_C2.log @@ -0,0 +1,185 @@ +0.00.244.374 I log_info: verbosity = 3 (adjust with the `-lv N` CLI arg) +0.00.244.393 I device_info: +0.00.244.653 I - ROCm0 : AMD Radeon Graphics (131072 MiB, 123682 MiB free) +0.00.245.094 I - Vulkan0 : AMD Radeon Graphics (RADV GFX1151) (132096 MiB, 131922 MiB free) +0.00.245.113 I - CPU : AMD RYZEN AI MAX+ 395 w/ Radeon 8060S (127438 MiB, 127438 MiB free) +0.00.245.304 I system_info: n_threads = 16 (n_threads_batch = 16) / 32 | ROCm : NO_VMM = 1 | PEER_MAX_BATCH_SIZE = 128 | FA_ALL_QUANTS = 1 | CPU : SSE3 = 1 | SSSE3 = 1 | AVX = 1 | AVX_VNNI = 1 | AVX2 = 1 | F16C = 1 | FMA = 1 | BMI2 = 1 | AVX512 = 1 | AVX512_VBMI = 1 | AVX512_VNNI = 1 | AVX512_BF16 = 1 | LLAMAFILE = 1 | OPENMP = 1 | REPACK = 1 | +0.00.245.404 I srv init: running without SSL +0.00.245.477 I srv init: using 31 threads for HTTP server +0.00.245.479 I srv init: the WebUI is disabled +0.00.245.698 I srv start: binding port with default address family +0.00.247.040 I srv main: loading model +0.00.247.045 I srv load_model: loading model '/mnt/models/nex-n2.5-mini/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf' +0.00.310.751 W llama_model_loader: direct I/O is enabled, disabling mmap +0.22.302.525 W llama_context: n_ctx_seq (16384) < n_ctx_train (262144) -- the full capacity of the model will not be utilized +0.22.428.373 W common_init_from_params: warming up the model with an empty run - please wait ... (--no-warmup to disable) +0.22.720.526 I srv load_model: initializing slots, n_slots = 1 +0.23.006.696 W srv load_model: speculative decoding will use checkpoints +0.23.006.713 W common_speculative_init: no implementations specified for speculative decoding +0.23.006.717 I slot load_model: id 0 | task -1 | new slot, n_ctx = 16384 +0.23.006.919 I srv load_model: prompt cache RAM enabled: limit_mib=8192 +0.23.006.923 I srv load_model: for more info see https://github.com/ggml-org/llama.cpp/pull/16391 +0.23.006.964 I srv init: idle slots will be saved to prompt cache upon starting a new task +0.23.066.942 I init: chat template, example_format: '<|im_start|>system +You are a helpful assistant<|im_end|> +<|im_start|>user +Hello<|im_end|> +<|im_start|>assistant + + + + +Hi there<|im_end|> +<|im_start|>user +How are you?<|im_end|> +<|im_start|>assistant +' +0.23.112.988 I srv init: init: chat template, thinking = 1 +0.23.113.079 I srv main: model loaded +0.23.113.093 I srv main: server is listening on http://127.0.0.1:18653 +0.23.113.102 I srv update_slots: all slots are idle +0.24.535.309 I srv params_from_: Chat format: peg-native +0.24.535.740 I slot get_availabl: id 0 | task -1 | selected slot by LRU, t_last = -1 +0.24.535.743 I srv get_availabl: updating prompt cache +0.24.535.749 I srv load: - looking for better prompt, base f_keep = -1.000, sim = 0.000 +0.24.535.754 I srv update: - cache state: 0 prompts, 0.000 MiB (limits: 8192.000 MiB, 16384 tokens, 8589934592 est) +0.24.535.757 I srv get_availabl: prompt cache update took 0.01 ms +0.24.536.006 I reasoning-budget: activated, budget=2147483647 tokens +0.24.536.022 I slot launch_slot_: id 0 | task 0 | processing task, is_child = 0 +0.25.228.510 I slot create_check: id 0 | task 0 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.25.312.278 I reasoning-budget: deactivated (natural end) +0.25.454.972 I slot print_timing: id 0 | task 0 | +prompt eval time = 749.65 ms / 424 tokens ( 1.77 ms per token, 565.60 tokens per second) + eval time = 169.27 ms / 7 tokens ( 24.18 ms per token, 41.35 tokens per second) + total time = 918.92 ms / 431 tokens +0.25.455.052 I slot release: id 0 | task 0 | stop processing: n_tokens = 430, truncated = 0 +0.25.455.062 I srv update_slots: all slots are idle +0.25.515.651 I srv params_from_: Chat format: peg-native +0.25.518.171 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.969 (> 0.100 thold), f_keep = 0.942 +0.25.518.665 I reasoning-budget: activated, budget=2147483647 tokens +0.25.518.762 I slot launch_slot_: id 0 | task 9 | processing task, is_child = 0 +0.25.518.786 W slot update_slots: id 0 | task 9 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.25.518.790 I slot update_slots: id 0 | task 9 | Checking checkpoint with [419, 419] against 405... +0.25.518.793 W slot update_slots: id 0 | task 9 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.25.518.809 W slot update_slots: id 0 | task 9 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.26.100.410 I slot create_check: id 0 | task 9 | created context checkpoint 1 of 32 (pos_min = 413, pos_max = 413, n_tokens = 414, size = 62.813 MiB) +0.26.190.265 I reasoning-budget: deactivated (natural end) +0.26.318.946 I slot print_timing: id 0 | task 9 | +prompt eval time = 636.97 ms / 418 tokens ( 1.52 ms per token, 656.23 tokens per second) + eval time = 163.18 ms / 5 tokens ( 32.64 ms per token, 30.64 tokens per second) + total time = 800.14 ms / 423 tokens +0.26.319.036 I slot release: id 0 | task 9 | stop processing: n_tokens = 422, truncated = 0 +0.26.319.067 I srv update_slots: all slots are idle +0.26.365.926 I srv params_from_: Chat format: peg-native +0.26.367.996 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.960 +0.26.368.525 I reasoning-budget: activated, budget=2147483647 tokens +0.26.368.597 I slot launch_slot_: id 0 | task 16 | processing task, is_child = 0 +0.26.368.617 W slot update_slots: id 0 | task 16 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.26.368.621 I slot update_slots: id 0 | task 16 | Checking checkpoint with [413, 413] against 405... +0.26.368.623 W slot update_slots: id 0 | task 16 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.26.368.628 W slot update_slots: id 0 | task 16 | erased invalidated context checkpoint (pos_min = 413, pos_max = 413, n_tokens = 414, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.26.955.113 I slot create_check: id 0 | task 16 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.27.041.682 I reasoning-budget: deactivated (natural end) +0.27.918.891 I slot print_timing: id 0 | task 16 | +prompt eval time = 644.55 ms / 421 tokens ( 1.53 ms per token, 653.16 tokens per second) + eval time = 905.71 ms / 42 tokens ( 21.56 ms per token, 46.37 tokens per second) + total time = 1550.26 ms / 463 tokens +0.27.918.973 I slot release: id 0 | task 16 | stop processing: n_tokens = 462, truncated = 0 +0.27.919.004 I srv update_slots: all slots are idle +0.27.975.579 I srv params_from_: Chat format: peg-native +0.27.977.357 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.958 (> 0.100 thold), f_keep = 0.879 +0.27.977.785 I reasoning-budget: activated, budget=2147483647 tokens +0.27.977.885 I slot launch_slot_: id 0 | task 60 | processing task, is_child = 0 +0.27.977.912 W slot update_slots: id 0 | task 60 | n_past = 406, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.27.977.913 I slot update_slots: id 0 | task 60 | Checking checkpoint with [416, 416] against 406... +0.27.977.915 W slot update_slots: id 0 | task 60 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.27.977.920 W slot update_slots: id 0 | task 60 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.28.575.881 I slot create_check: id 0 | task 60 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.28.639.525 I reasoning-budget: deactivated (natural end) +0.28.760.322 I slot print_timing: id 0 | task 60 | +prompt eval time = 636.60 ms / 424 tokens ( 1.50 ms per token, 666.03 tokens per second) + eval time = 145.80 ms / 7 tokens ( 20.83 ms per token, 48.01 tokens per second) + total time = 782.40 ms / 431 tokens +0.28.760.399 I slot release: id 0 | task 60 | stop processing: n_tokens = 430, truncated = 0 +0.28.760.426 I srv update_slots: all slots are idle +0.28.773.374 I srv params_from_: Chat format: peg-native +0.28.773.718 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.969 (> 0.100 thold), f_keep = 0.942 +0.28.773.919 I reasoning-budget: activated, budget=2147483647 tokens +0.28.773.955 I slot launch_slot_: id 0 | task 69 | processing task, is_child = 0 +0.28.773.965 W slot update_slots: id 0 | task 69 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.28.773.967 I slot update_slots: id 0 | task 69 | Checking checkpoint with [419, 419] against 405... +0.28.773.968 W slot update_slots: id 0 | task 69 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.28.773.971 W slot update_slots: id 0 | task 69 | erased invalidated context checkpoint (pos_min = 419, pos_max = 419, n_tokens = 420, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.29.251.924 I slot create_check: id 0 | task 69 | created context checkpoint 1 of 32 (pos_min = 413, pos_max = 413, n_tokens = 414, size = 62.813 MiB) +0.29.313.310 I reasoning-budget: deactivated (natural end) +0.29.391.023 I slot print_timing: id 0 | task 69 | +prompt eval time = 514.69 ms / 418 tokens ( 1.23 ms per token, 812.15 tokens per second) + eval time = 102.36 ms / 5 tokens ( 20.47 ms per token, 48.85 tokens per second) + total time = 617.04 ms / 423 tokens +0.29.391.120 I slot release: id 0 | task 69 | stop processing: n_tokens = 422, truncated = 0 +0.29.391.150 I srv update_slots: all slots are idle +0.29.426.233 I srv params_from_: Chat format: peg-native +0.29.426.716 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.960 +0.29.426.958 I reasoning-budget: activated, budget=2147483647 tokens +0.29.427.000 I slot launch_slot_: id 0 | task 76 | processing task, is_child = 0 +0.29.427.011 W slot update_slots: id 0 | task 76 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.29.427.013 I slot update_slots: id 0 | task 76 | Checking checkpoint with [413, 413] against 405... +0.29.427.014 W slot update_slots: id 0 | task 76 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.29.427.017 W slot update_slots: id 0 | task 76 | erased invalidated context checkpoint (pos_min = 413, pos_max = 413, n_tokens = 414, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.29.922.653 I slot create_check: id 0 | task 76 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.29.984.421 I reasoning-budget: deactivated (natural end) +0.30.786.431 I slot print_timing: id 0 | task 76 | +prompt eval time = 529.05 ms / 421 tokens ( 1.26 ms per token, 795.76 tokens per second) + eval time = 830.35 ms / 42 tokens ( 19.77 ms per token, 50.58 tokens per second) + total time = 1359.40 ms / 463 tokens +0.30.786.507 I slot release: id 0 | task 76 | stop processing: n_tokens = 462, truncated = 0 +0.30.786.534 I srv update_slots: all slots are idle +0.30.827.357 I srv params_from_: Chat format: peg-native +0.30.827.975 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.951 (> 0.100 thold), f_keep = 0.879 +0.30.828.455 I reasoning-budget: activated, budget=2147483647 tokens +0.30.828.460 I reasoning-budget: deactivated (natural end) +0.30.828.579 I slot launch_slot_: id 0 | task 120 | processing task, is_child = 0 +0.30.828.602 W slot update_slots: id 0 | task 120 | n_past = 406, slot.prompt.tokens.size() = 462, seq_id = 0, pos_min = 461, n_swa = 0 +0.30.828.605 I slot update_slots: id 0 | task 120 | Checking checkpoint with [416, 416] against 406... +0.30.828.608 W slot update_slots: id 0 | task 120 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.30.828.615 W slot update_slots: id 0 | task 120 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.31.403.843 I slot create_check: id 0 | task 120 | created context checkpoint 1 of 32 (pos_min = 422, pos_max = 422, n_tokens = 423, size = 62.813 MiB) +0.31.535.343 I slot print_timing: id 0 | task 120 | +prompt eval time = 610.22 ms / 427 tokens ( 1.43 ms per token, 699.75 tokens per second) + eval time = 96.52 ms / 4 tokens ( 24.13 ms per token, 41.44 tokens per second) + total time = 706.74 ms / 431 tokens +0.31.535.416 I slot release: id 0 | task 120 | stop processing: n_tokens = 430, truncated = 0 +0.31.535.442 I srv update_slots: all slots are idle +0.31.579.928 I srv params_from_: Chat format: peg-native +0.31.582.191 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.962 (> 0.100 thold), f_keep = 0.942 +0.31.582.782 I reasoning-budget: activated, budget=2147483647 tokens +0.31.582.789 I reasoning-budget: deactivated (natural end) +0.31.582.964 I slot launch_slot_: id 0 | task 126 | processing task, is_child = 0 +0.31.582.991 W slot update_slots: id 0 | task 126 | n_past = 405, slot.prompt.tokens.size() = 430, seq_id = 0, pos_min = 429, n_swa = 0 +0.31.582.995 I slot update_slots: id 0 | task 126 | Checking checkpoint with [422, 422] against 405... +0.31.582.997 W slot update_slots: id 0 | task 126 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.31.583.004 W slot update_slots: id 0 | task 126 | erased invalidated context checkpoint (pos_min = 422, pos_max = 422, n_tokens = 423, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.142.847 I slot create_check: id 0 | task 126 | created context checkpoint 1 of 32 (pos_min = 416, pos_max = 416, n_tokens = 417, size = 62.813 MiB) +0.32.249.462 I slot print_timing: id 0 | task 126 | +prompt eval time = 617.07 ms / 421 tokens ( 1.47 ms per token, 682.25 tokens per second) + eval time = 49.39 ms / 2 tokens ( 24.70 ms per token, 40.49 tokens per second) + total time = 666.46 ms / 423 tokens +0.32.249.559 I slot release: id 0 | task 126 | stop processing: n_tokens = 422, truncated = 0 +0.32.249.590 I srv update_slots: all slots are idle +0.32.262.258 I srv params_from_: Chat format: peg-native +0.32.262.613 I slot get_availabl: id 0 | task -1 | selected slot by LCP similarity, sim_best = 0.955 (> 0.100 thold), f_keep = 0.960 +0.32.262.803 I reasoning-budget: activated, budget=2147483647 tokens +0.32.262.808 I reasoning-budget: deactivated (natural end) +0.32.262.842 I slot launch_slot_: id 0 | task 130 | processing task, is_child = 0 +0.32.262.852 W slot update_slots: id 0 | task 130 | n_past = 405, slot.prompt.tokens.size() = 422, seq_id = 0, pos_min = 421, n_swa = 0 +0.32.262.852 I slot update_slots: id 0 | task 130 | Checking checkpoint with [416, 416] against 405... +0.32.262.853 W slot update_slots: id 0 | task 130 | forcing full prompt re-processing due to lack of cache data (likely due to SWA or hybrid/recurrent memory, see https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055) +0.32.262.856 W slot update_slots: id 0 | task 130 | erased invalidated context checkpoint (pos_min = 416, pos_max = 416, n_tokens = 417, n_swa = 0, pos_next = 0, size = 62.813 MiB) +0.32.810.656 I slot create_check: id 0 | task 130 | created context checkpoint 1 of 32 (pos_min = 419, pos_max = 419, n_tokens = 420, size = 62.813 MiB) +0.33.748.645 I slot print_timing: id 0 | task 130 | +prompt eval time = 596.89 ms / 424 tokens ( 1.41 ms per token, 710.35 tokens per second) + eval time = 888.89 ms / 39 tokens ( 22.79 ms per token, 43.88 tokens per second) + total time = 1485.78 ms / 463 tokens +0.33.748.736 I slot release: id 0 | task 130 | stop processing: n_tokens = 462, truncated = 0 +0.33.748.767 I srv update_slots: all slots are idle +0.33.749.943 I srv operator(): operator(): cleaning up before exit... diff --git a/recipe/pipeline/diag_bf16.sh b/recipe/pipeline/diag_bf16.sh new file mode 100644 index 0000000000000000000000000000000000000000..288f2ea1b39b322f3d663334437559d64ff171f2 --- /dev/null +++ b/recipe/pipeline/diag_bf16.sh @@ -0,0 +1,20 @@ +#!/bin/bash +# Which backend computes the Nex BF16 GGUF correctly? d3ca537 ROCm0 (-fa on) gave PPL 139-171 on the first chunks while +# the 4-bit STRIX_LEAN gave 6.28-7.84 on the same chunks. Same text, same -c/-b; only device / -fa change. +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; A=/mnt/models/agnes-3.0-flash +export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1 +cd $W; prlimit --pid $$ --core=1:1 +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +BF=gguf/Nex-N2.5-mini-BF16.gguf; Q=out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf +T=$A/calib/wikitext-2-raw/wiki.test.raw +run(){ # label chunks model args... + local lab=$1 ch=$2 m=$3; shift 3 + timeout 1800 $B/llama-perplexity -m $m -f $T -c 2048 -b 2048 --chunks $ch "$@" > logs/diag_$lab.log 2>&1 + log "$lab rc=$? $(grep -oE '\[[0-9]+\][0-9.]+' logs/diag_$lab.log | tr '\n' ' ')" +} +run bf16_rocm_faoff 4 $BF -ngl 999 -fa off -dev ROCm0 +run bf16_vk_faon 4 $BF -ngl 999 -fa on -dev Vulkan0 +run q106_vk_faon 4 $Q -ngl 999 -fa on -dev Vulkan0 +run bf16_cpu 2 $BF -ngl 0 -t 16 +log "DIAG_BF16_DONE" diff --git a/recipe/pipeline/diag_tools_run.sh b/recipe/pipeline/diag_tools_run.sh new file mode 100644 index 0000000000000000000000000000000000000000..6b36d2207c0ab90962b2374263d8c14348461ec2 --- /dev/null +++ b/recipe/pipeline/diag_tools_run.sh @@ -0,0 +1,8 @@ +#!/bin/bash +# start the tools diagnostic as soon as the N8 seat smoke tests are finished (so it does not disturb them), +# capped at 32G; OxCoder phase 2 waits for GPU-free and therefore starts after it. +cd /mnt/models/nex-n2.5-mini +until grep -q "nex_seats exit=" logs/N8_unice.log 2>/dev/null; do sleep 5; done +echo "[$(date -u +%FT%TZ)] seats done -> diag" +systemd-run --scope --quiet --unit=nex-tools-diag-$(date +%s) -p MemoryMax=32G -p MemorySwapMax=0 python3 nex_tools_diag.py +echo "[$(date -u +%FT%TZ)] diag exit=$?" diff --git a/recipe/pipeline/nex_aggregate.py b/recipe/pipeline/nex_aggregate.py new file mode 100644 index 0000000000000000000000000000000000000000..591ae89dbacd20921f5adb16d041e641931594ed --- /dev/null +++ b/recipe/pipeline/nex_aggregate.py @@ -0,0 +1,408 @@ +#!/usr/bin/env python3 +"""Collect every measured number for the Nex-N2.5-mini ROCmFP4 cards into results/summary.json. +Nothing is typed by hand except model facts and protocol constants. Each value is parsed from a log, +a results file, or the GGUF itself; missing => None.""" +import hashlib, json, os, re, sys +sys.path.insert(0, "/opt/llama-rocm/rocmfpx-724/gguf-py") +import gguf # noqa: E402 + +W = "/mnt/models/nex-n2.5-mini" +A = "/mnt/models/agnes-3.0-flash" +N = "Nex-N2.5-mini" +OB = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin" +FAST = os.environ.get("AGG_FAST") == "1" +OUT = os.environ.get("AGG_OUT", f"{W}/results/summary.json") +UB = 1024 # llama-server -ub; cache checkpoints land 4 + n_ubatch tokens before prompt end + + +def rd(p): + try: + return open(p, errors="replace").read() + except FileNotFoundError: + return "" + + +def f(rx, s, g=1, cast=float): + m = re.search(rx, s) + return cast(m.group(g)) if m else None + + +def jl(p): + return [json.loads(l) for l in rd(p).splitlines() if l.strip()] + + +def pubname(fn): + """Published filename: the Hub cannot parse an underscore-glued quant token.""" + return fn.replace("Q4_0_ROCMFP4_", "Q4_0-ROCmFP4-") + + +STAGED = {} +for _k in ("std", "imat"): + for _l in rd(f"{W}/hf-upload/{_k}/SHA256SUMS").splitlines(): + _h, _n = _l.split(maxsplit=1) + STAGED[_n.strip()] = _h + + +def sha_file(p): + if not os.path.exists(p): + return None + if pubname(os.path.basename(p)) in STAGED: + return STAGED[pubname(os.path.basename(p))] + if os.path.basename(p) in STAGED: + return STAGED[os.path.basename(p)] + if FAST and os.path.getsize(p) > 1 << 30: + return None + h = hashlib.sha256() + with open(p, "rb") as fh: + for c in iter(lambda: fh.read(64 << 20), b""): + h.update(c) + return h.hexdigest() + + +def tensors(p): + """name -> (type, bytes); {} if the file cannot be read (e.g. still being written by a running quantize).""" + out = {} + try: + reader = gguf.GGUFReader(p) + except (ValueError, OSError) as e: + print(f"WARN unreadable GGUF {p}: {e}", file=sys.stderr) + return out + for t in reader.tensors: + try: + tn = t.tensor_type.name + except Exception: + tn = f"TYPE_{int(t.tensor_type)}" + out[t.name] = (tn, int(t.n_bytes)) + return out + + +def rb_line(blob, fn): + return next((l for l in blob.splitlines() if len(l.split()) > 1 and l.split()[1] == fn), "") + + +def kld_stats(k): + return { + "ppl": f(r"Mean PPL\(Q\)\s*:\s*([\d.]+)", k), + "ppl_err": f(r"Mean PPL\(Q\)\s*:\s*[\d.]+\s*±\s*([\d.]+)", k), + "ppl_ratio": f(r"Mean PPL\(Q\)/PPL\(base\)\s*:\s*([\d.]+)", k), + "kld_mean": f(r"Mean\s+KLD:\s*([\d.]+)", k), + "kld_err": f(r"Mean\s+KLD:\s*[\d.]+\s*±\s*([\d.]+)", k), + "kld_p99": f(r"99\.0%\s+KLD:\s*([\d.]+)", k), + "kld_median": f(r"Median\s+KLD:\s*([\d.]+)", k), + "same_top_p": f(r"Same top p:\s*([\d.]+)", k), + "rms_dp": f(r"RMS Δp\s*:\s*([\d.]+)", k), + } + + +def exists_size(p): + return os.path.getsize(p) if os.path.exists(p) else None + + +S = { + "layout": "no MTP (checkpoint declares mtp_num_hidden_layers=1 but ships no mtp.* tensors)", + "model": N, + "model_repo_std": f"kingjones777/{N}-ROCmFP4-GGUF", + "model_repo_imat": f"kingjones777/{N}-ROCmFP4-imatrix-GGUF", + "source": { + "repo": "nex-agi/Nex-N2.5-mini", + "revision": "87420286149d9cce9bd46cd335ef9bda33c37c1b", + "license": "apache-2.0", + "gated": False, + "params": 35107181936, + }, + "arch": { + "name": "qwen3_5_moe", + "llama_cpp": "qwen35moe", + "layers": 40, + "linear_attn_layers": 30, + "full_attn_layers": 10, + "full_attention_interval": 4, + "hidden": 2048, + "num_experts": 256, + "num_experts_per_tok": 8, + "expert_width": 512, + "shared_expert_width": 512, + "attn_heads": 16, + "kv_heads": 2, + "head_dim": 256, + "linear_key_heads": 16, + "linear_value_heads": 32, + "linear_head_dim": 128, + "vocab": 248320, + "max_position_embeddings": 262144, + "tie_word_embeddings": False, + "vision_layers": 27, + "vision_width": 1152, + "hf_tensors": 1026, + "mtp_num_hidden_layers_declared": 1, + }, +} + +# ---------- BF16 reference ---------- +# phase 2b: this build's ROCm0 path computes the BF16 GGUF of this MoE wrong, so the reference ran on pure CPU +b1 = rd(f"{W}/logs/N1c_ppl_bf16_cpu.log") +k106 = rd(f"{W}/logs/N4_kld_q106.log") +crb = rd(f"{W}/logs/C_readback.log") +chunks = f(r"calculating perplexity over (\d+) chunks", b1, cast=int) +nctx = f(r"n_ctx=(\d+)", b1, cast=int) +bf_fn = f"{N}-BF16.gguf" +bf_line = rb_line(crb, bf_fn) +bf_path = f"{W}/gguf/{bf_fn}" +bf_ten = tensors(bf_path) if os.path.exists(bf_path) else {} +S["bf16"] = { + "ppl": f(r"Final estimate: PPL = ([\d.]+)", b1), + "ppl_err": f(r"Final estimate: PPL = [\d.]+ \+/- ([\d.]+)", b1), + "ppl_paired": f(r"Mean PPL\(base\)\s*:\s*([\d.]+)", k106), + "ppl_paired_err": f(r"Mean PPL\(base\)\s*:\s*[\d.]+\s*±\s*([\d.]+)", k106), + "chunks": chunks, + "n_ctx": nctx, + "scored_tokens": chunks * (nctx // 2 - 1) if chunks and nctx else None, + "size_bytes": exists_size(bf_path), + "file": bf_fn, + "readback": bf_line.split()[0] if bf_line else None, + "arch": f(r"arch=(\S+)", bf_line, cast=str), + "ftype": f(r"ftype=(\d+)", bf_line, cast=int), + "tensors": f(r"tensors=(\d+)", bf_line, cast=int) or (len(bf_ten) or None), + "nextn_tensors": f(r"nextn=(\d+)", bf_line, cast=int), + "output_weight": f(r"output\.weight=(\S+)", bf_line, cast=str), + "token_embd": f(r"token_embd\.weight=(\S+)", bf_line, cast=str), + "mtp_named_tensors": sum(1 for n in bf_ten if n.startswith("mtp.")) if bf_ten else None, + "nextn_named_tensors": sum(1 for n in bf_ten if ".nextn." in n) if bf_ten else None, +} + +# ---------- binaries ---------- +qlog = rd(f"{W}/logs/Q1_q106.log") +S["binary"] = { + "repo": "https://github.com/charlie12345/ROCmFPX", + "commit": f(r"llama_print_build_info: build = \d+ \(([0-9a-f]+)\)", qlog, cast=str), + "dir": OB, + "sha256": {x: sha_file(f"{OB}/{x}") for x in + ("llama-server", "llama-quantize", "llama-imatrix", "llama-perplexity")}, +} + +vp = f"{A}/calib/vision_probe.png" +if os.path.exists(vp): + raw = open(vp, "rb").read() + S["vision_probe"] = { + "width": int.from_bytes(raw[16:20], "big"), + "height": int.from_bytes(raw[20:24], "big"), + "bytes": len(raw), + "sha256": hashlib.sha256(raw).hexdigest(), + } + +imat = rd(f"{W}/logs/N2c_imatrix_cpu.log") +S["imatrix"] = { + "chunks": f(r"computing over (\d+) chunks", imat, cast=int), + "n_ctx": f(r"n_ctx=(\d+)", imat, cast=int), + "calibration": "bartowski calibration_datav3.txt", + "file": f"{N}.imatrix", + "sha256": (rd(f"{W}/imat/{N}.imatrix.sha256").split() or [None])[0], + "size_bytes": exists_size(f"{W}/imat/{N}.imatrix"), +} + +hc_raw = rd(f"{W}/results/hub_check.json").strip() +if hc_raw: + try: + S["hub_check"] = json.loads(hc_raw) + except json.JSONDecodeError: + S["hub_check"] = None +else: + S["hub_check"] = None + + +def type_name(i): + try: + return gguf.GGMLQuantizationType(int(i)).name + except (TypeError, ValueError): + return None + + +for _h in (S["hub_check"] or {}).get("header_checks") or []: + # ids were read from the other repo's GGUF header; names come from this fork's gguf-py enum + _h["output_weight_type"] = type_name(_h.get("output_weight_type_id")) + _h["token_embd_type"] = type_name(_h.get("token_embd_type_id")) + +REP = jl(f"{W}/results/nex_repeat.jsonl") +S["repeat"] = next((d for d in REP if d.get("step") == "N5"), None) # ROCm0 +S["repeat_vk"] = next((d for d in REP if d.get("step") == "N5v"), None) # Vulkan0 +S["reference"] = next((d for d in jl(f"{W}/results/nex_reference.jsonl") if d.get("step") == "N1c"), None) +S["reference_device"] = "CPU" if S["reference"] else None +S["imatrix"]["device"] = "CPU" if imat else None + +# ---------- tiers ---------- +TIERS = { + "q106": ("out", f"{N}-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "Q1_q106", "Q_readback.log", 106), + "q102": ("out", f"{N}-Q4_0_ROCMFP4_COHERENT.gguf", "Q1_q102", "Q_readback.log", 102), + "q103": ("out", f"{N}-Q4_0_ROCMFP4_FAST.gguf", "Q1_q103", "Q_readback.log", 103), + "q106i": ("out-imat", f"{N}-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "N3_q106i", "N3_readback.log", 106), + "q102i": ("out-imat", f"{N}-imatrix-Q4_0_ROCMFP4_COHERENT.gguf", "N3_q102i", "N3_readback.log", 102), + "q103i": ("out-imat", f"{N}-imatrix-Q4_0_ROCMFP4_FAST.gguf", "N3_q103i", "N3_readback.log", 103), +} +RB = {name: rd(f"{W}/logs/{name}") for name in ("Q_readback.log", "N3_readback.log")} + +S["tiers"] = {} +TEN = {} +for tag, (d, fn, qlog_stem, rblog, ft) in TIERS.items(): + p = f"{W}/{d}/{fn}" + q = rd(f"{W}/logs/{qlog_stem}.log") + k = rd(f"{W}/logs/N4_kld_{tag}.log") + line = rb_line(RB.get(rblog, ""), fn) + ten = tensors(p) if os.path.exists(p) else {} + TEN[tag] = ten + qs = f(r"quantize time\s*=\s*([\d.]+) ms", q) + t = { + "file": pubname(fn), + "local_file": fn, + "dir": d, + "ftype": ft, + "size_bytes": exists_size(p), + "quant_mib": f(r"quant size\s*=\s*([\d.]+) MiB", q), + "bpw": f(r"quant size\s*=\s*[\d.]+ MiB \(([\d.]+) BPW\)", q), + "quant_seconds": (qs / 1000) if qs is not None else None, + "imatrix_entries": f(r"loaded (\d+) importance matrix entries", q, cast=int), + "readback": line.split()[0] if line else None, + "arch": f(r"arch=(\S+)", line, cast=str), + "tensors": f(r"tensors=(\d+)", line, cast=int), + "nextn_tensors": f(r"nextn=(\d+)", line, cast=int), + "output_weight": f(r"output\.weight=(\S+)", line, cast=str), + "token_embd": f(r"token_embd\.weight=(\S+)", line, cast=str), + "quality_measured": f(r"Mean\s+KLD:\s*([\d.]+)", k) is not None, + } + t.update(kld_stats(k) if t["quality_measured"] else {x: None for x in kld_stats("")}) + kv = rd(f"{W}/logs/N4v_kld_{tag}.log") # the same file graded on Vulkan0 + t["vk"] = kld_stats(kv) if f(r"Mean\s+KLD:\s*([\d.]+)", kv) is not None else None + S["tiers"][tag] = t + +TWINS = (("q106i", "q106"), ("q102i", "q102"), ("q103i", "q103")) +for im, st in TWINS: + pa = f"{W}/{TIERS[st][0]}/{TIERS[st][1]}" + pb = f"{W}/{TIERS[im][0]}/{TIERS[im][1]}" + if TEN.get(st) and TEN.get(im): + S["tiers"][im]["same_tensor_types_as_standard"] = TEN[st] == TEN[im] + else: + S["tiers"][im]["same_tensor_types_as_standard"] = None + sa, sb = S["tiers"][st]["size_bytes"], S["tiers"][im]["size_bytes"] + S["tiers"][im]["file_size_delta_bytes"] = (sb - sa) if None not in (sa, sb) else None + ha, hb = (sha_file(pa), sha_file(pb)) if os.path.exists(pa) and os.path.exists(pb) else (None, None) + S["tiers"][im]["differs_from_standard"] = (ha != hb) if ha and hb else None + +mm_fn = f"mmproj-{N}-BF16.gguf" +mm_line = rb_line(crb, mm_fn) +S["aux"] = {mm_fn: exists_size(f"{W}/out/{mm_fn}")} +S["mmproj"] = { + "file": mm_fn, + "size_bytes": S["aux"][mm_fn], + "readback": mm_line.split()[0] if mm_line else None, + "arch": f(r"arch=(\S+)", mm_line, cast=str), + "ftype": f(r"ftype=(\d+)", mm_line, cast=int), + "tensors": f(r"tensors=(\d+)", mm_line, cast=int), +} + +# ---------- measurements ---------- +S["bench"], S["gates"] = [], [] +for d in jl(f"{W}/results/nex_bench.jsonl"): + if "tg_median" in d: + row = {k: v for k, v in d.items() if k != "runs"} + pns = [r.get("prompt_n") for r in d.get("runs") or [] if r.get("prompt_n") is not None] + row["prompt_n_min"], row["prompt_n_max"] = (min(pns), max(pns)) if pns else (None, None) + S["bench"].append(row) + else: + S["gates"].append(d) + +PE = re.compile(r"prompt eval time =\s*([\d.]+) ms /\s*(\d+) tokens") +for g_ in S["gates"]: + if g_.get("label") != "n-c3-q106" or not g_.get("rows"): + continue + ev = [(float(a), int(b)) for a, b in PE.findall(rd(f"{W}/logs/b_{g_['label']}.log"))] + per = 3 # warm-up, warm, cold — one server log line each, in order + for i, r_ in enumerate(g_["rows"]): + blk = ev[per * i: per * i + per] + if len(blk) == per and blk[1][1] == r_["warm_prompt_n"] and blk[2][1] == r_["cold_prompt_n"]: + r_["warm_prompt_ms"], r_["cold_prompt_ms"] = blk[1][0], blk[2][0] + else: + r_["warm_prompt_ms"] = r_["cold_prompt_ms"] = None + +S["sizing"] = jl(f"{W}/results/nex_sizing.jsonl") +S["n_ubatch"] = UB + + +def _jobj(p): + raw = rd(p).strip() + try: + return json.loads(raw) if raw else None + except json.JSONDecodeError: + return None + + +# ---------- chat template fix (tool calls / reasoning) ---------- +TPLF = "chat_template_enable_thinking.jinja" +S["aux"][TPLF] = exists_size(f"{W}/tpl/{TPLF}") +shim = {k: _jobj(f"{W}/results/nex_template_shim{k}.json") for k in ("", "_medium", "_c1", "_roff")} +_roff = shim["_roff"] or {} +S["template_fix"] = { + "file": TPLF, + "size_bytes": S["aux"][TPLF], + "sha256": _roff.get("sha256"), + "source_sha256": _roff.get("source_sha256"), + "shim": _roff.get("shim"), + "server_flags": ["--chat-template-file", TPLF, "--reasoning", "off"], + "probes_roff": _roff.get("probes"), # the recommended configuration + "probes_high_default_on": (shim[""] or {}).get("probes"), # same template, thinking on by default + "medium_mapping_label": (shim["_medium"] or {}).get("label"), +} +_diag = _jobj(f"{W}/results/nex_tools_diag.json") or {} +_v = _diag.get("variants") or {} +_on = [r.get("first") or {} for r in (_v.get("gate_on") or {}).values()] +_nested = [(_v.get("gate_off") or {}).get("nested-object", {}).get("first") or {}] + list(_v.get("nested_off_x3") or []) +_rp = _jobj(f"{W}/results/nex_reasoning_probe.json") or {} +S["tools_diag"] = { + # stock template, the gate's own sampling: replies with reasoning left in content, of all thinking-on replies + "stock_on_replies": len(_on) if _on else None, + "stock_on_leaks": sum(1 for r in _on if r.get("leaks_in_content")) if _on else None, + "stock_on_reasoning_extracted": sum(1 for r in _on if r.get("reasoning_len")) if _on else None, + # stock template, thinking off, the nested-object request repeated: HTTP 500 "does not match peg-native" + "nested_off_attempts": len(_nested) if _v else None, + "nested_off_http500": sum(1 for r in _nested if r.get("http_error") == 500) if _v else None, + "gate_http500_logged": "does not match the expected peg-native format" in rd(f"{W}/logs/b_n-tools-q106.log"), + # server-side switches that did NOT fix the stock template (reasoning still in content) + "flag_probes": {k: {"leaks": sum(1 for r in (c.get("results") or {}).values() if r.get("leaks")), + "n": len(c.get("results") or {})} + for k, c in (_rp.get("configs") or {}).items()}, +} +_seats = {} +for r_ in jl(f"{W}/results/nex_seats.jsonl"): + _seats[r_.get("unit")] = r_ # last row per unit = the current unit files +S["seats"] = _seats +S["measured"] = ( + f(r"\[(\d{4}-\d{2}-\d{2})T", rd(f"{W}/logs/N6_bench.log"), cast=str) + or f(r"\[(\d{4}-\d{2}-\d{2})T", b1, cast=str) + or f(r"\[(\d{4}-\d{2}-\d{2})T", rd(f"{W}/logs/N1_ppl_bf16.log"), cast=str) +) + +_dates = sorted(set(re.findall(r"(\d{4}-\d{2}-\d{2})T\d", rd(f"{W}/logs/N6_bench.log") + rd(f"{W}/logs/N6t_tools_roff.log") + + rd(f"{W}/results/nex_seats.jsonl")))) +S["measured_range"] = [_dates[0], _dates[-1]] if len(_dates) > 1 else (_dates or None) + + +def _kv_keys(p): + try: + return set(gguf.GGUFReader(p).fields) + except (ValueError, OSError): + return None + + +for im, st in TWINS: + ka = _kv_keys(f"{W}/{TIERS[st][0]}/{TIERS[st][1]}") if FAST is False else None + kb = _kv_keys(f"{W}/{TIERS[im][0]}/{TIERS[im][1]}") if FAST is False else None + S["tiers"][im]["header_keys_only_in_imatrix"] = sorted(kb - ka) if ka is not None and kb is not None else None + S["tiers"][im]["header_keys_only_in_standard"] = sorted(ka - kb) if ka is not None and kb is not None else None + +os.makedirs(os.path.dirname(OUT), exist_ok=True) +json.dump(S, open(OUT, "w"), indent=2) + +print("bench rows", len(S["bench"]), "| gate rows", len(S["gates"]), + "| sizing rows", len(S["sizing"]), "| hub_check", S["hub_check"] is not None, + "| repeat", (S["repeat"] or {}).get("result")) +for tg_, v in S["tiers"].items(): + print(tg_, {k: v[k] for k in ("size_bytes", "bpw", "readback", "tensors", "quality_measured", "kld_mean")}) +print("binary", json.dumps(S["binary"], indent=1)) diff --git a/recipe/pipeline/nex_bench.py b/recipe/pipeline/nex_bench.py new file mode 100644 index 0000000000000000000000000000000000000000..42a8048e386d727657b4651b9888771b22187181 --- /dev/null +++ b/recipe/pipeline/nex_bench.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Nex-N2.5-mini speed + gates. Every 4-bit file x backend, no draft head (the checkpoint ships no MTP weights), +same house protocol as Agnes (nex_harness.py). Then gates on STRIX_LEAN: aligned prompt-cache identity, tool calls +(thinking on + off), vision with -fa on and off. Sequential GPU; results/nex_bench.jsonl.""" +import fcntl, os, subprocess, time +W = "/mnt/models/nex-n2.5-mini"; N = "Nex-N2.5-mini"; A = "/mnt/models/agnes-3.0-flash" +J = f"{W}/results/nex_bench.jsonl" +OB = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin" +T = {"q106": f"{W}/out/{N}-Q4_0_ROCMFP4_STRIX_LEAN.gguf", "q102": f"{W}/out/{N}-Q4_0_ROCMFP4_COHERENT.gguf", + "q103": f"{W}/out/{N}-Q4_0_ROCMFP4_FAST.gguf", + "q106i": f"{W}/out-imat/{N}-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf", + "q102i": f"{W}/out-imat/{N}-imatrix-Q4_0_ROCMFP4_COHERENT.gguf", + "q103i": f"{W}/out-imat/{N}-imatrix-Q4_0_ROCMFP4_FAST.gguf"} +ENV = dict(os.environ, AGNES_BIN=OB) + + +def log(m): print(f"[{time.strftime('%FT%TZ', time.gmtime())}] {m}", flush=True) + + +def run(argv, label): + log(label) + r = subprocess.run(argv, capture_output=True, text=True, env=ENV) + print(r.stdout.strip()[-800:], flush=True) + if r.returncode != 0: + print(" RC", r.returncode, r.stderr.strip()[-600:], flush=True) + return r.returncode + + +def harness(cmd, label, model, dev="ROCm0", reps=3, workload="code", extra=()): + return run(["python3", f"{W}/nex_harness.py", cmd, "--model", model, "--dev", dev, "--label", label, + "--reps", str(reps), "--jsonl", J, "--serverlog", f"{W}/logs/b_{label}.log", "--workload", workload, + *extra], f"{cmd} {label}") + + +# One quiet box: publish_v2.sh upload takes the same lock, so an HF upload never overlaps the speed runs. +QUIET = open("/mnt/models/.quiet-box.lock", "w") +log("waiting for the quiet-box lock (no HF upload during speed runs)") +fcntl.flock(QUIET, fcntl.LOCK_EX) +log("quiet-box lock held") +for tag in ("q106", "q102", "q103", "q106i", "q102i", "q103i"): + for dev, sh in (("ROCm0", "rocm"), ("Vulkan0", "vk")): + harness("bench", f"n-{tag}-{sh}", T[tag], dev=dev) +harness("bench", "n-q106-rocm-prose", T["q106"], workload="prose") +harness("bench", "n-q106-vk-prose", T["q106"], dev="Vulkan0", workload="prose") +run(["python3", f"{W}/nex_cachegate.py", "--model", T["q106"], "--label", "n-c3-q106", "--reps", "5", + "--jsonl", J, "--serverlog", f"{W}/logs/b_n-c3-q106.log"], "cachegate n-c3-q106") +harness("tools", "n-tools-q106", T["q106"]) +for fa in ("on", "off"): + harness("vision", f"n-vision-q106-fa{fa}", T["q106"], + extra=("--fa", fa, "--mmproj", f"{W}/out/mmproj-{N}-BF16.gguf", "--image", f"{A}/calib/vision_probe.png", + "--question", "Describe the two shapes in this image and their colors.", + "--expect", "red,blue,circle,square")) +log("NEX_BENCH_DONE") diff --git a/recipe/pipeline/nex_cachegate.py b/recipe/pipeline/nex_cachegate.py new file mode 100644 index 0000000000000000000000000000000000000000..a4e5a071a4fd2034eed931987a33f7abddb964ce --- /dev/null +++ b/recipe/pipeline/nex_cachegate.py @@ -0,0 +1,121 @@ +#!/usr/bin/env python3 +"""Chunk-ALIGNED warm/cold identity gate (Nex-N2.5-mini: unpatched server, no draft head). +Written for the Agnes MTP prompt-cache patch; here it checks the stock server's own checkpoint restore for this +hybrid (gated-delta + attention) model. + +Why a third gate: llama-server splits every prompt so that context checkpoints land (4 + n_ubatch) and 4 tokens +before its end (tools/server/server-context.cpp `checkpoint_offsets`, upstream PR #20288). A warm request restores a +checkpoint whose position was fixed by the length of the request that CREATED it. When those lengths differ, the warm +tail is processed in different chunks than a cold run of the same prompt, so float rounding differs and a greedy +token can flip - with or without a draft head. cachegate2 mixed prompt lengths and hit exactly that. + +Here every prompt in the run is padded to ONE token length L, so warm and cold see identical chunking. What is left +under test is whether the restored checkpoint (attention KV + recurrent state) is exact. +Per variant: warm-up A (cache on) -> B warm (cache on, must restore L-(4+ub)) -> B cold (cache off). PASS = every +warm B reused the cache at the aligned position AND is byte-identical to its cold twin, over all variants.""" +import argparse, hashlib, json, os, sys +sys.path.insert(0, "/mnt/models/nex-n2.5-mini") +from nex_harness import Server, post, prompt_8k, THINK_OFF # noqa: E402 (reads AGNES_BIN at import) + +UB = 1024 # agnes_harness.Server passes -ub 1024 +WARM = "Summarise this file." +INSTR = ["Write a function that lists every tensor name in this file.", + "Write a function that counts the model classes registered in this file.", + "Write a function that finds the longest method in this file.", + "Write a function that returns every regular expression used in this file.", + "Write a function that maps each class in this file to its base classes.", + "Write a function that extracts all string constants from this file.", + "Write a function that reports which imports in this file are unused.", + "Write a function that lists every method that raises an exception in this file.", + "Write a function that counts the lines of code per class in this file.", + "Write a function that finds duplicate method names across classes in this file."] +KW = THINK_OFF + + +def plen(port, content): + p = post(port, "/apply-template", {"messages": [{"role": "user", "content": content}], + "chat_template_kwargs": KW})["prompt"] + return len(post(port, "/tokenize", {"content": p, "add_special": True, "parse_special": True})["tokens"]) + + +def pad(port, ctx, instr, L): + s = instr + n = plen(port, ctx + "\n\n" + s) + for filler in (" ok", ".", " x"): + while n < L: + t = s + filler + m = plen(port, ctx + "\n\n" + t) + if m > L: + break + s, n = t, m + if n == L: + return s + raise SystemExit(f"could not pad {instr!r} to {L} (stuck at {n})") + + +def req(port, ctx, instr, n, cache): + body = {"messages": [{"role": "user", "content": ctx + "\n\n" + instr}], "max_tokens": n, + "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": cache, "chat_template_kwargs": KW} + r = post(port, "/v1/chat/completions", body) + tm = r.get("timings", {}) + return (r["choices"][0]["message"].get("content") or ""), tm.get("prompt_n"), tm.get("cache_n"), \ + tm.get("draft_n"), tm.get("draft_n_accepted") + + +def first_diff(x, y): + if x == y: + return None + return next((i for i, (p, q) in enumerate(zip(x, y)) if p != q), min(len(x), len(y))) + + +def main(a): + s = Server(a, a.port) + rows = [] + try: + ctx, _ = prompt_8k("code") + instr = INSTR[:a.reps] + warm = [f"{WARM} (warm-up {i})" for i in range(len(instr))] + # L over the FULL prompt set, whatever --reps is: runs with different --reps then share byte-identical + # prompts, so their outputs can be compared across configs (e.g. MTP vs no draft head). + allp = INSTR + [f"{WARM} (warm-up {i})" for i in range(len(INSTR))] + L = max(plen(a.port, ctx + "\n\n" + t) for t in allp) + 1 + instr = [pad(a.port, ctx, t, L) for t in instr] + warm = [pad(a.port, ctx, t, L) for t in warm] + expect_cache = L - (4 + UB) + for i, (wa, vb) in enumerate(zip(warm, instr)): + req(a.port, ctx, wa, 16, True) + tw, pw, cw, dw, aw = req(a.port, ctx, vb, 192, True) + tc, pc, cc, dc, ac = req(a.port, ctx, vb, 192, False) + rows.append({"variant": i, "L": L, "warm_prompt_n": pw, "warm_cache_n": cw, "cold_prompt_n": pc, + "cold_cache_n": cc, "aligned": (cw == expect_cache and pc == L), + "warm_draft": [aw, dw], "cold_draft": [ac, dc], "identical": tw == tc, + "first_diff_char": first_diff(tw, tc), + "warm_sha": hashlib.sha256(tw.encode()).hexdigest()[:12], + "cold_sha": hashlib.sha256(tc.encode()).hexdigest()[:12], "instr": vb, "warm_instr": wa, + "warm_text": tw, "cold_text": tc}) + print(" ", json.dumps({k: v for k, v in rows[-1].items() if not k.endswith("_text")}), flush=True) + finally: + s.stop() + reused = all((r["warm_cache_n"] or 0) > 0 for r in rows) + aligned = all(r["aligned"] for r in rows) + ident = all(r["identical"] for r in rows) + res = {"label": a.label, "gate": "cachegate3-aligned", "n": len(rows), "L": rows[0]["L"] if rows else None, + "expect_cache_n": expect_cache if rows else None, "all_reused": reused, "all_aligned": aligned, + "identical": sum(r["identical"] for r in rows), "rows": rows, + "result": "PASS" if (rows and reused and aligned and ident) else "FAIL"} + print(json.dumps({k: v for k, v in res.items() if k != "rows"}), flush=True) + with open(a.jsonl, "a") as f: + f.write(json.dumps(res) + "\n") + sys.exit(0 if res["result"] == "PASS" else 1) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("--model", required=True); ap.add_argument("--draft") + ap.add_argument("--nmax", type=int, default=4); ap.add_argument("--pmin", type=float, default=0.0) + ap.add_argument("--dev", default="ROCm0"); ap.add_argument("--ctx", type=int, default=65536) + ap.add_argument("--port", type=int, default=18600); ap.add_argument("--reps", type=int, default=10) + ap.add_argument("--label", required=True); ap.add_argument("--jsonl", required=True) + ap.add_argument("--serverlog", required=True) + ap.add_argument("--mtp-infile", action="store_true"); ap.add_argument("--strict", action="store_true") + main(ap.parse_args()) diff --git a/recipe/pipeline/nex_cards.sh b/recipe/pipeline/nex_cards.sh new file mode 100644 index 0000000000000000000000000000000000000000..b78f22d0567471acb2b004cdd0b394126e767268 --- /dev/null +++ b/recipe/pipeline/nex_cards.sh @@ -0,0 +1,41 @@ +#!/bin/bash +# nex_cards.sh cards|upload|verify — Nex-N2.5-mini cards on real data, then the two SEPARATE repos. +# cards : summary (fast, to satisfy stage's recipe check) -> stage std+imat (hashes) -> summary with staged hashes +# -> refresh the staged summary copy -> judge -> render (prints pending judgments) +# upload: behind the quiet-box lock; each repo through upload_watchdog.sh (40G cap, niced, stall kill + retry). verify: size + sha256 + recipe + separation. +set -euo pipefail +W=/mnt/models/nex-n2.5-mini; cd $W +export HF_XET_CACHE=/mnt/models/.hf-xet-cache TMPDIR=/mnt/models/.tmp HF_HUB_DISABLE_TELEMETRY=1 PYTHONUNBUFFERED=1 +mkdir -p "$HF_XET_CACHE" "$TMPDIR" +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +case "${1:-}" in + cards) + grep -q NEX_PHASE2_DONE logs/phase2.log || { log "phase 2 not done -> refuse"; exit 3; } + AGG_FAST=1 nice -n 10 python3 nex_aggregate.py > logs/aggregate_fast.log 2>&1 + for k in std imat; do log "stage $k"; nice -n 10 python3 nex_publish.py stage $k; done + nice -n 10 python3 nex_aggregate.py > logs/aggregate.log 2>&1; tail -12 logs/aggregate.log + for k in std imat; do cp results/summary.json hf-upload/$k/recipe/results_summary.json; done + python3 - <<'PY' +import json +d = {} +for k in ("std", "imat"): + d.update(json.load(open(f"hf-upload/staged_sizes_{k}.json"))) +json.dump(d, open("hf-upload/staged_sizes.json", "w"), indent=1) +print("staged sizes:", len(d)) +PY + python3 nex_judge.py results/summary.json results/judgments.json > logs/judge.log 2>&1; head -1 logs/judge.log + rm -rf cards + python3 nex_render.py results/summary.json cards results/judgments.json \ + hf-upload/std/SHA256SUMS hf-upload/imat/SHA256SUMS hf-upload/staged_sizes.json + ls -la cards; log "NEX_CARDS_DONE" ;; + upload) + exec 9>/mnt/models/.quiet-box.lock; log "waiting for the quiet-box lock"; flock 9; log "quiet-box lock held" + for k in std imat; do + bash upload_watchdog.sh nex-upload-$k $W python3 nex_publish.py upload $k || { log "upload $k FAILED"; exit 1; } + done + log "NEX_UPLOAD_ALL_DONE" ;; + verify) + rc=0; for k in std imat; do python3 nex_publish.py verify $k || rc=1; done + log "NEX_VERIFY rc=$rc"; exit $rc ;; + *) echo "usage: $0 cards|upload|verify"; exit 2 ;; +esac diff --git a/recipe/pipeline/nex_download.sh b/recipe/pipeline/nex_download.sh new file mode 100644 index 0000000000000000000000000000000000000000..29dbdab7ffa76d94498b65e422b7273da93bc28c --- /dev/null +++ b/recipe/pipeline/nex_download.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Nex-N2.5-mini: pinned download + byte/sha256 gate. Runs inside a capped scope (2 cores, 16 GiB, idle I/O) so the +# Agnes measurements still running on this box are not disturbed. HF_HOME stays default (token lives there). +set -uo pipefail +R=nex-agi/Nex-N2.5-mini; REV=87420286149d9cce9bd46cd335ef9bda33c37c1b +W=/mnt/models/nex-n2.5-mini; mkdir -p $W/hf $W/logs +# 8 parallel files x default range concurrency grew hf to 5.6 GiB RSS and the 6 GiB cap OOM-killed it (20:56Z). +export HF_XET_CACHE=/mnt/models/.hf-xet-cache TMPDIR=/mnt/models/.tmp HF_HUB_DISABLE_TELEMETRY=1 HF_XET_NUM_CONCURRENT_RANGE_GETS=8 +mkdir -p "$HF_XET_CACHE" "$TMPDIR" +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +t0=$(date +%s) +log "download $R @ $REV -> $W/hf" +hf download "$R" --revision "$REV" --local-dir "$W/hf" --max-workers 4 +rc=$? +log "DONE_RC=$rc elapsed=$(( $(date +%s) - t0 ))s on disk: $(du -sh --apparent-size $W/hf | cut -f1)" +if [ $rc -eq 0 ]; then + log "verify (size + sha256 vs the tree API lfs.oid at the pinned revision)" + python3 $W/verify_download.py "$R" "$REV" "$W/hf" > $W/logs/D2_verify_download.log 2>&1; vrc=$? + tail -3 $W/logs/D2_verify_download.log + log "VERIFY_RC=$vrc" + echo "$vrc" > $W/logs/DOWNLOAD_VERIFY_RC +fi +echo "$rc" > $W/logs/DOWNLOAD_RC +log "NEX_DOWNLOAD_END" diff --git a/recipe/pipeline/nex_harness.py b/recipe/pipeline/nex_harness.py new file mode 100644 index 0000000000000000000000000000000000000000..227eb655de76fc86a9710a7d2059e34acbfb24b3 --- /dev/null +++ b/recipe/pipeline/nex_harness.py @@ -0,0 +1,326 @@ +#!/usr/bin/env python3 +"""Measurement harness for Nex-N2.5-mini on MAX-1 (stdlib only) - derived from the Agnes harness. +Nex's template has no thinking on/off flag: thinking is `reasoning_effort` = none (off) | high (on) | anything else +(adaptive). Tool calls use the upstream sampling (temp 0.7, top_p 0.95, top_k 40). +House protocol (Qwen3.8-27B card): ctx 65536, batch 1, greedy (temp 0, top_k 1), ignore_eos -> exactly +256 tokens, unique nonce + cache_prompt:false (cached tokens asserted 0), median of reps after 1 warm-up. +Subcommands: bench | cachegate | tools | vision (see argparse)""" +import argparse, base64, json, os, signal, statistics, subprocess, sys, time, urllib.request, uuid + +W = "/mnt/models/nex-n2.5-mini" +CAL = "/mnt/models/agnes-3.0-flash/calib" # shared calibration / prose corpora +THINK_OFF = {"reasoning_effort": "none"} +THINK_ON = {"reasoning_effort": "high"} +BIN = os.environ.get("AGNES_BIN", "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin") +ENV = dict(os.environ, LD_LIBRARY_PATH=f"{BIN}:/opt/rocm-7.2.4/lib", HSA_OVERRIDE_GFX_VERSION="11.5.1", + GGML_HIP_ENABLE_UNIFIED_MEMORY="1") + +def post(port, path, body, timeout=1800): + req = urllib.request.Request(f"http://127.0.0.1:{port}{path}", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + return json.load(urllib.request.urlopen(req, timeout=timeout)) + +class Server: + def __init__(self, a, port, mmproj=None, fa="on"): + self.port = port + cmd = [f"{BIN}/llama-server", "-m", a.model, "-dev", a.dev, "-ngl", "999", "-fa", fa, "-dio", + "--jinja", "-fit", "off", "--parallel", "1", "-c", str(a.ctx), "-b", "2048", "-ub", "1024", + "--host", "127.0.0.1", "--port", str(port), "--no-webui"] + if a.draft or a.mtp_infile: + cmd += ["--spec-type", "draft-mtp"] + if a.draft: + cmd += ["--model-draft", a.draft, "--spec-draft-ngl", "99", "--spec-draft-device", a.dev] + cmd += ["--spec-draft-n-max", str(a.nmax), "--spec-draft-n-min", "0", "--spec-draft-p-min", str(a.pmin)] + if a.strict: + cmd += ["--spec-mtp-strict-qwen"] + if mmproj: + cmd += ["--mmproj", mmproj] + self.cmd = cmd + self.logf = open(a.serverlog, "w") + self.t0 = time.time() + self.p = subprocess.Popen(cmd, env=ENV, stdout=self.logf, stderr=subprocess.STDOUT, start_new_session=True) + while True: + if self.p.poll() is not None: + raise SystemExit(f"SERVER DIED rc={self.p.returncode} see {a.serverlog}") + try: + if json.load(urllib.request.urlopen(f"http://127.0.0.1:{port}/health", timeout=3)).get("status") == "ok": + break + except Exception: + pass + if time.time() - self.t0 > 900: + self.stop(); raise SystemExit("SERVER LOAD TIMEOUT") + time.sleep(2) + self.load_s = time.time() - self.t0 + def stop(self): + try: os.killpg(self.p.pid, signal.SIGTERM); self.p.wait(60) + except Exception: + try: os.killpg(self.p.pid, signal.SIGKILL) + except Exception: pass + self.logf.close() + +def prompt_8k(kind): + if kind == "code": + src = open("/opt/llama-rocm/rocmfpx-724/convert_hf_to_gguf.py").read()[:30000] + return src, "Above is part of a model converter. Write a new, complete Python function that validates a GGUF tensor-name map against a list of HF tensor names and reports unmapped names. Code only." + txt = open(f"{CAL}/wikitext-2-raw/wiki.train.raw").read()[:34000] + return txt, "Above are encyclopedia excerpts. Write a long, detailed new encyclopedia article in the same style about the history of lighthouses." + +def one_request(port, ctx_text, instr, n=256, cache=False, nonce=True): + tag = f"[req {uuid.uuid4()}]\n" if nonce else "" + body = {"messages": [{"role": "user", "content": tag + ctx_text + "\n\n" + instr}], + "max_tokens": n, "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": cache, + "chat_template_kwargs": THINK_OFF} + t = time.time(); r = post(port, "/v1/chat/completions", body); el = time.time() - t + tm = r.get("timings", {}) + return {"pred_n": tm.get("predicted_n"), "tg": tm.get("predicted_per_second"), + "pp": tm.get("prompt_per_second"), "prompt_n": tm.get("prompt_n"), "cache_n": tm.get("cache_n"), + "draft_n": tm.get("draft_n"), "draft_acc": tm.get("draft_n_accepted"), "wall": el, + "usage": r.get("usage", {})} + +def cmd_bench(a): + s = Server(a, a.port) + out = {"label": a.label, "model": os.path.basename(a.model), + "draft": os.path.basename(a.draft) if a.draft else ("in-file" if a.mtp_infile else None), + "nmax": a.nmax if (a.draft or a.mtp_infile) else None, "strict": bool(a.strict), "bin": BIN, "dev": a.dev, "ctx": a.ctx, "workload": a.workload, + "load_s": round(s.load_s, 1), "cmd": " ".join(s.cmd), "runs": []} + try: + ctx_text, instr = prompt_8k(a.workload) + one_request(a.port, ctx_text, instr) # warm-up, discarded + for _ in range(a.reps): + r = one_request(a.port, ctx_text, instr) + cached = (r["usage"].get("prompt_tokens_details") or {}).get("cached_tokens", 0) or 0 + if r["pred_n"] != 256: raise SystemExit(f"GATE FAIL: predicted_n={r['pred_n']} != 256") + if (r["cache_n"] or 0) != 0 or cached != 0: raise SystemExit(f"GATE FAIL: cache hit cache_n={r['cache_n']} cached={cached}") + out["runs"].append(r) + finally: + s.stop() + tg = [r["tg"] for r in out["runs"]]; pp = [r["pp"] for r in out["runs"]] + dn = sum(r["draft_n"] or 0 for r in out["runs"]); da = sum(r["draft_acc"] or 0 for r in out["runs"]) + out.update(tg_median=round(statistics.median(tg), 2), tg_min=round(min(tg), 2), tg_max=round(max(tg), 2), + pp_median=round(statistics.median(pp), 1), prompt_n=out["runs"][0]["prompt_n"], + accept=(round(da / dn, 3) if dn else None)) + print(json.dumps({k: v for k, v in out.items() if k != "runs"})) + with open(a.jsonl, "a") as f: f.write(json.dumps(out) + "\n") + +def cmd_identity(a): + """Fixed prompt, no nonce, no cache, greedy: return the exact generated text for cross-config diffing.""" + s = Server(a, a.port) + texts = [] + try: + ctx_text, instr = prompt_8k(a.workload) + for _ in range(a.reps): + body = {"messages": [{"role": "user", "content": ctx_text + "\n\n" + instr}], "max_tokens": 256, + "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": False, + "chat_template_kwargs": THINK_OFF} + r = post(a.port, "/v1/chat/completions", body) + texts.append(r["choices"][0]["message"].get("content") or "") + finally: + s.stop() + import hashlib + res = {"label": a.label, "reps": a.reps, "sha256": [hashlib.sha256(t.encode()).hexdigest()[:16] for t in texts], + "self_consistent": len(set(texts)) == 1, "text": texts[0]} + print(json.dumps({k: v for k, v in res.items() if k != "text"})) + with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n") + +def cmd_cachegate2(a): + """Partial-prefix reuse WITH output identity: warm on A, run B warm (partial reuse), run B cold, diff outputs.""" + import hashlib + s = Server(a, a.port) + rows = [] + try: + ctx_text, _ = prompt_8k("code") + variants = ["Write a function that lists every tensor name in this file.", + "Write a function that counts the model classes registered in this file.", + "Write a function that finds the longest method in this file."] + for i, vb in enumerate(variants[:a.reps]): + def req(instr, n, cache): + body = {"messages": [{"role": "user", "content": ctx_text + "\n\n" + instr}], "max_tokens": n, + "temperature": 0, "top_k": 1, "ignore_eos": True, "cache_prompt": cache, + "chat_template_kwargs": THINK_OFF} + r = post(a.port, "/v1/chat/completions", body) + tm = r.get("timings", {}) + return (r["choices"][0]["message"].get("content") or ""), tm.get("prompt_n"), tm.get("cache_n") + req(f"Summarise this file. (warm-up {i})", 16, True) + tw, pw, cw = req(vb, 192, True) + tc, pc, cc = req(vb, 192, False) + rows.append({"variant": i, "warm_prompt_n": pw, "warm_cache_n": cw, "cold_prompt_n": pc, "cold_cache_n": cc, + "identical": tw == tc, "warm_sha": hashlib.sha256(tw.encode()).hexdigest()[:12], + "cold_sha": hashlib.sha256(tc.encode()).hexdigest()[:12]}) + print(" ", json.dumps(rows[-1]), flush=True) + finally: + s.stop() + reused = all((r["warm_cache_n"] or 0) > 0 for r in rows) + ident = all(r["identical"] for r in rows) + res = {"label": a.label, "rows": rows, "all_reused": reused, "all_identical": ident, + "result": "PASS" if (reused and ident) else "FAIL"} + print(json.dumps({k: v for k, v in res.items() if k != "rows"})) + with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n") + sys.exit(0 if res["result"] == "PASS" else 1) + +def cmd_cachegate(a): + """Repeat-prompt test WITH the draft head loaded: turn 2 must reuse turn 1's prefix.""" + s = Server(a, a.port) + try: + ctx_text, instr = prompt_8k("code") + r1 = one_request(a.port, ctx_text, instr, n=32, cache=True, nonce=False) + r2 = one_request(a.port, ctx_text, instr + " Also add type hints.", n=32, cache=True, nonce=False) + finally: + s.stop() + res = {"label": a.label, "turn1_prompt_n": r1["prompt_n"], "turn2_prompt_n": r2["prompt_n"], + "turn2_cache_n": r2["cache_n"]} + # hybrid recurrent models resume only from context checkpoints (~1024-token spacing), so the test is + # "any prefix reuse" -- report the fraction rather than demand near-total reuse. + ok = isinstance(r2["cache_n"], int) and r2["cache_n"] > 0 + res["reuse_fraction"] = round(r2["cache_n"] / r1["prompt_n"], 3) if ok and r1["prompt_n"] else 0.0 + res["result"] = "PASS" if ok else "FAIL" + print(json.dumps(res)) + with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n") + sys.exit(0 if ok else 1) + +TOOLS = [ + {"type": "function", "function": {"name": "get_weather", "description": "Current weather for a city", + "parameters": {"type": "object", "properties": {"city": {"type": "string"}, + "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, "required": ["city", "unit"]}}}, + {"type": "function", "function": {"name": "create_event", "description": "Create a calendar event", + "parameters": {"type": "object", "properties": {"title": {"type": "string"}, + "when": {"type": "object", "properties": {"date": {"type": "string"}, "time": {"type": "string"}}, + "required": ["date", "time"]}, + "attendees": {"type": "array", "items": {"type": "string"}}}, "required": ["title", "when", "attendees"]}}}, +] +LEAK = ("", "", "") + +def chat(port, msgs, think, stream=False, tools=TOOLS): + body = {"messages": msgs, "tools": tools, "tool_choice": "auto", "temperature": 0.7, "top_p": 0.95, + "top_k": 40, "max_tokens": 8192, + "chat_template_kwargs": (THINK_ON if think else THINK_OFF)} + if not stream: + return post(port, "/v1/chat/completions", body)["choices"][0]["message"] + body["stream"] = True + req = urllib.request.Request(f"http://127.0.0.1:{port}/v1/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + calls, content = {}, "" + for line in urllib.request.urlopen(req, timeout=1800): + line = line.decode().strip() + if not line.startswith("data:") or line.endswith("[DONE]"): continue + d = json.loads(line[5:])["choices"][0]["delta"] + content += d.get("content") or "" + for tc in d.get("tool_calls") or []: + c = calls.setdefault(tc["index"], {"name": "", "arguments": ""}) + c["name"] += (tc.get("function") or {}).get("name") or "" + c["arguments"] += (tc.get("function") or {}).get("arguments") or "" + return {"content": content, "tool_calls": [{"function": v} for _, v in sorted(calls.items())]} + +def args_of(m, i=0): + return json.loads(m["tool_calls"][i]["function"]["arguments"]) + +def cmd_tools(a): + s = Server(a, a.port) + results = {} + try: + for think in (True, False): + def check(name, fn): + try: ok, why = fn() + except Exception as e: ok, why = False, f"exception {e!r}"[:160] + results[f"{name}|think={think}"] = (ok, why) + print(f" {'PASS' if ok else 'FAIL'} think={think!s:5} {name}: {why}", flush=True) + def clean(m): return not any(x in (m.get("content") or "") for x in LEAK) + def t1(): + m = chat(a.port, [{"role": "user", "content": "What's the weather in Paris in celsius?"}], think) + ag = args_of(m); return (m["tool_calls"][0]["function"]["name"] == "get_weather" and ag.get("city", "").lower().startswith("paris") + and ag.get("unit") == "celsius" and clean(m)), f"args={ag}" + def t2(): + m = chat(a.port, [{"role": "user", "content": "Book 'Design review' on 2026-10-02 at 14:00 with ana@x.io and bo@x.io."}], think) + ag = args_of(m); return (isinstance(ag.get("when"), dict) and ag["when"].get("date") == "2026-10-02" + and sorted(ag.get("attendees", [])) == ["ana@x.io", "bo@x.io"] and clean(m)), f"args={ag}" + def t3(): + m = chat(a.port, [{"role": "user", "content": "Weather in Denver, and give it to me in fahrenheit."}], think) + return args_of(m).get("unit") == "fahrenheit" and clean(m), f"unit={args_of(m).get('unit')}" + def t4(): + m = chat(a.port, [{"role": "user", "content": "What is 17 times 23? Answer directly."}], think) + c = m.get("content") or "" + return (not m.get("tool_calls")) and "391" in c and clean(m), f"content={c[:60]!r}" + def t5(): + msgs = [{"role": "user", "content": "What's the weather in Tokyo in celsius?"}] + m = chat(a.port, msgs, think) + tc = m["tool_calls"][0] + msgs += [{"role": "assistant", "content": m.get("content") or "", "tool_calls": [ + {"id": "call_1", "type": "function", "function": tc["function"]}]}, + {"role": "tool", "tool_call_id": "call_1", "content": json.dumps({"temp_c": 21, "sky": "clear"})}] + m2 = chat(a.port, msgs, think) + c = m2.get("content") or "" + return ("21" in c and not m2.get("tool_calls") and clean(m2)), f"final={c[:70]!r}" + def t6(): + m = chat(a.port, [{"role": "user", "content": "What's the weather in Rome in celsius?"}], think, stream=True) + ag = args_of(m); return (m["tool_calls"][0]["function"]["name"] == "get_weather" + and ag.get("city", "").lower().startswith("rome") and clean(m)), f"stream args={ag}" + def t7(): + m = chat(a.port, [{"role": "user", "content": "Get the weather in Oslo AND in Lima, both in celsius. Call the tool for each city."}], think) + cities = sorted(args_of(m, i).get("city", "").lower() for i in range(len(m.get("tool_calls") or []))) + return (len(cities) == 2 and cities[0].startswith("lima") and cities[1].startswith("oslo") and clean(m)), f"calls={cities}" + for nm, fn in (("multi-arg", t1), ("nested-object", t2), ("enum", t3), ("correct-decline", t4), + ("multi-turn", t5), ("streaming", t6), ("parallel", t7)): + check(nm, fn) + finally: + s.stop() + n_ok = sum(v[0] for v in results.values()) + summary = {"label": a.label, "passed": n_ok, "total": len(results), + "detail": {k: v[0] for k, v in results.items()}} + print(json.dumps(summary)) + with open(a.jsonl, "a") as f: f.write(json.dumps(summary) + "\n") + +def cmd_vision(a): + """Image gate. A server that fails to load or dies on the image is a RESULT (FAIL row), not a harness crash.""" + fa = a.fa or "off" + res = {"label": a.label, "fa": fa, "mtp": bool(a.draft or a.mtp_infile), "expected": a.expect, + "answer": "", "hits": [], "error": None, "server_died": False, "server_log_errors": []} + c = "" + try: + s = Server(a, a.port, mmproj=a.mmproj, fa=fa) + except SystemExit as e: + res.update(error=f"server did not start: {e}", server_died=True); s = None + if s is not None: + try: + img = base64.b64encode(open(a.image, "rb").read()).decode() + body = {"messages": [{"role": "user", "content": [ + {"type": "image_url", "image_url": {"url": f"data:image/png;base64,{img}"}}, + {"type": "text", "text": a.question}]}], + "temperature": 0, "top_k": 1, "max_tokens": 300, "chat_template_kwargs": THINK_OFF} + try: + r = post(a.port, "/v1/chat/completions", body, timeout=900) + c = r["choices"][0]["message"].get("content") or "" + except Exception as e: + res["error"] = f"{type(e).__name__}: {e}"[:300] + time.sleep(1) + res["server_died"] = s.p.poll() is not None + finally: + s.stop() + try: + res["server_log_errors"] = [l.strip()[-200:] for l in open(a.serverlog, errors="replace") + if any(k in l for k in ("GGML_ABORT", "abort", "failed to process", " E "))][-5:] + except OSError: + pass + res["answer"] = c[:300] + res["hits"] = [w for w in a.expect.split(",") if w.lower() in c.lower()] + ok = res["error"] is None and not res["server_died"] and len(res["hits"]) == len(a.expect.split(",")) + res["result"] = "PASS" if ok else "FAIL" + print(json.dumps(res)) + with open(a.jsonl, "a") as f: f.write(json.dumps(res) + "\n") + sys.exit(0 if ok else 1) + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("cmd", choices=["bench", "cachegate", "cachegate2", "tools", "vision", "identity"]) + ap.add_argument("--model", required=True); ap.add_argument("--draft") + ap.add_argument("--nmax", type=int, default=4); ap.add_argument("--pmin", type=float, default=0.0) + ap.add_argument("--dev", default="ROCm0"); ap.add_argument("--ctx", type=int, default=65536) + ap.add_argument("--port", type=int, default=18600); ap.add_argument("--reps", type=int, default=3) + ap.add_argument("--workload", default="code", choices=["code", "prose"]) + ap.add_argument("--label", default=""); ap.add_argument("--jsonl", default=f"{W}/results/phase_b.jsonl") + ap.add_argument("--serverlog", default=f"{W}/logs/server_last.log") + ap.add_argument("--mmproj"); ap.add_argument("--image"); ap.add_argument("--question"); ap.add_argument("--expect") + ap.add_argument("--mtp-infile", action="store_true"); ap.add_argument("--strict", action="store_true") + ap.add_argument("--fa", choices=["on", "off", "auto"], help="vision only; default off") + a = ap.parse_args() + os.makedirs(os.path.dirname(a.jsonl), exist_ok=True) + {"bench": cmd_bench, "cachegate": cmd_cachegate, "tools": cmd_tools, "vision": cmd_vision, + "identity": cmd_identity, "cachegate2": cmd_cachegate2}[a.cmd](a) diff --git a/recipe/pipeline/nex_judge.py b/recipe/pipeline/nex_judge.py new file mode 100644 index 0000000000000000000000000000000000000000..5876ef9068c949c0c68088fc87c02072df888e96 --- /dev/null +++ b/recipe/pipeline/nex_judge.py @@ -0,0 +1,584 @@ +#!/usr/bin/env python3 +"""Write judgments.json for the Nex-N2.5-mini cards. Sentence structure is authored; every number is +computed from summary.json. A judgment whose inputs are missing is NOT emitted (the renderer then +shows JUDGMENT PENDING). usage: nex_judge.py """ +import json, math, statistics, sys + +S = json.load(open(sys.argv[1])); T = S.get("tiers") or {}; J = {} +MiB = 1024 ** 2 +UB = S.get("n_ubatch") if S.get("n_ubatch") is not None else 1024 +SPEED_EQ = 3.0 # % — tiers closer than this are called speed-equivalent +TWINS = (("q106i", "q106"), ("q102i", "q102"), ("q103i", "q103")) +STD_TAGS = ("q106", "q102", "q103") +IMAT_TAGS = ("q106i", "q102i", "q103i") +NAMES = {"q106": "STRIX_LEAN", "q102": "COHERENT", "q103": "FAST", + "q106i": "STRIX_LEAN", "q102i": "COHERENT", "q103i": "FAST"} + + +def pct(a, b): + return (b - a) / a * 100 + + +def sig(a, ea, b, eb): + return abs(b - a) / math.sqrt(ea * ea + eb * eb) + + +def have(*v): + return all(x is not None for x in v) + + +def bench(label): # last row wins: a re-run supersedes an earlier row with the same label + return next((x for x in reversed(S.get("bench") or []) if x["label"] == label), None) + + +def gate(label): + return next((x for x in reversed(S.get("gates") or []) if x.get("label") == label), None) + + +def bn(tag, dev): + return bench(f"n-{tag}-{dev}") + + +def twin_spread(key): + """Largest |gap| between an imatrix file and its standard twin: a measured noise floor.""" + if not all(T.get(i, {}).get("same_tensor_types_as_standard") is True for i, _ in TWINS): + return None + vals = [] + for ti, ts in TWINS: + for dev in ("rocm", "vk"): + x, y = bn(ti, dev), bn(ts, dev) + if not (x and y and x.get(key) is not None and y.get(key) is not None): + return None + vals.append(abs(pct(y[key], x[key]))) + return max(vals) if vals else None + + +TG_NOISE, PP_NOISE = twin_spread("tg_median"), twin_spread("pp_median") +DEC_EQ = max(SPEED_EQ, TG_NOISE) if TG_NOISE is not None else None +PRE_EQ = max(SPEED_EQ, PP_NOISE) if PP_NOISE is not None else None + + +def rel(g, what, eq): + if abs(g) < eq: + return f"{what} within {abs(g):.1f} % of" + return f"{what} {abs(g):.1f} % {'faster' if g > 0 else 'slower'} than" + + +def cmp_line(a_tag, b_tag, a_nm, b_nm): + """-> (text, a_advantage, a_disadvantage) or (None, None, None).""" + ra, rb, va, vb = bn(a_tag, "rocm"), bn(b_tag, "rocm"), bn(a_tag, "vk"), bn(b_tag, "vk") + if not all((ra, rb, va, vb)) or DEC_EQ is None or PRE_EQ is None: + return None, None, None + if not have(ra.get("tg_median"), rb.get("tg_median"), va.get("tg_median"), vb.get("tg_median"), + ra.get("pp_median"), rb.get("pp_median"), + ra.get("tg_min"), ra.get("tg_max"), rb.get("tg_min"), rb.get("tg_max")): + return None, None, None + g_r, g_v = pct(rb["tg_median"], ra["tg_median"]), pct(vb["tg_median"], va["tg_median"]) + g_p = pct(rb["pp_median"], ra["pp_median"]) + txt = (f"`{a_nm}` {rel(g_r, 'decodes', DEC_EQ)} `{b_nm}` on ROCm0 ({ra['tg_median']:.2f} vs " + f"{rb['tg_median']:.2f} tok/s; per-run ranges {ra['tg_min']:.2f}–{ra['tg_max']:.2f} and " + f"{rb['tg_min']:.2f}–{rb['tg_max']:.2f}) and {rel(g_v, 'decodes', DEC_EQ)[len('decodes '):]} it on Vulkan0 " + f"({va['tg_median']:.2f} vs {vb['tg_median']:.2f}), and {rel(g_p, 'prefills', PRE_EQ)} it on ROCm0 " + f"({ra['pp_median']:.0f} vs {rb['pp_median']:.0f} tok/s)") + adv = g_p >= PRE_EQ or g_r >= DEC_EQ or g_v >= DEC_EQ + dis = g_p <= -PRE_EQ or g_r <= -DEC_EQ or g_v <= -DEC_EQ + return txt, adv, dis + + +NOISE_NOTE = ("" if DEC_EQ is None or PRE_EQ is None else + f"Speed gaps below {DEC_EQ:.1f} % (decode) and {PRE_EQ:.1f} % (prefill) are called a tie: the larger of " + f"{SPEED_EQ:.0f} % and the widest gap measured between files that do identical work per token (each imatrix " + f"file and its standard twin: decode {TG_NOISE:.1f} %, prefill {PP_NOISE:.1f} %).") + + +def kld_cmp(a, b): + """KLD of tag a against tag b -> (pct change b->a, sigma, word). 'lower'/'higher' only at >= 2 sigma.""" + x, y = T[a], T[b] + d = pct(y["kld_mean"], x["kld_mean"]) + sg = sig(y["kld_mean"], y["kld_err"], x["kld_mean"], x["kld_err"]) + word = "within noise of" if sg < 2 else ("lower than" if d < 0 else "higher than") + return d, sg, word + + +def kld_ready(tags): + return all(have(T.get(k, {}).get("kld_mean"), T.get(k, {}).get("kld_err"), + T.get(k, {}).get("size_bytes")) for k in tags) + + +def kld_rank_text(tags): + items = sorted(tags, key=lambda t: T[t]["kld_mean"]) + parts = [] + for i, tag in enumerate(items): + x = T[tag] + bit = f"`{NAMES[tag]}` {x['kld_mean']:.4f}" + if i > 0: + lo = T[items[0]]["kld_mean"] + parts.append(f"{bit} ({pct(lo, x['kld_mean']):+.1f} % vs `{NAMES[items[0]]}`)") + else: + parts.append(bit) + return ", ".join(parts) + + +def size_vs(a, b): + da, db = T[a]["size_bytes"], T[b]["size_bytes"] + if da == db: + return f"`{NAMES[a]}` and `{NAMES[b]}` are the same size ({da / MiB:.0f} MiB)" + smaller, larger = (a, b) if da < db else (b, a) + return (f"`{NAMES[smaller]}` is {(T[larger]['size_bytes'] - T[smaller]['size_bytes']) / MiB:.0f} MiB " + f"smaller than `{NAMES[larger]}`") + + +# ---------- quality provenance ---------- +rep, rep_v, ref = S.get("repeat") or {}, S.get("repeat_vk") or {}, S.get("reference") or {} +if rep.get("result") == "MATCH" and rep.get("rows") is not None and have(ref.get("cpu_chunk1"), ref.get("vulkan0_chunk1")): + both = rep_v.get("result") == "MATCH" and rep_v.get("rows") is not None + J["quality_provenance"] = ( + f"Measured directly on these files, against BF16 logits computed **on the CPU** in the same session " + f"(first-window perplexity {ref['cpu_chunk1']:.4f}; Vulkan0 gave {ref['vulkan0_chunk1']:.4f} for the same " + f"window). Every file was graded on ROCm0 (the columns above) and again on Vulkan0. The STRIX_LEAN grade was run " + f"twice {'on each backend' if both else 'on ROCm0'} and every per-chunk row matched: {rep['rows']} of " + f"{rep['rows']} on ROCm0" + (f", {rep_v['rows']} of {rep_v['rows']} on Vulkan0." if both else ".") + + " Why not the GPU for the reference: see [Known issues](#known-issues-and-limits).") + +# ---------- quality by backend ---------- +vk_rows = [(t, T[t]["vk"]) for t in STD_TAGS + IMAT_TAGS if (T.get(t) or {}).get("vk") and have( + T[t].get("kld_mean"), T[t].get("kld_err"), T[t]["vk"].get("kld_mean"), T[t]["vk"].get("kld_err"))] +if len(vk_rows) == 6: + parts_b, lower_vk, lower_rocm = [], 0, 0 + for t, v in vk_rows: + a_, b_ = T[t]["kld_mean"], v["kld_mean"] + sg_ = sig(a_, T[t]["kld_err"], b_, v["kld_err"]) + if sg_ >= 2: + lower_vk += b_ < a_; lower_rocm += a_ < b_ + pc_ = f"{pct(a_, b_):+.1f}" + pc_ = "0.0" if pc_ in ("+0.0", "-0.0") else pc_ + parts_b.append(f"{'imatrix ' if t.endswith('i') else ''}{NAMES[t]} {a_:.4f} / {b_:.4f} ({pc_} %, {sg_:.1f}σ)") + verdict = ("The two backends agree within noise on every file." if not (lower_vk or lower_rocm) else + f"Vulkan0's output is measurably closer to BF16 on {lower_vk} of 6 files and ROCm0's on {lower_rocm}.") + J["backend_quality_note"] = ( + f"**Same files, same reference, graded on each backend** — KLD ROCm0 / Vulkan0: " + "; ".join(parts_b) + f". {verdict}") + + +# ---------- imatrix verdict ---------- +def twin_kld_ok(st, im): + keys = ("kld_mean", "kld_err", "kld_median", "kld_p99", "same_top_p") + return have(*(T.get(st, {}).get(k) for k in keys), *(T.get(im, {}).get(k) for k in keys)) + + +if all(twin_kld_ok(st, im) for im, st in TWINS): + bits = [] + improved, worse = [], [] + for im, st in TWINS: + nm = NAMES[st] + a, b = T[st], T[im] + dk = pct(a["kld_mean"], b["kld_mean"]) + sg = sig(a["kld_mean"], a["kld_err"], b["kld_mean"], b["kld_err"]) + dmed = pct(a["kld_median"], b["kld_median"]) + dp99 = pct(a["kld_p99"], b["kld_p99"]) + dtop = b["same_top_p"] - a["same_top_p"] + if dk < 0 and sg >= 2: + verb = "improves" + improved.append(nm) + elif dk > 0 and sg >= 2: + verb = "is measurably worse than" + worse.append(nm) + else: + verb = "is within noise of" + bits.append( + f"**{nm}** {verb} the standard file on mean KLD " + f"({dk:+.1f} %, {sg:.1f}σ; {a['kld_mean']:.4f} → {b['kld_mean']:.4f}); " + f"median {dmed:+.1f} %, 99th-pct {dp99:+.1f} %, top-1 {dtop:+.2f} pp") + if len(improved) == 3: + head = "**The imatrix measurably improves all three tiers.** " + elif improved: + head = f"**The imatrix measurably improves {', '.join(improved)}.** " + else: + head = "**The imatrix does not measurably improve any of the three tiers on this corpus.** " + if worse: + head += f"**It is measurably worse on {', '.join(worse)}.** " + J["imat_verdict"] = head + "; ".join(bits) + "." + + +# ---------- speed_note (standard card) ---------- +def backend_line(tag): + r, v = bn(tag, "rocm"), bn(tag, "vk") + if not have(r, v) or DEC_EQ is None or PRE_EQ is None: + return None + if not have(r.get("tg_median"), v.get("tg_median"), r.get("pp_median"), v.get("pp_median"), + r.get("tg_min"), r.get("tg_max"), v.get("tg_min"), v.get("tg_max")): + return None + g_t, g_p = pct(v["tg_median"], r["tg_median"]), pct(v["pp_median"], r["pp_median"]) + return (f"`{NAMES[tag]}`: ROCm0 {rel(g_t, 'decodes', DEC_EQ)} Vulkan0 " + f"({r['tg_median']:.2f} vs {v['tg_median']:.2f} tok/s; ranges " + f"{r['tg_min']:.2f}–{r['tg_max']:.2f} / {v['tg_min']:.2f}–{v['tg_max']:.2f}) and " + f"{rel(g_p, 'prefills', PRE_EQ)} it ({r['pp_median']:.0f} vs {v['pp_median']:.0f} tok/s)") + + +if DEC_EQ is not None and PRE_EQ is not None: + lines = [backend_line(t) for t in STD_TAGS] + cr, pr = bench("n-q106-rocm"), bench("n-q106-rocm-prose") + cv, pv = bench("n-q106-vk"), bench("n-q106-vk-prose") + prose = None + if have(cr, pr, cr and cr.get("tg_median"), pr and pr.get("tg_median")): + g = pct(cr["tg_median"], pr["tg_median"]) + prose = (f"STRIX_LEAN workload range on ROCm0: code {cr['tg_median']:.2f} tok/s vs prose " + f"{pr['tg_median']:.2f} ({rel(g, 'prose decodes', DEC_EQ)} code)") + if have(cv, pv, cv and cv.get("tg_median"), pv and pv.get("tg_median")): + gv = pct(cv["tg_median"], pv["tg_median"]) + prose += (f"; Vulkan0 code {cv['tg_median']:.2f} vs prose {pv['tg_median']:.2f} " + f"({rel(gv, 'prose decodes', DEC_EQ)} code)") + prose += "." + + def _span(wl): + v = [(b_.get("prompt_n_min") if b_.get("prompt_n_min") is not None else b_.get("prompt_n"), + b_.get("prompt_n_max") if b_.get("prompt_n_max") is not None else b_.get("prompt_n")) + for b_ in S.get("bench") or [] if b_.get("workload") == wl] + v = [x for x in v if None not in x] + if not v: + return None + lo, hi = min(a for a, _ in v), max(b for _, b in v) + return f"{lo:,}" if lo == hi else f"{lo:,}–{hi:,}" + sc, sp = _span("code"), _span("prose") + if sc and sp: + prose += (f" Prompt lengths: code {sc} tokens (the first 30,000 characters of `convert_hf_to_gguf.py` plus " + f"an instruction), prose {sp} tokens (the first 34,000 characters of wikitext-2 *train* plus a " + f"writing instruction).") + if all(lines) and prose: + J["speed_note"] = " ".join(x + "." for x in lines) + " " + prose + (f" {NOISE_NOTE}" if NOISE_NOTE else "") + + +# ---------- cache ---------- +d = gate("n-c3-q106") +if d and d.get("all_reused") and d.get("all_aligned") and have(d.get("n"), d.get("L"), d.get("expect_cache_n"), + d.get("identical")): + wm = [r_["warm_prompt_ms"] for r_ in d.get("rows") or [] if r_.get("warm_prompt_ms") is not None] + cm = [r_["cold_prompt_ms"] for r_ in d.get("rows") or [] if r_.get("cold_prompt_ms") is not None] + t_line = "" + if wm and cm: + t_line = (f" — median prefill **{statistics.median(wm) / 1000:.1f} s instead of " + f"{statistics.median(cm) / 1000:.1f} s** cold " + f"({statistics.median(cm) / statistics.median(wm):.1f}× faster)") + J["cache_note"] = ( + f"In {d['n']} request pairs sharing a long prefix at one fixed prompt length of {d['L']:,} tokens, every " + f"second request resumed from the checkpoint the first one left {4 + UB:,} tokens before its end — " + f"**{d['expect_cache_n']:,} tokens reused " + f"({d['expect_cache_n'] / d['L'] * 100:.0f} %), {d['L'] - d['expect_cache_n']:,} processed**{t_line}. " + f"Each warm reply was byte-identical to a cold run of the same prompt in **{d['identical']}/{d['n']}** " + f"exchanges.\n\n" + f"llama-server processes the last `n_ubatch` + 4 tokens of every prompt as two batches so it can checkpoint " + f"there ({UB:,} + 4 = {4 + UB:,} tokens with the `-ub {UB}` used in these measurements, where the server " + f"default is `-ub 512`; [upstream PR #20288](https://github.com/ggml-org/llama.cpp/pull/20288)). A turn that " + f"resumes from a checkpoint left by a prompt of a *different* length therefore splits its tail differently " + f"from a cold run, and float rounding can flip a greedy token.") + + +# ---------- tools ---------- +def _tools_ok(x): + return bool(x) and have(x.get("passed"), x.get("total"), x.get("detail")) + + +tl = gate("n-tools-q106") # stock template +FX = [gate(l) for l in ("n-tools-q106-roff", "n-tools-q106-roff-r2", "n-tools-q106-roff-r3")] # quick-start config +TD = S.get("tools_diag") or {} +TF = S.get("template_fix") or {} +if _tools_ok(tl) and all(_tools_ok(x) for x in FX) and have( + TD.get("stock_on_leaks"), TD.get("stock_on_replies"), TD.get("stock_on_reasoning_extracted"), + TD.get("nested_off_http500"), TD.get("nested_off_attempts")): + misses, on_n, on_ok, off_n, off_ok = [], 0, 0, 0, 0 + for i, x in enumerate(FX, 1): + for k, v in x["detail"].items(): + name, think = k.split("|think=") + if think == "True": + on_n += 1; on_ok += bool(v) + else: + off_n += 1; off_ok += bool(v) + if not v: + misses.append("`%s` with thinking %s (pass %d)" % (name, "on" if think == "True" else "off", i)) + fp = TD.get("flag_probes") or {} + flag_txt = "; ".join("%s: %d of %d replies still had reasoning in `content`" % (lab, fp[key]["leaks"], fp[key]["n"]) + for key, lab in (("fmt-deepseek", "`--reasoning-format deepseek`"), + ("srv-kwargs-high", "`--chat-template-kwargs` with `reasoning_effort`"), + ("reasoning-on", "`--reasoning on`")) if key in fp) + J["tools_note"] = ( + "**Stock chat template: %d/%d.** Every thinking-on check failed. Re-run with the raw replies kept, %d of %d " + "thinking-on replies carried the reasoning and a `` in `content`, and %d had any " + "`reasoning_content`. llama-server builds its reasoning parser by rendering the template with " + "`enable_thinking` on and off; this template ignores `enable_thinking` (it switches on `reasoning_effort`), " + "so the parser finds no reasoning markers and extracts nothing. Server switches did not help (%s). With " + "thinking off, `nested-object` failed on an HTTP 500 — see [Known issues](#known-issues-and-limits).\n\n" + "**With the included `%s` and `--reasoning off` (the quick start): %s over three passes of the same suite " + "(%d/%d)** — thinking off %d/%d, thinking on %d/%d; the misses were %s. A pass requires a native " + "`tool_calls` entry with the right arguments and no think tags in `content`. Each check is a single sample at " + "the recommended temperature 0.7." % ( + tl["passed"], tl["total"], TD["stock_on_leaks"], TD["stock_on_replies"], + TD["stock_on_reasoning_extracted"], flag_txt or "not measured", TF.get("file") or "—", + ", ".join("%d/%d" % (x["passed"], x["total"]) for x in FX), + sum(x["passed"] for x in FX), sum(x["total"] for x in FX), off_ok, off_n, on_ok, on_n, + ", ".join(misses) if misses else "none")) + + +# ---------- the template fix: what each request option does (recommended configuration) ---------- +PR = TF.get("probes_roff") or {} +if PR and all(_tools_ok(x) for x in FX): + def _probe(prefix): + rows_ = [v for k, v in PR.items() if k.split("|")[0] == prefix] + return len(rows_), sum(1 for v in rows_ if v.get("leaks")), sum(1 for v in rows_ if v.get("reasoning_len")) + on_ok = sum(bool(v) for x in FX for k, v in x["detail"].items() if k.endswith("|think=True")) + on_n = sum(1 for x in FX for k in x["detail"] if k.endswith("|think=True")) + off_ok = sum(bool(v) for x in FX for k, v in x["detail"].items() if k.endswith("|think=False")) + off_n = sum(1 for x in FX for k in x["detail"] if k.endswith("|think=False")) + lines = [] + for label, prefix, think, dest in ( + ("no `chat_template_kwargs`", "no-kwargs", "off (the server default with `--reasoning off`)", "—"), + ("`\"enable_thinking\": false`", "enable_thinking=false", "off", "—"), + ("`\"reasoning_effort\": \"none\"`", "reasoning_effort=none", "off", "—"), + ("`\"reasoning_effort\": \"high\"`", "reasoning_effort=high", "on", "**`content`** — do not use"), + ("`\"reasoning_effort\": \"medium\"`", "reasoning_effort=medium", "adaptive", "**`content`** — do not use")): + n, leaks, _ = _probe(prefix) + if n: + lines.append("| %s | %s | %s | %d of %d replies with think tags in `content` |" % (label, think, dest, leaks, n)) + lines.insert(1 if lines else 0, + "| `\"enable_thinking\": true` | on | `reasoning_content` | tool suite with thinking on: %d/%d " + "(a pass requires no think tags in `content`) |" % (on_ok, on_n)) + J["template_note"] = ( + "Measured on the standard STRIX_LEAN file with the included template file and `--reasoning off` (greedy " + "probes: a direct question, a " + "one-word instruction and a tool request, each with the tool schema attached; tool suite: 3 passes):\n\n" + "| request | thinking | reasoning ends up in | measured |\n| --- | --- | --- | --- |\n" + "\n".join(lines) + + "\n\nSo: switch thinking with `enable_thinking` only. Thinking-off tool checks: %d/%d." % (off_ok, off_n)) + + +# ---------- vision ---------- +def vwhy(x): + if x.get("server_died"): + import re as _re + errs = [_re.sub(r"^[0-9.]+ [IWE] (srv +)?", "", e).strip() for e in x.get("server_log_errors") or []] + first = next((e for e in errs if "failed" in e.lower() or "error" in e.lower() or "abort" in e.lower()), + None) or (errs[0] if errs else None) + return ("the server aborted" + (f" (`{first[:90]}`)" if first else "")) + if x.get("error"): + return f"the request failed ({x['error'][:80]})" + exp = x.get("expected") or "" + nexp = len(exp.split(",")) if exp else None + hits = x.get("hits") or [] + if nexp: + return f"the reply named {len(hits)} of {nexp} expected terms" + return "the reply did not pass" + + +on, off = gate("n-vision-q106-faon"), gate("n-vision-q106-faoff") +if on and off: + def vok(x): + return x.get("result") == "PASS" + if vok(on) and vok(off): + J["vision_note"] = "✅ **Images work with `-fa on` and `-fa off`.**" + elif vok(off) and not vok(on): + J["vision_note"] = (f"⛔ **With `-fa on`, image requests fail** — {vwhy(on)}. With `-fa off` they work. " + f"**For image input, serve with `-fa off`.**") + J["vision_quickstart_warning"] = ("> ⛔ **Images:** with `-fa on`, image requests fail on this build " + "([measured](#vision)). If you send images, use `-fa off`.") + elif vok(on) and not vok(off): + J["vision_note"] = (f"⛔ **With `-fa off`, image requests fail** — {vwhy(off)}. With `-fa on` they work. " + f"**For image input, keep `-fa on` (the quick-start default).**") + else: + J["vision_note"] = (f"⛔ **Images failed in both `-fa` settings.** `-fa on`: {vwhy(on)}; " + f"`-fa off`: {vwhy(off)}.") + vfx = gate("n-vision-q106-roff-faon") + if vfx and "vision_note" in J: + J["vision_note"] += (" The `-fa on` image test was repeated with the included template file and " + f"`--reasoning off` (the quick start): {'✅ passed' if vok(vfx) else '❌ ' + vwhy(vfx)}.") + + +# ---------- memory ---------- +rows_sz = [r for r in (S.get("sizing") or []) if r.get("label") == "strix-lean"] +if rows_sz: + def mem_cell(r, key): + if r.get("result") == "LOAD_FAIL": + return "did not load" + v = r.get(key) + return f"{v:.2f} GiB" if v is not None else "—" + body = "\n".join( + f"| {r['ctx']:,} | {mem_cell(r, 'footprint_loaded_gib')} | {mem_cell(r, 'footprint_after_8k_gib')} |" + for r in rows_sz if r.get("ctx") is not None) + J["memory_note"] = ( + "Measured footprint (drop in `MemAvailable`) of STRIX_LEAN with the vision projector, q8_0 KV cache, " + "`-cram 512`, one slot, no draft head:\n\n" + "| context | after load | after one request (30,000-character code prompt) |\n" + "| ---: | ---: | ---: |\n" + body + + "\n\nNo row was decode-benchmarked beyond that one request. Nothing beyond these rows was measured.") + + +# ---------- recommendations ---------- +def recommend(lean, coh, fast): + """STRIX_LEAN is the flagship tier; the data decides whether COHERENT's quality or FAST's speed is worth taking + instead. -> (default tag, markdown) or (None, None) when an input is missing.""" + lc, lc_adv, lc_dis = cmp_line(lean, coh, "STRIX_LEAN", "COHERENT") + fl, fl_adv, fl_dis = cmp_line(fast, lean, "FAST", "STRIX_LEAN") + if lc is None or fl is None: + return None, None + L, C, F = T[lean], T[coh], T[fast] + d_c, s_c, w_c = kld_cmp(coh, lean) + more = (C["size_bytes"] - L["size_bytes"]) / MiB + size_c = f"{abs(more):.0f} MiB {'more' if more > 0 else 'less'}" + if w_c == "lower than" and lc_adv and not lc_dis: + default = lean + head = (f"**Start with `STRIX_LEAN`; take `COHERENT` if quality matters more than speed.** {lc}. `COHERENT`'s " + f"KLD is {abs(d_c):.1f} % lower ({s_c:.1f}σ) for {size_c}.") + elif w_c == "lower than": + default = coh + head = (f"**Start with `COHERENT`.** Its KLD is {abs(d_c):.1f} % lower than `STRIX_LEAN`'s ({s_c:.1f}σ) for " + f"{size_c}, and the speed comparison does not clearly favour `STRIX_LEAN`: {lc}.") + elif w_c == "within noise of" and lc_dis and not lc_adv: + default = coh + head = (f"**Start with `COHERENT`.** Its KLD is {w_c} `STRIX_LEAN`'s ({d_c:+.1f} %, {s_c:.1f}σ) and it is " + f"measurably faster: {lc}. It costs {size_c}.") + else: + default = lean + head = (f"**Start with `STRIX_LEAN`.** `COHERENT`'s KLD is {w_c} it ({d_c:+.1f} %, {s_c:.1f}σ) for " + f"{size_c}; {lc}.") + d_f, s_f, w_f = kld_cmp(fast, lean) + kf = f"its KLD is {w_f} `STRIX_LEAN`'s ({F['kld_mean']:.4f} vs {L['kld_mean']:.4f}, {d_f:+.1f} %, {s_f:.1f}σ)" + if fl_adv and not fl_dis: + fast_txt = f"**Take `FAST` for speed:** {fl}; {kf}." + else: + fast_txt = f"`FAST` does not buy a clear speed gain here: {fl}; {kf}." + sizes = f"{size_vs(lean, coh)}; {size_vs(lean, fast)}." + body = (f"{head}\n\n{fast_txt}\n\nKLD order (lower is closer to BF16): {kld_rank_text((lean, coh, fast))}. " + f"{sizes} {NOISE_NOTE}") + return default, body + + +def imat_pointer(): + """Point the standard card at the imatrix repo; name only measurable (>= 2 sigma) twin improvements.""" + repo = S.get("model_repo_imat") + if not repo: + return "" + link = f"[imatrix build](https://huggingface.co/{repo})" + bits = [] + for im, st in TWINS: + if not twin_kld_ok(st, im): + return "" + d, sg, w = kld_cmp(im, st) + if w == "lower than": + bits.append(f"`{NAMES[st]}` {T[st]['kld_mean']:.4f} → {T[im]['kld_mean']:.4f} ({d:+.1f} %, {sg:.1f}σ)") + if bits: + return f"**The {link} is measurably closer to BF16 at the same size:** " + "; ".join(bits) + "." + return f"Importance-matrix twins of all three files: **{link}** (not measurably closer to BF16 on this corpus)." + + +if kld_ready(STD_TAGS) and DEC_EQ is not None and PRE_EQ is not None: + default, body = recommend("q106", "q102", "q103") + if default: + ptr = imat_pointer() + J["std_recommendation"] = body + (f"\n\n{ptr}" if ptr else "") + J["std_default"] = T[default]["file"] + +if kld_ready(IMAT_TAGS) and DEC_EQ is not None and PRE_EQ is not None: + default_i, body_i = recommend("q106i", "q102i", "q103i") + if default_i: + cross = "" + if kld_ready(("q102",)): + d, sg, w = kld_cmp("q106i", "q102") + cross = (f"\n\nFor scale: the imatrix `STRIX_LEAN`'s KLD is {w} the *standard* `COHERENT`'s " + f"({T['q106i']['kld_mean']:.4f} vs {T['q102']['kld_mean']:.4f}, {d:+.1f} %, {sg:.1f}σ), " + f"{size_vs('q106i', 'q102')}.") + J["imat_recommendation"] = body_i + cross + J["imat_default"] = T[default_i]["file"] + +parts = [] +for nm, ti, ts in (("STRIX_LEAN", "q106i", "q106"), ("COHERENT", "q102i", "q102"), ("FAST", "q103i", "q103")): + for dev, dn in (("rocm", "ROCm0"), ("vk", "Vulkan0")): + a_, b_ = bn(ti, dev), bn(ts, dev) + if a_ and b_ and have(a_.get("tg_median"), b_.get("tg_median")): + parts.append(f"{nm} on {dn} {a_['tg_median']:.2f} vs {b_['tg_median']:.2f} tok/s") +same = [T.get(i, {}).get("same_tensor_types_as_standard") for i, _ in TWINS] +_hk = [T.get(i, {}).get("header_keys_only_in_imatrix") for i, _ in TWINS] +_hs = [T.get(i, {}).get("header_keys_only_in_standard") for i, _ in TWINS] +HDR = "" +if all(x is not None for x in _hk + _hs) and not any(_hs) and len({tuple(x) for x in _hk}) == 1 and _hk[0]: + HDR = ", all in the GGUF header, which carries %d extra keys in each imatrix file: %s" % ( + len(_hk[0]), ", ".join("`%s`" % k for k in _hk[0])) +deltas = [T.get(i, {}).get("file_size_delta_bytes") for i, _ in TWINS] +if len(parts) == 6 and all(x is True for x in same) and all(d is not None for d in deltas): + J["imat_speed_note"] = ( + f"The imatrix changes scale values, not tensor types or sizes: tensor by tensor, each imatrix file has the " + f"same names, types and byte sizes as its standard twin, so the work per token is the same (the files differ " + f"in size by {deltas[0]} / {deltas[1]} / {deltas[2]} bytes{HDR})." + " " + f"Measured speed still differs — imatrix vs standard: " + "; ".join(parts) + "." + + (f" The largest of these decode gaps is {TG_NOISE:.1f} %; both cards call decode gaps below {DEC_EQ:.1f} % " + f"a tie." if TG_NOISE is not None and DEC_EQ is not None else "")) + + +# ---------- known issues (model facts + protocol; always emit) ---------- +ki = [ + "- **No MTP head.** The checkpoint's `config.json` declares `mtp_num_hidden_layers: 1`, but the weights contain " + "**no** `mtp.*` tensors (1,026 tensors total). There is no multi-token-prediction head and no speculative decoding " + "on these files — do not pass `--spec-type draft-mtp`.", + "- **The stock chat template ignores `enable_thinking`, and llama-server cannot separate its reasoning** " + "(see [Tool calling](#tool-calling)). Serve with the included `chat_template_enable_thinking.jinja` and " + "`--reasoning off`, and switch thinking per request with `enable_thinking` — see " + "[Reasoning controls](#reasoning-controls).", + "- **Earlier assistant turns are re-rendered with their reasoning**, so multi-turn contexts grow faster than with " + "templates that drop it.", + "- **`llama-server`'s host-RAM prompt cache defaults to 8 GiB** (`-cram 8192`). On a shared box, set `-cram` explicitly.", + "- Measured on Linux only (Ryzen AI Max+ 395, ROCm 7.2.4, unpatched `d3ca537`).", +] +_pn_vals = [b_.get("prompt_n_max") if b_.get("prompt_n_max") is not None else b_.get("prompt_n") + for b_ in (S.get("bench") or [])] +_pn = max((v for v in _pn_vals if v is not None), default=None) # longest prompt any timed decode followed +_sz = {r_.get("ctx"): r_ for r_ in (S.get("sizing") or []) if r_.get("label") == "strix-lean"} +_big = max(_sz) if _sz else None +_ctx_txt = ("" if _big is None else + f" (a {_big:,}-token context was loaded in the memory test, not benchmarked)" + if _sz[_big].get("result") != "LOAD_FAIL" else "") +for _c, _r in sorted(_sz.items()): + if _r.get("result") == "LOAD_FAIL": + ki.append(f"- **A {_c:,}-token context did not load** in the memory test (STRIX_LEAN + vision projector, " + f"q8_0 KV cache, one slot) on this box.") +ki.append(f"- **Not measured:** decode beyond a {_pn:,}-token prompt{_ctx_txt}, " if _pn is not None else + "- **Not measured:** decode at long context, ") +ki[-1] += ("long-context quality, video input, concurrency above 1, and task-level accuracy. Perplexity/KLD measure " + "next-token fidelity to BF16 on prose, not reasoning or code correctness.") +if have(ref.get("hip_rocm0_chunk1"), ref.get("cpu_chunk1"), ref.get("hip_rocm0_final")) and have(S.get("bf16", {}).get("ppl")): + ki.insert(0, f"- **Do not run the BF16 GGUF of this model on ROCm0 with this build.** `d3ca537` computes it wrong on " + f"that path: wikitext-2 perplexity {ref['hip_rocm0_final']:.1f} on ROCm0 vs {S['bf16']['ppl']:.2f} on the CPU " + f"(first window {ref['hip_rocm0_chunk1']:.1f} vs {ref['cpu_chunk1']:.2f}; also wrong with `-fa off`). " + f"The 4-bit files are not affected — their ROCm0 grades are in the table — and the BF16 file is not " + f"published here; the quality reference was computed on the CPU instead.") +if have(TD.get("nested_off_http500"), TD.get("nested_off_attempts")): + ki.insert(1, "- **llama-server rejects a tool call whose required arguments are not in the order the schema lists " + "them** — HTTP 500, *The model produced output that does not match the expected peg-native format*: " + "this build's parser for the XML tool-call format expects required arguments in definition order. " + "On the standard STRIX_LEAN file with the stock template and thinking off, the `nested-object` " + "request (three required arguments) " + "hit it in the tool suite%s and in %d of %d repeats; the model had written a well-formed call with " + "the arguments reordered. Be ready to retry on this error." % ( + "" if TD.get("gate_http500_logged") else " (not confirmed in the suite's server log)", + TD["nested_off_http500"], TD["nested_off_attempts"])) +_PR = (S.get("template_fix") or {}).get("probes_roff") or {} +_lk = [v for k, v in _PR.items() if k.split("|")[0] in ("reasoning_effort=high", "reasoning_effort=medium")] +if _lk: + ki.insert(2, "- **With the included template, do not set `reasoning_effort` to `high` or `medium`** — the reasoning " + "goes back into `content` (%d of %d probe replies). `enable_thinking: true` is the way to turn " + "thinking on." % (sum(1 for v in _lk if v.get("leaks")), len(_lk))) +_seat = [r_ for r_ in (S.get("seats") or {}).values() if r_.get("thinking_reasoning_len") is not None] +_short = [r_ for r_ in _seat if not r_.get("thinking_reply") and r_.get("thinking_reasoning_len")] +if _short: + ki.insert(3, "- **With thinking on, a very short answer can stay inside the think block.** Served as in the quick " + "start, `Reply with the single word: ready` (no tools, `enable_thinking: true`, temperature 0) came " + "back with the word in `reasoning_content` and an empty `content` on %d of %d FAST files tested.%s" % ( + len(_short), len(_seat), + " With thinking off the same request returned `ready` in `content`." + if all((r_.get("default_reply") or "").strip().lower().startswith("ready") for r_ in _seat) else "")) +J["std_known_issues"] = "\n".join(ki) +J["imat_known_issues"] = "\n".join([ + "- Calibration text is general-purpose English/code (bartowski `calibration_datav3.txt`); an imatrix built from " + "your own domain can do better on that domain.", + "- Graded on wikitext-2 *test*, a different corpus from the calibration text. Improvements on reasoning/code " + "tasks were not measured.", +] + ki) + +json.dump(J, open(sys.argv[2], "w"), indent=2) +print("judgments written:", sorted(J)) +for k, v in J.items(): + print(f"\n[{k}]\n{v}") diff --git a/recipe/pipeline/nex_phase1.sh b/recipe/pipeline/nex_phase1.sh new file mode 100644 index 0000000000000000000000000000000000000000..032e2b9fac9ed06b5257ec9f4bc715bbd1fb8c1c --- /dev/null +++ b/recipe/pipeline/nex_phase1.sh @@ -0,0 +1,37 @@ +#!/bin/bash +# Nex-N2.5-mini phase 1 (CPU): verified download -> BF16 GGUF + vision projector -> the three standard 4-bit tiers +# (King: STRIX_LEAN + COHERENT + FAST; no Q8/Q6) with head protection, read back by exact tensor name. +# Runs in the capped scope `nex-conv` (Agnes memory sizing waits for nex-* scopes). Waits until the Agnes BF16 +# re-grade (60 GiB on the GPU) is finished so the two large memory users never overlap. +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; T=/opt/llama-rocm/rocmfpx-724; B=$T/build-hipvk/bin; N=Nex-N2.5-mini +A=/mnt/models/agnes-3.0-flash +export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib TMPDIR=/mnt/models/.tmp PYTHONUNBUFFERED=1 +cd $W; mkdir -p gguf out logs +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +until [ -f logs/DOWNLOAD_RC ]; do sleep 20; done +if [ "$(cat logs/DOWNLOAD_RC)" != 0 ] || [ "$(cat logs/DOWNLOAD_VERIFY_RC 2>/dev/null)" != 0 ]; then + log "download or verify failed -> stop"; log "NEX_PHASE1_FAILED"; exit 1 +fi +log "download verified; waiting for the Agnes BF16 re-grade to leave the GPU" +until grep -q "R3 grade" $A/logs/regrade.log 2>/dev/null; do sleep 20; done + +log "C1 convert BF16 (the checkpoint has no mtp.* tensors, so no MTP block is emitted)" +python3 $T/convert_hf_to_gguf.py hf --outtype bf16 --model-name "$N" --outfile gguf/$N-BF16.gguf > logs/C1_convert.log 2>&1 +rc=$?; log "C1 exit=$rc"; [ $rc -eq 0 ] || { tail -30 logs/C1_convert.log; log "NEX_PHASE1_FAILED"; exit 2; } +log "C2 convert vision projector" +python3 $T/convert_hf_to_gguf.py hf --outtype bf16 --mmproj --model-name "$N" --outfile out/mmproj-$N-BF16.gguf > logs/C2_mmproj.log 2>&1 +rc=$?; log "C2 exit=$rc"; [ $rc -eq 0 ] || { tail -30 logs/C2_mmproj.log; log "NEX_PHASE1_FAILED"; exit 3; } +python3 readback.py - - gguf/$N-BF16.gguf | tee logs/C_readback.log +python3 readback.py - - out/mmproj-$N-BF16.gguf | tee -a logs/C_readback.log + +BF=gguf/$N-BF16.gguf; Q=$B/llama-quantize +log "Q1 standard tiers" +$Q --output-tensor-type q6_K $BF out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/Q1_q106.log 2>&1; log " q106 exit=$?" +$Q --output-tensor-type q6_K --token-embedding-type q6_K $BF out/$N-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/Q1_q102.log 2>&1; log " q102 exit=$?" +$Q --output-tensor-type q6_K $BF out/$N-Q4_0_ROCMFP4_FAST.gguf Q4_0_ROCMFP4_FAST 16 > logs/Q1_q103.log 2>&1; log " q103 exit=$?" +python3 readback.py Q6_K Q5_K out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf | tee logs/Q_readback.log +python3 readback.py Q6_K Q6_K out/$N-Q4_0_ROCMFP4_COHERENT.gguf | tee -a logs/Q_readback.log +python3 readback.py Q6_K - out/$N-Q4_0_ROCMFP4_FAST.gguf | tee -a logs/Q_readback.log +for l in Q1_q106 Q1_q102 Q1_q103; do printf "%-8s " $l; grep -oE "quant size\s*=\s*[0-9.]+ MiB \([0-9.]+ BPW\)" logs/$l.log; done | tee logs/Q_sizes.log +log "NEX_PHASE1_DONE" diff --git a/recipe/pipeline/nex_phase2.sh b/recipe/pipeline/nex_phase2.sh new file mode 100644 index 0000000000000000000000000000000000000000..35f24631425c86c5b84b67d3ce6469fc7675f165 --- /dev/null +++ b/recipe/pipeline/nex_phase2.sh @@ -0,0 +1,74 @@ +#!/bin/bash +# Nex-N2.5-mini phase 2 (GPU, box iced): BF16 reference + imatrix -> imatrix tiers (CPU, alongside the standard grades) +# -> KLD grade of all six 4-bit files + a repeat run -> speed on both backends + gates -> memory sizing -> un-ice. +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; A=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Nex-N2.5-mini +export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1 +cd $W; mkdir -p kld imat out-imat results +ulimit -c 1 # a crashing server must not pipe a multi-GiB core into apport on the full root disk +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +table(){ + python3 - "$1" <<'EOF' +import sys +out = [] +for l in open(sys.argv[1], errors="replace"): + t = l.split() + if len(t) > 2 and t[0].isdigit() and l.rstrip().endswith("%") and l.count("%") == 2: + out.append(" ".join(t)) +print("\n".join(out)) +EOF +} +until grep -q -E "NEX_PHASE1_(DONE|FAILED)" logs/phase1.log 2>/dev/null; do sleep 30; done +grep -q NEX_PHASE1_DONE logs/phase1.log || { log "phase 1 failed -> stop (box stays iced)"; log "NEX_PHASE2_FAILED"; exit 1; } +until grep -q AGNES_FINISH_DONE $A/logs/agnes_finish.log 2>/dev/null; do sleep 30; done +log "box handed over from Agnes" +PPL="$B/llama-perplexity -c 2048 -b 2048 -ngl 999 -fa on -dio -dev ROCm0" +BF=gguf/$N-BF16.gguf; Q=$B/llama-quantize + +log "N1 BF16 perplexity + base logits (wikitext-2 test, 40 chunks)" +$PPL -m $BF -f $A/calib/wikitext-2-raw/wiki.test.raw --chunks 40 --kl-divergence-base kld/bf16.kld > logs/N1_ppl_bf16.log 2>&1 +log "N1 exit=$? $(grep -E 'Final estimate' logs/N1_ppl_bf16.log)" + +log "N2 imatrix (BF16, bartowski calibration_datav3, c=512)" +$B/llama-imatrix -m $BF -f $A/calib/calibration_datav3.txt -o imat/$N.imatrix -ngl 999 -c 512 -b 512 -fa on -dio -dev ROCm0 \ + > logs/N2_imatrix.log 2>&1 +rc=$?; log "N2 exit=$rc" +[ $rc -eq 0 ] && [ -s imat/$N.imatrix ] || { log "imatrix failed -> stop (box stays iced)"; log "NEX_PHASE2_FAILED"; exit 2; } +sha256sum imat/$N.imatrix > imat/$N.imatrix.sha256 + +log "N3 imatrix tiers on CPU (background) while N4a grades the standard tiers on the GPU" +( + systemd-run --scope --quiet -p MemoryMax=24G -p MemorySwapMax=0 nice -n 5 bash -c " + $Q --imatrix imat/$N.imatrix --output-tensor-type q6_K $BF out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/N3_q106i.log 2>&1 + $Q --imatrix imat/$N.imatrix --output-tensor-type q6_K --token-embedding-type q6_K $BF out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/N3_q102i.log 2>&1 + $Q --imatrix imat/$N.imatrix --output-tensor-type q6_K $BF out-imat/$N-imatrix-Q4_0_ROCMFP4_FAST.gguf Q4_0_ROCMFP4_FAST 16 > logs/N3_q103i.log 2>&1" + python3 readback.py Q6_K Q5_K out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf > logs/N3_readback.log + python3 readback.py Q6_K Q6_K out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf >> logs/N3_readback.log + python3 readback.py Q6_K - out-imat/$N-imatrix-Q4_0_ROCMFP4_FAST.gguf >> logs/N3_readback.log + echo N3_DONE >> logs/N3_readback.log +) & +n3=$! +grade(){ $PPL -m "$2" --kl-divergence-base kld/bf16.kld --kl-divergence > logs/N4_kld_$1.log 2>&1 + log " $1 exit=$? $(grep -E 'Mean +KLD' logs/N4_kld_$1.log | tr -s ' ')"; } +grade q106 out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf +grade q102 out/$N-Q4_0_ROCMFP4_COHERENT.gguf +grade q103 out/$N-Q4_0_ROCMFP4_FAST.gguf +wait $n3; log "N3 done: $(grep -c ^PASS logs/N3_readback.log)/3 read-back PASS"; cat logs/N3_readback.log +for l in N3_q106i N3_q102i N3_q103i; do printf "%-9s " $l; grep -oE "quant size\s*=\s*[0-9.]+ MiB \([0-9.]+ BPW\)" logs/$l.log; done | tee -a logs/Q_sizes.log +log "N4b grade the imatrix tiers" +grade q106i out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf +grade q102i out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf +grade q103i out-imat/$N-imatrix-Q4_0_ROCMFP4_FAST.gguf + +log "N5 repeat the STRIX_LEAN grade; every per-chunk row must match" +$PPL -m out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf --kl-divergence-base kld/bf16.kld --kl-divergence > logs/N5_kld_q106_repeat.log 2>&1 +a=$(table logs/N5_kld_q106_repeat.log); b=$(table logs/N4_kld_q106.log); n=$(printf '%s\n' "$a" | grep -c .) +echo "{\"step\":\"N5\",\"rows\":$n,\"result\":\"$( [ "$n" -ge 40 ] && [ "$a" = "$b" ] && echo MATCH || echo MISMATCH )\"}" | tee results/nex_repeat.jsonl + +log "N6 speed + gates" +python3 nex_bench.py > logs/N6_bench.log 2>&1; log "N6 exit=$?" +log "N7 memory sizing" +bash nex_sizing.sh > logs/N7_sizing.log 2>&1; log "N7 exit=$?" +log "N8 un-ice" +bash $A/unice.sh > logs/N8_unice.log 2>&1; log "N8 exit=$? $(tail -1 logs/N8_unice.log)" +log "NEX_PHASE2_DONE" diff --git a/recipe/pipeline/nex_phase2b.sh b/recipe/pipeline/nex_phase2b.sh new file mode 100644 index 0000000000000000000000000000000000000000..34e672f75cd81513b7829c814a23773a24810c9a --- /dev/null +++ b/recipe/pipeline/nex_phase2b.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Nex-N2.5-mini phase 2b. Phase 2 stopped at 22:24Z: this build's ROCm0 path computes the BF16 MoE GGUF wrong +# (wikitext PPL 106 vs 7.8 for its own 4-bit quant; -fa off and "-ngl 0" with the default op offload are wrong too). +# Controls, chunk 1: pure CPU (-dev none --no-op-offload) 5.6964, Vulkan0 5.6953. +# => BF16 reference logits and the imatrix are computed on PURE CPU; every 4-bit file is graded on BOTH GPU backends +# against that reference; then speed + gates -> memory sizing -> Nex FAST seats -> un-ice (as phase 2). +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; A=/mnt/models/agnes-3.0-flash; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Nex-N2.5-mini +export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1 +cd $W; mkdir -p kld imat out-imat results +prlimit --pid $$ --core=1:1 # RLIMIT_CORE of exactly 1 BYTE (bash `ulimit -c 1` is 1 KiB and does not stop apport) +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +table(){ + python3 - "$1" <<'EOF' +import sys +out = [] +for l in open(sys.argv[1], errors="replace"): + t = l.split() + if len(t) > 2 and t[0].isdigit() and l.rstrip().endswith("%") and l.count("%") == 2: + out.append(" ".join(t)) +print("\n".join(out)) +EOF +} +chunk1(){ grep -oE '\[1\][0-9.]+' "$1" | head -1 | cut -d']' -f2; } +CPU="-dev none -ngl 0 --no-op-offload -t 16" +PPL="$B/llama-perplexity -c 2048 -b 2048" +BF=gguf/$N-BF16.gguf; Q=$B/llama-quantize; TXT=$A/calib/wikitext-2-raw/wiki.test.raw + +log "N1c BF16 perplexity + base logits on pure CPU (wikitext-2 test, 40 chunks)" +$PPL -m $BF -f $TXT --chunks 40 --kl-divergence-base kld/bf16.kld $CPU > logs/N1c_ppl_bf16_cpu.log 2>&1 +log "N1c exit=$? $(grep -E 'Final estimate' logs/N1c_ppl_bf16_cpu.log)" +a=$(chunk1 logs/N1c_ppl_bf16_cpu.log); b=$(chunk1 logs/diag_bf16_vk_faon.log) +if ! python3 -c "import sys; a, b = float('${a:-nan}'), float('${b:-nan}'); sys.exit(0 if abs(a - b) / b < 0.005 else 1)"; then + log "GATE FAIL: CPU chunk-1 PPL ${a:-missing} vs Vulkan0 control ${b:-missing} -> stop (box stays iced)"; log "NEX_PHASE2B_FAILED"; exit 1 +fi +echo "{\"step\":\"N1c\",\"cpu_chunk1\":$a,\"vulkan0_chunk1\":$b,\"hip_rocm0_chunk1\":$(chunk1 logs/N1_ppl_bf16.log),\"hip_rocm0_faoff_chunk1\":$(chunk1 logs/diag_bf16_rocm_faoff.log),\"hip_rocm0_final\":$(grep -oE 'Final estimate: PPL = [0-9.]+' logs/N1_ppl_bf16.log | grep -oE '[0-9.]+$'),\"result\":\"PASS\"}" | tee results/nex_reference.jsonl +log "N1c gate PASS (CPU $a vs Vulkan0 $b)" + +log "N2c imatrix on pure CPU (BF16, bartowski calibration_datav3, c=512)" +$B/llama-imatrix -m $BF -f $A/calib/calibration_datav3.txt -o imat/$N.imatrix -c 512 -b 512 $CPU > logs/N2c_imatrix_cpu.log 2>&1 +rc=$?; log "N2c exit=$rc" +[ $rc -eq 0 ] && [ -s imat/$N.imatrix ] || { log "imatrix failed -> stop (box stays iced)"; log "NEX_PHASE2B_FAILED"; exit 2; } +sha256sum imat/$N.imatrix > imat/$N.imatrix.sha256 + +log "N3 imatrix tiers on CPU (background) while the standard tiers are graded on the GPU" +( + systemd-run --scope --quiet -p MemoryMax=24G -p MemorySwapMax=0 nice -n 5 bash -c " + $Q --imatrix imat/$N.imatrix --output-tensor-type q6_K $BF out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 > logs/N3_q106i.log 2>&1 + $Q --imatrix imat/$N.imatrix --output-tensor-type q6_K --token-embedding-type q6_K $BF out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 > logs/N3_q102i.log 2>&1 + $Q --imatrix imat/$N.imatrix --output-tensor-type q6_K $BF out-imat/$N-imatrix-Q4_0_ROCMFP4_FAST.gguf Q4_0_ROCMFP4_FAST 16 > logs/N3_q103i.log 2>&1" + python3 readback.py Q6_K Q5_K out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf > logs/N3_readback.log + python3 readback.py Q6_K Q6_K out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf >> logs/N3_readback.log + python3 readback.py Q6_K - out-imat/$N-imatrix-Q4_0_ROCMFP4_FAST.gguf >> logs/N3_readback.log + echo N3_DONE >> logs/N3_readback.log +) & +n3=$! +grade(){ # tag file device logprefix + $PPL -m "$2" --kl-divergence-base kld/bf16.kld --kl-divergence -ngl 999 -fa on -dio -dev $3 > logs/$4_kld_$1.log 2>&1 + log " $4 $1 exit=$? $(grep -E 'Mean +KLD' logs/$4_kld_$1.log | tr -s ' ')" +} +for t in "q106 out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf" "q102 out/$N-Q4_0_ROCMFP4_COHERENT.gguf" "q103 out/$N-Q4_0_ROCMFP4_FAST.gguf"; do + set -- $t; grade $1 $2 ROCm0 N4; grade $1 $2 Vulkan0 N4v +done +wait $n3; log "N3 done: $(grep -c ^PASS logs/N3_readback.log)/3 read-back PASS"; cat logs/N3_readback.log +for l in N3_q106i N3_q102i N3_q103i; do printf "%-9s " $l; grep -oE "quant size\s*=\s*[0-9.]+ MiB \([0-9.]+ BPW\)" logs/$l.log; done | tee -a logs/Q_sizes.log +log "N4 grade the imatrix tiers (both backends)" +for t in "q106i out-imat/$N-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf" "q102i out-imat/$N-imatrix-Q4_0_ROCMFP4_COHERENT.gguf" "q103i out-imat/$N-imatrix-Q4_0_ROCMFP4_FAST.gguf"; do + set -- $t; grade $1 $2 ROCm0 N4; grade $1 $2 Vulkan0 N4v +done + +log "N5 repeat the STRIX_LEAN grade on both backends; every per-chunk row must match" +: > results/nex_repeat.jsonl +for pair in "ROCm0 N4 N5" "Vulkan0 N4v N5v"; do + set -- $pair + $PPL -m out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf --kl-divergence-base kld/bf16.kld --kl-divergence -ngl 999 -fa on -dio -dev $1 \ + > logs/$3_kld_q106_repeat.log 2>&1 + x=$(table logs/$3_kld_q106_repeat.log); y=$(table logs/$2_kld_q106.log); n=$(printf '%s\n' "$x" | grep -c .) + echo "{\"step\":\"$3\",\"device\":\"$1\",\"rows\":$n,\"result\":\"$( [ "$n" -ge 40 ] && [ "$x" = "$y" ] && echo MATCH || echo MISMATCH )\"}" | tee -a results/nex_repeat.jsonl +done + +log "N6 speed + gates" +python3 nex_bench.py > logs/N6_bench.log 2>&1; log "N6 exit=$?" +log "N7 memory sizing" +bash nex_sizing.sh > logs/N7_sizing.log 2>&1; log "N7 exit=$?" +log "N8 Nex FAST seats + un-ice" +bash $A/unice.sh > logs/N8_unice.log 2>&1; log "N8 exit=$? $(tail -1 logs/N8_unice.log)" +log "NEX_PHASE2_DONE" diff --git a/recipe/pipeline/nex_publish.py b/recipe/pipeline/nex_publish.py new file mode 100644 index 0000000000000000000000000000000000000000..5f7ff66c7b7b2966610fba57126ec35e419a8c92 --- /dev/null +++ b/recipe/pipeline/nex_publish.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""Stage (hardlinks), upload, and byte-verify the two Nex-N2.5-mini repos. +usage: nex_publish.py stage|upload|verify|readme [README path] +Upload through upload_watchdog.sh (40G cap, stall kill + retry) - see nex_cards.sh upload +Env: default HF_HOME (token); HF_XET_CACHE / TMPDIR pinned to /mnt/models by the caller.""" +import hashlib, json, os, shutil, sys, urllib.request + +W = "/mnt/models/nex-n2.5-mini"; N = "Nex-N2.5-mini"; U = "kingjones777" +REPOS = {"std": f"{U}/{N}-ROCmFP4-GGUF", "imat": f"{U}/{N}-ROCmFP4-imatrix-GGUF"} + + +def pubname(fn): + """Published filename: the quant token must be hyphen-delimited or the Hub cannot parse the variant + (`Q4_0_ROCMFP4_STRIX_LEAN` reads as the garbage label `Q4_0_ROCMFP`).""" + return fn.replace("Q4_0_ROCMFP4_", "Q4_0-ROCmFP4-") + + +MMPROJ = {f"mmproj-{N}-BF16.gguf": (f"{W}/out", f"mmproj-{N}-BF16.gguf")} +# the stock chat template plus one line that maps enable_thinking onto reasoning_effort (served with +# --chat-template-file; see the card's Reasoning controls) - shipped in both repos +TPL = {"chat_template_enable_thinking.jinja": (f"{W}/tpl", "chat_template_enable_thinking.jinja")} +TIERS = ("Q4_0_ROCMFP4_STRIX_LEAN", "Q4_0_ROCMFP4_COHERENT", "Q4_0_ROCMFP4_FAST") +FILES = { + "std": {**{pubname(f"{N}-{t}.gguf"): (f"{W}/out", f"{N}-{t}.gguf") for t in TIERS}, **MMPROJ, **TPL}, + "imat": {**{pubname(f"{N}-imatrix-{t}.gguf"): (f"{W}/out-imat", f"{N}-imatrix-{t}.gguf") for t in TIERS}, + f"{N}.imatrix": (f"{W}/imat", f"{N}.imatrix"), **MMPROJ, **TPL}, +} +P = "recipe/pipeline/" +SCRIPTS = ("nex_download.sh", "verify_download.py", "nex_phase1.sh", "nex_phase2.sh", "diag_bf16.sh", "nex_phase2b.sh", + "nex_harness.py", "nex_bench.py", "nex_sizing.sh", + "nex_cachegate.py", "readback.py", "nex_aggregate.py", "nex_judge.py", "nex_render.py", "nex_publish.py", + "nex_cards.sh", "upload_watchdog.sh", "nex_seats.sh", + # tool-call / reasoning diagnosis (stock template 6/14) and the template fix + "diag_tools_run.sh", "nex_tools_diag.py", "nex_reasoning_probe.py", "nex_tools_tpl.py", + "nex_seat_default_probe.py", "run_tools_c1.sh", "run_tools_roff.sh", "nex_refresh_cards.sh", "nex_publish_finish.sh") +LOGS = ( + "D2_verify_download.log", "phase1.log", "phase2.log", "Q_sizes.log", "N6_bench.log", "N7_sizing.log", + "b_n-vision-q106-faon.log", "b_n-vision-q106-faoff.log", + "C1_convert.log", "C2_mmproj.log", "C_readback.log", + "Q1_q106.log", "Q1_q102.log", "Q1_q103.log", "Q_readback.log", + "N1c_ppl_bf16_cpu.log", "N2c_imatrix_cpu.log", + # the stopped first attempt on ROCm0 and the backend diagnosis that followed (the Known issues evidence) + "N1_ppl_bf16.log", "diag_ppl_q106_rocm_c4.log", "diag_bf16.log", "diag_bf16_rocm_faoff.log", "diag_bf16_vk_faon.log", + "diag_q106_vk_faon.log", "diag_bf16_cpu.log", "diag_bf16_purecpu_c1.log", + "N3_q106i.log", "N3_q102i.log", "N3_q103i.log", "N3_readback.log", + *(f"{p}_kld_{t}.log" for p in ("N4", "N4v") for t in ("q106", "q102", "q103", "q106i", "q102i", "q103i")), + "N5_kld_q106_repeat.log", "N5v_kld_q106_repeat.log", "b_n-c3-q106.log", + "b_n-tools-q106.log", "diag_tools.log", "diag_tools_server.log", + "probe_reasoning.log", *(f"probe_reasoning_{c}.log" for c in + ("default", "fmt-deepseek", "srv-kwargs-high", "reasoning-on", "tpl-enable-thinking")), + "N6t_tools_tpl.log", "b_n-tools-q106-tpl.log", "b_n-tools-q106-tpl-probe.log", + "N6t_tools_tpl_medium.log", "b_n-tools-q106-tpl-medium.log", "b_n-tools-q106-tpl-medium-probe.log", + "probe_seat_default.log", "probe_seat_default_C1.log", "probe_seat_default_C2.log", + "N6t_tools_c1.log", "b_n-tools-q106-c1.log", "b_n-tools-q106-c1-probe.log", "b_n-vision-q106-c1-faon.log", + "N6t_tools_roff.log", "b_n-tools-q106-roff.log", "b_n-tools-q106-roff-r2.log", "b_n-tools-q106-roff-r3.log", + "b_n-tools-q106-roff-probe.log", "b_n-vision-q106-roff-faon.log", + "N8_unice.log", "N8a_seats.log", "N8b_seats.log", "N8c_seats.log", "N8d_seats.log", +) +RAW = ("nex_repeat.jsonl", "nex_reference.jsonl", "nex_bench.jsonl", "nex_sizing.jsonl", "hub_check.json", + "nex_tools_diag.json", "nex_reasoning_probe.json", "nex_seat_default_probe.json", "nex_template_shim.json", + "nex_template_shim_medium.json", "nex_template_shim_c1.json", "nex_template_shim_roff.json", + "nex_seats.jsonl", "nex_seats_plan.json") +TEMPLATES = ("chat_template_enable_thinking_medium.jinja", "chat_template_enable_thinking_v2.jinja") # tested, not used +RECIPE = { + "recipe/results_summary.json": "results/summary.json", + **{P + x: x for x in SCRIPTS}, + **{f"recipe/raw/{x}": f"results/{x}" for x in RAW}, + **{f"recipe/logs/{x}": f"logs/{x}" for x in LOGS}, + **{f"recipe/templates/{x}": f"tpl/{x}" for x in TEMPLATES}, + "recipe/templates/chat_template_stock.jinja": "hf/chat_template.jinja", +} + + +def separation_errors(kind, names): + """The imatrix build is its OWN repo (King): no imatrix weights in the standard repo, no standard weights in the + imatrix repo. The vision projector is the only model file both carry.""" + errs = [] + for fn in names: + if not fn.endswith(".gguf") or fn.startswith("mmproj-"): + continue + is_imat = "-imatrix-" in fn + if kind == "std" and is_imat: + errs.append(f"imatrix model file in the standard repo: {fn}") + if kind == "imat" and not is_imat: + errs.append(f"standard model file in the imatrix repo: {fn}") + if kind == "imat" and f"{N}.imatrix" not in names: + errs.append("imatrix repo is missing the .imatrix file") + return errs + + +assert REPOS["std"] != REPOS["imat"] +assert set(FILES["std"]) & set(FILES["imat"]) == set(MMPROJ) | set(TPL), "only the projector + template may be shared" +for _k in FILES: + assert not separation_errors(_k, list(FILES[_k])), separation_errors(_k, list(FILES[_k])) + + +def git_blob_sha1(p): + """What the Hub reports as `oid` for a file stored in plain git (no LFS/Xet pointer).""" + h = hashlib.sha1(b"blob %d\0" % os.path.getsize(p)) + with open(p, "rb") as fh: + for b in iter(lambda: fh.read(64 << 20), b""): + h.update(b) + return h.hexdigest() + + +def sha(p): + h = hashlib.sha256() + with open(p, "rb") as fh: + for b in iter(lambda: fh.read(64 << 20), b""): + h.update(b) + return h.hexdigest() + + +kind = sys.argv[2]; stage = f"{W}/hf-upload/{kind}"; repo = REPOS[kind] +if sys.argv[1] == "stage": + missing = [f"{d}/{src}" for fn, (d, src) in FILES[kind].items() if not os.path.exists(f"{d}/{src}")] + \ + [src for src in RECIPE.values() if not os.path.exists(f"{W}/{src}")] + if missing: + raise SystemExit(f"STAGE ABORT - missing: {missing}") + shutil.rmtree(stage, ignore_errors=True) + for fn, (d, src) in FILES[kind].items(): + os.makedirs(stage, exist_ok=True); os.link(f"{d}/{src}", f"{stage}/{fn}") + for dst, src in RECIPE.items(): + os.makedirs(os.path.dirname(f"{stage}/{dst}"), exist_ok=True); shutil.copy2(f"{W}/{src}", f"{stage}/{dst}") + sums = {fn: sha(f"{stage}/{fn}") for fn in sorted(FILES[kind])} + with open(f"{stage}/SHA256SUMS", "w") as fh: + for fn, h in sums.items(): + fh.write(f"{h} {fn}\n") + sizes = {x: os.path.getsize(f"{stage}/{x}") for x in sums} + json.dump(sizes, open(f"{stage}/../staged_sizes_{kind}.json", "w")) + print(json.dumps({"stage": stage, "files": len(sums), "bytes": sum(sizes.values()), "recipe_files": len(RECIPE)})) +elif sys.argv[1] == "upload": + from huggingface_hub import HfApi + api = HfApi() + print("whoami:", api.whoami()["name"], flush=True) + api.create_repo(repo, repo_type="model", private=False, exist_ok=True) + api.upload_large_folder(repo_id=repo, repo_type="model", folder_path=stage, + ignore_patterns=["README.md", ".cache/**"], num_workers=2) + print("UPLOAD_DONE", repo, flush=True) +elif sys.argv[1] == "verify": + tree = json.load(urllib.request.urlopen(f"https://huggingface.co/api/models/{repo}/tree/main?recursive=true", timeout=60)) + remote = {t["path"]: t for t in tree if t.get("type") == "file"} + local = dict(l.split()[::-1] for l in open(f"{stage}/SHA256SUMS").read().splitlines()) + bad = [] + for fn, h in local.items(): + r = remote.get(fn) + if not r: + bad.append((fn, "MISSING remote")); continue + if r["size"] != os.path.getsize(f"{stage}/{fn}"): + bad.append((fn, f"size {r['size']}")); continue + lfs = r.get("lfs") + if lfs: + if lfs.get("oid") != h: + bad.append((fn, f"sha {lfs.get('oid')} != {h[:12]}")) + elif r.get("oid") != git_blob_sha1(f"{stage}/{fn}"): + bad.append((fn, f"git oid {r.get('oid')} does not match the staged bytes")) + for fn in list(RECIPE) + ["SHA256SUMS"]: + if fn not in remote: + bad.append((fn, "MISSING remote")) + stale = sorted(x for x in remote if x not in local and x not in RECIPE and x not in ("SHA256SUMS", "README.md", ".gitattributes")) + bad += [(x, "SEPARATION") for x in separation_errors(kind, list(remote))] + print(json.dumps({"repo": repo, "checked": len(local), "bad": bad, "unexpected_remote_files": stale, + "result": "PASS" if not bad and not stale else "FAIL"})) + sys.exit(0 if not bad and not stale else 1) +elif sys.argv[1] == "readme": + from huggingface_hub import HfApi + HfApi().upload_file(path_or_fileobj=sys.argv[3], path_in_repo="README.md", repo_id=repo, repo_type="model", + commit_message="Model card: measured results, reproduction, known issues") + print("README_DONE", repo) +else: + raise SystemExit(f"usage: {sys.argv[0]} stage|upload|verify|readme [README path]") diff --git a/recipe/pipeline/nex_publish_finish.sh b/recipe/pipeline/nex_publish_finish.sh new file mode 100644 index 0000000000000000000000000000000000000000..154027e13653b16b1debc1c75da3e9091ce7f1d3 --- /dev/null +++ b/recipe/pipeline/nex_publish_finish.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Publish the two Nex repos (Jay, 2026-09-17). Order: +# 1. wait until the OxCoder speed + memory runs are finished (its phase-2 log reaches "O7 exit"), so this upload never +# holds the quiet-box lock they need; +# 2. upload std, then imat, through upload_watchdog.sh (40G cap, stall kill + retry) under the quiet-box lock; +# 3. verify both repos: model files by size + sha256, every recipe file by content (git sha1 / LFS sha256); +# 4. push the READMEs only once cards/.factcheck_ok exists (created by hand after the lane fact-checks are reviewed). +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; O=/mnt/models/oxcoder-9b; cd $W +export HF_XET_CACHE=/mnt/models/.hf-xet-cache TMPDIR=/mnt/models/.tmp HF_HUB_DISABLE_TELEMETRY=1 PYTHONUNBUFFERED=1 +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +log "waiting for the OxCoder speed + memory runs to finish (phase-2 log reaches O7)" +until grep -q -E "O7 exit=|OXCODER_PHASE2_(DONE|FAILED)" $O/logs/phase2.log 2>/dev/null; do sleep 30; done +exec 9>/mnt/models/.quiet-box.lock; log "waiting for the quiet-box lock"; flock 9; log "quiet-box lock held" +ok=1 +for k in std imat; do + bash upload_watchdog.sh nex-upload-$k $W python3 nex_publish.py upload $k || { log "upload $k FAILED"; ok=0; } +done +exec 9>&-; log "quiet-box lock released" +for k in std imat; do + python3 nex_publish.py verify $k > logs/verify_$k.log 2>&1 || ok=0 + log "verify $k: $(tail -c 300 logs/verify_$k.log)" +done +python3 - > logs/verify_recipe_content.log 2>&1 <<'PY' || ok=0 +import hashlib, json, os, sys, urllib.request +W = "/mnt/models/nex-n2.5-mini" +REPOS = {"std": "kingjones777/Nex-N2.5-mini-ROCmFP4-GGUF", "imat": "kingjones777/Nex-N2.5-mini-ROCmFP4-imatrix-GGUF"} +bad, n = [], 0 +for k, repo in REPOS.items(): + tree = json.load(urllib.request.urlopen( + "https://huggingface.co/api/models/%s/tree/main?recursive=true" % repo, timeout=60)) + remote = {t["path"]: t for t in tree if t.get("type") == "file"} + root = "%s/hf-upload/%s" % (W, k) + for dp, _, fns in os.walk(root + "/recipe"): + for fn in fns: + p = os.path.join(dp, fn) + rel = os.path.relpath(p, root) + r = remote.get(rel) + n += 1 + if r is None: + bad.append((k, rel, "missing remote")) + continue + data = open(p, "rb").read() + if r.get("lfs"): + same = r["lfs"].get("oid") == hashlib.sha256(data).hexdigest() + else: + same = r.get("oid") == hashlib.sha1(b"blob %d\0" % len(data) + data).hexdigest() + if not same: + bad.append((k, rel, "content differs")) +print(json.dumps({"recipe_files_checked": n, "bad": bad, "result": "PASS" if not bad else "FAIL"})) +sys.exit(1 if bad else 0) +PY +log "recipe content: $(tail -c 300 logs/verify_recipe_content.log)" +if [ $ok -ne 1 ]; then log "NEX_PUBLISH_FAILED (files)"; exit 1; fi +log "files verified; waiting for cards/.factcheck_ok before the READMEs" +until [ -f cards/.factcheck_ok ]; do sleep 30; done +for k in std imat; do + python3 nex_publish.py readme $k cards/README_$k.md > logs/readme_$k.log 2>&1 || ok=0 + log "readme $k: $(tail -1 logs/readme_$k.log)" +done +[ $ok -eq 1 ] && log "NEX_PUBLISH_DONE" || log "NEX_PUBLISH_FAILED (readme)" diff --git a/recipe/pipeline/nex_reasoning_probe.py b/recipe/pipeline/nex_reasoning_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..dfdca2f5b0d6bac00f073b2d3b4fd4b2294f30a3 --- /dev/null +++ b/recipe/pipeline/nex_reasoning_probe.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +"""Find a llama-server configuration that separates Nex's reasoning from `content` when thinking is on. +Diagnostic follow-up to nex_tools_diag.py (reasoning_content empty, `...` left in content for +reasoning_effort=high and for the template's default). Each config: start the unpatched server with extra flags +(and, for the last one, a template copy in which enable_thinking drives reasoning_effort), send the same three +thinking-on requests, record content / reasoning_content / tool calls. Diagnostic only -> results/nex_reasoning_probe.json.""" +import json, os, re, sys, time +from types import SimpleNamespace + +os.environ["AGNES_BIN"] = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin" +sys.path.insert(0, "/mnt/models/nex-n2.5-mini") +import nex_harness as H # noqa: E402 + +W = H.W +PORT = 18651 +OUT = f"{W}/results/nex_reasoning_probe.json" +TPL_SRC = f"{W}/hf/chat_template.jinja" +TPL_PATCHED = f"{W}/results/nex_template_enable_thinking.jinja" + +# template copy: if the caller passes enable_thinking (and no reasoning_effort), map it onto reasoning_effort +src = open(TPL_SRC).read() +shim = ("{%- if reasoning_effort is not defined and enable_thinking is defined %}" + "{%- set reasoning_effort = 'high' if enable_thinking else 'none' %}{%- endif %}\n") +open(TPL_PATCHED, "w").write(shim + src) + +CONFIGS = [ + ("default", [], {"reasoning_effort": "high"}), + ("fmt-deepseek", ["--reasoning-format", "deepseek"], {"reasoning_effort": "high"}), + ("srv-kwargs-high", ["--chat-template-kwargs", json.dumps({"reasoning_effort": "high"})], {}), + ("reasoning-on", ["--reasoning", "on"], {"reasoning_effort": "high"}), + ("tpl-enable-thinking", ["--chat-template-file", TPL_PATCHED], {"enable_thinking": True}), +] +REQS = [ + ("correct-decline", "What is 17 times 23? Answer directly."), + ("multi-arg", "What's the weather in Paris in celsius?"), + ("parallel", "Get the weather in Oslo AND in Lima, both in celsius. Call the tool for each city."), +] + + +def patched_popen_cmd(a, port, extra): + """H.Server builds and launches its argv in __init__; append the extra flags through a temporary Popen hook.""" + import subprocess + orig = subprocess.Popen + + def popen(cmd, *args, **kw): + if cmd and str(cmd[0]).endswith("llama-server"): + cmd = list(cmd) + list(extra) + return orig(cmd, *args, **kw) + subprocess.Popen = popen + try: + return H.Server(a, port) + finally: + subprocess.Popen = orig + + +report = {"template_shim": shim, "configs": {}} +for name, extra, kwargs in CONFIGS: + a = SimpleNamespace(model=f"{W}/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", dev="ROCm0", ctx=16384, + draft=None, mtp_infile=False, nmax=4, pmin=0.0, strict=False, + serverlog=f"{W}/logs/probe_reasoning_{name}.log") + rec = {"extra": extra, "kwargs": kwargs, "results": {}} + try: + s = patched_popen_cmd(a, PORT, extra) + except SystemExit as e: + rec["error"] = str(e) + report["configs"][name] = rec + print(name, "SERVER FAILED", e, flush=True) + continue + try: + for rn, prompt in REQS: + body = {"messages": [{"role": "user", "content": prompt}], "tools": H.TOOLS, "tool_choice": "auto", + "temperature": 0, "top_k": 1, "max_tokens": 2048, "chat_template_kwargs": kwargs} + try: + m = H.post(PORT, "/v1/chat/completions", body)["choices"][0]["message"] + c = m.get("content") or "" + rec["results"][rn] = {"content": c[:300], "reasoning": (m.get("reasoning_content") or "")[:300], + "tool_calls": [(t.get("function") or {}).get("name") for t in m.get("tool_calls") or []], + "leaks": [x for x in H.LEAK if x in c]} + except Exception as e: # noqa: BLE001 + rec["results"][rn] = {"error": repr(e)[:300]} + print(name, rn, json.dumps(rec["results"][rn])[:300], flush=True) + log = open(a.serverlog, errors="replace").read() + rec["chat_format_lines"] = sorted(set(re.findall(r"Chat format: \S+", log))) + rec["thinking_line"] = next((l.strip()[-80:] for l in log.splitlines() if "thinking =" in l), None) + finally: + s.stop() + report["configs"][name] = rec + time.sleep(3) +json.dump(report, open(OUT, "w"), indent=1) +print("NEX_REASONING_PROBE_DONE", OUT) diff --git a/recipe/pipeline/nex_refresh_cards.sh b/recipe/pipeline/nex_refresh_cards.sh new file mode 100644 index 0000000000000000000000000000000000000000..f4c063ab8699e6152ed5095cc25806b1b7ff955e --- /dev/null +++ b/recipe/pipeline/nex_refresh_cards.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Re-render the Nex cards after script or text fixes without re-staging the model files: summary (reuses the staged +# sha256), refresh every recipe copy in both stages with a FRESH mtime (plain cp; `cp -p` can hide a change from +# upload_large_folder's cache), judge, render. Light work, niced; no HF traffic. +set -euo pipefail +W=/mnt/models/nex-n2.5-mini; cd $W +log(){ echo "[$(date -u +%FT%TZ)] $*"; } +[ -s hf-upload/std/SHA256SUMS ] && [ -s hf-upload/imat/SHA256SUMS ] || { log "not staged yet -> run nex_cards.sh cards"; exit 3; } +nice -n 19 ionice -c3 python3 nex_aggregate.py > logs/aggregate_refresh.log 2>&1; tail -3 logs/aggregate_refresh.log +python3 - <<'PY' +import os, shutil, sys +sys.argv = ["nex_publish.py", "noop", "std"] +src = open("nex_publish.py").read() +ns = {} +exec(compile(src[:src.index("kind = sys.argv[2]")], "nex_publish_head", "exec"), ns) +W = ns["W"] +changed = 0 +for kind in ("std", "imat"): + stage = f"{W}/hf-upload/{kind}" + for dst, s in ns["RECIPE"].items(): + a, b = f"{W}/{s}", f"{stage}/{dst}" + if not os.path.exists(b) or open(a, "rb").read() != open(b, "rb").read(): + os.makedirs(os.path.dirname(b), exist_ok=True) + shutil.copyfile(a, b) # fresh mtime on purpose + changed += 1 + print("refreshed", kind, dst) +print("recipe files refreshed:", changed) +PY +python3 nex_judge.py results/summary.json results/judgments.json > logs/judge.log 2>&1; head -1 logs/judge.log | cut -c1-200 +rm -rf cards +python3 nex_render.py results/summary.json cards results/judgments.json \ + hf-upload/std/SHA256SUMS hf-upload/imat/SHA256SUMS hf-upload/staged_sizes.json +log "NEX_CARDS_REFRESHED" diff --git a/recipe/pipeline/nex_render.py b/recipe/pipeline/nex_render.py new file mode 100644 index 0000000000000000000000000000000000000000..465bf5fea99d5a67a59e6e8485e6b7d2218e4a52 --- /dev/null +++ b/recipe/pipeline/nex_render.py @@ -0,0 +1,752 @@ +#!/usr/bin/env python3 +"""Render both HF cards from summary.json + judgments.json. Every number comes from the summary; +absent => '—'. Judgment prose lives in nex_judge.py and only ever references computed values. +usage: nex_render.py """ +import json, math, os, sys + +S = json.load(open(sys.argv[1])); OUT = sys.argv[2]; os.makedirs(OUT, exist_ok=True) +JUDGE = json.load(open(sys.argv[3])) +SUMS = {} +for sf in sys.argv[4:6]: + if os.path.exists(sf): + for line in open(sf): + h, fn = line.split(maxsplit=1); SUMS[fn.strip()] = h +SIZES = json.load(open(sys.argv[6])) if len(sys.argv) > 6 and os.path.exists(sys.argv[6]) else {} +N = S["model"]; STD, IMAT = S["model_repo_std"], S["model_repo_imat"] +T = S.get("tiers") or {}; B = S.get("binary") or {}; SRC = S.get("source") or {}; AR = S.get("arch") or {} +GiB, MiB = 1024 ** 3, 1024 ** 2 +UB = S.get("n_ubatch") if S.get("n_ubatch") is not None else 1024 +CTX, GEN, REPS = 65536, 256, 3 +STD_TAGS, IMAT_TAGS = ("q106", "q102", "q103"), ("q106i", "q102i", "q103i") +NAMES = {"q106": "STRIX_LEAN", "q102": "COHERENT", "q103": "FAST", + "q106i": "STRIX_LEAN", "q102i": "COHERENT", "q103i": "FAST"} + + +def g(v, fmt="{:.2f}"): + return "—" if v is None else fmt.format(v) + + +def gib(b): + return g(None if b is None else b / GiB, "{:.2f} GiB") + + +def pm(v, e, fmt="{:.4f}"): + return "—" if v is None else (fmt.format(v) + ("" if e is None else " ± " + fmt.format(e))) + + +def bench(label): + return next((x for x in S.get("bench") or [] if x["label"] == label), None) + + +def tg(label): + x = bench(label) + return None if x is None else x.get("tg_median") + + +def pp(label): + x = bench(label) + return None if x is None else x.get("pp_median") + + +def gate(label): + return next((x for x in S.get("gates") or [] if x.get("label") == label), None) + + +def J(k): + return JUDGE.get(k, f"**[JUDGMENT PENDING: {k}]**") + + +def Jopt(k): + return JUDGE.get(k, "") + + +def speed_label(tag, dev): + return f"n-{tag}-{dev}" + + +def prompt_range(workload="code"): + """Prompt tokens processed by the timed requests of one workload (each carries a unique nonce).""" + lo, hi = [], [] + for b in S.get("bench") or []: + if b.get("workload") != workload: + continue + a_ = b.get("prompt_n_min") if b.get("prompt_n_min") is not None else b.get("prompt_n") + z_ = b.get("prompt_n_max") if b.get("prompt_n_max") is not None else b.get("prompt_n") + if a_ is not None and z_ is not None: + lo.append(a_); hi.append(z_) + if not lo: + return "—" + return f"{min(lo):,}" if min(lo) == max(hi) else f"{min(lo):,}–{max(hi):,}" + + +def tier(tag): + return T.get(tag) or {} + + +def tier_row(tag): + t = tier(tag); ratio = t.get("ppl_ratio") + r = [f"`{t.get('file') or '—'}`", g(t.get("ftype"), "{}"), gib(t.get("size_bytes")), g(t.get("bpw")), + pm(t.get("kld_mean"), t.get("kld_err")), g(t.get("same_top_p"), "{:.2f} %"), + pm(t.get("ppl"), t.get("ppl_err")) + ("" if ratio is None else f" (×{ratio:.4f})"), + g(tg(speed_label(tag, "rocm"))), g(tg(speed_label(tag, "vk"))), + g(pp(speed_label(tag, "rocm")), "{:.0f}")] + return "| " + " | ".join(r) + " |" + + +PN_TXT = prompt_range("code") +PROTOCOL = ( + f"Ryzen AI Max+ 395 (MAX-1), ROCm 7.2.4, unpatched `llama-server` at `d3ca537` (see [Quick start](#quick-start)), " + f"`-c {CTX}`, one request at a time (`--parallel 1`), greedy (`temp 0`, `top_k 1`), `ignore_eos` so every arm " + f"generates exactly {GEN} tokens " + f"after a code prompt of {PN_TXT} tokens (the first 30,000 characters of `convert_hf_to_gguf.py` plus an " + f"instruction), a unique nonce per request and `cache_prompt: false` (`cache_n = 0` asserted on " + f"every timed request), 1 warm-up then the median of {REPS}. Decode numbers are the server's own " + f"`predicted_per_second`. Box iced: no other model loaded.") +TABLE_HEAD = ("| File | ftype | Size | BPW | KLD vs BF16 ↓² | Same top-1 ↑ | PPL (× BF16) | TG ROCm0 | TG Vulkan0 | PP ROCm0 |\n" + "| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |") +TG_NOTE = f"TG = decode tokens/s after the {PN_TXT}-token code prompt, no draft head. PP = prefill tokens/s on ROCm0." + + +def quality_blurb(): + b = S.get("bf16") or {} + ch, nc, st = b.get("chunks"), b.get("n_ctx"), b.get("scored_tokens") + if ch is not None and nc is not None and st is not None: + scored = (f"{ch} chunks × {nc // 2 - 1:,} scored tokens each — the second half of every window, less its first token — = {st:,}") + else: + scored = "— chunks × — scored tokens" + return (f"Quality is graded against the **BF16 GGUF** (reference logits computed on the CPU) on a **held-out** corpus (wikitext-2 *test*, `-c 2048`, " + f"{scored}), never on the imatrix calibration text. **KLD** is the per-token KL divergence of each " + "quant's next-token distribution from BF16's on the same tokens — far more sensitive than perplexity.") + + +def bf16_row(): + b = S.get("bf16") or {} + return (f"| *BF16 reference* | {g(b.get('ftype'), '{}')} | {gib(b.get('size_bytes'))}³ | 16 | 0 | 100 % | " + f"{pm(b.get('ppl_paired'), b.get('ppl_paired_err'))}¹ | — | — | — |") + + +def footnotes(where="below"): + b = S.get("bf16") or {} + return (f"{TG_NOTE}\n" + f"¹ The BF16 PPL shown is the paired base every \"×\" ratio is computed against (averaged over the same scored tokens " + f"in the KL-divergence runs). The standalone BF16 run's own summary line reads {pm(b.get('ppl'), b.get('ppl_err'))}.\n" + f"² Quality columns: see *Where the quality numbers come from* {where}.\n" + f"³ BF16 conversion of the checkpoint; not published.") + + +YAML = """--- +license: apache-2.0 +base_model: nex-agi/Nex-N2.5-mini +base_model_relation: quantized +pipeline_tag: image-text-to-text +library_name: gguf +tags: + - gguf + - llama.cpp + - rocm + - amd + - rocmfp4 + - rocmfpx + - strix-halo + - amd-strix-halo + - gfx1151 + - ryzen-ai-max + - ryzen-ai-max-395 + - radeon-8060s + - moe + - reasoning + - multimodal + - vision + - nex + - qwen3.5 + - quantized{extra} +--- +""" + + +def cmake_block(): + commit = B.get("commit") or "d3ca537" + return f"""```bash +git clone https://github.com/charlie12345/ROCmFPX.git && cd ROCmFPX +git checkout {commit} +HIPCXX="$(hipconfig -l)/clang" HIP_PATH="$(hipconfig -R)" \\ +cmake -S . -B build -DCMAKE_BUILD_TYPE=Release \\ + -DGGML_HIP=ON -DGGML_VULKAN=ON -DGPU_TARGETS=gfx1151 \\ + -DGGML_HIP_GRAPHS=ON -DGGML_HIP_NO_VMM=ON -DLLAMA_CURL=OFF +cmake --build build --target llama-server -j +```""" + + +def serve_block(model_file): + env = ("env LD_LIBRARY_PATH=$PWD/build/bin:/opt/rocm/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 " + "GGML_HIP_ENABLE_UNIFIED_MEMORY=1 \\\n") + mm = f" --mmproj ~/models/nex/mmproj-{N}-BF16.gguf \\\n" + tpl = (" --chat-template-file ~/models/nex/chat_template_enable_thinking.jinja --reasoning off \\\n") + tail = f" -ngl 999 -fa on -dio --jinja -fit off --parallel 1 -dev ROCm0 \\\n -c {CTX} --host 127.0.0.1 --port 8080" + head = f"build/bin/llama-server \\\n -m ~/models/nex/{model_file} \\\n" + cmd = f"```bash\n{env}{head}{mm}{tpl}{tail}\n```" + w = Jopt("vision_quickstart_warning") + return cmd + ("\n\n" + w if w else "") + + +def curl_block(): + return """```bash +curl http://127.0.0.1:8080/v1/chat/completions \\ + -H 'Content-Type: application/json' \\ + -d '{ + "messages": [{"role": "user", "content": "Hello"}], + "temperature": 0.7, + "top_p": 0.95, + "top_k": 40, + "chat_template_kwargs": {"enable_thinking": true} + }' +```""" + + +def quick_start(model_file, repo): + return f"""**1. Download** + +```bash +hf download {repo} --local-dir ~/models/nex +``` + +**2. Build `llama-server`** — ROCmFPX at the measured commit (ROCm and Vulkan +prerequisites: the project's [build guide](https://github.com/charlie12345/ROCmFPX/blob/{B.get('commit') or 'd3ca537'}/docs/build.md)). No patch. + +{cmake_block()} + +(The CMake options of the measured build are listed in [Reproduction](#reproduction).) + +`d3ca537` is also in the history of the official [ROCmFPX/ROCmFPX](https://github.com/ROCmFPX/ROCmFPX) repository. + +**3. Serve** + +{serve_block(model_file)} + +(`LD_LIBRARY_PATH` avoids a soname clash on machines that also have a Vulkan-only llama.cpp build.) The exact measured +argv is in [Reproduction](#reproduction). + +**4. Call** — upstream sampling. Thinking is off unless the request sets `"enable_thinking": true` (as here; drop that +line for a direct answer): + +{curl_block()} + +| Flag | Why | +| --- | --- | +| `--chat-template-file …/chat_template_enable_thinking.jinja` | The model's own template plus one line (see [Reasoning controls](#reasoning-controls)). Without it llama-server leaves the reasoning in `content` and thinking-on tool calls fail ([measured](#tool-calling)). | +| `--reasoning off` | Thinking stays off unless a request passes `"enable_thinking": true`. | +| `--jinja` | Already on by default in this build; keep it on — the reasoning controls (`chat_template_kwargs`) and tool calling rely on the Jinja chat template. | +| `-fit off` | Autofit reads `MemAvailable` on integrated GPUs and can silently shrink context or push tensors to CPU. | +| `-cram ` | Not set above (default 8 GiB of host RAM for saved prompts). Set it on a shared box — see [Known issues](#known-issues-and-limits). | +| `--mmproj` | Loads the {AR.get('vision_layers', '—')}-layer vision tower. Drop the flag for text-only. | + +Requires a llama.cpp build with ROCmFP4 / ROCmFPX tensor-type support; stock llama.cpp rejects these tensor types.""" + + +def reasoning_block(): + tf = S.get("template_fix") or {} + shim = (tf.get("shim") or "").rstrip("\n") + shim_md = ("```jinja\n" + shim + "\n```") if shim else "**[JUDGMENT PENDING: template shim]**" + return f"""## Reasoning controls + +The model's own chat template switches thinking with `chat_template_kwargs.reasoning_effort` and ignores +`enable_thinking`: + +| `reasoning_effort` | What the stock template emits | +| --- | --- | +| `"none"` | empty `\\n\\n` (no thinking) | +| `"high"` | opens `\\n` (always think) | +| `"medium"`, unset, or anything else | opens `` and lets the model decide (adaptive; upstream default is `"medium"`) | + +llama-server decides how to split reasoning from the answer by rendering the template with `enable_thinking` on and +off. With this template both renders are the same, so it never extracts the reasoning ([measured](#tool-calling)). +`chat_template_enable_thinking.jinja` in this repo is the model's `chat_template.jinja` (sha256 +`{tf.get('source_sha256') or '—'}`) with one line added at the top (file sha256 `{tf.get('sha256') or '—'}`): + +{shim_md} + +Serve it with `--chat-template-file` and `--reasoning off`. + +{J('template_note')} + +Upstream serving (SGLang) uses `--reasoning-parser qwen3 --tool-call-parser qwen3_coder`. Recommended sampling: +temperature 0.7, top_p 0.95, top_k 40. + +Earlier assistant turns are re-rendered **with** their reasoning (contexts grow faster than with templates that drop +it). With thinking on and a small `max_tokens`, the whole budget can go to reasoning and `content` comes back empty — +raise `max_tokens` before concluding the model is broken. + +Tool calls use the XML-style `` format, which llama.cpp parses natively +through the Jinja chat template (on by default).""" + + +def speed_table(): + rows = ["| File | Backend | Workload | Decode tok/s (min–max) | Prefill tok/s |", + "| --- | --- | --- | ---: | ---: |"] + for tag in STD_TAGS + IMAT_TAGS: + t = tier(tag) + fn = t.get("file") or "—" + for dev, dn in (("rocm", "ROCm0"), ("vk", "Vulkan0")): + b = bench(speed_label(tag, dev)) + if b is None or b.get("tg_median") is None: + rows.append(f"| `{fn}` | {dn} | code | — | — |") + else: + rows.append(f"| `{fn}` | {dn} | code | {b['tg_median']:.2f} ({g(b.get('tg_min'))}–{g(b.get('tg_max'))}) | " + f"{g(b.get('pp_median'), '{:.0f}')} |") + for lab, dn in (("n-q106-rocm-prose", "ROCm0"), ("n-q106-vk-prose", "Vulkan0")): + b = bench(lab) + fn = tier("q106").get("file") or "—" + if b is None or b.get("tg_median") is None: + rows.append(f"| `{fn}` | {dn} | prose | — | — |") + else: + rows.append(f"| `{fn}` | {dn} | prose | {b['tg_median']:.2f} ({g(b.get('tg_min'))}–{g(b.get('tg_max'))}) | " + f"{g(b.get('pp_median'), '{:.0f}')} |") + return "\n".join(rows) + + +def cache_table(): + x = gate("n-c3-q106") + rows = ["| server | second-request prompt tokens reused | processed | warm reply = cold reply |", + "| --- | ---: | ---: | :---: |"] + if not x: + rows.append("| d3ca537, unpatched | — | — | — |") + return "\n".join(rows) + n, L, ident = x.get("n"), x.get("L"), x.get("identical") + got = sorted({r_.get("warm_cache_n") for r_ in x.get("rows") or []}, key=lambda v: (v is None, v)) + if n is None or L is None or not got or None in got: + rows.append("| d3ca537, unpatched | — | — | — |") + return "\n".join(rows) + if len(got) == 1: + reused, proc = f"**{got[0]:,}** of {L:,} (all {n} pairs)", f"{L - got[0]:,}" + else: + reused, proc = f"{got[0]:,}–{got[-1]:,} of {L:,} (varies across {n} pairs)", f"{L - got[-1]:,}–{L - got[0]:,}" + rows.append(f"| d3ca537, unpatched | {reused} | {proc} | {ident if ident is not None else '—'}/{n} |") + return "\n".join(rows) + + +def tools_block(): + t = gate("n-tools-q106") + fx = [gate(l) for l in ("n-tools-q106-roff", "n-tools-q106-roff-r2", "n-tools-q106-roff-r3")] + if not t and not any(fx): + return "_Not measured._" + names = ["multi-arg", "nested-object", "enum", "correct-decline", "multi-turn", "streaming", "parallel"] + rows = ["| check | quick start, thinking ON | quick start, thinking OFF | stock template, thinking ON | " + "stock template, thinking OFF |", + "| --- | :---: | :---: | :---: | :---: |"] + detail = (t or {}).get("detail") or {} + mk = lambda x: "—" if x is None else ("✅" if x else "❌") + + def count(n, think): + vals = [((x or {}).get("detail") or {}).get(f"{n}|think={think}") for x in fx] + if any(v is None for v in vals): + return "—" + return f"{sum(bool(v) for v in vals)}/{len(vals)}" + for n in names: + rows.append(f"| {n} | {count(n, True)} | {count(n, False)} | {mk(detail.get(f'{n}|think=True'))} | " + f"{mk(detail.get(f'{n}|think=False'))} |") + fn = os.path.basename(tier("q106").get("file") or "—") + tot = (f"**{sum(x['passed'] for x in fx)}/{sum(x['total'] for x in fx)}** over three passes with the quick-start " + f"configuration, **{t['passed']}/{t['total']}** with the stock template" + if all(fx) and t and t.get("passed") is not None else "Tool-calling suite") + return (f"{tot}, run on `{fn}`. Quick start = the included template file + `--reasoning off`, thinking switched " + f"with `enable_thinking`; stock = the model's own template, thinking switched with `reasoning_effort` " + f"(`high` / `none`). A check passes only with a native `tool_calls` entry carrying the right arguments " + f"and no raw XML or think tags left in `content`.\n\n" + "\n".join(rows)) + + +def vision_block(): + on, off = gate("n-vision-q106-faon"), gate("n-vision-q106-faoff") + if not on and not off: + return "_Not measured._" + vp = S.get("vision_probe") or {} + exp_row = next((x for x in (on, off) if x and x.get("expected")), None) + exp_txt = ", ".join(f"`{e.strip()}`" for e in exp_row["expected"].split(",")) if exp_row else "—" + def cell(x): + if not x: + return "—" + exp = x.get("expected") or "" + nexp = len(exp.split(",")) if exp else None + hits = x.get("hits") or [] + n = f"{len(hits)}/{nexp} terms" if nexp is not None else "—" + if x.get("result") == "PASS": + return f"✅ {n}" + return "❌ " + ("server stopped" if x.get("server_died") else ("request failed" if x.get("error") else n)) + rows = ["| | `-fa on` | `-fa off` |", "| --- | :---: | :---: |", + f"| STRIX_LEAN + projector | {cell(on)} | {cell(off)} |"] + txt = (f"Probe: a synthetic {vp.get('width', '—')}×{vp.get('height', '—')} image with a red circle and a blue square " + f"(a model that ignores the image cannot name both), sent to `{tier('q106').get('file') or '—'}` with " + f"`--mmproj`, temperature 0. Pass = the reply names every expected term ({exp_txt}).\n\n" + + "\n".join(rows) + "\n\n" + J("vision_note")) + ans = next((x for x in (on, off) if x and x.get("result") == "PASS" and x.get("answer")), None) + if ans: + a_ = (ans.get("answer") or "").strip() + which = "`-fa on`" if ans is on else "`-fa off`" + cut = a_[:300] + txt += f"\n\nReply ({which}):\n\n> {cut}" + (" …" if len(a_) >= 300 else "") + return txt + + +def files_table(names): + rows = ["| File | Size | sha256 |", "| --- | ---: | --- |"] + for fn, size in names: + size = SIZES.get(fn, size) + sz = ("—" if size is None else gib(size) if size >= GiB // 10 else + f"{size / MiB:.1f} MiB" if size >= MiB else f"{size / 1024:.1f} KiB") + rows.append(f"| `{fn}` | {sz} | `{SUMS.get(fn, '—')}` |") + return "\n".join(rows) + + +def receipts_table(tags, imat=False): + head = ("| File | `output.weight` | `token_embd.weight` | tensors |" + + (" imatrix entries | bytes differ from standard |" if imat else "")) + sep = "| --- | --- | --- | ---: |" + (" ---: | :---: |" if imat else "") + rows = [head, sep] + for k in tags: + t = tier(k) + r = (f"| `{t.get('file') or '—'}` | {t.get('output_weight') or '—'} | {t.get('token_embd') or '—'} | " + f"{g(t.get('tensors'), '{}')} |") + if imat: + dfs = t.get("differs_from_standard") + r += f" {t.get('imatrix_entries') if t.get('imatrix_entries') is not None else '—'} | " + r += f"{'yes' if dfs else ('no' if dfs is False else '—')} |" + rows.append(r) + return "\n".join(rows) + + +def repro(model_file, label): + b = bench(label) + sha = B.get("sha256") or {} + cmd = (b.get("cmd") if b else None) or "—" + return f"""``` +server : {B.get('repo') or '—'} @ {B.get('commit') or '—'} + unpatched; build dir {os.path.dirname(B['dir']) if B.get('dir') else '—'}, Release, Unix Makefiles, GGML_HIP=ON GGML_VULKAN=ON + GGML_HIP_GRAPHS=ON GGML_HIP_NO_VMM=ON GGML_NATIVE=ON AMDGPU_TARGETS=gfx1151 LLAMA_CURL=OFF + CMAKE_HIP_COMPILER=/opt/rocm-7.2.4/lib/llvm/bin/clang + sha256 llama-quantize {sha.get('llama-quantize') or '—'} + sha256 llama-imatrix {sha.get('llama-imatrix') or '—'} + sha256 llama-perplexity {sha.get('llama-perplexity') or '—'} + sha256 llama-server {sha.get('llama-server') or '—'} +source : {SRC.get('repo') or '—'} revision {SRC.get('revision') or '—'} +model : {model_file} (the argv below; every file was measured the same way) +argv : {cmd} +template : the quick-start tool-suite and image rows add --chat-template-file chat_template_enable_thinking.jinja + --reasoning off to this argv (recipe/pipeline/run_tools_roff.sh -> nex_tools_tpl.py; their server logs + read "chat template, thinking = 0"); the speed rows use the stock template +env : LD_LIBRARY_PATH=/bin:/opt/rocm-7.2.4/lib + HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1 +box : aimax — AMD Ryzen AI Max+ 395 / Radeon 8060S (gfx1151), 124 GiB, GTT 131072 MiB, + kernel 6.17.6-061706-generic, ROCm 7.2.4 +protocol : {PN_TXT}-token code prompt, {GEN} generated tokens, temp 0 / top_k 1, ignore_eos, cache_prompt false, + 1 warm-up + median of {REPS}, no co-resident models (box iced) +measured : {' to '.join(S.get('measured_range') or []) or S.get('measured') or '—'}, by the pipeline in recipe/ (every raw number in recipe/results_summary.json and recipe/raw/) +```""" + + +def methodology_std(): + return f"""```bash +# 1. convert: text model and the vision projector (the checkpoint has no mtp.* tensors) +python convert_hf_to_gguf.py hf --outtype bf16 --model-name {N} --outfile {N}-BF16.gguf +python convert_hf_to_gguf.py hf --outtype bf16 --mmproj --model-name {N} --outfile mmproj-{N}-BF16.gguf + +# 2. quantize from BF16 only; the LM head is forced up on every tier and read back by exact tensor name +llama-quantize --output-tensor-type q6_K {N}-BF16.gguf OUT Q4_0_ROCMFP4_STRIX_LEAN 16 +llama-quantize --output-tensor-type q6_K --token-embedding-type q6_K {N}-BF16.gguf OUT Q4_0_ROCMFP4_COHERENT 16 +llama-quantize --output-tensor-type q6_K {N}-BF16.gguf OUT Q4_0_ROCMFP4_FAST 16 + +# 3. BF16 reference logits on the CPU only (this build's ROCm0 path computes the BF16 MoE wrong — Known issues) +llama-perplexity -m {N}-BF16.gguf -f wikitext-2-raw/wiki.test.raw -c 2048 -b 2048 --chunks 40 --kl-divergence-base bf16.kld \\ + -dev none -ngl 0 --no-op-offload -t 16 +# 4. grade each shipped file against those logits, on each GPU backend +llama-perplexity -m OUT --kl-divergence-base bf16.kld --kl-divergence -c 2048 -b 2048 -ngl 999 -fa on -dio -dev ROCm0 +llama-perplexity -m OUT --kl-divergence-base bf16.kld --kl-divergence -c 2048 -b 2048 -ngl 999 -fa on -dio -dev Vulkan0 +``` + +Receipts (the built file is the receipt — exact tensor names, never a substring match; `recipe/logs/`): + +{receipts_table(STD_TAGS)}""" + + +def first_rocm_bullet(): + hc = S.get("hub_check") + others = [h for h in (hc or {}).get("header_checks") or [] if h.get("output_weight_type")] + if others: + return "".join( + f"- Another public ROCmFP4 build of this model exists — [{h['repo']}](https://huggingface.co/{h['repo']}): " + f"its `{h['file']}` stores `output.weight` as `{h['output_weight_type']}`" + f"{' and carries no imatrix metadata' if h.get('imatrix_keys') == [] else ''}. Every tier here keeps " + f"`output.weight` at `Q6_K`, and the imatrix builds are a separate repo.\n" for h in others) + if isinstance(hc, dict) and hc.get("rocm_builds_found") == 0: + return ("- **First ROCmFP4 build of this model** — no ROCm or Strix Halo build of Nex-N2.5-mini was on the Hub " + "at publication.\n") + return "" + + +def intro_arch(): + p = SRC.get("params") + ptxt = f"{p:,} parameters (BF16)" if p is not None else "— parameters" + return (f"{AR.get('layers', '—')}-layer Qwen3.5 MoE ({AR.get('linear_attn_layers', '—')} Gated DeltaNet linear-attention + " + f"{AR.get('full_attn_layers', '—')} full-attention layers), {AR.get('num_experts', '—')} routed experts / " + f"{AR.get('num_experts_per_tok', '—')} active, {AR.get('max_position_embeddings', '—'):,}-token context" + if AR.get("max_position_embeddings") is not None else + f"{AR.get('layers', '—')}-layer Qwen3.5 MoE, {ptxt}") + + +def std_card(): + L = tier("q106") + bf = S.get("bf16") or {} + nextn = bf.get("nextn_tensors") + no_mtp = ("the checkpoint ships no `mtp.*` weights" + + (f" (the converted BF16 GGUF reads back {nextn} `nextn` tensors)" if nextn is not None else "")) + return YAML.format(extra="") + f""" +# Nex-N2.5-mini — ROCmFP4 for AMD Strix Halo (gfx1151) + +ROCmFP4 / ROCmFPX quantizations of **[nex-agi/Nex-N2.5-mini](https://huggingface.co/nex-agi/Nex-N2.5-mini)** — +{g(SRC.get('params'), '{:,}')} parameters (BF16), {intro_arch()}, text + image — built and measured on an AMD Ryzen AI +Max+ 395 (Radeon 8060S, `gfx1151`). Upstream publishes no GGUF. + +{first_rocm_bullet()}- **Vision projector included.** +- **No MTP head.** `mtp_num_hidden_layers: 1` is declared in `config.json`, but {no_mtp}. There is no speculative + decoding on these files. +- Importance-matrix builds of the same three 4-bit tiers: **[{IMAT}](https://huggingface.co/{IMAT})**. + +## Which file should I use? + +{PROTOCOL} + +{TABLE_HEAD} +{chr(10).join(tier_row(k) for k in STD_TAGS)} +{bf16_row()} + +{footnotes()} + +{J('std_recommendation')} + +{quality_blurb()} + +**Where the quality numbers come from.** {J('quality_provenance')} + +{J('backend_quality_note')} + +## Quick start + +{quick_start(JUDGE.get('std_default') or L.get('file') or '—', STD)} + +{reasoning_block()} + +## Speed + +{speed_table()} + +{J('speed_note')} + +## Prompt caching + +Measured: pairs of requests that share a long code prefix and differ only in the closing instruction. The second +request of each pair runs warm (`cache_prompt: true`, resuming from what the first one left) and then cold +(`cache_prompt: false`), and the two replies are compared byte for byte. Every prompt is padded to one token length so +warm and cold see identical chunking. + +{cache_table()} + +{J('cache_note')} + +## Tool calling + +The template emits the XML-style `` format, which llama.cpp parses natively +through the Jinja chat template (on by default). Suite run through `llama-server`, at the checkpoint's recommended +sampling (temperature 0.7, top-p 0.95, top-k 40): + +{tools_block()} + +{J('tools_note')} + +## Vision + +`mmproj-{N}-BF16.gguf` is the {AR.get('vision_layers', '—')}-layer vision tower (width {AR.get('vision_width', '—')}), +loaded with `--mmproj`. Its attention follows the server's `-fa` setting, so both settings were checked. + +{vision_block()} + +## Memory + +{J('memory_note')} + +## Quantization methodology + +{methodology_std()} + +`tie_word_embeddings` is false, so the output head is a real tensor and `--output-tensor-type q6_K` does real work. +All three tiers pin `output.weight` to `q6_K`; COHERENT also pins `token_embd.weight` to `q6_K`, while STRIX_LEAN and +FAST keep their tier's own embedding type (shown in the receipts). + +## Reproduction + +{repro(L.get('file') or '—', 'n-q106-rocm')} + +## Files + +{files_table([(tier(k).get('file'), tier(k).get('size_bytes')) for k in STD_TAGS] + list((S.get('aux') or {}).items()))} + +`SHA256SUMS` covers every model file and the chat template file. `recipe/` holds the measurement pipeline (`recipe/pipeline/`), raw per-run +results (`recipe/raw/`), build and receipt logs (`recipe/logs/`), and `results_summary.json` with every measured value +on this card. Architecture facts (layer counts, vocabulary, vision depth) come from the checkpoint's `config.json` at +revision `{SRC.get('revision') or '—'}`. + +## Known issues and limits + +{J('std_known_issues')} + +## License and attribution + +Apache-2.0, inherited from the base model. Weights and architecture: **Nex-AGI** +([nex-agi/Nex-N2.5-mini](https://huggingface.co/nex-agi/Nex-N2.5-mini)). ROCmFP4 / ROCmFPX quantization format and +runtime: the ROCmFPX project. Quantization and measurements: kingjones777. +""" + + +def imat_effect_table(): + rows = ["| Tier | Build | Size | KLD vs BF16 ↓ | Same top-1 ↑ | PPL (× BF16) | 99th-pct KLD |", + "| --- | --- | ---: | ---: | ---: | ---: | ---: |"] + def d(a, b, k, ek): + x, y = tier(a).get(k), tier(b).get(k) + ex, ey = (tier(a).get(ek), tier(b).get(ek)) if ek else (None, None) + if None in (x, y): + return "—" + s = f"{(y - x) / x * 100:+.1f} %" + if ex is not None and ey is not None: + s += f" ({abs(y - x) / math.sqrt(ex * ex + ey * ey):.1f}σ)" + return s + for base, imat in (("q106", "q106i"), ("q102", "q102i"), ("q103", "q103i")): + for tag, lab in ((base, "standard"), (imat, "**imatrix**")): + x = tier(tag) + ratio = x.get("ppl_ratio") + rows.append(f"| {NAMES[tag]} | {lab} | {gib(x.get('size_bytes'))} | {pm(x.get('kld_mean'), x.get('kld_err'))} | " + f"{g(x.get('same_top_p'), '{:.2f} %')} | {pm(x.get('ppl'), x.get('ppl_err'))} " + f"({'—' if ratio is None else '×%.4f' % ratio}) | {g(x.get('kld_p99'), '{:.4f}')} |") + dpp = ("—" if None in (tier(base).get("same_top_p"), tier(imat).get("same_top_p")) + else f"{tier(imat)['same_top_p'] - tier(base)['same_top_p']:+.2f} pp") + rows.append(f"| | *Δ imatrix* | | {d(base, imat, 'kld_mean', 'kld_err')} | {dpp} | " + f"{d(base, imat, 'ppl', 'ppl_err')} | {d(base, imat, 'kld_p99', None)} |") + return "\n".join(rows) + + +def imat_card(): + im = S.get("imatrix") or {} + Li = tier("q106i") + entries = [tier(k).get("imatrix_entries") for k in IMAT_TAGS] + ent = next((e for e in entries if e is not None), None) + ent_txt = g(ent, "{:,}") if (ent is None or len(set(e for e in entries if e is not None)) <= 1) else \ + " / ".join(g(e, "{:,}") for e in entries) + return YAML.format(extra="\n - imatrix") + f""" +# Nex-N2.5-mini — ROCmFP4 **imatrix** for AMD Strix Halo (gfx1151) + +Importance-matrix-calibrated ROCmFP4 quantizations of +**[nex-agi/Nex-N2.5-mini](https://huggingface.co/nex-agi/Nex-N2.5-mini)** ({g(SRC.get('params'), '{:,}')} parameters, +{intro_arch()}, text + image). Companion to the standard build **[{STD}](https://huggingface.co/{STD})** — the same +three 4-bit tiers, same vision projector, same unpatched `d3ca537` server; the only difference is how each 4-bit +block's scale was chosen. There is no MTP head on either repo. + +## What the imatrix changes + +ROCmFP4 has an importance-weighted quantizer path: with `--imatrix`, each block's scale is chosen by an exhaustive +search that minimises error **weighted by how strongly the calibration activations use each weight**, instead of the +unweighted default. It changes **which** scales are picked at the **same** bit width and tensor types — so it moves +quality, not size, and per-token compute is identical. + +| | | +| --- | --- | +| calibration text | {im.get('calibration') or '—'} (the widely used community calibration set) | +| computed on | BF16 GGUF, {g(im.get('chunks'), '{}')} chunks × {g(im.get('n_ctx'), '{}')} tokens, {im.get('device') or '—'} | +| entries loaded | {ent_txt} (from the N3 quantize logs) | +| file | `{im.get('file') or '—'}` (GGUF format), sha256 `{im.get('sha256') or '—'}` | + +## Measured effect + +{quality_blurb()} The calibration text and the grading text are different corpora. + +{imat_effect_table()} + +σ = difference divided by the two runs' combined standard error. The two runs score the **same** tokens, so this is +conservative (paired noise is smaller). + +{J('imat_verdict')} + +**Where the quality numbers come from.** {J('quality_provenance')} + +{J('backend_quality_note')} + +## Which file should I use? + +{J('imat_recommendation')} + +{TABLE_HEAD} +{chr(10).join(tier_row(k) for k in IMAT_TAGS)} +{bf16_row()} + +{footnotes("above")} + +{J('imat_speed_note')} + +Full speed tables (both backends, prose vs code), prompt-cache, tool-calling and vision results are on +the [standard card](https://huggingface.co/{STD}). + +## Quick start + +{quick_start(JUDGE.get('imat_default') or Li.get('file') or '—', IMAT)} + +{reasoning_block()} + +## Quantization methodology + +```bash +llama-imatrix -m {N}-BF16.gguf -f calibration_datav3.txt -o {N}.imatrix \\ + -c 512 -b 512 -dev none -ngl 0 --no-op-offload -t 16 +llama-quantize --imatrix {N}.imatrix --output-tensor-type q6_K \\ + {N}-BF16.gguf {N}-imatrix-Q4_0_ROCMFP4_STRIX_LEAN.gguf Q4_0_ROCMFP4_STRIX_LEAN 16 +llama-quantize --imatrix {N}.imatrix --output-tensor-type q6_K --token-embedding-type q6_K \\ + {N}-BF16.gguf {N}-imatrix-Q4_0_ROCMFP4_COHERENT.gguf Q4_0_ROCMFP4_COHERENT 16 +llama-quantize --imatrix {N}.imatrix --output-tensor-type q6_K \\ + {N}-BF16.gguf {N}-imatrix-Q4_0_ROCMFP4_FAST.gguf Q4_0_ROCMFP4_FAST 16 +``` + +Receipts that the weighted path was actually taken, and that each shipped file differs from its standard twin: + +{receipts_table(IMAT_TAGS, imat=True)} + +## Reproduction + +{repro(Li.get('file') or '—', 'n-q106i-rocm')} + +## Files + +{files_table([(tier(k).get('file'), tier(k).get('size_bytes')) for k in IMAT_TAGS] + [(im.get('file'), im.get('size_bytes'))] + list((S.get('aux') or {}).items()))} + +## Known issues and limits + +{J('imat_known_issues')} + +## License and attribution + +Apache-2.0, inherited from the base model. Weights and architecture: **Nex-AGI** +([nex-agi/Nex-N2.5-mini](https://huggingface.co/nex-agi/Nex-N2.5-mini)). Calibration text: bartowski's +`calibration_datav3`. ROCmFP4 / ROCmFPX: the ROCmFPX project. Imatrix, quantization and measurements: kingjones777. +""" + + +open(os.path.join(OUT, "README_std.md"), "w").write(std_card()) +def fix_anchors(md, other_repo): + """Links to sections that exist only on the other card point there instead of to a missing anchor.""" + import re + slugs = {re.sub(r"[^a-z0-9 -]", "", h.strip().lower()).replace(" ", "-") + for h in re.findall(r"^#{1,6} (.+)$", md, flags=re.M)} + return re.sub(r"\]\(#([a-z0-9-]+)\)", + lambda m: m.group(0) if m.group(1) in slugs else "](https://huggingface.co/%s#%s)" % ( + other_repo, m.group(1)), md) + + +open(os.path.join(OUT, "README_imat.md"), "w").write(fix_anchors(imat_card(), STD)) +cards = open(os.path.join(OUT, "README_std.md")).read() + open(os.path.join(OUT, "README_imat.md")).read() +pend = sorted(set(x.split("JUDGMENT PENDING: ")[1].split("]")[0] for x in cards.split("**[")[1:] if "JUDGMENT PENDING" in x)) +dash_cells = cards.count("| — |") +print("rendered | bench rows =", len(S.get("bench") or []), "| gates =", len(S.get("gates") or []), + "| pending judgments:", pend, "| '—' cells:", dash_cells) diff --git a/recipe/pipeline/nex_seat_default_probe.py b/recipe/pipeline/nex_seat_default_probe.py new file mode 100644 index 0000000000000000000000000000000000000000..8e8c1a312ee41200e9c8ddf7c24c1b133c776707 --- /dev/null +++ b/recipe/pipeline/nex_seat_default_probe.py @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 +"""Which server-side default makes Nex behave well for clients that pass no chat_template_kwargs (the gateway case)? +Follow-up to nex_tools_tpl.py: with the `high` shim, a no-kwargs one-word request came back entirely in +reasoning_content (the model never closed its forced-open think block). Two candidate defaults, greedy probes: + C1 high shim + `--chat-template-kwargs {"enable_thinking": false}` (thinking off unless a client asks) + C2 v2 shim (explicit enable_thinking=false wins; otherwise enable_thinking maps to high unless reasoning_effort is + set) + `--chat-template-kwargs {"reasoning_effort": "medium"}` (upstream's adaptive default) +Diagnostic only -> results/nex_seat_default_probe.json.""" +import json, os, subprocess, sys, time +from types import SimpleNamespace + +os.environ["AGNES_BIN"] = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin" +sys.path.insert(0, "/mnt/models/nex-n2.5-mini") +import nex_harness as H # noqa: E402 + +W = H.W +SRC = open(f"{W}/hf/chat_template.jinja", "rb").read() +SHIM_V2 = ("{%- if enable_thinking is defined and not enable_thinking %}{%- set reasoning_effort = 'none' %}" + "{%- elif reasoning_effort is not defined and enable_thinking is defined %}" + "{%- set reasoning_effort = 'high' %}{%- endif %}\n") +TPL_V2 = f"{W}/tpl/chat_template_enable_thinking_v2.jinja" +open(TPL_V2, "wb").write(SHIM_V2.encode() + SRC) +CONFIGS = [ + ("C1", [f"--chat-template-file", f"{W}/tpl/chat_template_enable_thinking.jinja", + "--chat-template-kwargs", json.dumps({"enable_thinking": False})]), + ("C2", ["--chat-template-file", TPL_V2, "--chat-template-kwargs", json.dumps({"reasoning_effort": "medium"})]), +] +PROMPTS = [("correct-decline", "What is 17 times 23? Answer directly."), + ("single-word", "Reply with the single word: ready"), + ("multi-arg", "What's the weather in Paris in celsius?")] +KW = [("no-kwargs", None), ("enable_thinking=true", {"enable_thinking": True}), + ("enable_thinking=false", {"enable_thinking": False})] +_orig = subprocess.Popen +report = {"shim_v2": SHIM_V2, "configs": {}} +for name, extra in CONFIGS: + def _popen(cmd, *a, _extra=extra, **k): + if cmd and str(cmd[0]).endswith("llama-server"): + cmd = list(cmd) + list(_extra) + return _orig(cmd, *a, **k) + subprocess.Popen = _popen + a = SimpleNamespace(model=f"{W}/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", dev="ROCm0", ctx=16384, + draft=None, mtp_infile=False, nmax=4, pmin=0.0, strict=False, + serverlog=f"{W}/logs/probe_seat_default_{name}.log") + try: + s = H.Server(a, 18653) + finally: + subprocess.Popen = _orig + rec = {"extra": extra, "results": {}} + try: + for kn, kw in KW: + for pn, prompt in PROMPTS: + body = {"messages": [{"role": "user", "content": prompt}], "tools": H.TOOLS, "tool_choice": "auto", + "temperature": 0, "top_k": 1, "max_tokens": 2048} + if kw is not None: + body["chat_template_kwargs"] = kw + try: + m = H.post(18653, "/v1/chat/completions", body)["choices"][0]["message"] + c = m.get("content") or "" + r = {"content": c[:120], "reasoning_len": len(m.get("reasoning_content") or ""), + "tool_calls": [(t.get("function") or {}).get("name") for t in m.get("tool_calls") or []], + "leaks": [x for x in H.LEAK if x in c]} + except Exception as e: # noqa: BLE001 + r = {"error": repr(e)[:300]} + rec["results"][f"{kn}|{pn}"] = r + print(name, kn, pn, json.dumps(r)[:220], flush=True) + finally: + s.stop() + report["configs"][name] = rec + time.sleep(3) +json.dump(report, open(f"{W}/results/nex_seat_default_probe.json", "w"), indent=1) +print("NEX_SEAT_DEFAULT_PROBE_DONE") diff --git a/recipe/pipeline/nex_seats.sh b/recipe/pipeline/nex_seats.sh new file mode 100644 index 0000000000000000000000000000000000000000..febbadf10f9a85dce171da696a5ddc6e0acf4565 --- /dev/null +++ b/recipe/pipeline/nex_seats.sh @@ -0,0 +1,177 @@ +#!/bin/bash +# King 2026-09-16: "leave the fast of the models on this machine for swap i want to test them" -> the two Nex-N2.5-mini +# FAST files (standard + imatrix) become ON-DEMAND seats: DISABLED systemd units King starts by hand (llama-swap is +# masked on MAX-1). Every setting comes from this build's measurements (backend, -fa, context, memory cap); a missing +# measurement stops the script instead of guessing. Each seat is smoke-tested once (start -> /health -> one real +# completion -> stop) while the box is still iced, and through the gateway when a route already exists. +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Nex-N2.5-mini +# the stock template switches thinking only via reasoning_effort, which llama-server's parser analysis cannot see +# (reasoning leaks into content); this copy maps enable_thinking onto it, and --reasoning off makes thinking opt-in +# per request (enable_thinking: true). Measured: tool suite 40/42 over 3 passes vs 6/14 stock (nex_tools_tpl.py). +TPL=$W/tpl/chat_template_enable_thinking.jinja +GW=http://192.168.0.87:4000; GWKEY=sk-homecloud-master +cd $W +prlimit --pid $$ --core=1:1 # 1 BYTE (bash `ulimit -c 1` is 1 KiB and does not stop apport) +log(){ echo "[$(date -u +%FT%TZ)] $*"; } + +python3 - > results/nex_seats_plan.json <<'PY' || { log "PLAN FAILED -> no seats"; exit 2; } +import json, math, sys +W = "/mnt/models/nex-n2.5-mini"; N = "Nex-N2.5-mini" +rows = [json.loads(l) for l in open(f"{W}/results/nex_bench.jsonl") if l.strip()] +bench = {r["label"]: r for r in rows if "tg_median" in r} +gate = {r["label"]: r for r in rows if "tg_median" not in r} +siz = [json.loads(l) for l in open(f"{W}/results/nex_sizing.jsonl") if l.strip()] +EQ = 0.03 # a backend must win decode by more than this AND not lose prefill by more than this to replace ROCm0 + +def backend(tag): + r, v = bench.get(f"n-{tag}-rocm"), bench.get(f"n-{tag}-vk") + if not (r and v) or None in (r.get("tg_median"), v.get("tg_median"), r.get("pp_median"), v.get("pp_median")): + sys.exit(f"missing bench rows for {tag}") + vk = v["tg_median"] > r["tg_median"] * (1 + EQ) and v["pp_median"] >= r["pp_median"] * (1 - EQ) + return ("Vulkan0" if vk else "ROCm0"), {"rocm": [r["tg_median"], r["pp_median"]], "vk": [v["tg_median"], v["pp_median"]]} + +on, off = gate.get("n-vision-q106-faon"), gate.get("n-vision-q106-faoff") +if not (on and off): + sys.exit("missing vision gates") +fa, mmproj = ("on", True) if on.get("result") == "PASS" else (("off", True) if off.get("result") == "PASS" else ("on", False)) + +ok = [s for s in siz if s.get("label") == "strix-lean" and s.get("footprint_after_8k_gib") is not None] +if not ok: + sys.exit("missing sizing rows") +best = max(ok, key=lambda s: s["ctx"]) # largest context that loaded and served a request +cap = int(math.ceil((best["footprint_after_8k_gib"] * 1.15 + 2) * 1024 / 1000.0) * 1000) + +seats = [] +for unit, port, tag, path, label in ( + ("max1-nex-fast", 8097, "q103", f"{W}/out/{N}-Q4_0_ROCMFP4_FAST.gguf", "FAST"), + ("max1-nex-fast-imat", 8098, "q103i", f"{W}/out-imat/{N}-imatrix-Q4_0_ROCMFP4_FAST.gguf", "FAST imatrix")): + dev, speeds = backend(tag) + seats.append({"unit": unit, "port": port, "tag": tag, "model": path, "label": label, "dev": dev, "speeds": speeds, + "fa": fa, "mmproj": f"{W}/out/mmproj-{N}-BF16.gguf" if mmproj else None, "ctx": best["ctx"], + "memory_max_mib": cap, "sized_from": best, + "gateway_model": "nex-n2.5-mini-fast@max1" if tag == "q103" else "nex-n2.5-mini-fast-imatrix@max1"}) +print(json.dumps({"seats": seats, "vision": {"faon": on.get("result"), "faoff": off.get("result")}}, indent=1)) +PY +log "plan: $(python3 -c "import json; d=json.load(open('results/nex_seats_plan.json')); print([(s['unit'], s['dev'], 'fa '+s['fa'], s['ctx'], str(s['memory_max_mib'])+'M', bool(s['mmproj'])) for s in d['seats']])")" + +n=$(python3 -c "import json; print(len(json.load(open('results/nex_seats_plan.json'))['seats']))") +fail=0 +for i in $(seq 0 $((n - 1))); do + eval "$(python3 - $i <<'PY' +import json, shlex, sys +s = json.load(open("/mnt/models/nex-n2.5-mini/results/nex_seats_plan.json"))["seats"][int(sys.argv[1])] +for k in ("unit", "port", "model", "label", "dev", "fa", "ctx", "memory_max_mib", "gateway_model"): + print(f"{k.upper()}={shlex.quote(str(s[k]))}") +print(f"MMPROJ={shlex.quote(s['mmproj'] or '')}") +PY +)" + [ -s "$MODEL" ] || { log "$UNIT: model file missing ($MODEL)"; fail=1; continue; } + [ -s "$TPL" ] || { log "$UNIT: template file missing ($TPL)"; fail=1; continue; } + if ss -tln | grep -q ":$PORT "; then log "$UNIT: port $PORT already in use -> skip"; fail=1; continue; fi + MM=""; [ -n "$MMPROJ" ] && MM="--mmproj $MMPROJ" + tmp=$(mktemp /tmp/$UNIT.XXXX.service) + cat > $tmp < /tmp/$UNIT.verify 2>&1; vrc=$? + if [ $vrc -ne 0 ] || grep -qiE "error|invalid|unknown|ignoring" /tmp/$UNIT.verify; then + log "$UNIT: unit did not verify (rc=$vrc):"; cat /tmp/$UNIT.verify; rm -f $tmp; fail=1; continue + fi + [ -f /etc/systemd/system/$UNIT.service ] && cp /etc/systemd/system/$UNIT.service /etc/systemd/system/.$UNIT.service.bak-$(date +%s) + install -m 0644 $tmp /etc/systemd/system/$UNIT.service; rm -f $tmp /tmp/$UNIT.service /tmp/$UNIT.verify + systemctl daemon-reload + systemctl disable $UNIT.service >/dev/null 2>&1 # on-demand: never part of boot + log "$UNIT written ($DEV, -fa $FA, ctx $CTX, MemoryMax ${MEMORY_MAX_MIB}M) -> smoke test" + t0=$(date +%s); systemctl start $UNIT.service + ok=0 + for k in $(seq 1 180); do + curl -s -m 3 http://127.0.0.1:$PORT/health | grep -q '"ok"' && { ok=1; break; } + systemctl is-active --quiet $UNIT.service || { log " $UNIT not active"; break; } + sleep 5 + done + load_s=$(( $(date +%s) - t0 )) + if [ $ok -ne 1 ]; then + log " FAIL $UNIT never healthy"; journalctl -u $UNIT.service -n 20 --no-pager | tail -20 + systemctl stop $UNIT.service; fail=1; continue + fi + python3 - "$UNIT" "$PORT" "$GATEWAY_MODEL" "$load_s" "$GW" "$GWKEY" <<'PY' | tee -a results/nex_seats.jsonl +import json, sys, time, urllib.request +unit, port, gwm, load_s, gw, key = sys.argv[1], int(sys.argv[2]), sys.argv[3], int(sys.argv[4]), sys.argv[5], sys.argv[6] +body = {"messages": [{"role": "user", "content": "Reply with the single word: ready"}], "max_tokens": 64, + "temperature": 0, "chat_template_kwargs": {"enable_thinking": False}} +def call(url, b, hdr): + req = urllib.request.Request(url, data=json.dumps(b).encode(), headers={"Content-Type": "application/json", **hdr}) + return json.load(urllib.request.urlopen(req, timeout=600)) +row = {"unit": unit, "port": port, "load_s": load_s, "time": time.strftime("%FT%TZ", time.gmtime())} +try: + r = call(f"http://127.0.0.1:{port}/v1/chat/completions", body, {}) + row["direct_reply"] = (r["choices"][0]["message"].get("content") or "")[:60] + row["direct_tg"] = r.get("timings", {}).get("predicted_per_second") +except Exception as e: + row["direct_error"] = str(e)[:200] +try: # default request (no kwargs): thinking off by default; nothing of a think block may reach content + r = call(f"http://127.0.0.1:{port}/v1/chat/completions", + {"messages": body["messages"], "max_tokens": 1024, "temperature": 0}, {}) + m = r["choices"][0]["message"] + row["default_reply"] = (m.get("content") or "")[:60] + row["default_reasoning_len"] = len(m.get("reasoning_content") or "") + row["default_leak"] = any(x in (m.get("content") or "") for x in ("", "")) +except Exception as e: + row["default_error"] = str(e)[:200] +try: # thinking requested: nothing of the think block may reach content. With thinking forced on and no tools, a + # one-word answer can stay inside the think block (answer in reasoning_content, content empty) - model behaviour + r = call(f"http://127.0.0.1:{port}/v1/chat/completions", + {"messages": body["messages"], "max_tokens": 1024, "temperature": 0, + "chat_template_kwargs": {"enable_thinking": True}}, {}) + m = r["choices"][0]["message"] + row["thinking_reply"] = (m.get("content") or "")[:60] + row["thinking_reasoning_len"] = len(m.get("reasoning_content") or "") + row["thinking_leak"] = any(x in (m.get("content") or "") for x in ("", "")) +except Exception as e: + row["thinking_error"] = str(e)[:200] +try: + names = [m["id"] for m in json.load(urllib.request.urlopen(urllib.request.Request( + f"{gw}/v1/models", headers={"Authorization": f"Bearer {key}"}), timeout=30))["data"]] + if gwm in names: + r = call(f"{gw}/v1/chat/completions", {**body, "model": gwm}, {"Authorization": f"Bearer {key}"}) + row["gateway_model"] = gwm + row["gateway_reply"] = (r["choices"][0]["message"].get("content") or "")[:60] + else: + row["gateway_model"] = None # route not added yet +except Exception as e: + row["gateway_error"] = str(e)[:200] +row["result"] = "PASS" if row.get("direct_reply") and not row.get("gateway_error") and \ + row.get("default_reply") and row.get("default_leak") is False and \ + (row.get("thinking_reply") or row.get("thinking_reasoning_len")) and row.get("thinking_leak") is False and \ + (row.get("gateway_model") is None or row.get("gateway_reply")) else "FAIL" +print(json.dumps(row)) +PY + grep -q "\"unit\": \"$UNIT\".*\"result\": \"PASS\"" <(tail -1 results/nex_seats.jsonl) || fail=1 + systemctl stop $UNIT.service + for k in $(seq 1 30); do systemctl is-active --quiet $UNIT.service || break; sleep 2; done + log " $UNIT stopped (enabled: $(systemctl is-enabled $UNIT.service 2>&1))" + sleep 5 +done +log "NEX_SEATS_DONE fail=$fail" +exit $fail diff --git a/recipe/pipeline/nex_sizing.sh b/recipe/pipeline/nex_sizing.sh new file mode 100644 index 0000000000000000000000000000000000000000..4949a96081c77a00754a4a0124812f2f6c14ae02 --- /dev/null +++ b/recipe/pipeline/nex_sizing.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Nex-N2.5-mini memory footprint (drop in MemAvailable): STRIX_LEAN + vision projector, q8_0 KV, one slot, no draft head. +set -uo pipefail +W=/mnt/models/nex-n2.5-mini; B=/opt/llama-rocm/rocmfpx-724/build-hipvk/bin; N=Nex-N2.5-mini; cd $W +export LD_LIBRARY_PATH=$B:/opt/rocm-7.2.4/lib HSA_OVERRIDE_GFX_VERSION=11.5.1 GGML_HIP_ENABLE_UNIFIED_MEMORY=1 +prlimit --pid $$ --core=1:1 # 1 BYTE (bash `ulimit -c 1` is 1 KiB and does not stop apport) +# One quiet box: publish_v2.sh upload takes the same lock, so an HF upload never skews MemAvailable here. +exec 9>/mnt/models/.quiet-box.lock +echo "[$(date -u +%FT%TZ)] waiting for the quiet-box lock"; flock 9; echo "[$(date -u +%FT%TZ)] quiet-box lock held" +avail(){ awk '/MemAvailable/{printf "%.2f", $2/1048576}' /proc/meminfo; } +sync; sleep 5 +measure(){ # $1 label $2 ctx $3 model + local a0=$(avail) + $B/llama-server -m "$3" -dev ROCm0 -ngl 999 -fa on --no-mmap -fit off -c $2 -ctk q8_0 -ctv q8_0 -np 1 -b 2048 -ub 1024 -t 16 \ + --jinja --mmproj out/mmproj-$N-BF16.gguf -cram 512 --host 127.0.0.1 --port 18720 9>&- > logs/N7_sizing_$1_c$2.log 2>&1 & + local pid=$! ok=0 + for i in $(seq 1 120); do curl -s -m 2 http://127.0.0.1:18720/health | grep -q ok && { ok=1; break; }; kill -0 $pid 2>/dev/null || break; sleep 3; done + if [ $ok -ne 1 ]; then echo "{\"label\":\"$1\",\"ctx\":$2,\"result\":\"LOAD_FAIL\"}" | tee -a results/nex_sizing.jsonl; kill $pid 2>/dev/null; sleep 5; return; fi + local a1=$(avail) + python3 - </dev/null; sleep 8 +} +for C in 65536 262144; do measure strix-lean $C out/$N-Q4_0_ROCMFP4_STRIX_LEAN.gguf; done +echo NEX_SIZING_DONE diff --git a/recipe/pipeline/nex_tools_diag.py b/recipe/pipeline/nex_tools_diag.py new file mode 100644 index 0000000000000000000000000000000000000000..a880564a717107819ff6f4e9fb6993785614d20d --- /dev/null +++ b/recipe/pipeline/nex_tools_diag.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Diagnose the Nex tool-call gate (n-tools-q106: 6/14 - every thinking-on check failed, and nested-object failed +with thinking off on an HTTP 500 "does not match the expected peg-native format"). nex_bench.py keeps only the last +800 characters of the harness output, so the per-check reasons were lost. + +Same binary, file, server flags and tool schema as the gate, but every raw reply is kept (content, reasoning_content, +tool_calls, HTTP error body). Variants, one server session: + gate_on reasoning_effort=high, gate sampling (temp 0.7, top_p 0.95, top_k 40) - the failing half + gate_off reasoning_effort=none, gate sampling - the passing half + on_greedy reasoning_effort=high, temp 0 / top_k 1 - is it sampling? + adaptive no reasoning_effort (the template's adaptive mode), gate sampling + nested_off_x3 the nested-object request three more times with thinking off - is the 500 repeatable? +Diagnostic only (not a card measurement) -> results/nex_tools_diag.json.""" +import json, os, sys, time, urllib.error, urllib.request +from types import SimpleNamespace + +os.environ["AGNES_BIN"] = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin" # the gate ran on the unpatched server +sys.path.insert(0, "/mnt/models/nex-n2.5-mini") +import nex_harness as H # noqa: E402 + +W = H.W +PORT = 18650 +OUT = f"{W}/results/nex_tools_diag.json" +a = SimpleNamespace(model=f"{W}/out/Nex-N2.5-mini-Q4_0_ROCMFP4_STRIX_LEAN.gguf", dev="ROCm0", ctx=65536, + draft=None, mtp_infile=False, nmax=4, pmin=0.0, strict=False, + serverlog=f"{W}/logs/diag_tools_server.log") +PROMPTS = { + "multi-arg": "What's the weather in Paris in celsius?", + "nested-object": "Book 'Design review' on 2026-10-02 at 14:00 with ana@x.io and bo@x.io.", + "enum": "Weather in Denver, and give it to me in fahrenheit.", + "correct-decline": "What is 17 times 23? Answer directly.", + "multi-turn": "What's the weather in Tokyo in celsius?", + "streaming": "What's the weather in Rome in celsius?", + "parallel": "Get the weather in Oslo AND in Lima, both in celsius. Call the tool for each city.", +} +GATE = dict(temperature=0.7, top_p=0.95, top_k=40) +GREEDY = dict(temperature=0, top_k=1) + + +def call(msgs, kwargs, sampling, stream=False): + body = {"messages": msgs, "tools": H.TOOLS, "tool_choice": "auto", "max_tokens": 8192, **sampling} + if kwargs is not None: + body["chat_template_kwargs"] = kwargs + t0 = time.time() + try: + if not stream: + r = H.post(PORT, "/v1/chat/completions", body) + ch = r["choices"][0] + return {"message": ch["message"], "finish": ch.get("finish_reason"), "s": round(time.time() - t0, 2)} + body["stream"] = True + req = urllib.request.Request(f"http://127.0.0.1:{PORT}/v1/chat/completions", data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + calls, content, reasoning, finish = {}, "", "", None + for line in urllib.request.urlopen(req, timeout=1800): + line = line.decode().strip() + if not line.startswith("data:") or line.endswith("[DONE]"): + continue + ch = json.loads(line[5:])["choices"][0] + d = ch.get("delta") or {} + finish = ch.get("finish_reason") or finish + content += d.get("content") or "" + reasoning += d.get("reasoning_content") or "" + for tc in d.get("tool_calls") or []: + c = calls.setdefault(tc["index"], {"name": "", "arguments": ""}) + c["name"] += (tc.get("function") or {}).get("name") or "" + c["arguments"] += (tc.get("function") or {}).get("arguments") or "" + msg = {"content": content, "reasoning_content": reasoning, + "tool_calls": [{"function": v} for _, v in sorted(calls.items())]} + return {"message": msg, "finish": finish, "s": round(time.time() - t0, 2)} + except urllib.error.HTTPError as e: + return {"http_error": e.code, "body": e.read().decode(errors="replace")[:600], "s": round(time.time() - t0, 2)} + except Exception as e: # noqa: BLE001 - a diagnostic records every failure mode + return {"exception": repr(e)[:300], "s": round(time.time() - t0, 2)} + + +def summarize(res): + if "message" not in res: + return res + m = res["message"] + c = m.get("content") or "" + return {"finish": res.get("finish"), "s": res["s"], + "tool_calls": [(t.get("function") or {}).get("name") for t in (m.get("tool_calls") or [])], + "args": [(t.get("function") or {}).get("arguments") for t in (m.get("tool_calls") or [])], + "leaks_in_content": [x for x in H.LEAK if x in c], + "content": c[:600], "reasoning_len": len(m.get("reasoning_content") or ""), + "reasoning_head": (m.get("reasoning_content") or "")[:300]} + + +def run_variant(name, kwargs, sampling, stream_ok=True): + out = {} + for k, p in PROMPTS.items(): + msgs = [{"role": "user", "content": p}] + r = call(msgs, kwargs, sampling, stream=(k == "streaming" and stream_ok)) + rec = {"first": summarize(r)} + if k == "multi-turn" and "message" in r and r["message"].get("tool_calls"): + tc = r["message"]["tool_calls"][0] + msgs += [{"role": "assistant", "content": r["message"].get("content") or "", + "tool_calls": [{"id": "call_1", "type": "function", "function": tc["function"]}]}, + {"role": "tool", "tool_call_id": "call_1", "content": json.dumps({"temp_c": 21, "sky": "clear"})}] + rec["second"] = summarize(call(msgs, kwargs, sampling)) + out[k] = rec + print(name, k, json.dumps(rec)[:400], flush=True) + return out + + +s = H.Server(a, PORT) +report = {"model": os.path.basename(a.model), "bin": H.BIN, "cmd": " ".join(s.cmd), "variants": {}} +try: + report["variants"]["gate_on"] = run_variant("gate_on", H.THINK_ON, GATE) + report["variants"]["gate_off"] = run_variant("gate_off", H.THINK_OFF, GATE) + report["variants"]["on_greedy"] = run_variant("on_greedy", H.THINK_ON, GREEDY, stream_ok=False) + report["variants"]["adaptive"] = run_variant("adaptive", None, GATE, stream_ok=False) + report["variants"]["nested_off_x3"] = [ + summarize(call([{"role": "user", "content": PROMPTS["nested-object"]}], H.THINK_OFF, GATE)) for _ in range(3)] + print("nested_off_x3", json.dumps(report["variants"]["nested_off_x3"])[:600], flush=True) +finally: + s.stop() +json.dump(report, open(OUT, "w"), indent=1) +print("NEX_TOOLS_DIAG_DONE", OUT) diff --git a/recipe/pipeline/nex_tools_tpl.py b/recipe/pipeline/nex_tools_tpl.py new file mode 100644 index 0000000000000000000000000000000000000000..8e08d4a3234e8d11a5e277d96be435029977a4ab --- /dev/null +++ b/recipe/pipeline/nex_tools_tpl.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +"""Tool-call gate on the Nex STRIX_LEAN file with a one-line template shim. + +Why: the stock template switches thinking with `reasoning_effort` only. llama-server's template analysis probes +`enable_thinking`, finds no difference, and builds a parser that never extracts reasoning: with thinking on, +`reasoning_content` stays empty and the reasoning plus `` land in `content` (n-tools-q106: 0/7 with thinking +on; results/nex_tools_diag.json, results/nex_reasoning_probe.json). `--reasoning-format deepseek`, server-level +`--chat-template-kwargs` and `--reasoning on` do not change that. Prepending one line that maps `enable_thinking` +onto `reasoning_effort` does. + +This run: the same suite, sampling, file and server as n-tools-q106, plus `--chat-template-file` with the shim, and +thinking switched with `enable_thinking` true / false -> label n-tools-q106-tpl in results/nex_bench.jsonl (full +per-check output in logs/N6t_tools_tpl.log). Then probes of the shim's other paths (no kwargs; an explicit +reasoning_effort) -> results/nex_template_shim.json.""" +import argparse, hashlib, json, os, subprocess, sys +from types import SimpleNamespace + +os.environ["AGNES_BIN"] = "/opt/llama-rocm/rocmfpx-724/build-hipvk/bin" # same unpatched server as n-tools-q106 +sys.path.insert(0, "/mnt/models/nex-n2.5-mini") +import nex_harness as H # noqa: E402 + +ap = argparse.ArgumentParser() +ap.add_argument("--on", default="high", choices=["high", "medium"], help="reasoning_effort that enable_thinking=true maps to") +ap.add_argument("--label", default="n-tools-q106-tpl") +ap.add_argument("--tpl", default="chat_template_enable_thinking.jinja") +ap.add_argument("--out", default="nex_template_shim.json") +ap.add_argument("--extra", default="[]", help="JSON list of extra llama-server flags") +ap.add_argument("--vision", action="store_true", help="also run the -fa on image gate in this configuration") +ap.add_argument("--repeats", type=int, default=1, help="tool-suite passes (labels \n\n391", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [ + "" + ] + }, + "reasoning_effort=high|single-word": { + "content": "We need need output exactly ready.\n\n\nready", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [ + "" + ] + }, + "reasoning_effort=high|multi-arg": { + "content": "We need need tool. Current weather Paris celsius.\n\n\n", + "reasoning_len": 0, + "tool_calls": [ + "get_weather" + ], + "leaks": [ + "" + ] + }, + "reasoning_effort=medium|correct-decline": { + "content": "\n\n\n\n391", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [ + "" + ] + }, + "reasoning_effort=medium|single-word": { + "content": "\n\n\n\nready", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [ + "" + ] + }, + "reasoning_effort=medium|multi-arg": { + "content": "\n\n\n\n", + "reasoning_len": 0, + "tool_calls": [ + "get_weather" + ], + "leaks": [ + "" + ] + }, + "reasoning_effort=none|correct-decline": { + "content": "391", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [] + }, + "reasoning_effort=none|single-word": { + "content": "ready", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [] + }, + "reasoning_effort=none|multi-arg": { + "content": "", + "reasoning_len": 0, + "tool_calls": [ + "get_weather" + ], + "leaks": [] + } + }, + "probes_high_default_on": { + "no-kwargs|correct-decline": { + "content": "391", + "reasoning_len": 30, + "tool_calls": [], + "leaks": [] + }, + "no-kwargs|multi-arg": { + "content": "", + "reasoning_len": 50, + "tool_calls": [ + "get_weather" + ], + "leaks": [] + }, + "reasoning_effort=medium|correct-decline": { + "content": "391", + "reasoning_len": 0, + "tool_calls": [], + "leaks": [] + }, + "reasoning_effort=medium|multi-arg": { + "content": "", + "reasoning_len": 0, + "tool_calls": [ + "get_weather" + ], + "leaks": [] + }, + "reasoning_effort=none|correct-decline": { + "content": "", + "reasoning_len": 3, + "tool_calls": [], + "leaks": [] + }, + "reasoning_effort=none|multi-arg": { + "content": "", + "reasoning_len": 133, + "tool_calls": [], + "leaks": [] + } + }, + "medium_mapping_label": "n-tools-q106-tpl-medium" + }, + "tools_diag": { + "stock_on_replies": 7, + "stock_on_leaks": 7, + "stock_on_reasoning_extracted": 0, + "nested_off_attempts": 4, + "nested_off_http500": 2, + "gate_http500_logged": true, + "flag_probes": { + "default": { + "leaks": 3, + "n": 3 + }, + "fmt-deepseek": { + "leaks": 3, + "n": 3 + }, + "srv-kwargs-high": { + "leaks": 3, + "n": 3 + }, + "reasoning-on": { + "leaks": 3, + "n": 3 + }, + "tpl-enable-thinking": { + "leaks": 0, + "n": 3 + } + } + }, + "seats": { + "max1-nex-fast": { + "unit": "max1-nex-fast", + "port": 8097, + "load_s": 25, + "time": "2026-09-17T01:21:10Z", + "direct_reply": "ready", + "direct_tg": 41.41386950489719, + "default_reply": "ready", + "default_reasoning_len": 0, + "default_leak": false, + "thinking_reply": "", + "thinking_reasoning_len": 5, + "thinking_leak": false, + "gateway_model": "nex-n2.5-mini-fast@max1", + "gateway_reply": "ready", + "result": "PASS" + }, + "max1-nex-fast-imat": { + "unit": "max1-nex-fast-imat", + "port": 8098, + "load_s": 25, + "time": "2026-09-17T01:21:43Z", + "direct_reply": "ready", + "direct_tg": 41.54290343352097, + "default_reply": "ready", + "default_reasoning_len": 0, + "default_leak": false, + "thinking_reply": "", + "thinking_reasoning_len": 5, + "thinking_leak": false, + "gateway_model": "nex-n2.5-mini-fast-imatrix@max1", + "gateway_reply": "ready", + "result": "PASS" + } + }, + "measured": "2026-09-16", + "measured_range": [ + "2026-09-16", + "2026-09-17" + ] +} \ No newline at end of file diff --git a/recipe/templates/chat_template_enable_thinking_medium.jinja b/recipe/templates/chat_template_enable_thinking_medium.jinja new file mode 100644 index 0000000000000000000000000000000000000000..aab1b9de6d5743788f0aa72024a0177df045a7e3 --- /dev/null +++ b/recipe/templates/chat_template_enable_thinking_medium.jinja @@ -0,0 +1,155 @@ +{%- if reasoning_effort is not defined and enable_thinking is defined %}{%- set reasoning_effort = 'medium' if enable_thinking else 'none' %}{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if reasoning_effort is not defined or reasoning_effort is none %} + {{- '' }} + {%- elif reasoning_effort == 'none' %} + {{- '\n\n\n\n' }} + {%- elif reasoning_effort == 'high' %} + {{- '\n' }} + {%- else %} + {{- '' }} + {%- endif %} +{%- endif %} diff --git a/recipe/templates/chat_template_enable_thinking_v2.jinja b/recipe/templates/chat_template_enable_thinking_v2.jinja new file mode 100644 index 0000000000000000000000000000000000000000..477b8ffede84f116d27eed2764d263aeeac62e60 --- /dev/null +++ b/recipe/templates/chat_template_enable_thinking_v2.jinja @@ -0,0 +1,155 @@ +{%- if enable_thinking is defined and not enable_thinking %}{%- set reasoning_effort = 'none' %}{%- elif reasoning_effort is not defined and enable_thinking is defined %}{%- set reasoning_effort = 'high' %}{%- endif %} +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if reasoning_effort is not defined or reasoning_effort is none %} + {{- '' }} + {%- elif reasoning_effort == 'none' %} + {{- '\n\n\n\n' }} + {%- elif reasoning_effort == 'high' %} + {{- '\n' }} + {%- else %} + {{- '' }} + {%- endif %} +{%- endif %} diff --git a/recipe/templates/chat_template_stock.jinja b/recipe/templates/chat_template_stock.jinja new file mode 100644 index 0000000000000000000000000000000000000000..34398e4315897b110ab12fcd0837af47ed95c790 --- /dev/null +++ b/recipe/templates/chat_template_stock.jinja @@ -0,0 +1,154 @@ +{%- set image_count = namespace(value=0) %} +{%- set video_count = namespace(value=0) %} +{%- macro render_content(content, do_vision_count, is_system_content=false) %} + {%- if content is string %} + {{- content }} + {%- elif content is iterable and content is not mapping %} + {%- for item in content %} + {%- if 'image' in item or 'image_url' in item or item.type == 'image' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain images.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set image_count.value = image_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Picture ' ~ image_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|image_pad|><|vision_end|>' }} + {%- elif 'video' in item or item.type == 'video' %} + {%- if is_system_content %} + {{- raise_exception('System message cannot contain videos.') }} + {%- endif %} + {%- if do_vision_count %} + {%- set video_count.value = video_count.value + 1 %} + {%- endif %} + {%- if add_vision_id %} + {{- 'Video ' ~ video_count.value ~ ': ' }} + {%- endif %} + {{- '<|vision_start|><|video_pad|><|vision_end|>' }} + {%- elif 'text' in item %} + {{- item.text }} + {%- else %} + {{- raise_exception('Unexpected item type in content.') }} + {%- endif %} + {%- endfor %} + {%- elif content is none or content is undefined %} + {{- '' }} + {%- else %} + {{- raise_exception('Unexpected content type.') }} + {%- endif %} +{%- endmacro %} +{%- if not messages %} + {{- raise_exception('No messages provided.') }} +{%- endif %} +{%- if tools and tools is iterable and tools is not mapping %} + {{- '<|im_start|>system\n' }} + {{- "# Tools\n\nYou have access to the following functions:\n\n" }} + {%- for tool in tools %} + {{- "\n" }} + {{- tool | tojson }} + {%- endfor %} + {{- "\n" }} + {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n' }} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {%- if content %} + {{- '\n\n' + content }} + {%- endif %} + {%- endif %} + {{- '<|im_end|>\n' }} +{%- else %} + {%- if messages[0].role == 'system' %} + {%- set content = render_content(messages[0].content, false, true)|trim %} + {{- '<|im_start|>system\n' + content + '<|im_end|>\n' }} + {%- endif %} +{%- endif %} +{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %} +{%- for message in messages[::-1] %} + {%- set index = (messages|length - 1) - loop.index0 %} + {%- if ns.multi_step_tool and message.role == "user" %} + {%- set content = render_content(message.content, false)|trim %} + {%- if not(content.startswith('') and content.endswith('')) %} + {%- set ns.multi_step_tool = false %} + {%- set ns.last_query_index = index %} + {%- endif %} + {%- endif %} +{%- endfor %} +{%- if ns.multi_step_tool %} + {{- raise_exception('No user query found in messages.') }} +{%- endif %} +{%- for message in messages %} + {%- set content = render_content(message.content, true)|trim %} + {%- if message.role == "system" %} + {%- if not loop.first %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- endif %} + {%- elif message.role == "user" %} + {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }} + {%- elif message.role == "assistant" %} + {%- set reasoning_content = '' %} + {%- if message.reasoning_content is string %} + {%- set reasoning_content = message.reasoning_content %} + {%- else %} + {%- if '' in content %} + {%- set reasoning_content = content.split('')[0].rstrip('\n').split('')[-1].lstrip('\n') %} + {%- set content = content.split('')[-1].lstrip('\n') %} + {%- endif %} + {%- endif %} + {%- set reasoning_content = reasoning_content|trim %} + {{- '<|im_start|>' + message.role + '\n\n' + reasoning_content + '\n\n\n' + content }} + {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %} + {%- for tool_call in message.tool_calls %} + {%- if tool_call.function is defined %} + {%- set tool_call = tool_call.function %} + {%- endif %} + {%- if loop.first %} + {%- if content|trim %} + {{- '\n\n\n\n' }} + {%- else %} + {{- '\n\n' }} + {%- endif %} + {%- else %} + {{- '\n\n\n' }} + {%- endif %} + {%- if tool_call.arguments is defined %} + {%- for args_name, args_value in tool_call.arguments|items %} + {{- '\n' }} + {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %} + {{- args_value }} + {{- '\n\n' }} + {%- endfor %} + {%- endif %} + {{- '\n' }} + {%- endfor %} + {%- endif %} + {{- '<|im_end|>\n' }} + {%- elif message.role == "tool" %} + {%- if loop.previtem and loop.previtem.role != "tool" %} + {{- '<|im_start|>user' }} + {%- endif %} + {{- '\n\n' }} + {{- content }} + {{- '\n' }} + {%- if not loop.last and loop.nextitem.role != "tool" %} + {{- '<|im_end|>\n' }} + {%- elif loop.last %} + {{- '<|im_end|>\n' }} + {%- endif %} + {%- else %} + {{- raise_exception('Unexpected message role.') }} + {%- endif %} +{%- endfor %} +{%- if add_generation_prompt %} + {{- '<|im_start|>assistant\n' }} + {%- if reasoning_effort is not defined or reasoning_effort is none %} + {{- '' }} + {%- elif reasoning_effort == 'none' %} + {{- '\n\n\n\n' }} + {%- elif reasoning_effort == 'high' %} + {{- '\n' }} + {%- else %} + {{- '' }} + {%- endif %} +{%- endif %}