| { |
| "ciru": { |
| "launcher_expansion": { |
| "argv": [ |
| "<bundle>/bin/llama-server", |
| "--model", |
| "/srv/llm/models/Qwen3.8-Flash-CIRU-STRIX-IU4/Qwen3.8-Flash-CIRU-STRIX-IU4.gguf", |
| "--alias", |
| "qwen38-mtp-probe", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18186", |
| "--jinja", |
| "--ple-sidecar", |
| "/srv/llm/models/Qwen3.8-Flash-CIRU-STRIX-IU4/ple", |
| "--ple-cache-mib", |
| "4096", |
| "-ngl", |
| "all", |
| "-sm", |
| "none", |
| "--fit", |
| "off", |
| "-c", |
| "16384", |
| "-b", |
| "2048", |
| "-ub", |
| "512", |
| "--parallel", |
| "1", |
| "-t", |
| "8", |
| "-tb", |
| "8", |
| "-ctk", |
| "f16", |
| "-ctv", |
| "f16", |
| "-fa", |
| "on", |
| "--cont-batching", |
| "--cache-prompt", |
| "--cache-ram", |
| "8192", |
| "--cache-idle-slots", |
| "--ctx-checkpoints", |
| "32", |
| "--checkpoint-min-step", |
| "8192", |
| "--metrics", |
| "--slots", |
| "--spec-type", |
| "draft-mtp", |
| "--spec-draft-model", |
| "/srv/llm/models/Qwen3.8-Flash-CIRU-STRIX-IU4/mtp/Qwen3.8-Flash-CIRU-STRIX-IU4-MTP-Q8_0.gguf", |
| "--spec-draft-ngl", |
| "all", |
| "--spec-draft-device", |
| "ROCm0", |
| "--spec-draft-type-k", |
| "q8_0", |
| "--spec-draft-type-v", |
| "q8_0", |
| "--spec-draft-threads", |
| "8", |
| "--spec-draft-threads-batch", |
| "8", |
| "--spec-draft-n-max", |
| "6", |
| "--spec-draft-n-min", |
| "0", |
| "--spec-draft-p-min", |
| "0", |
| "--spec-draft-p-split", |
| "0.10" |
| ], |
| "env": { |
| "GGML_CUDA_Q41_MOE_FORCE_J": "32", |
| "GGML_QWEN4EXP_PLE_WORKERS": "16", |
| "GGML_QWEN4EXP_PLE_STRICT_SHA": "0", |
| "ROCBLAS_USE_HIPBLASLT": "1", |
| "GGML_QSA_LONG_TOPK": "1", |
| "GGML_QSA_RESTORE_FAST": "1", |
| "CIRU_MTP_TOPK10": "1", |
| "CIRU_MTP_SHORTLIST": "32768" |
| }, |
| "unset": [ |
| "GGML_HIP_GRAPH_EXEC_UPDATE", |
| "CIRU_MTP_GPU_CONFIDENCE", |
| "CIRU_MTP_GPU_ADAPTIVE", |
| "CIRU_MTP_GPU_CONF_MIN", |
| "CIRU_MOE_EXPERT_REUSE", |
| "CIRU_MTP_TRACE", |
| "CIRU_MTP_CONF_TRACE", |
| "LD_PRELOAD" |
| ], |
| "library_path": "<bundle>/bin plus installed ROCm10 libraries", |
| "cwd": "<bundle>" |
| }, |
| "mtp_off": { |
| "command": [ |
| "/srv/llm/work/sozo-adaptive-diagnosis-20260905/locked/qwen38-ciru-rocm10-20260905-rc2/bin/llama-server", |
| "--model", |
| "/srv/llm/models/Qwen3.8-Flash-CIRU-STRIX-IU4/Qwen3.8-Flash-CIRU-STRIX-IU4.gguf", |
| "--alias", |
| "qwen38-mtp-probe", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18186", |
| "--jinja", |
| "--ple-sidecar", |
| "/srv/llm/models/Qwen3.8-Flash-CIRU-STRIX-IU4/ple", |
| "--ple-cache-mib", |
| "4096", |
| "-ngl", |
| "all", |
| "-sm", |
| "none", |
| "--fit", |
| "off", |
| "-c", |
| "16384", |
| "-b", |
| "2048", |
| "-ub", |
| "512", |
| "--parallel", |
| "1", |
| "-t", |
| "8", |
| "-tb", |
| "8", |
| "-ctk", |
| "f16", |
| "-ctv", |
| "f16", |
| "-fa", |
| "on", |
| "--cont-batching", |
| "--cache-prompt", |
| "--cache-ram", |
| "8192", |
| "--cache-idle-slots", |
| "--ctx-checkpoints", |
| "32", |
| "--checkpoint-min-step", |
| "8192", |
| "--metrics", |
| "--slots", |
| "-c", |
| "262144", |
| "--temp", |
| "1.0", |
| "--top-p", |
| "0.95", |
| "--top-k", |
| "20", |
| "--min-p", |
| "0.0", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18184", |
| "--parallel", |
| "1", |
| "--metrics", |
| "--slots", |
| "--slot-save-path", |
| "/srv/llm/work/qwen38-v2-release-20260905/sweep-sozo/ciru-rc2-mtp-off-sweep/slot-state" |
| ], |
| "slots": [ |
| { |
| "id": 0, |
| "n_ctx": 262144, |
| "speculative": false, |
| "is_processing": false |
| } |
| ], |
| "props": { |
| "default_generation_settings": { |
| "params": { |
| "seed": 4294967295, |
| "temperature": 1.0, |
| "dynatemp_range": 0.0, |
| "dynatemp_exponent": 1.0, |
| "top_k": 20, |
| "top_p": 0.949999988079071, |
| "min_p": 0.0, |
| "top_n_sigma": -1.0, |
| "xtc_probability": 0.0, |
| "xtc_threshold": 0.10000000149011612, |
| "typical_p": 1.0, |
| "repeat_last_n": 64, |
| "repeat_penalty": 1.0, |
| "presence_penalty": 0.0, |
| "frequency_penalty": 0.0, |
| "dry_multiplier": 0.0, |
| "dry_base": 1.75, |
| "dry_allowed_length": 2, |
| "dry_penalty_last_n": 64, |
| "mirostat": 0, |
| "mirostat_tau": 5.0, |
| "mirostat_eta": 0.10000000149011612, |
| "adaptive_target": -1.0, |
| "adaptive_decay": 0.8999999761581421, |
| "max_tokens": -1, |
| "n_predict": -1, |
| "n_keep": 0, |
| "n_discard": 0, |
| "ignore_eos": false, |
| "stream": false, |
| "n_probs": 0, |
| "min_keep": 0, |
| "chat_format": "Content-only", |
| "reasoning_format": "none", |
| "reasoning_in_content": false, |
| "generation_prompt": "", |
| "samplers": [ |
| "penalties", |
| "dry", |
| "top_n_sigma", |
| "top_k", |
| "typ_p", |
| "top_p", |
| "min_p", |
| "xtc", |
| "temperature" |
| ], |
| "speculative.types": "none", |
| "timings_per_token": false, |
| "post_sampling_probs": false, |
| "backend_sampling": false, |
| "lora": [] |
| }, |
| "n_ctx": 262144 |
| }, |
| "total_slots": 1, |
| "model_alias": "qwen38-mtp-probe", |
| "model_ftype": "Q5_K - Medium", |
| "model_path": "/srv/llm/models/Qwen3.8-Flash-CIRU-STRIX-IU4/Qwen3.8-Flash-CIRU-STRIX-IU4.gguf", |
| "modalities": { |
| "vision": false, |
| "video": false, |
| "audio": false |
| }, |
| "media_marker": "<__media_BSN5RPeNDNKtyQ0UdZFxMrljTfk28EPK__>", |
| "endpoint_slots": true, |
| "endpoint_props": false, |
| "endpoint_metrics": true, |
| "ui": true, |
| "ui_settings": {}, |
| "chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '\\n\\n' + content }}\n {%- endif %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if messages[0].role == 'system' %}\n {%- set content = render_content(messages[0].content, false, true)|trim %}\n {%- if content %}\n {{- '<|im_start|>system\\n' + (reasoning_instructions + '\\n\\n' if reasoning_instructions else '') + content + '<|im_end|>\\n' }}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if ns.multi_step_tool %}\n {{- raise_exception('No user query found in messages.') }}\n{%- endif %}\n{%- for message in messages %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" %}\n {%- if not loop.first %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- endif %}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is defined and tool_call.arguments != '' %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- else %}\n {{- '<think>\\n' }}\n {%- endif %}\n{%- endif %}", |
| "chat_template_caps": { |
| "supports_object_arguments": true, |
| "supports_parallel_tool_calls": true, |
| "supports_preserve_reasoning": true, |
| "supports_reasoning_effort": true, |
| "supports_string_content": true, |
| "supports_system_role": true, |
| "supports_tool_calls": true, |
| "supports_tools": true, |
| "supports_typed_content": false |
| }, |
| "bos_token": "<|endoftext|>", |
| "eos_token": "<|im_end|>", |
| "build_info": "b1-baba5e0", |
| "is_sleeping": false, |
| "cors_proxy_enabled": false |
| } |
| } |
| }, |
| "agention": { |
| "mtp_off": { |
| "command": [ |
| "/srv/llm/work/agention-baseline-20260904/bin-native/llama-server", |
| "-m", |
| "/srv/llm/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST-ad4c5717254a/Qwen3.8-Flash-Next-ROCmFP4-FAST-v2-ple16.gguf", |
| "--alias", |
| "agention-baseline", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18184", |
| "-ngl", |
| "99", |
| "-ctk", |
| "q8_0", |
| "-ctv", |
| "q8_0", |
| "-fa", |
| "on", |
| "--parallel", |
| "1", |
| "--metrics", |
| "--slots", |
| "--slot-save-path", |
| "/srv/llm/bench-results/agention-card-recipe-20260905/target-only-card/slot-state" |
| ], |
| "slots": [ |
| { |
| "id": 0, |
| "is_processing": false, |
| "n_ctx": 262144, |
| "speculative": false |
| } |
| ], |
| "props": { |
| "bos_token": "<|endoftext|>", |
| "build_info": "b1-5e085d1", |
| "chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set sysns = namespace(count=0, text='') %}\n{%- for message in messages %}\n {%- if sysns.count == loop.index0 and (message.role == 'system' or message.role == 'developer') %}\n {%- set sys_content = render_content(message.content, false, true)|trim %}\n {%- if sys_content %}\n {%- set sysns.text = sysns.text + ('\\n' if sysns.text else '') + sys_content %}\n {%- endif %}\n {%- set sysns.count = sysns.count + 1 %}\n {%- endif %}\n{%- endfor %}\n{%- set num_sys = sysns.count %}\n{%- set merged_system = sysns.text %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort == 'high' %}\n {%- set resolved_reasoning_effort = 'xhigh' %}\n {%- endif %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if merged_system %}\n {{- '\\n\\n' + merged_system }}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if merged_system %}\n {{- '<|im_start|>system\\n' + (reasoning_instructions + '\\n\\n' if reasoning_instructions else '') + merged_system + '<|im_end|>\\n' }}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if loop.index0 >= num_sys %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" or message.role == \"developer\" %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if tool_call.name is not defined or tool_call.name is none %}\n {{- raise_exception('Tool call is missing a function name.') }}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is mapping %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- elif tool_call.arguments is string %}\n {%- if tool_call.arguments|trim %}\n {{- raise_exception('Tool call arguments for function \"' + (tool_call.name | string) + '\" were passed as a JSON string. Parse them into an object before calling apply_chat_template.') }}\n {%- endif %}\n {%- elif tool_call.arguments is defined and tool_call.arguments is not none %}\n {{- raise_exception('Tool call arguments for function \"' + (tool_call.name | string) + '\" must be an object/mapping or a JSON string.') }}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- else %}\n {{- '<think>\\n' }}\n {%- endif %}\n{%- endif %}\n{#- Unsloth fixes - developer role, merged system messages, tool calling #}", |
| "chat_template_caps": { |
| "supports_object_arguments": true, |
| "supports_parallel_tool_calls": true, |
| "supports_preserve_reasoning": true, |
| "supports_reasoning_effort": true, |
| "supports_string_content": true, |
| "supports_system_role": true, |
| "supports_tool_calls": true, |
| "supports_tools": true, |
| "supports_typed_content": false |
| }, |
| "cors_proxy_enabled": false, |
| "default_generation_settings": { |
| "n_ctx": 262144, |
| "params": { |
| "adaptive_decay": 0.8999999761581421, |
| "adaptive_target": -1.0, |
| "backend_sampling": false, |
| "chat_format": "Content-only", |
| "dry_allowed_length": 2, |
| "dry_base": 1.75, |
| "dry_multiplier": 0.0, |
| "dry_penalty_last_n": 64, |
| "dynatemp_exponent": 1.0, |
| "dynatemp_range": 0.0, |
| "frequency_penalty": 0.0, |
| "generation_prompt": "", |
| "ignore_eos": false, |
| "lora": [], |
| "max_tokens": -1, |
| "min_keep": 0, |
| "min_p": 0.05000000074505806, |
| "mirostat": 0, |
| "mirostat_eta": 0.10000000149011612, |
| "mirostat_tau": 5.0, |
| "n_discard": 0, |
| "n_keep": 0, |
| "n_predict": -1, |
| "n_probs": 0, |
| "post_sampling_probs": false, |
| "presence_penalty": 0.0, |
| "reasoning_format": "none", |
| "reasoning_in_content": false, |
| "repeat_last_n": 64, |
| "repeat_penalty": 1.0, |
| "samplers": [ |
| "penalties", |
| "dry", |
| "top_n_sigma", |
| "top_k", |
| "typ_p", |
| "top_p", |
| "min_p", |
| "xtc", |
| "temperature" |
| ], |
| "seed": 4294967295, |
| "speculative.types": "none", |
| "stream": false, |
| "temperature": 1.0, |
| "timings_per_token": false, |
| "top_k": 20, |
| "top_n_sigma": -1.0, |
| "top_p": 0.949999988079071, |
| "typical_p": 1.0, |
| "xtc_probability": 0.0, |
| "xtc_threshold": 0.10000000149011612 |
| } |
| }, |
| "endpoint_metrics": true, |
| "endpoint_props": false, |
| "endpoint_slots": true, |
| "eos_token": "<|im_end|>", |
| "is_sleeping": false, |
| "media_marker": "<__media_UNyIr3ttHWFHGGPrU6xFkGLvA13eUauH__>", |
| "modalities": { |
| "audio": false, |
| "video": false, |
| "vision": false |
| }, |
| "model_alias": "agention-baseline", |
| "model_ftype": "Q4_0_ROCMFP4_FAST", |
| "model_path": "/srv/llm/models/agentionai-Qwen3.8-Flash-Next-ROCmFP4-FAST-ad4c5717254a/Qwen3.8-Flash-Next-ROCmFP4-FAST-v2-ple16.gguf", |
| "total_slots": 1, |
| "ui": true, |
| "ui_settings": {} |
| } |
| } |
| }, |
| "unsloth": { |
| "mtp_off": { |
| "command": [ |
| "/srv/llm/work/unsloth-runtime-compare-20260905/bin-unsloth/llama-server", |
| "-m", |
| "/srv/llm/models/unsloth-Qwen3.8-Flash-Next-38bb39ee9782/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf", |
| "--alias", |
| "unsloth-iq4xs-unsloth", |
| "--host", |
| "127.0.0.1", |
| "--port", |
| "18184", |
| "-ngl", |
| "99", |
| "-sm", |
| "none", |
| "--fit", |
| "off", |
| "-c", |
| "262144", |
| "-b", |
| "2048", |
| "-ub", |
| "512", |
| "-t", |
| "16", |
| "-tb", |
| "16", |
| "-ctk", |
| "q8_0", |
| "-ctv", |
| "q8_0", |
| "-fa", |
| "on", |
| "--parallel", |
| "1", |
| "--metrics", |
| "--slots", |
| "--lazy-mode", |
| "off", |
| "-ot", |
| "per_layer_token_embd.weight=CPU", |
| "--slot-save-path", |
| "/srv/llm/bench-results/unsloth-runtime-compare-20260905/unsloth-no-mtp/slot-state" |
| ], |
| "slots": [ |
| { |
| "id": 0, |
| "n_ctx": 262144, |
| "speculative": false, |
| "is_processing": false |
| } |
| ], |
| "props": { |
| "default_generation_settings": { |
| "params": { |
| "seed": 4294967295, |
| "temperature": 1.0, |
| "dynatemp_range": 0.0, |
| "dynatemp_exponent": 1.0, |
| "top_k": 20, |
| "top_p": 0.949999988079071, |
| "min_p": 0.05000000074505806, |
| "top_n_sigma": -1.0, |
| "xtc_probability": 0.0, |
| "xtc_threshold": 0.10000000149011612, |
| "typical_p": 1.0, |
| "repeat_last_n": 64, |
| "repeat_penalty": 1.0, |
| "presence_penalty": 0.0, |
| "frequency_penalty": 0.0, |
| "dry_multiplier": 0.0, |
| "dry_base": 1.75, |
| "dry_allowed_length": 2, |
| "dry_penalty_last_n": 64, |
| "mirostat": 0, |
| "mirostat_tau": 5.0, |
| "mirostat_eta": 0.10000000149011612, |
| "adaptive_target": -1.0, |
| "adaptive_decay": 0.8999999761581421, |
| "max_tokens": -1, |
| "n_predict": -1, |
| "n_keep": 0, |
| "n_discard": 0, |
| "ignore_eos": false, |
| "stream": false, |
| "n_probs": 0, |
| "min_keep": 0, |
| "chat_format": "Content-only", |
| "reasoning_format": "none", |
| "reasoning_in_content": false, |
| "generation_prompt": "", |
| "samplers": [ |
| "penalties", |
| "dry", |
| "top_n_sigma", |
| "top_k", |
| "typ_p", |
| "top_p", |
| "min_p", |
| "xtc", |
| "temperature" |
| ], |
| "speculative.types": "none", |
| "timings_per_token": false, |
| "post_sampling_probs": false, |
| "backend_sampling": false, |
| "lora": [] |
| }, |
| "n_ctx": 262144 |
| }, |
| "total_slots": 1, |
| "model_alias": "unsloth-iq4xs-unsloth", |
| "model_ftype": "IQ4_XS - 4.25 bpw", |
| "model_path": "/srv/llm/models/unsloth-Qwen3.8-Flash-Next-38bb39ee9782/UD-IQ4_XS/Qwen3.8-Flash-Next-UD-IQ4_XS-00001-of-00003.gguf", |
| "modalities": { |
| "vision": false, |
| "video": false, |
| "audio": false |
| }, |
| "media_marker": "<__media_P9hmgfcA5xJQIkTb96SAQTq61DAOI7Ag__>", |
| "endpoint_slots": true, |
| "endpoint_props": false, |
| "endpoint_metrics": true, |
| "ui": true, |
| "ui_settings": {}, |
| "chat_template": "{%- set image_count = namespace(value=0) %}\n{%- set video_count = namespace(value=0) %}\n{%- macro render_content(content, do_vision_count, is_system_content=false) %}\n {%- if content is string %}\n {{- content }}\n {%- elif content is iterable and content is not mapping %}\n {%- for item in content %}\n {%- if 'image' in item or 'image_url' in item or item.type == 'image' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain images.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set image_count.value = image_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Picture ' ~ image_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|image_pad|><|vision_end|>' }}\n {%- elif 'video' in item or item.type == 'video' %}\n {%- if is_system_content %}\n {{- raise_exception('System message cannot contain videos.') }}\n {%- endif %}\n {%- if do_vision_count %}\n {%- set video_count.value = video_count.value + 1 %}\n {%- endif %}\n {%- if add_vision_id %}\n {{- 'Video ' ~ video_count.value ~ ': ' }}\n {%- endif %}\n {{- '<|vision_start|><|video_pad|><|vision_end|>' }}\n {%- elif 'text' in item %}\n {{- item.text }}\n {%- else %}\n {{- raise_exception('Unexpected item type in content.') }}\n {%- endif %}\n {%- endfor %}\n {%- elif content is none or content is undefined %}\n {{- '' }}\n {%- else %}\n {{- raise_exception('Unexpected content type.') }}\n {%- endif %}\n{%- endmacro %}\n{%- if not messages %}\n {{- raise_exception('No messages provided.') }}\n{%- endif %}\n{%- set sysns = namespace(count=0, text='') %}\n{%- for message in messages %}\n {%- if sysns.count == loop.index0 and (message.role == 'system' or message.role == 'developer') %}\n {%- set sys_content = render_content(message.content, false, true)|trim %}\n {%- if sys_content %}\n {%- set sysns.text = sysns.text + ('\\n' if sysns.text else '') + sys_content %}\n {%- endif %}\n {%- set sysns.count = sysns.count + 1 %}\n {%- endif %}\n{%- endfor %}\n{%- set num_sys = sysns.count %}\n{%- set merged_system = sysns.text %}\n{%- set reasoning_instructions = '' %}\n{%- if enable_thinking is undefined or enable_thinking is true %}\n {%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}\n {%- if resolved_reasoning_effort == 'high' %}\n {%- set resolved_reasoning_effort = 'xhigh' %}\n {%- endif %}\n {%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}\n {{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}\n {%- endif %}\n {%- if resolved_reasoning_effort == 'xhigh' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}\n {%- elif resolved_reasoning_effort == 'low' %}\n {%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}\n {%- endif %}\n{%- endif %}\n{%- if tools and tools is iterable and tools is not mapping %}\n {{- '<|im_start|>system\\n' }}\n {%- if reasoning_instructions %}\n {{- reasoning_instructions + '\\n\\n' }}\n {%- endif %}\n {{- \"# Tools\\n\\nYou have access to the following functions:\\n\\n<tools>\" }}\n {%- for tool in tools %}\n {{- \"\\n\" }}\n {{- tool | tojson }}\n {%- endfor %}\n {{- \"\\n</tools>\" }}\n {{- '\\n\\nIf you choose to call a function ONLY reply in the following format with NO suffix:\\n\\n<tool_call>\\n<function=example_function_name>\\n<parameter=example_parameter_1>\\nvalue_1\\n</parameter>\\n<parameter=example_parameter_2>\\nThis is the value for the second parameter\\nthat can span\\nmultiple lines\\n</parameter>\\n</function>\\n</tool_call>\\n\\n<IMPORTANT>\\nReminder:\\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\\n- Required parameters MUST be specified\\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\\n</IMPORTANT>' }}\n {%- if merged_system %}\n {{- '\\n\\n' + merged_system }}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n{%- else %}\n {%- if merged_system %}\n {{- '<|im_start|>system\\n' + (reasoning_instructions + '\\n\\n' if reasoning_instructions else '') + merged_system + '<|im_end|>\\n' }}\n {%- elif reasoning_instructions %}\n {{- '<|im_start|>system\\n' + reasoning_instructions + '<|im_end|>\\n' }}\n {%- endif %}\n{%- endif %}\n{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}\n{%- for message in messages[::-1] %}\n {%- set index = (messages|length - 1) - loop.index0 %}\n {%- if ns.multi_step_tool and message.role == \"user\" %}\n {%- set content = render_content(message.content, false)|trim %}\n {%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}\n {%- set ns.multi_step_tool = false %}\n {%- set ns.last_query_index = index %}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- for message in messages %}\n {%- if loop.index0 >= num_sys %}\n {%- set content = render_content(message.content, true)|trim %}\n {%- if message.role == \"system\" or message.role == \"developer\" %}\n {{- raise_exception('System message must be at the beginning.') }}\n {%- elif message.role == \"user\" %}\n {{- '<|im_start|>' + message.role + '\\n' + content + '<|im_end|>' + '\\n' }}\n {%- elif message.role == \"assistant\" %}\n {%- set reasoning_content = '' %}\n {%- if message.reasoning_content is string %}\n {%- set reasoning_content = message.reasoning_content %}\n {%- endif %}\n {%- set reasoning_content = reasoning_content|trim %}\n {%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}\n {{- '<|im_start|>' + message.role + '\\n<think>\\n' + reasoning_content + '\\n</think>\\n\\n' + content }}\n {%- else %}\n {{- '<|im_start|>' + message.role + '\\n' + content }}\n {%- endif %}\n {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}\n {%- for tool_call in message.tool_calls %}\n {%- if tool_call.function is defined %}\n {%- set tool_call = tool_call.function %}\n {%- endif %}\n {%- if tool_call.name is not defined or tool_call.name is none %}\n {{- raise_exception('Tool call is missing a function name.') }}\n {%- endif %}\n {%- if loop.first %}\n {%- if content|trim %}\n {{- '\\n\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- else %}\n {{- '<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- else %}\n {{- '\\n<tool_call>\\n<function=' + tool_call.name + '>\\n' }}\n {%- endif %}\n {%- if tool_call.arguments is mapping %}\n {%- for args_name, args_value in tool_call.arguments|items %}\n {{- '<parameter=' + args_name + '>\\n' }}\n {%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}\n {{- args_value }}\n {{- '\\n</parameter>\\n' }}\n {%- endfor %}\n {%- elif tool_call.arguments is string %}\n {%- if tool_call.arguments|trim %}\n {{- raise_exception('Tool call arguments for function \"' + (tool_call.name | string) + '\" were passed as a JSON string. Parse them into an object before calling apply_chat_template.') }}\n {%- endif %}\n {%- elif tool_call.arguments is defined and tool_call.arguments is not none %}\n {{- raise_exception('Tool call arguments for function \"' + (tool_call.name | string) + '\" must be an object/mapping or a JSON string.') }}\n {%- endif %}\n {{- '</function>\\n</tool_call>' }}\n {%- endfor %}\n {%- endif %}\n {{- '<|im_end|>\\n' }}\n {%- elif message.role == \"tool\" %}\n {%- if loop.previtem and loop.previtem.role != \"tool\" %}\n {{- '<|im_start|>user' }}\n {%- endif %}\n {{- '\\n<tool_response>\\n' }}\n {{- content }}\n {{- '\\n</tool_response>' }}\n {%- if not loop.last and loop.nextitem.role != \"tool\" %}\n {{- '<|im_end|>\\n' }}\n {%- elif loop.last %}\n {{- '<|im_end|>\\n' }}\n {%- endif %}\n {%- else %}\n {{- raise_exception('Unexpected message role.') }}\n {%- endif %}\n {%- endif %}\n{%- endfor %}\n{%- if add_generation_prompt %}\n {{- '<|im_start|>assistant\\n' }}\n {%- if enable_thinking is defined and enable_thinking is false %}\n {{- '<think>\\n\\n</think>\\n\\n' }}\n {%- else %}\n {{- '<think>\\n' }}\n {%- endif %}\n{%- endif %}\n{#- Unsloth fixes - developer role, merged system messages, tool calling #}", |
| "chat_template_caps": { |
| "supports_object_arguments": true, |
| "supports_parallel_tool_calls": true, |
| "supports_preserve_reasoning": true, |
| "supports_reasoning_effort": true, |
| "supports_string_content": true, |
| "supports_system_role": true, |
| "supports_tool_calls": true, |
| "supports_tools": true, |
| "supports_typed_content": false |
| }, |
| "bos_token": "<|endoftext|>", |
| "eos_token": "<|im_end|>", |
| "build_info": "b1-d1a9235", |
| "is_sleeping": false, |
| "cors_proxy_enabled": false |
| } |
| } |
| } |
| } |
|
|
