inference_metadata.yaml

# Faithful onnx-genai v1 inference_metadata for google/gemma-4-E2B-it text decoder.
# Authored to the onnx-genai #1716 schema (example 23 shape) using the REAL exported
# graph port names (past_key_values.N.key / present.N.key) and REAL owner layers:
#   full_attention owners (head_dim 512): layers [4, 9, 14]
#   sliding_attention owners (head_dim 256): layers [0, 1, 2, 3, 5, 6, 7, 8, 10, 11, 12, 13]
# The 20 shared-KV layers borrow an owner's buffer inside the graph and expose no ports.
# MoE is DISABLED in this checkpoint (enable_moe_block=false) -> dense MLP, not invented.
# final_logit_softcapping / tie_word_embeddings are graph-internal. Validates against
# onnx-genai PR #1716 schema/inference_metadata.schema.json.
schema_version: v1
pipeline:
  workflow:
    manifest:
      capabilities: [workflow_ssa, typed_emit, serving_service_contract]
    inputs:
      request.active:
        contract:
          dtype: bool
          rank: 1
          shape: [batch]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: active}
        required: true
      request.done:
        contract:
          dtype: bool
          rank: 1
          shape: [batch]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: done}
        required: true
      request.accepted_len:
        contract:
          dtype: int64
          rank: 1
          shape: [batch]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: accepted_len}
        required: true
      request.input_ids:
        contract:
          dtype: int64
          rank: 2
          shape: [batch, sequence]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: runtime, version: '1.0', role: prompt_tokens}
        source: {kind: request}
        required: true
      request.attention_mask:
        contract:
          dtype: int64
          rank: 2
          shape: [batch, sequence]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: attention_mask}
        required: true
      request.past_key_values.0.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.0.key}
        required: true
      request.past_key_values.0.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.0.value}
        required: true
      request.past_key_values.1.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.1.key}
        required: true
      request.past_key_values.1.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.1.value}
        required: true
      request.past_key_values.2.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.2.key}
        required: true
      request.past_key_values.2.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.2.value}
        required: true
      request.past_key_values.3.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.3.key}
        required: true
      request.past_key_values.3.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.3.value}
        required: true
      request.past_key_values.4.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.4.key}
        required: true
      request.past_key_values.4.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.4.value}
        required: true
      request.past_key_values.5.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.5.key}
        required: true
      request.past_key_values.5.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.5.value}
        required: true
      request.past_key_values.6.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.6.key}
        required: true
      request.past_key_values.6.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.6.value}
        required: true
      request.past_key_values.7.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.7.key}
        required: true
      request.past_key_values.7.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.7.value}
        required: true
      request.past_key_values.8.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.8.key}
        required: true
      request.past_key_values.8.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.8.value}
        required: true
      request.past_key_values.9.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.9.key}
        required: true
      request.past_key_values.9.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.9.value}
        required: true
      request.past_key_values.10.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.10.key}
        required: true
      request.past_key_values.10.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.10.value}
        required: true
      request.past_key_values.11.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.11.key}
        required: true
      request.past_key_values.11.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.11.value}
        required: true
      request.past_key_values.12.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.12.key}
        required: true
      request.past_key_values.12.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.12.value}
        required: true
      request.past_key_values.13.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.13.key}
        required: true
      request.past_key_values.13.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.13.value}
        required: true
      request.past_key_values.14.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.14.key}
        required: true
      request.past_key_values.14.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        role: {kind: opaque}
        source: {kind: application, name: past_key_values.14.value}
        required: true
    outputs:
      logits:
        contract:
          dtype: float16
          rank: 3
          shape: [batch, sequence, vocab]
          batch_layout: {kind: request_aligned, axis: 0}
        role: tensor
        stage: pre_adapter
    components:
      decoder:
        implementation: {kind: onnx, artifact: model.onnx}
        ports:
          inputs:
            input_ids:
              dtype: int64
              rank: 2
              shape: [batch, sequence]
              batch_layout: {kind: request_aligned, axis: 0}
            attention_mask:
              dtype: int64
              rank: 2
              shape: [batch, sequence]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.0.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.0.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.1.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.1.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.2.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.2.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.3.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.3.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.4.key:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, past_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.4.value:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, past_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.5.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.5.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.6.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.6.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.7.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.7.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.8.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.8.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.9.key:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, past_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.9.value:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, past_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.10.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.10.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.11.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.11.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.12.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.12.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.13.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.13.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.14.key:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, past_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            past_key_values.14.value:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, past_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
          outputs:
            logits:
              dtype: float16
              rank: 3
              shape: [batch, sequence, vocab]
              batch_layout: {kind: request_aligned, axis: 0}
            present.0.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.0.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.1.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.1.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.2.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.2.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.3.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.3.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.4.key:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, total_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.4.value:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, total_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.5.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.5.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.6.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.6.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.7.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.7.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.8.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.8.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.9.key:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, total_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.9.value:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, total_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.10.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.10.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.11.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.11.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.12.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.12.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.13.key:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.13.value:
              dtype: float16
              rank: 4
              shape: [batch, sliding_kv_heads, total_sequence, sliding_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.14.key:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, total_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            present.14.value:
              dtype: float16
              rank: 4
              shape: [batch, full_kv_heads, total_sequence, full_head_dim]
              batch_layout: {kind: request_aligned, axis: 0}
            # Post-final-norm hidden state (the lm_head input == HF hidden_states[-1]).
            # Declared for truthful graph contract; consumed only by a borrowed-KV
            # speculative drafter as its folded_carry_seed (see the assistant/speculative
            # packages). Standalone generation does not route it, so the port is
            # optional: this workflow leaves it unbound while the speculative
            # workflow binds it.
            hidden_states.34:
              dtype: float16
              rank: 3
              shape: [batch, sequence, hidden]
              optional: true
              batch_layout: {kind: request_aligned, axis: 0}
          roles: {input_ids: token_ids, logits: logits}
    state:
      past_key_values.0.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.0.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.0.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.0.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.1.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.1.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.1.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.1.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.2.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.2.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.2.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.2.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.3.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.3.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.3.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.3.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.4.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.4.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: full_attention
      past_key_values.4.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.4.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: full_attention
      past_key_values.5.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.5.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.5.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.5.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.6.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.6.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.6.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.6.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.7.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.7.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.7.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.7.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.8.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.8.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.8.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.8.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.9.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.9.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: full_attention
      past_key_values.9.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.9.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: full_attention
      past_key_values.10.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.10.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.10.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.10.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.11.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.11.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.11.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.11.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.12.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.12.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.12.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.12.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.13.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.13.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.13.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, sliding_kv_heads, past_sequence, sliding_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.13.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: sliding_attention
      past_key_values.14.key:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.14.key
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: full_attention
      past_key_values.14.value:
        contract:
          dtype: float16
          rank: 4
          shape: [batch, full_kv_heads, past_sequence, full_head_dim]
          batch_layout: {kind: request_aligned, axis: 0}
        scope: invocation
        initializer: request.past_key_values.14.value
        recurrence: {kind: invariant}
        management: runtime
        release_boundary: invocation
        service_group: full_attention
    steps:
    - kind: invoke
      component: decoder
      inputs: {input_ids: request.input_ids, attention_mask: request.attention_mask, past_key_values.0.key: request.past_key_values.0.key, past_key_values.0.value: request.past_key_values.0.value, past_key_values.1.key: request.past_key_values.1.key, past_key_values.1.value: request.past_key_values.1.value, past_key_values.2.key: request.past_key_values.2.key, past_key_values.2.value: request.past_key_values.2.value, past_key_values.3.key: request.past_key_values.3.key, past_key_values.3.value: request.past_key_values.3.value, past_key_values.4.key: request.past_key_values.4.key, past_key_values.4.value: request.past_key_values.4.value, past_key_values.5.key: request.past_key_values.5.key, past_key_values.5.value: request.past_key_values.5.value, past_key_values.6.key: request.past_key_values.6.key, past_key_values.6.value: request.past_key_values.6.value, past_key_values.7.key: request.past_key_values.7.key, past_key_values.7.value: request.past_key_values.7.value, past_key_values.8.key: request.past_key_values.8.key,
        past_key_values.8.value: request.past_key_values.8.value, past_key_values.9.key: request.past_key_values.9.key, past_key_values.9.value: request.past_key_values.9.value, past_key_values.10.key: request.past_key_values.10.key, past_key_values.10.value: request.past_key_values.10.value, past_key_values.11.key: request.past_key_values.11.key, past_key_values.11.value: request.past_key_values.11.value, past_key_values.12.key: request.past_key_values.12.key, past_key_values.12.value: request.past_key_values.12.value, past_key_values.13.key: request.past_key_values.13.key, past_key_values.13.value: request.past_key_values.13.value, past_key_values.14.key: request.past_key_values.14.key, past_key_values.14.value: request.past_key_values.14.value}
      outputs: {logits: decoder.logits, present.0.key: decoder.past_key_values.0.key, present.0.value: decoder.past_key_values.0.value, present.1.key: decoder.past_key_values.1.key, present.1.value: decoder.past_key_values.1.value, present.2.key: decoder.past_key_values.2.key, present.2.value: decoder.past_key_values.2.value, present.3.key: decoder.past_key_values.3.key, present.3.value: decoder.past_key_values.3.value, present.4.key: decoder.past_key_values.4.key, present.4.value: decoder.past_key_values.4.value, present.5.key: decoder.past_key_values.5.key, present.5.value: decoder.past_key_values.5.value, present.6.key: decoder.past_key_values.6.key, present.6.value: decoder.past_key_values.6.value, present.7.key: decoder.past_key_values.7.key, present.7.value: decoder.past_key_values.7.value, present.8.key: decoder.past_key_values.8.key, present.8.value: decoder.past_key_values.8.value, present.9.key: decoder.past_key_values.9.key, present.9.value: decoder.past_key_values.9.value, present.10.key: decoder.past_key_values.10.key,
        present.10.value: decoder.past_key_values.10.value, present.11.key: decoder.past_key_values.11.key, present.11.value: decoder.past_key_values.11.value, present.12.key: decoder.past_key_values.12.key, present.12.value: decoder.past_key_values.12.value, present.13.key: decoder.past_key_values.13.key, present.13.value: decoder.past_key_values.13.value, present.14.key: decoder.past_key_values.14.key, present.14.value: decoder.past_key_values.14.value}
    - {kind: emit, value: decoder.logits, output: logits, mode: replace}
    serving:
      active: request.active
      done: request.done
      accepted_len: request.accepted_len
      state_service:
        groups:
          full_attention:
            kind: full_attention
            sequence_axis: 2
            layout: bnsh
            update: {kind: append}
            reuse: {prefix_reusable: true, evictable_prefix: false}
            ports:
              decoder:
                past_key_values.4.key: {input: past_key_values.4.key, output: present.4.key, role: key, layer: 4}
                past_key_values.4.value: {input: past_key_values.4.value, output: present.4.value, role: value, layer: 4}
                past_key_values.9.key: {input: past_key_values.9.key, output: present.9.key, role: key, layer: 9}
                past_key_values.9.value: {input: past_key_values.9.value, output: present.9.value, role: value, layer: 9}
                past_key_values.14.key: {input: past_key_values.14.key, output: present.14.key, role: key, layer: 14}
                past_key_values.14.value: {input: past_key_values.14.value, output: present.14.value, role: value, layer: 14}
          sliding_attention:
            kind: sliding_attention
            sequence_axis: 2
            layout: bnsh
            update: {kind: append}
            reuse: {prefix_reusable: true, evictable_prefix: true}
            ports:
              decoder:
                past_key_values.0.key: {input: past_key_values.0.key, output: present.0.key, role: key, layer: 0}
                past_key_values.0.value: {input: past_key_values.0.value, output: present.0.value, role: value, layer: 0}
                past_key_values.1.key: {input: past_key_values.1.key, output: present.1.key, role: key, layer: 1}
                past_key_values.1.value: {input: past_key_values.1.value, output: present.1.value, role: value, layer: 1}
                past_key_values.2.key: {input: past_key_values.2.key, output: present.2.key, role: key, layer: 2}
                past_key_values.2.value: {input: past_key_values.2.value, output: present.2.value, role: value, layer: 2}
                past_key_values.3.key: {input: past_key_values.3.key, output: present.3.key, role: key, layer: 3}
                past_key_values.3.value: {input: past_key_values.3.value, output: present.3.value, role: value, layer: 3}
                past_key_values.5.key: {input: past_key_values.5.key, output: present.5.key, role: key, layer: 5}
                past_key_values.5.value: {input: past_key_values.5.value, output: present.5.value, role: value, layer: 5}
                past_key_values.6.key: {input: past_key_values.6.key, output: present.6.key, role: key, layer: 6}
                past_key_values.6.value: {input: past_key_values.6.value, output: present.6.value, role: value, layer: 6}
                past_key_values.7.key: {input: past_key_values.7.key, output: present.7.key, role: key, layer: 7}
                past_key_values.7.value: {input: past_key_values.7.value, output: present.7.value, role: value, layer: 7}
                past_key_values.8.key: {input: past_key_values.8.key, output: present.8.key, role: key, layer: 8}
                past_key_values.8.value: {input: past_key_values.8.value, output: present.8.value, role: value, layer: 8}
                past_key_values.10.key: {input: past_key_values.10.key, output: present.10.key, role: key, layer: 10}
                past_key_values.10.value: {input: past_key_values.10.value, output: present.10.value, role: value, layer: 10}
                past_key_values.11.key: {input: past_key_values.11.key, output: present.11.key, role: key, layer: 11}
                past_key_values.11.value: {input: past_key_values.11.value, output: present.11.value, role: value, layer: 11}
                past_key_values.12.key: {input: past_key_values.12.key, output: present.12.key, role: key, layer: 12}
                past_key_values.12.value: {input: past_key_values.12.value, output: present.12.value, role: value, layer: 12}
                past_key_values.13.key: {input: past_key_values.13.key, output: present.13.key, role: key, layer: 13}
                past_key_values.13.value: {input: past_key_values.13.value, output: present.13.value, role: value, layer: 13}