diff --git "a/inference_metadata.yaml" "b/inference_metadata.yaml" new file mode 100644--- /dev/null +++ "b/inference_metadata.yaml" @@ -0,0 +1,4066 @@ +schema_version: '1.0' +pipeline: + workflow: + manifest: + capabilities: + - workflow_ssa + - linear_effects + - nested_control_flow + - typed_emit + - emit_valid_length + - loop_induction_values + - serving_service_contract + - bounded_state_recurrence + inputs: + request.input_ids: + contract: + dtype: int64 + rank: 2 + shape: + - batch + - sequence_len + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: runtime + version: '1.0' + role: prompt_tokens + source: + kind: request + required: true + request.max_iterations: + contract: + dtype: int64 + rank: 1 + shape: + - 1 + role: + kind: runtime + version: '1.0' + role: max_output_tokens + source: + kind: request + required: true + package.eos_ids: + contract: + dtype: int64 + rank: 1 + shape: + - E + role: + kind: opaque + source: + kind: literal + required: true + default: 2 + package.one_token: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: literal + required: false + default: 1 + package.one_step: + contract: + dtype: int64 + rank: 1 + shape: + - 1 + role: + kind: opaque + source: + kind: literal + required: false + default: 1 + package.max_context: + contract: + dtype: int64 + rank: 1 + shape: + - 1 + role: + kind: opaque + source: + kind: literal + required: false + default: 32768 + request.prompt_lengths: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: application + name: prompt_lengths + required: false + default: -1 + request.eos_ids: + contract: + dtype: int64 + rank: 2 + shape: + - batch + - num_eos + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: application + name: eos_ids + required: false + default: 2 + request.eos_lengths: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: application + name: eos_lengths + required: false + default: 1 + request.row_max_iterations: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: application + name: row_max_iterations + required: false + default: -1 + request.temperature: + contract: + dtype: float32 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: runtime + version: '1.0' + role: sampling_temperature + source: + kind: request + required: false + default: 1.0 + request.top_k: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: runtime + version: '1.0' + role: sampling_top_k + source: + kind: request + required: false + default: 1 + request.top_p: + contract: + dtype: float32 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: runtime + version: '1.0' + role: sampling_top_p + source: + kind: request + required: false + default: 1.0 + request.min_p: + contract: + dtype: float32 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: runtime + version: '1.0' + role: sampling_min_p + source: + kind: request + required: false + default: 0.0 + request.seed: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: runtime + version: '1.0' + role: seed + source: + kind: request + required: false + default: 0 + request.rng_counter: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: application + name: rng_counter + required: false + default: 0 + package.active: + contract: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: literal + required: false + default: true + package.not_done: + contract: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: literal + required: false + default: false + package.cache_lengths: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: literal + required: false + default: 0 + package.zero_batch: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + role: + kind: opaque + source: + kind: literal + required: false + default: 0 + outputs: + tokens: + contract: + dtype: int64 + rank: 2 + shape: + - batch + - generated_sequence + batch_layout: + kind: request_aligned + axis: 0 + role: tokens + stage: pre_adapter + components: + model: + implementation: + kind: onnx + artifact: model.onnx + ports: + roles: + input_ids: token_ids + attention_mask: attention_mask + logits: logits + token_sampler: + implementation: + kind: onnx + artifact: policies/token_sampler.onnx + ports: + inputs: + logits: + dtype: float32 + rank: 2 + shape: + - batch + - vocabulary + batch_layout: + kind: request_aligned + axis: 0 + temperature: + dtype: float32 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + top_k: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + top_p: + dtype: float32 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + min_p: + dtype: float32 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + seed: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + counter: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + done: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + token: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + next_counter: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + contract: + id: onnx-genai.token-sampler + version: '2' + bindings: + logits: logits + token: token + temperature: temperature + top_k: top_k + top_p: top_p + min_p: min_p + active: active + done: done + seed: seed + counter: counter + next_counter: next_counter + parameters: + mode: seeded_stochastic + batching: per_row + inactive_rows: preserve + application_overridable: true + termination: + implementation: + kind: onnx + artifact: policies/termination.onnx + ports: + inputs: + tokens: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + eos_ids: + dtype: int64 + rank: 2 + shape: + - batch + - num_eos + batch_layout: + kind: request_aligned + axis: 0 + eos_lengths: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + iteration: + dtype: int64 + rank: 1 + shape: + - 1 + max_iterations: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + done: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + next_active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + continue: + dtype: bool + rank: 1 + shape: + - 1 + contract: + id: onnx-genai.termination-predicate + version: '2' + bindings: + tokens: tokens + eos_ids: eos_ids + iteration: iteration + max_iterations: max_iterations + eos_lengths: eos_lengths + active: active + done: done + next_active: next_active + continue: continue + parameters: + batching: per_row + inactive_rows: preserve + token_state_update: + implementation: + kind: onnx + artifact: policies/token_state_update.onnx + ports: + inputs: + current: + dtype: int64 + rank: 2 + shape: + - batch + - 1 + batch_layout: + kind: request_aligned + axis: 0 + update: + dtype: int64 + rank: 2 + shape: + - batch + - 1 + batch_layout: + kind: request_aligned + axis: 0 + active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + done: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + next: + dtype: int64 + rank: 2 + shape: + - batch + - 1 + batch_layout: + kind: request_aligned + axis: 0 + contract: + id: onnx-genai.state-update + version: '2' + bindings: + current: current + update: update + active: active + done: done + next: next + parameters: + batching: per_row + inactive_rows: preserve + last_token_logits: + implementation: + kind: onnx + artifact: policies/last_token_logits.onnx + ports: + inputs: + logits: + dtype: float16 + rank: 3 + shape: + - batch + - sequence + - vocabulary + batch_layout: + kind: request_aligned + axis: 0 + outputs: + last_logits: + dtype: float32 + rank: 2 + shape: + - batch + - vocabulary + batch_layout: + kind: request_aligned + axis: 0 + decoder_state_initializer: + implementation: + kind: onnx + artifact: policies/decoder_state_initializer.onnx + ports: + inputs: + prompt_tokens: + dtype: int64 + rank: 2 + shape: + - batch + - prompt_sequence + batch_layout: + kind: request_aligned + axis: 0 + prompt_lengths: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + max_iterations: + dtype: int64 + rank: 1 + shape: + - 1 + outputs: + attention_mask: + dtype: int64 + rank: 2 + shape: + - batch + - capacity + batch_layout: + kind: request_aligned + axis: 0 + body_attention_mask: + dtype: int64 + rank: 2 + shape: + - batch + - capacity + batch_layout: + kind: request_aligned + axis: 0 + token_slot: + dtype: int64 + rank: 2 + shape: + - batch + - 1 + batch_layout: + kind: request_aligned + axis: 0 + generated_lengths: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + cache_lengths: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.0.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.0.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.1.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.1.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.10.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.10.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.11.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.11.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.12.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.12.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.13.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.13.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.14.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.14.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.15.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.15.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.16.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.16.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.17.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.17.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.18.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.18.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.19.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.19.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.2.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.2.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.20.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.20.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.21.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.21.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.22.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.22.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.23.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.23.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.24.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.24.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.25.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.25.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.26.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.26.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.27.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.27.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.28.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.28.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.29.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.29.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.3.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.3.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.30.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.30.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.31.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.31.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.4.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.4.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.5.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.5.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.6.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.6.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.7.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.7.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.8.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.8.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.9.key: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + past_key_values.9.value: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - capacity + - 128 + batch_layout: + kind: request_aligned + axis: 0 + decoder_step_update: + implementation: + kind: onnx + artifact: policies/decoder_step_update.onnx + ports: + inputs: + attention_mask: + dtype: int64 + rank: 2 + shape: + - batch + - context + batch_layout: + kind: request_aligned + axis: 0 + logical_length: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + next_attention_mask: + dtype: int64 + rank: 2 + shape: + - batch + - context + batch_layout: + kind: request_aligned + axis: 0 + cache_length_update: + implementation: + kind: onnx + artifact: policies/cache_length_update.onnx + ports: + inputs: + left: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + right: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + done: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + total: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + termination_batch_initializer: + implementation: + kind: onnx + artifact: policies/termination_batch_initializer.onnx + ports: + inputs: + input_eos_ids: + dtype: int64 + rank: 2 + shape: + - batch + - num_eos + batch_layout: + kind: request_aligned + axis: 0 + input_eos_lengths: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + input_max_iterations: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + fallback_max_iterations: + dtype: int64 + rank: 1 + shape: + - 1 + active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + row_eos_ids: + dtype: int64 + rank: 2 + shape: + - batch + - num_eos + batch_layout: + kind: request_aligned + axis: 0 + eos_lengths: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + max_iterations: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + token_to_slot: + implementation: + kind: onnx + artifact: policies/token_to_slot.onnx + ports: + inputs: + token: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + slot: + dtype: int64 + rank: 2 + shape: + - batch + - 1 + batch_layout: + kind: request_aligned + axis: 0 + generated_length_update: + implementation: + kind: onnx + artifact: policies/generated_length_update.onnx + ports: + inputs: + left: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + right: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + active: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + done: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + outputs: + total: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + state: + token: + contract: + dtype: int64 + rank: 2 + shape: + - batch + - 1 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: initializer.token_slot + recurrence: + kind: invariant + logits: + contract: + dtype: float32 + rank: 2 + shape: + - batch + - 32000 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.last_logits + recurrence: + kind: invariant + generated_lengths: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + class: semantic + scope: invocation + initializer: initializer.generated_lengths + recurrence: + kind: invariant + active: + contract: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + class: semantic + scope: invocation + initializer: package.active + recurrence: + kind: invariant + done: + contract: + dtype: bool + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + class: semantic + scope: invocation + initializer: package.not_done + recurrence: + kind: invariant + accepted_len: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + class: semantic + scope: invocation + initializer: package.zero_batch + recurrence: + kind: invariant + cache_lengths: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + class: semantic + scope: invocation + initializer: initializer.cache_lengths + recurrence: + kind: invariant + rng_counter: + contract: + dtype: int64 + rank: 1 + shape: + - batch + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + class: semantic + initializer: request.rng_counter + recurrence: + kind: invariant + attention_mask: + contract: + dtype: int64 + rank: 2 + shape: + - batch + - context + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: initializer.attention_mask + recurrence: + kind: invariant + cache_0: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.0.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_1: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.0.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_2: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.1.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_3: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.1.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_4: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.2.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_5: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.2.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_6: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.3.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_7: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.3.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_8: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.4.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_9: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.4.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_10: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.5.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_11: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.5.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_12: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.6.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_13: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.6.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_14: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.7.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_15: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.7.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_16: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.8.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_17: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.8.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_18: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.9.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_19: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.9.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_20: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.10.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_21: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.10.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_22: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.11.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_23: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.11.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_24: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.12.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_25: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.12.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_26: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.13.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_27: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.13.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_28: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.14.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_29: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.14.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_30: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.15.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_31: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.15.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_32: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.16.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_33: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.16.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_34: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.17.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_35: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.17.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_36: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.18.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_37: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.18.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_38: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.19.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_39: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.19.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_40: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.20.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_41: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.20.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_42: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.21.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_43: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.21.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_44: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.22.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_45: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.22.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_46: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.23.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_47: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.23.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_48: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.24.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_49: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.24.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_50: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.25.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_51: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.25.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_52: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.26.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_53: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.26.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_54: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.27.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_55: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.27.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_56: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.28.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_57: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.28.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_58: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.29.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_59: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.29.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_60: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.30.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_61: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.30.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_62: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.31.key + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + cache_63: + contract: + dtype: float16 + rank: 4 + shape: + - batch + - 8 + - past_sequence_len + - 128 + batch_layout: + kind: request_aligned + axis: 0 + scope: invocation + initializer: decoder.setup.present.31.value + recurrence: + kind: bounded + axis: 2 + max: package.max_context + management: runtime + release_boundary: invocation + service_group: decoder_cache + serving: + active: active + done: done + accepted_len: accepted_len + state_service: + groups: + decoder_cache: + kind: sliding_attention + sequence_axis: 2 + layout: bnsh + logical_lengths: cache_lengths + aliasing: permitted + reuse: + prefix_reusable: true + evictable_prefix: true + ports: + model: + cache_0: + input: past_key_values.0.key + output: present.0.key + role: key + layer: 0 + cache_1: + input: past_key_values.0.value + output: present.0.value + role: value + layer: 0 + cache_2: + input: past_key_values.1.key + output: present.1.key + role: key + layer: 1 + cache_3: + input: past_key_values.1.value + output: present.1.value + role: value + layer: 1 + cache_4: + input: past_key_values.2.key + output: present.2.key + role: key + layer: 2 + cache_5: + input: past_key_values.2.value + output: present.2.value + role: value + layer: 2 + cache_6: + input: past_key_values.3.key + output: present.3.key + role: key + layer: 3 + cache_7: + input: past_key_values.3.value + output: present.3.value + role: value + layer: 3 + cache_8: + input: past_key_values.4.key + output: present.4.key + role: key + layer: 4 + cache_9: + input: past_key_values.4.value + output: present.4.value + role: value + layer: 4 + cache_10: + input: past_key_values.5.key + output: present.5.key + role: key + layer: 5 + cache_11: + input: past_key_values.5.value + output: present.5.value + role: value + layer: 5 + cache_12: + input: past_key_values.6.key + output: present.6.key + role: key + layer: 6 + cache_13: + input: past_key_values.6.value + output: present.6.value + role: value + layer: 6 + cache_14: + input: past_key_values.7.key + output: present.7.key + role: key + layer: 7 + cache_15: + input: past_key_values.7.value + output: present.7.value + role: value + layer: 7 + cache_16: + input: past_key_values.8.key + output: present.8.key + role: key + layer: 8 + cache_17: + input: past_key_values.8.value + output: present.8.value + role: value + layer: 8 + cache_18: + input: past_key_values.9.key + output: present.9.key + role: key + layer: 9 + cache_19: + input: past_key_values.9.value + output: present.9.value + role: value + layer: 9 + cache_20: + input: past_key_values.10.key + output: present.10.key + role: key + layer: 10 + cache_21: + input: past_key_values.10.value + output: present.10.value + role: value + layer: 10 + cache_22: + input: past_key_values.11.key + output: present.11.key + role: key + layer: 11 + cache_23: + input: past_key_values.11.value + output: present.11.value + role: value + layer: 11 + cache_24: + input: past_key_values.12.key + output: present.12.key + role: key + layer: 12 + cache_25: + input: past_key_values.12.value + output: present.12.value + role: value + layer: 12 + cache_26: + input: past_key_values.13.key + output: present.13.key + role: key + layer: 13 + cache_27: + input: past_key_values.13.value + output: present.13.value + role: value + layer: 13 + cache_28: + input: past_key_values.14.key + output: present.14.key + role: key + layer: 14 + cache_29: + input: past_key_values.14.value + output: present.14.value + role: value + layer: 14 + cache_30: + input: past_key_values.15.key + output: present.15.key + role: key + layer: 15 + cache_31: + input: past_key_values.15.value + output: present.15.value + role: value + layer: 15 + cache_32: + input: past_key_values.16.key + output: present.16.key + role: key + layer: 16 + cache_33: + input: past_key_values.16.value + output: present.16.value + role: value + layer: 16 + cache_34: + input: past_key_values.17.key + output: present.17.key + role: key + layer: 17 + cache_35: + input: past_key_values.17.value + output: present.17.value + role: value + layer: 17 + cache_36: + input: past_key_values.18.key + output: present.18.key + role: key + layer: 18 + cache_37: + input: past_key_values.18.value + output: present.18.value + role: value + layer: 18 + cache_38: + input: past_key_values.19.key + output: present.19.key + role: key + layer: 19 + cache_39: + input: past_key_values.19.value + output: present.19.value + role: value + layer: 19 + cache_40: + input: past_key_values.20.key + output: present.20.key + role: key + layer: 20 + cache_41: + input: past_key_values.20.value + output: present.20.value + role: value + layer: 20 + cache_42: + input: past_key_values.21.key + output: present.21.key + role: key + layer: 21 + cache_43: + input: past_key_values.21.value + output: present.21.value + role: value + layer: 21 + cache_44: + input: past_key_values.22.key + output: present.22.key + role: key + layer: 22 + cache_45: + input: past_key_values.22.value + output: present.22.value + role: value + layer: 22 + cache_46: + input: past_key_values.23.key + output: present.23.key + role: key + layer: 23 + cache_47: + input: past_key_values.23.value + output: present.23.value + role: value + layer: 23 + cache_48: + input: past_key_values.24.key + output: present.24.key + role: key + layer: 24 + cache_49: + input: past_key_values.24.value + output: present.24.value + role: value + layer: 24 + cache_50: + input: past_key_values.25.key + output: present.25.key + role: key + layer: 25 + cache_51: + input: past_key_values.25.value + output: present.25.value + role: value + layer: 25 + cache_52: + input: past_key_values.26.key + output: present.26.key + role: key + layer: 26 + cache_53: + input: past_key_values.26.value + output: present.26.value + role: value + layer: 26 + cache_54: + input: past_key_values.27.key + output: present.27.key + role: key + layer: 27 + cache_55: + input: past_key_values.27.value + output: present.27.value + role: value + layer: 27 + cache_56: + input: past_key_values.28.key + output: present.28.key + role: key + layer: 28 + cache_57: + input: past_key_values.28.value + output: present.28.value + role: value + layer: 28 + cache_58: + input: past_key_values.29.key + output: present.29.key + role: key + layer: 29 + cache_59: + input: past_key_values.29.value + output: present.29.value + role: value + layer: 29 + cache_60: + input: past_key_values.30.key + output: present.30.key + role: key + layer: 30 + cache_61: + input: past_key_values.30.value + output: present.30.value + role: value + layer: 30 + cache_62: + input: past_key_values.31.key + output: present.31.key + role: key + layer: 31 + cache_63: + input: past_key_values.31.value + output: present.31.value + role: value + layer: 31 + steps: + - kind: loop + setup: + - kind: invoke + component: decoder_state_initializer + inputs: + prompt_tokens: request.input_ids + prompt_lengths: request.prompt_lengths + max_iterations: request.max_iterations + outputs: + attention_mask: initializer.attention_mask + body_attention_mask: initializer.body_attention_mask + token_slot: initializer.token_slot + generated_lengths: initializer.generated_lengths + cache_lengths: initializer.cache_lengths + past_key_values.0.key: initializer.past_key_values.0.key + past_key_values.0.value: initializer.past_key_values.0.value + past_key_values.1.key: initializer.past_key_values.1.key + past_key_values.1.value: initializer.past_key_values.1.value + past_key_values.10.key: initializer.past_key_values.10.key + past_key_values.10.value: initializer.past_key_values.10.value + past_key_values.11.key: initializer.past_key_values.11.key + past_key_values.11.value: initializer.past_key_values.11.value + past_key_values.12.key: initializer.past_key_values.12.key + past_key_values.12.value: initializer.past_key_values.12.value + past_key_values.13.key: initializer.past_key_values.13.key + past_key_values.13.value: initializer.past_key_values.13.value + past_key_values.14.key: initializer.past_key_values.14.key + past_key_values.14.value: initializer.past_key_values.14.value + past_key_values.15.key: initializer.past_key_values.15.key + past_key_values.15.value: initializer.past_key_values.15.value + past_key_values.16.key: initializer.past_key_values.16.key + past_key_values.16.value: initializer.past_key_values.16.value + past_key_values.17.key: initializer.past_key_values.17.key + past_key_values.17.value: initializer.past_key_values.17.value + past_key_values.18.key: initializer.past_key_values.18.key + past_key_values.18.value: initializer.past_key_values.18.value + past_key_values.19.key: initializer.past_key_values.19.key + past_key_values.19.value: initializer.past_key_values.19.value + past_key_values.2.key: initializer.past_key_values.2.key + past_key_values.2.value: initializer.past_key_values.2.value + past_key_values.20.key: initializer.past_key_values.20.key + past_key_values.20.value: initializer.past_key_values.20.value + past_key_values.21.key: initializer.past_key_values.21.key + past_key_values.21.value: initializer.past_key_values.21.value + past_key_values.22.key: initializer.past_key_values.22.key + past_key_values.22.value: initializer.past_key_values.22.value + past_key_values.23.key: initializer.past_key_values.23.key + past_key_values.23.value: initializer.past_key_values.23.value + past_key_values.24.key: initializer.past_key_values.24.key + past_key_values.24.value: initializer.past_key_values.24.value + past_key_values.25.key: initializer.past_key_values.25.key + past_key_values.25.value: initializer.past_key_values.25.value + past_key_values.26.key: initializer.past_key_values.26.key + past_key_values.26.value: initializer.past_key_values.26.value + past_key_values.27.key: initializer.past_key_values.27.key + past_key_values.27.value: initializer.past_key_values.27.value + past_key_values.28.key: initializer.past_key_values.28.key + past_key_values.28.value: initializer.past_key_values.28.value + past_key_values.29.key: initializer.past_key_values.29.key + past_key_values.29.value: initializer.past_key_values.29.value + past_key_values.3.key: initializer.past_key_values.3.key + past_key_values.3.value: initializer.past_key_values.3.value + past_key_values.30.key: initializer.past_key_values.30.key + past_key_values.30.value: initializer.past_key_values.30.value + past_key_values.31.key: initializer.past_key_values.31.key + past_key_values.31.value: initializer.past_key_values.31.value + past_key_values.4.key: initializer.past_key_values.4.key + past_key_values.4.value: initializer.past_key_values.4.value + past_key_values.5.key: initializer.past_key_values.5.key + past_key_values.5.value: initializer.past_key_values.5.value + past_key_values.6.key: initializer.past_key_values.6.key + past_key_values.6.value: initializer.past_key_values.6.value + past_key_values.7.key: initializer.past_key_values.7.key + past_key_values.7.value: initializer.past_key_values.7.value + past_key_values.8.key: initializer.past_key_values.8.key + past_key_values.8.value: initializer.past_key_values.8.value + past_key_values.9.key: initializer.past_key_values.9.key + past_key_values.9.value: initializer.past_key_values.9.value + - kind: invoke + component: model + inputs: + input_ids: request.input_ids + past_key_values.0.key: initializer.past_key_values.0.key + past_key_values.0.value: initializer.past_key_values.0.value + past_key_values.1.key: initializer.past_key_values.1.key + past_key_values.1.value: initializer.past_key_values.1.value + past_key_values.2.key: initializer.past_key_values.2.key + past_key_values.2.value: initializer.past_key_values.2.value + past_key_values.3.key: initializer.past_key_values.3.key + past_key_values.3.value: initializer.past_key_values.3.value + past_key_values.4.key: initializer.past_key_values.4.key + past_key_values.4.value: initializer.past_key_values.4.value + past_key_values.5.key: initializer.past_key_values.5.key + past_key_values.5.value: initializer.past_key_values.5.value + past_key_values.6.key: initializer.past_key_values.6.key + past_key_values.6.value: initializer.past_key_values.6.value + past_key_values.7.key: initializer.past_key_values.7.key + past_key_values.7.value: initializer.past_key_values.7.value + past_key_values.8.key: initializer.past_key_values.8.key + past_key_values.8.value: initializer.past_key_values.8.value + past_key_values.9.key: initializer.past_key_values.9.key + past_key_values.9.value: initializer.past_key_values.9.value + past_key_values.10.key: initializer.past_key_values.10.key + past_key_values.10.value: initializer.past_key_values.10.value + past_key_values.11.key: initializer.past_key_values.11.key + past_key_values.11.value: initializer.past_key_values.11.value + past_key_values.12.key: initializer.past_key_values.12.key + past_key_values.12.value: initializer.past_key_values.12.value + past_key_values.13.key: initializer.past_key_values.13.key + past_key_values.13.value: initializer.past_key_values.13.value + past_key_values.14.key: initializer.past_key_values.14.key + past_key_values.14.value: initializer.past_key_values.14.value + past_key_values.15.key: initializer.past_key_values.15.key + past_key_values.15.value: initializer.past_key_values.15.value + past_key_values.16.key: initializer.past_key_values.16.key + past_key_values.16.value: initializer.past_key_values.16.value + past_key_values.17.key: initializer.past_key_values.17.key + past_key_values.17.value: initializer.past_key_values.17.value + past_key_values.18.key: initializer.past_key_values.18.key + past_key_values.18.value: initializer.past_key_values.18.value + past_key_values.19.key: initializer.past_key_values.19.key + past_key_values.19.value: initializer.past_key_values.19.value + past_key_values.20.key: initializer.past_key_values.20.key + past_key_values.20.value: initializer.past_key_values.20.value + past_key_values.21.key: initializer.past_key_values.21.key + past_key_values.21.value: initializer.past_key_values.21.value + past_key_values.22.key: initializer.past_key_values.22.key + past_key_values.22.value: initializer.past_key_values.22.value + past_key_values.23.key: initializer.past_key_values.23.key + past_key_values.23.value: initializer.past_key_values.23.value + past_key_values.24.key: initializer.past_key_values.24.key + past_key_values.24.value: initializer.past_key_values.24.value + past_key_values.25.key: initializer.past_key_values.25.key + past_key_values.25.value: initializer.past_key_values.25.value + past_key_values.26.key: initializer.past_key_values.26.key + past_key_values.26.value: initializer.past_key_values.26.value + past_key_values.27.key: initializer.past_key_values.27.key + past_key_values.27.value: initializer.past_key_values.27.value + past_key_values.28.key: initializer.past_key_values.28.key + past_key_values.28.value: initializer.past_key_values.28.value + past_key_values.29.key: initializer.past_key_values.29.key + past_key_values.29.value: initializer.past_key_values.29.value + past_key_values.30.key: initializer.past_key_values.30.key + past_key_values.30.value: initializer.past_key_values.30.value + past_key_values.31.key: initializer.past_key_values.31.key + past_key_values.31.value: initializer.past_key_values.31.value + attention_mask: initializer.attention_mask + outputs: + logits: decoder.setup.logits + present.0.key: decoder.setup.present.0.key + present.0.value: decoder.setup.present.0.value + present.1.key: decoder.setup.present.1.key + present.1.value: decoder.setup.present.1.value + present.2.key: decoder.setup.present.2.key + present.2.value: decoder.setup.present.2.value + present.3.key: decoder.setup.present.3.key + present.3.value: decoder.setup.present.3.value + present.4.key: decoder.setup.present.4.key + present.4.value: decoder.setup.present.4.value + present.5.key: decoder.setup.present.5.key + present.5.value: decoder.setup.present.5.value + present.6.key: decoder.setup.present.6.key + present.6.value: decoder.setup.present.6.value + present.7.key: decoder.setup.present.7.key + present.7.value: decoder.setup.present.7.value + present.8.key: decoder.setup.present.8.key + present.8.value: decoder.setup.present.8.value + present.9.key: decoder.setup.present.9.key + present.9.value: decoder.setup.present.9.value + present.10.key: decoder.setup.present.10.key + present.10.value: decoder.setup.present.10.value + present.11.key: decoder.setup.present.11.key + present.11.value: decoder.setup.present.11.value + present.12.key: decoder.setup.present.12.key + present.12.value: decoder.setup.present.12.value + present.13.key: decoder.setup.present.13.key + present.13.value: decoder.setup.present.13.value + present.14.key: decoder.setup.present.14.key + present.14.value: decoder.setup.present.14.value + present.15.key: decoder.setup.present.15.key + present.15.value: decoder.setup.present.15.value + present.16.key: decoder.setup.present.16.key + present.16.value: decoder.setup.present.16.value + present.17.key: decoder.setup.present.17.key + present.17.value: decoder.setup.present.17.value + present.18.key: decoder.setup.present.18.key + present.18.value: decoder.setup.present.18.value + present.19.key: decoder.setup.present.19.key + present.19.value: decoder.setup.present.19.value + present.20.key: decoder.setup.present.20.key + present.20.value: decoder.setup.present.20.value + present.21.key: decoder.setup.present.21.key + present.21.value: decoder.setup.present.21.value + present.22.key: decoder.setup.present.22.key + present.22.value: decoder.setup.present.22.value + present.23.key: decoder.setup.present.23.key + present.23.value: decoder.setup.present.23.value + present.24.key: decoder.setup.present.24.key + present.24.value: decoder.setup.present.24.value + present.25.key: decoder.setup.present.25.key + present.25.value: decoder.setup.present.25.value + present.26.key: decoder.setup.present.26.key + present.26.value: decoder.setup.present.26.value + present.27.key: decoder.setup.present.27.key + present.27.value: decoder.setup.present.27.value + present.28.key: decoder.setup.present.28.key + present.28.value: decoder.setup.present.28.value + present.29.key: decoder.setup.present.29.key + present.29.value: decoder.setup.present.29.value + present.30.key: decoder.setup.present.30.key + present.30.value: decoder.setup.present.30.value + present.31.key: decoder.setup.present.31.key + present.31.value: decoder.setup.present.31.value + - kind: invoke + component: termination_batch_initializer + inputs: + input_eos_ids: request.eos_ids + input_eos_lengths: request.eos_lengths + input_max_iterations: request.row_max_iterations + fallback_max_iterations: request.max_iterations + active: package.active + outputs: + row_eos_ids: termination.eos_ids + eos_lengths: termination.eos_lengths + max_iterations: termination.max_iterations + - kind: invoke + component: last_token_logits + inputs: + logits: decoder.setup.logits + outputs: + last_logits: decoder.setup.last_logits + steps: + - kind: invoke + component: token_sampler + inputs: + logits: logits + temperature: request.temperature + top_k: request.top_k + top_p: request.top_p + min_p: request.min_p + seed: request.seed + counter: rng_counter + active: active + done: done + outputs: + token: sample.body + next_counter: sample.next_counter + - kind: invoke + component: token_to_slot + inputs: + token: sample.body + outputs: + slot: sample.slot + - kind: invoke + component: generated_length_update + inputs: + left: generated_lengths + right: package.one_token + active: active + done: done + outputs: + total: token.next_lengths + - kind: invoke + component: generated_length_update + inputs: + left: package.zero_batch + right: package.one_token + active: active + done: done + outputs: + total: token.emitted_length + - kind: invoke + component: token_state_update + inputs: + current: token + update: sample.slot + active: active + done: done + outputs: + next: token.body + - kind: invoke + component: termination + inputs: + tokens: sample.body + eos_ids: termination.eos_ids + eos_lengths: termination.eos_lengths + iteration: loop.iteration + max_iterations: termination.max_iterations + active: active + outputs: + done: loop.done + continue: loop.continue + next_active: loop.next_active + - kind: invoke + component: cache_length_update + inputs: + left: cache_lengths + right: package.one_token + active: active + done: done + outputs: + total: cache_lengths.next + - kind: invoke + component: cache_length_update + inputs: + left: package.zero_batch + right: package.one_token + active: active + done: done + outputs: + total: accepted_len.next + - kind: emit + value: token.body + output: tokens + mode: append + valid_length: token.emitted_length + when: active + - kind: invoke + component: decoder_step_update + inputs: + attention_mask: attention_mask + logical_length: cache_lengths + outputs: + next_attention_mask: decoder_step.body_attention_mask + - kind: invoke + component: model + inputs: + input_ids: token.body + past_key_values.0.key: cache_0 + past_key_values.0.value: cache_1 + past_key_values.1.key: cache_2 + past_key_values.1.value: cache_3 + past_key_values.2.key: cache_4 + past_key_values.2.value: cache_5 + past_key_values.3.key: cache_6 + past_key_values.3.value: cache_7 + past_key_values.4.key: cache_8 + past_key_values.4.value: cache_9 + past_key_values.5.key: cache_10 + past_key_values.5.value: cache_11 + past_key_values.6.key: cache_12 + past_key_values.6.value: cache_13 + past_key_values.7.key: cache_14 + past_key_values.7.value: cache_15 + past_key_values.8.key: cache_16 + past_key_values.8.value: cache_17 + past_key_values.9.key: cache_18 + past_key_values.9.value: cache_19 + past_key_values.10.key: cache_20 + past_key_values.10.value: cache_21 + past_key_values.11.key: cache_22 + past_key_values.11.value: cache_23 + past_key_values.12.key: cache_24 + past_key_values.12.value: cache_25 + past_key_values.13.key: cache_26 + past_key_values.13.value: cache_27 + past_key_values.14.key: cache_28 + past_key_values.14.value: cache_29 + past_key_values.15.key: cache_30 + past_key_values.15.value: cache_31 + past_key_values.16.key: cache_32 + past_key_values.16.value: cache_33 + past_key_values.17.key: cache_34 + past_key_values.17.value: cache_35 + past_key_values.18.key: cache_36 + past_key_values.18.value: cache_37 + past_key_values.19.key: cache_38 + past_key_values.19.value: cache_39 + past_key_values.20.key: cache_40 + past_key_values.20.value: cache_41 + past_key_values.21.key: cache_42 + past_key_values.21.value: cache_43 + past_key_values.22.key: cache_44 + past_key_values.22.value: cache_45 + past_key_values.23.key: cache_46 + past_key_values.23.value: cache_47 + past_key_values.24.key: cache_48 + past_key_values.24.value: cache_49 + past_key_values.25.key: cache_50 + past_key_values.25.value: cache_51 + past_key_values.26.key: cache_52 + past_key_values.26.value: cache_53 + past_key_values.27.key: cache_54 + past_key_values.27.value: cache_55 + past_key_values.28.key: cache_56 + past_key_values.28.value: cache_57 + past_key_values.29.key: cache_58 + past_key_values.29.value: cache_59 + past_key_values.30.key: cache_60 + past_key_values.30.value: cache_61 + past_key_values.31.key: cache_62 + past_key_values.31.value: cache_63 + attention_mask: decoder_step.body_attention_mask + outputs: + logits: decoder.body.logits + present.0.key: decoder.body.present.0.key + present.0.value: decoder.body.present.0.value + present.1.key: decoder.body.present.1.key + present.1.value: decoder.body.present.1.value + present.2.key: decoder.body.present.2.key + present.2.value: decoder.body.present.2.value + present.3.key: decoder.body.present.3.key + present.3.value: decoder.body.present.3.value + present.4.key: decoder.body.present.4.key + present.4.value: decoder.body.present.4.value + present.5.key: decoder.body.present.5.key + present.5.value: decoder.body.present.5.value + present.6.key: decoder.body.present.6.key + present.6.value: decoder.body.present.6.value + present.7.key: decoder.body.present.7.key + present.7.value: decoder.body.present.7.value + present.8.key: decoder.body.present.8.key + present.8.value: decoder.body.present.8.value + present.9.key: decoder.body.present.9.key + present.9.value: decoder.body.present.9.value + present.10.key: decoder.body.present.10.key + present.10.value: decoder.body.present.10.value + present.11.key: decoder.body.present.11.key + present.11.value: decoder.body.present.11.value + present.12.key: decoder.body.present.12.key + present.12.value: decoder.body.present.12.value + present.13.key: decoder.body.present.13.key + present.13.value: decoder.body.present.13.value + present.14.key: decoder.body.present.14.key + present.14.value: decoder.body.present.14.value + present.15.key: decoder.body.present.15.key + present.15.value: decoder.body.present.15.value + present.16.key: decoder.body.present.16.key + present.16.value: decoder.body.present.16.value + present.17.key: decoder.body.present.17.key + present.17.value: decoder.body.present.17.value + present.18.key: decoder.body.present.18.key + present.18.value: decoder.body.present.18.value + present.19.key: decoder.body.present.19.key + present.19.value: decoder.body.present.19.value + present.20.key: decoder.body.present.20.key + present.20.value: decoder.body.present.20.value + present.21.key: decoder.body.present.21.key + present.21.value: decoder.body.present.21.value + present.22.key: decoder.body.present.22.key + present.22.value: decoder.body.present.22.value + present.23.key: decoder.body.present.23.key + present.23.value: decoder.body.present.23.value + present.24.key: decoder.body.present.24.key + present.24.value: decoder.body.present.24.value + present.25.key: decoder.body.present.25.key + present.25.value: decoder.body.present.25.value + present.26.key: decoder.body.present.26.key + present.26.value: decoder.body.present.26.value + present.27.key: decoder.body.present.27.key + present.27.value: decoder.body.present.27.value + present.28.key: decoder.body.present.28.key + present.28.value: decoder.body.present.28.value + present.29.key: decoder.body.present.29.key + present.29.value: decoder.body.present.29.value + present.30.key: decoder.body.present.30.key + present.30.value: decoder.body.present.30.value + present.31.key: decoder.body.present.31.key + present.31.value: decoder.body.present.31.value + - kind: invoke + component: last_token_logits + inputs: + logits: decoder.body.logits + outputs: + last_logits: decoder.body.last_logits + continue_when: active + max_iterations: request.max_iterations + carried: + - cell: token + next: token.body + - cell: logits + next: decoder.body.last_logits + - cell: generated_lengths + next: token.next_lengths + - cell: active + next: loop.next_active + - cell: done + next: loop.done + - cell: cache_lengths + next: cache_lengths.next + - cell: accepted_len + next: accepted_len.next + - cell: rng_counter + next: sample.next_counter + - cell: attention_mask + next: decoder_step.body_attention_mask + - cell: cache_0 + next: decoder.body.present.0.key + - cell: cache_1 + next: decoder.body.present.0.value + - cell: cache_2 + next: decoder.body.present.1.key + - cell: cache_3 + next: decoder.body.present.1.value + - cell: cache_4 + next: decoder.body.present.2.key + - cell: cache_5 + next: decoder.body.present.2.value + - cell: cache_6 + next: decoder.body.present.3.key + - cell: cache_7 + next: decoder.body.present.3.value + - cell: cache_8 + next: decoder.body.present.4.key + - cell: cache_9 + next: decoder.body.present.4.value + - cell: cache_10 + next: decoder.body.present.5.key + - cell: cache_11 + next: decoder.body.present.5.value + - cell: cache_12 + next: decoder.body.present.6.key + - cell: cache_13 + next: decoder.body.present.6.value + - cell: cache_14 + next: decoder.body.present.7.key + - cell: cache_15 + next: decoder.body.present.7.value + - cell: cache_16 + next: decoder.body.present.8.key + - cell: cache_17 + next: decoder.body.present.8.value + - cell: cache_18 + next: decoder.body.present.9.key + - cell: cache_19 + next: decoder.body.present.9.value + - cell: cache_20 + next: decoder.body.present.10.key + - cell: cache_21 + next: decoder.body.present.10.value + - cell: cache_22 + next: decoder.body.present.11.key + - cell: cache_23 + next: decoder.body.present.11.value + - cell: cache_24 + next: decoder.body.present.12.key + - cell: cache_25 + next: decoder.body.present.12.value + - cell: cache_26 + next: decoder.body.present.13.key + - cell: cache_27 + next: decoder.body.present.13.value + - cell: cache_28 + next: decoder.body.present.14.key + - cell: cache_29 + next: decoder.body.present.14.value + - cell: cache_30 + next: decoder.body.present.15.key + - cell: cache_31 + next: decoder.body.present.15.value + - cell: cache_32 + next: decoder.body.present.16.key + - cell: cache_33 + next: decoder.body.present.16.value + - cell: cache_34 + next: decoder.body.present.17.key + - cell: cache_35 + next: decoder.body.present.17.value + - cell: cache_36 + next: decoder.body.present.18.key + - cell: cache_37 + next: decoder.body.present.18.value + - cell: cache_38 + next: decoder.body.present.19.key + - cell: cache_39 + next: decoder.body.present.19.value + - cell: cache_40 + next: decoder.body.present.20.key + - cell: cache_41 + next: decoder.body.present.20.value + - cell: cache_42 + next: decoder.body.present.21.key + - cell: cache_43 + next: decoder.body.present.21.value + - cell: cache_44 + next: decoder.body.present.22.key + - cell: cache_45 + next: decoder.body.present.22.value + - cell: cache_46 + next: decoder.body.present.23.key + - cell: cache_47 + next: decoder.body.present.23.value + - cell: cache_48 + next: decoder.body.present.24.key + - cell: cache_49 + next: decoder.body.present.24.value + - cell: cache_50 + next: decoder.body.present.25.key + - cell: cache_51 + next: decoder.body.present.25.value + - cell: cache_52 + next: decoder.body.present.26.key + - cell: cache_53 + next: decoder.body.present.26.value + - cell: cache_54 + next: decoder.body.present.27.key + - cell: cache_55 + next: decoder.body.present.27.value + - cell: cache_56 + next: decoder.body.present.28.key + - cell: cache_57 + next: decoder.body.present.28.value + - cell: cache_58 + next: decoder.body.present.29.key + - cell: cache_59 + next: decoder.body.present.29.value + - cell: cache_60 + next: decoder.body.present.30.key + - cell: cache_61 + next: decoder.body.present.30.value + - cell: cache_62 + next: decoder.body.present.31.key + - cell: cache_63 + next: decoder.body.present.31.value + termination: generation_eos + iteration: + value: loop.iteration + contract: + dtype: int64 + rank: 1 + shape: + - 1