Struct DecodeInput 

pub struct DecodeInput {
    pub request_id: Option<RequestId>,
    pub input_ids: TensorRef,
    pub kv_cache: Arc<dyn KvCacheHandle>,
    pub recurrent_state: Option<Arc<dyn RecurrentStateHandle>>,
    pub position_ids: Option<TensorRef>,
    pub metadata: HashMap<String, Value>,
    pub logits_policy: LogitsReturnPolicy,
}

Input for decode phase (generating one token at a time)

Fields

request_id: Option<RequestId>

Stable product request identity for plan-runtime resources.

input_ids: TensorRef

Input token ID for current step [batch_size, 1]

kv_cache: Arc<dyn KvCacheHandle>

Existing KV cache from previous steps

recurrent_state: Option<Arc<dyn RecurrentStateHandle>>

Existing recurrent state from previous steps, when used.

position_ids: Option<TensorRef>

Position IDs for current step [batch_size, 1] (optional)

metadata: HashMap<String, Value>

Request metadata that can affect model execution.

logits_policy: LogitsReturnPolicy

How the model may return final-position logits for this request.

Implementations

impl DecodeInput

pub fn new(input_ids: TensorRef, kv_cache: Arc<dyn KvCacheHandle>) -> Self

Create new decode input

pub fn with_request_id(self, request_id: RequestId) -> Self

Attach the product request identity to this decode step.

pub fn with_position_ids(self, positions: TensorRef) -> Self

Add position IDs

pub fn with_recurrent_state( self, recurrent_state: Arc<dyn RecurrentStateHandle>, ) -> Self

Attach recurrent state for state-space or hybrid layers.

pub fn with_metadata(self, metadata: HashMap<String, Value>) -> Self

Attach request metadata.

pub fn with_logits_policy(self, policy: LogitsReturnPolicy) -> Self

pub fn batch_size(&self) -> usize

Get batch size

Trait Implementations

impl Clone for DecodeInput

fn clone(&self) -> DecodeInput

Returns a duplicate of the value.

fn clone_from(&mut self, source: &Self)

Performs copy-assignment from source.

impl Debug for DecodeInput

fn fmt(&self, f: &mut Formatter<'_>) -> Result

Formats the value using the given formatter.