Trait ModelExecutor 

pub trait ModelExecutor: Send + Sync {
    // Required methods
    fn info(&self) -> &ModelInfo;
    fn prefill<'life0, 'life1, 'async_trait>(
        &'life0 self,
        input: &'life1 PrefillInput,
    ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;
    fn decode<'life0, 'life1, 'async_trait>(
        &'life0 self,
        input: &'life1 DecodeInput,
    ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait;
    fn capabilities(&self) -> ExecutorCapabilities;
    fn status(&self) -> ExecutorStatus;

    // Provided methods
    fn execution_resource_authority(&self) -> ExecutionResourceAuthority { ... }
    fn admission_limits(
        &self,
    ) -> Result<Option<ExecutorAdmissionLimits>, FerrumError> { ... }
    fn resolved_model_plan(&self) -> Option<&ResolvedModelPlan> { ... }
    fn plan_runtime_resource_snapshot(
        &self,
    ) -> Result<Option<PlanRuntimeResourceSnapshot>, FerrumError> { ... }
    fn supports_native_unified_decode(&self) -> bool { ... }
    fn kv_capacity(&self) -> Option<usize> { ... }
    fn attach_execution_event_sink(&self, _sink: Arc<dyn ExecutionEventSink>) { ... }
    fn execution_capacity_epochs(
        &self,
    ) -> Result<Option<ExecutorAdmissionEpochs>, FerrumError> { ... }
    fn write_execution_capacity_snapshot(
        &self,
        availability: &mut Vec<CapacityAvailabilityEpoch>,
    ) -> Result<Option<ExecutorAdmissionEpochs>, FerrumError> { ... }
    fn register_execution_capacity_waiter(
        &self,
        _observed: &CapacityWaitCondition,
    ) -> Result<Option<ExecutorCapacityWaitRegistration>, FerrumError> { ... }
    fn try_admit_prefill(
        &self,
        _input: ExecutorPrefillAdmission<'_>,
    ) -> Result<ExecutorPrefillAdmissionDecision, FerrumError> { ... }
    fn cancel_prefill_admission(&self, _request_id: &RequestId) -> bool { ... }
    fn write_execution_capacity_release_sources(
        &self,
        _preemption: &ExecutorExecutionCapacityPreemption,
        sources: &mut Vec<CapacityAvailabilitySource>,
    ) -> Result<bool, FerrumError> { ... }
    fn preempt_execution_capacity<'life0, 'async_trait>(
        &'life0 self,
        _preemption: ExecutorExecutionCapacityPreemption,
    ) -> Pin<Box<dyn Future<Output = Result<ExecutorExecutionCapacityPreemptionReceipt, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait { ... }
    fn maintain_prefill_backing(
        &self,
        _request_id: &RequestId,
    ) -> Result<ExecutorPrefillMaintenanceOutcome, FerrumError> { ... }
    fn reserve_kv_slots(
        &self,
        _requests: &[KvSlotRequest],
    ) -> Result<Option<KvSlotReservation>, FerrumError> { ... }
    fn kv_slot_capacity_snapshot(&self) -> Option<KvSlotCapacitySnapshot> { ... }
    fn recurrent_state_spec(
        &self,
        _request_id: &RequestId,
        _input_tokens: &[TokenId],
    ) -> Result<Option<RecurrentStateSpec>, FerrumError> { ... }
    fn prefill_with_capacity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        input: &'life1 PrefillInput,
    ) -> Pin<Box<dyn Future<Output = Result<ExecutorPrefillOutcome, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn batch_prefill<'life0, 'life1, 'async_trait>(
        &'life0 self,
        inputs: &'life1 [PrefillInput],
    ) -> Pin<Box<dyn Future<Output = Result<Vec<PrefillOutput>, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn batch_prefill_with_capacity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _inputs: &'life1 [PrefillInput],
    ) -> Pin<Box<dyn Future<Output = Result<ExecutorBatchPrefillOutcome, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn plan_runtime_prefill_with_capacity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _input: &'life1 PlanRuntimePrefillInput,
    ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimePrefillOutcome, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn plan_runtime_batch_prefill_with_capacity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _inputs: &'life1 [PlanRuntimePrefillInput],
    ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimeBatchPrefillOutcome, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn discard_plan_runtime_prefill(
        &self,
        authority: PlanRuntimePrefillAuthority,
    ) -> Result<(), FerrumError> { ... }
    fn batch_decode<'life0, 'life1, 'async_trait>(
        &'life0 self,
        inputs: &'life1 [DecodeInput],
    ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn batch_decode_with_capacity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        inputs: &'life1 [DecodeInput],
    ) -> Pin<Box<dyn Future<Output = Result<ExecutorBatchDecodeOutcome, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn plan_runtime_batch_decode_with_capacity<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _inputs: &'life1 [PlanRuntimeDecodeInput],
    ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimeBatchDecodeOutcome, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn unified_decode<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _batch: &'life1 UnifiedBatch,
    ) -> Pin<Box<dyn Future<Output = Result<Vec<Option<Vec<f32>>>, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn forward<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _input: &'life1 Arc<dyn TensorLike>,
    ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn TensorLike>, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn truncate_kv<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _kv_cache: &'life1 Arc<dyn KvCacheHandle>,
        _new_len: usize,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn forward_verify<'life0, 'life1, 'async_trait>(
        &'life0 self,
        inputs: &'life1 [DecodeInput],
    ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             'life1: 'async_trait,
             Self: 'async_trait { ... }
    fn cache_metrics_snapshot(&self) -> Option<Value> { ... }
    fn lora_metrics_snapshot(&self) -> Option<Value> { ... }
    fn prepare_startup<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait { ... }
    fn warmup<'life0, 'async_trait>(
        &'life0 mut self,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait { ... }
    fn shutdown<'life0, 'async_trait>(
        &'life0 mut self,
    ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>
       where 'life0: 'async_trait,
             Self: 'async_trait { ... }
    fn complete_cache(
        &self,
        completion: ExecutorSequenceCompletion,
    ) -> Result<(), FerrumError> { ... }
    fn release_cache(&self, _cache_id: &str) { ... }
}

Core model executor trait focusing on tensor operations

Required Methods

fn info(&self) -> &ModelInfo

Get model information and metadata

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Execute prefill phase (process initial prompt)

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Execute decode phase (generate next token)

fn capabilities(&self) -> ExecutorCapabilities

Get executor capabilities

fn status(&self) -> ExecutorStatus

Get current executor status

Provided Methods

fn execution_resource_authority(&self) -> ExecutionResourceAuthority

Selects the single authority for request-lifetime model resources. Existing executors remain on the transitional legacy-engine path by default. A runtime that returns PlanRuntime must return the shared runtime’s opaque cache handle from prefill/decode and delegate release of that authority from release_cache.

fn admission_limits( &self, ) -> Result<Option<ExecutorAdmissionLimits>, FerrumError>

Immutable admission limits compiled into this executor’s runtime plan. A PlanRuntime executor must return Some; legacy executors may defer to the engine-owned scheduler and recurrent-state limits.

fn resolved_model_plan(&self) -> Option<&ResolvedModelPlan>

Returns the immutable product plan that owns planning, provider selection, and resource authority for a plan-runtime executor. Legacy executors return None; PlanRuntime executors must expose the exact plan used for provisioning and dispatch.

fn plan_runtime_resource_snapshot( &self, ) -> Result<Option<PlanRuntimeResourceSnapshot>, FerrumError>

Returns the authoritative memory breakdown for a shared plan runtime. LegacyEngine executors return None; PlanRuntime executors must return Some while they are ready.

fn supports_native_unified_decode(&self) -> bool

Whether this executor’s backend can run the unified mixed prefill+decode forward natively. When false, the engine routes Qwen3-MoE batches through the legacy split path. Reported by the (backend-aware) executor so the engine stays backend-agnostic — replaces a cfg(target_os) branch that previously hard-coded “Metal/CPU lack native unified” in the hot path.

Default false (conservative legacy path); accelerators with a native unified forward override to true.

fn kv_capacity(&self) -> Option<usize>

Per-request KV capacity in tokens when the executor owns a smaller runtime cache window than the model’s declared context length.

fn attach_execution_event_sink(&self, _sink: Arc<dyn ExecutionEventSink>)

Installs the product-owned execution event sink before requests start.

Legacy executors have no typed execution journal and keep the default no-op. Executors backed by the vNext runtime retain this sink with each admitted request so node/operation events share the product artifact.

fn execution_capacity_epochs( &self, ) -> Result<Option<ExecutorAdmissionEpochs>, FerrumError>

Current plan-local capacity evidence for scheduler wake suppression. Legacy-engine executors return None; an executor declaring ExecutionResourceAuthority::PlanRuntime must return Some.

fn write_execution_capacity_snapshot( &self, availability: &mut Vec<CapacityAvailabilityEpoch>, ) -> Result<Option<ExecutorAdmissionEpochs>, FerrumError>

Writes the canonical per-source availability generations into caller-owned storage and returns the matching global audit epochs. Executors with typed dynamic admission override this to avoid allocating on steady scheduler ticks.

fn register_execution_capacity_waiter( &self, _observed: &CapacityWaitCondition, ) -> Result<Option<ExecutorCapacityWaitRegistration>, FerrumError>

Synchronously subscribes to every source named by one passive capacity wait. The returned registration must remain alive until it is awaited or deliberately cancelled by being dropped.

Legacy-engine executors return None. An executor declaring ExecutionResourceAuthority::PlanRuntime must return Some for a wait condition issued by its own admission coordinator.

fn try_admit_prefill( &self, _input: ExecutorPrefillAdmission<'_>, ) -> Result<ExecutorPrefillAdmissionDecision, FerrumError>

Probe and retain the exact request/sequence authority needed by a future prefill. No provider encode, kernel launch, or device submit may occur in this method.

fn cancel_prefill_admission(&self, _request_id: &RequestId) -> bool

Release an admitted but not yet active prefill authority.

Returns true only when a retained authority was found and released.

fn write_execution_capacity_release_sources( &self, _preemption: &ExecutorExecutionCapacityPreemption, sources: &mut Vec<CapacityAvailabilitySource>, ) -> Result<bool, FerrumError>

Writes the exact availability sources advanced when this request authority is preempted for recompute.

true proves that preemption still identifies a live, quiescently releasable authority and that sources is its complete release footprint. Callers must treat false as not releasable; a generic “owns some cache” observation is not evidence that another request can advance the source on which the current frontier is blocked.

fn preempt_execution_capacity<'life0, 'async_trait>( &'life0 self, _preemption: ExecutorExecutionCapacityPreemption, ) -> Pin<Box<dyn Future<Output = Result<ExecutorExecutionCapacityPreemptionReceipt, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Retire one exact request-scoped runtime authority for recompute.

Success is a terminal release fence: all provider/device work that can access the authority is quiescent and the request can be admitted as a new sequence incarnation. Implementations must fail closed on identity mismatch or an in-flight authority they cannot terminalize.

fn maintain_prefill_backing( &self, _request_id: &RequestId, ) -> Result<ExecutorPrefillMaintenanceOutcome, FerrumError>

Consume one retained logical/physical backing deferral after the scheduler waiting lock has been released. Implementations must perform at most one bounded maintenance attempt and publish capacity epochs only after real backing is installed.

fn reserve_kv_slots( &self, _requests: &[KvSlotRequest], ) -> Result<Option<KvSlotReservation>, FerrumError>

Reserve model-owned KV slots before a forward is dispatched.

This is the executor-level admission hook for vLLM-style paged KV. The engine calls it at the batch boundary so a request that cannot grow its KV cache is delayed or preempted before kernel launch instead of panicking inside attention.

fn kv_slot_capacity_snapshot(&self) -> Option<KvSlotCapacitySnapshot>

Snapshot model-owned paged-KV capacity without allocating slots.

Executors without model-owned paged KV return None.

fn recurrent_state_spec( &self, _request_id: &RequestId, _input_tokens: &[TokenId], ) -> Result<Option<RecurrentStateSpec>, FerrumError>

Recurrent-state allocation spec for this request, when the model has state-space or hybrid layers that need per-request recurrent state.

Attention-only models return None. If this returns Some, the engine must allocate a recurrent-state handle before prefill and pass it through prefill/decode inputs. The default keeps existing executors KV-only.

fn prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<ExecutorPrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Execute one exact prefill chunk with an explicit pre-submit capacity deferral edge. Legacy executors inherit full-prefill behavior.

fn batch_prefill<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [PrefillInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<PrefillOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Batch prefill: process multiple prompts’ prefill in ONE forward pass.

Default implementation falls back to per-request prefill() (serial, which is the current behavior the engine sees today). Executors that support unified mixed-batch forward (e.g. via model.unified_forward over a varlen QKV path) should override this to amortize launch / kernel-overhead across all inputs items in one call.

Used by the continuous-batching engine to coalesce a cohort of new prefills (apples M3 c=32 sees 32 simultaneous prefills as one logical batch; the serial fallback runs each in ~47 ms while a true batched path runs all 32 in ~100 ms).

fn batch_prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, _inputs: &'life1 [PrefillInput], ) -> Pin<Box<dyn Future<Output = Result<ExecutorBatchPrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Attempt one physical, capacity-aware prefill batch.

Implementations must either complete every input in original order or return NotSubmitted after restoring every retained prefill authority to a retryable state. Partial device submission is an ordinary error, never a NotSubmitted result.

fn plan_runtime_prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 PlanRuntimePrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimePrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Tensor-free prefill for executors that declare ExecutionResourceAuthority::PlanRuntime.

The default fails closed because adapting through PrefillInput would silently restore a host tensor boundary.

fn plan_runtime_batch_prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, _inputs: &'life1 [PlanRuntimePrefillInput], ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimeBatchPrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Attempt one physical tensor-free prefill batch.

Unsupported is an optimization fallback to the typed single-request method. NotSubmitted proves that no participant reached provider encode or device submission.

fn discard_plan_runtime_prefill( &self, authority: PlanRuntimePrefillAuthority, ) -> Result<(), FerrumError>

Discard an exact prefill authority after engine-side validation, sampling, or scheduler commit fails.

Plan runtimes should override this to remove both retained-prefill and active state by the exact opaque handle, not by a reusable request id.

fn batch_decode<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Batch decode: process multiple sequences in one forward pass.

A successful result must contain exactly one output per input, in the original input order, and each output cache must retain the identity of its corresponding input cache. Implementations must not expose partial success as a shorter or reordered vector.

The default implementation falls back to serial per-request decode(). Executors with a typed batch submission path should override this so one call maps to one resource step and one terminal submission fence.

fn batch_decode_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<ExecutorBatchDecodeOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Batch decode with an explicit pre-submit capacity deferral edge.

Legacy executors inherit the successful/error-only behavior. A runtime with typed resource authority overrides this method so temporary capacity pressure is never flattened into a stringly resource error.

fn plan_runtime_batch_decode_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, _inputs: &'life1 [PlanRuntimeDecodeInput], ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimeBatchDecodeOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Tensor-free batch decode for executors that declare ExecutionResourceAuthority::PlanRuntime.

Implementations must preserve input ordering and cache identity exactly. Temporary pressure may return Deferred only before device submission. The default fails closed because adapting through DecodeInput would silently restore the host-tensor boundary this contract removes.

fn unified_decode<'life0, 'life1, 'async_trait>( &'life0 self, _batch: &'life1 UnifiedBatch, ) -> Pin<Box<dyn Future<Output = Result<Vec<Option<Vec<f32>>>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Unified mixed-batch forward: process a UnifiedBatch containing any combination of prefill chunks (one or more q_tokens per item, possibly continuing from pos_offset > 0) and decode steps (q_tokens.len() == 1, is_final_chunk = true) in a single model forward pass.

Returns one element per batch.items[i]:

• Some(logits) for items with is_final_chunk = true (the request’s final-position logits, ready for sampling)
• None for intermediate prefill chunks (no lm_head executed — model only updates KV state)

Default implementation returns Err(unsupported). Concrete LLM executors should override with either:

• A behavioral fallback that dispatches each chunk via existing prefill() and groups decode items into batch_decode() (this preserves current behavior; no perf change), OR
• A real unified-forward path that runs all items through one [M_total, hidden] GEMM chain with a varlen attention kernel (this is the chunked-prefill perf unlock).

fn forward<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 Arc<dyn TensorLike>, ) -> Pin<Box<dyn Future<Output = Result<Arc<dyn TensorLike>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Optional: full forward pass (for non-autoregressive use cases)

fn truncate_kv<'life0, 'life1, 'async_trait>( &'life0 self, _kv_cache: &'life1 Arc<dyn KvCacheHandle>, _new_len: usize, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Roll the KV cache for this executor’s sequence back to new_len. Used by speculative decoding on partial rejection so the next iteration sees a KV prefix that matches the accepted token stream. Default: Ok(()) — executors that don’t cache per-sequence state (stub, mock) are inherently tolerant; real LLM executors override.

fn forward_verify<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, Self: 'async_trait,

Multi-position decode-verify: one forward over N+1 tokens, producing one logits row per position. Used by speculative decoding’s target path so we don’t pay N+1 sequential forwards.

Default falls back to N+1 sequential decode() calls — correct but slow; real LLM executors override.

Returns a Vec<DecodeOutput> of length inputs.len() with the final KV handle attached to the last element.

fn cache_metrics_snapshot(&self) -> Option<Value>

Optional model/executor cache metrics.

Concrete LLM executors use this for model-level paged KV prefix reuse counters. Default implementations keep non-autoregressive executors and tests from needing cache-specific plumbing.

fn lora_metrics_snapshot(&self) -> Option<Value>

Optional LoRA runtime metrics.

fn prepare_startup<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Complete executor-owned startup preparation before either product entrypoint can accept a request.

Implementations use this cold-path hook for work that requires the fully constructed executor but must not be charged to a user’s first request, such as compiling reusable execution shapes. The default is a no-op so existing executors remain source compatible.

fn warmup<'life0, 'async_trait>( &'life0 mut self, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Warm up executor (load model, allocate memory, etc.)

fn shutdown<'life0, 'async_trait>( &'life0 mut self, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, Self: 'async_trait,

Shutdown executor gracefully

fn complete_cache( &self, completion: ExecutorSequenceCompletion, ) -> Result<(), FerrumError>

Complete and release one cache authority with product-authoritative terminal token counts.

Legacy executors only need physical release and inherit that behavior. Plan runtimes with terminal journals override this method so completion cannot be inferred from a generic release operation.

fn release_cache(&self, _cache_id: &str)

Release KV cache and state without asserting successful completion.

Called for cancellation, failure, recompute, and legacy cleanup. The cache_id matches the value embedded in the KvCacheHandle returned by prefill/decode. Successful product completion uses Self::complete_cache.

Dyn Compatibility

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementations on Foreign Types

impl ModelExecutor for BertModelExecutor

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, BertModelExecutor: 'async_trait,

For BERT, prefill returns the embeddings (not logits)

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, BertModelExecutor: 'async_trait,

BERT doesn’t support decode (it’s an encoder model)

fn info(&self) -> &ModelInfo

fn capabilities(&self) -> ExecutorCapabilities

fn status(&self) -> ExecutorStatus

impl ModelExecutor for ClipModelExecutor

fn info(&self) -> &ModelInfo

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, ClipModelExecutor: 'async_trait,

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, ClipModelExecutor: 'async_trait,

fn capabilities(&self) -> ExecutorCapabilities

fn release_cache(&self, _cache_id: &str)

fn status(&self) -> ExecutorStatus

impl ModelExecutor for LlmExecutor

fn batch_prefill<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [PrefillInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<PrefillOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

Batched prefill: combine all prompts into ONE model.unified_forward call so launch / kernel-overhead is amortized across the cohort.

Falls back to the trait default (serial per-item) when the model returns Err(unsupported) from unified_forward — e.g. Qwen3MoeModel today, until Phase 2 adds its native unified path.

fn batch_decode<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

Override default fallback to acquire the model lock ONCE for the whole batch, avoiding N round-trips through parking_lot. Does not yet do true attention batching (each cache has its own kv_len), but removes mutex churn that was serialising concurrent requests at async level.

fn unified_decode<'life0, 'life1, 'async_trait>( &'life0 self, batch: &'life1 UnifiedBatch, ) -> Pin<Box<dyn Future<Output = Result<Vec<Option<Vec<f32>>>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

Unified mixed-batch dispatch (chunked-prefill API).

This impl is a behavior-preserving FALLBACK over the existing trait methods on DecoderOnlyLLM: prefill items go through model.prefill(seq_id, &q_tokens) (one at a time, mirroring the engine’s current sequential prefill loop), decode items (q_len == 1 && is_final_chunk) are grouped into a single model.decode_batch(...) call. Net behavior is identical to the engine’s pre-Phase-13 path; this just changes WHO orchestrates the prefill/decode split (caller → unified_decode) so the engine can converge on a single call.

The real performance unlock comes in Step 5 when models override this with a true unified-forward (one [M_total, hidden] forward

• varlen attention) — at that point the kernel-level mix replaces the host-side serial dispatch here.

fn info(&self) -> &ModelInfo

fn supports_native_unified_decode(&self) -> bool

fn kv_capacity(&self) -> Option<usize>

fn reserve_kv_slots( &self, requests: &[KvSlotRequest], ) -> Result<Option<KvSlotReservation>, FerrumError>

fn kv_slot_capacity_snapshot(&self) -> Option<KvSlotCapacitySnapshot>

fn recurrent_state_spec( &self, request_id: &RequestId, input_tokens: &[TokenId], ) -> Result<Option<RecurrentStateSpec>, FerrumError>

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

fn truncate_kv<'life0, 'life1, 'async_trait>( &'life0 self, kv_cache: &'life1 Arc<dyn KvCacheHandle>, new_len: usize, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

fn forward_verify<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, LlmExecutor: 'async_trait,

fn release_cache(&self, cache_id: &str)

fn capabilities(&self) -> ExecutorCapabilities

fn status(&self) -> ExecutorStatus

fn cache_metrics_snapshot(&self) -> Option<Value>

fn lora_metrics_snapshot(&self) -> Option<Value>

impl ModelExecutor for StubModelExecutor

fn info(&self) -> &ModelInfo

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, StubModelExecutor: 'async_trait,

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, StubModelExecutor: 'async_trait,

fn capabilities(&self) -> ExecutorCapabilities

fn status(&self) -> ExecutorStatus

impl ModelExecutor for TtsModelExecutor

fn info(&self) -> &ModelInfo

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, TtsModelExecutor: 'async_trait,

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, TtsModelExecutor: 'async_trait,

fn capabilities(&self) -> ExecutorCapabilities

fn release_cache(&self, _: &str)

fn status(&self) -> ExecutorStatus

impl ModelExecutor for WhisperModelExecutor

fn info(&self) -> &ModelInfo

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, WhisperModelExecutor: 'async_trait,

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, _input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, WhisperModelExecutor: 'async_trait,

fn capabilities(&self) -> ExecutorCapabilities

fn release_cache(&self, _: &str)

fn status(&self) -> ExecutorStatus

impl<R> ModelExecutor for VNextModelExecutor<R>

where R: DeviceRuntime,

fn info(&self) -> &ModelInfo

fn prepare_startup<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Result<(), FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn execution_resource_authority(&self) -> ExecutionResourceAuthority

fn admission_limits( &self, ) -> Result<Option<ExecutorAdmissionLimits>, FerrumError>

fn resolved_model_plan(&self) -> Option<&ResolvedModelPlan>

fn plan_runtime_resource_snapshot( &self, ) -> Result<Option<PlanRuntimeResourceSnapshot>, FerrumError>

fn kv_capacity(&self) -> Option<usize>

fn attach_execution_event_sink(&self, sink: Arc<dyn ExecutionEventSink>)

fn execution_capacity_epochs( &self, ) -> Result<Option<ExecutorAdmissionEpochs>, FerrumError>

fn write_execution_capacity_snapshot( &self, availability: &mut Vec<CapacityAvailabilityEpoch>, ) -> Result<Option<ExecutorAdmissionEpochs>, FerrumError>

fn register_execution_capacity_waiter( &self, observed: &CapacityWaitCondition, ) -> Result<Option<ExecutorCapacityWaitRegistration>, FerrumError>

fn try_admit_prefill( &self, input: ExecutorPrefillAdmission<'_>, ) -> Result<ExecutorPrefillAdmissionDecision, FerrumError>

fn cancel_prefill_admission(&self, request_id: &RequestId) -> bool

fn write_execution_capacity_release_sources( &self, preemption: &ExecutorExecutionCapacityPreemption, sources: &mut Vec<CapacityAvailabilitySource>, ) -> Result<bool, FerrumError>

fn preempt_execution_capacity<'life0, 'async_trait>( &'life0 self, preemption: ExecutorExecutionCapacityPreemption, ) -> Pin<Box<dyn Future<Output = Result<ExecutorExecutionCapacityPreemptionReceipt, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn maintain_prefill_backing( &self, request_id: &RequestId, ) -> Result<ExecutorPrefillMaintenanceOutcome, FerrumError>

fn prefill<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PrefillOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PrefillInput, ) -> Pin<Box<dyn Future<Output = Result<ExecutorPrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn batch_prefill<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [PrefillInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<PrefillOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn batch_prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [PrefillInput], ) -> Pin<Box<dyn Future<Output = Result<ExecutorBatchPrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn plan_runtime_prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 PlanRuntimePrefillInput, ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimePrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn plan_runtime_batch_prefill_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [PlanRuntimePrefillInput], ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimeBatchPrefillOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn decode<'life0, 'life1, 'async_trait>( &'life0 self, input: &'life1 DecodeInput, ) -> Pin<Box<dyn Future<Output = Result<DecodeOutput, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn batch_decode<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<Vec<DecodeOutput>, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn batch_decode_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [DecodeInput], ) -> Pin<Box<dyn Future<Output = Result<ExecutorBatchDecodeOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn plan_runtime_batch_decode_with_capacity<'life0, 'life1, 'async_trait>( &'life0 self, inputs: &'life1 [PlanRuntimeDecodeInput], ) -> Pin<Box<dyn Future<Output = Result<PlanRuntimeBatchDecodeOutcome, FerrumError>> + Send + 'async_trait>>

where 'life0: 'async_trait, 'life1: 'async_trait, VNextModelExecutor<R>: 'async_trait,

fn discard_plan_runtime_prefill( &self, authority: PlanRuntimePrefillAuthority, ) -> Result<(), FerrumError>

fn release_cache(&self, cache_id: &str)

fn complete_cache( &self, completion: ExecutorSequenceCompletion, ) -> Result<(), FerrumError>

fn capabilities(&self) -> ExecutorCapabilities

fn status(&self) -> ExecutorStatus

fn cache_metrics_snapshot(&self) -> Option<Value>

Implementors