Skip to main content

LocalGenerationOffload

car_inference::offload

Trait LocalGenerationOffload 

Source 
pub trait LocalGenerationOffload: Send + Sync {
    // Required methods
    fn generate<'life0, 'async_trait>(
        &'life0 self,
        request: GenerateRequest,
    ) -> Pin<Box<dyn Future<Output = Result<InferenceResult, InferenceError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;
    fn stream<'life0, 'async_trait>(
        &'life0 self,
        request: GenerateRequest,
    ) -> Pin<Box<dyn Future<Output = Result<Receiver<StreamEvent>, InferenceError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait;

    // Provided methods
    fn generate_admitted<'life0, 'async_trait>(
        &'life0 self,
        request: GenerateRequest,
        _admission: LocalWorkerAdmission,
    ) -> Pin<Box<dyn Future<Output = Result<LocalOffloadResult, InferenceError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
    fn stream_admitted<'life0, 'async_trait>(
        &'life0 self,
        request: GenerateRequest,
        _admission: LocalWorkerAdmission,
    ) -> Pin<Box<dyn Future<Output = Result<LocalOffloadStream, InferenceError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
    fn refresh_resource_policy(&self, _generation: u64) { ... }
    fn resident_models<'life0, 'async_trait>(
        &'life0 self,
    ) -> Pin<Box<dyn Future<Output = Vec<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
    fn resident_allocation_id(&self, _model_id: &str) -> Option<String> { ... }
    fn release_model<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _model_id: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
    fn release_idle_models<'life0, 'async_trait>(
        &'life0 self,
        _idle_for: Duration,
    ) -> Pin<Box<dyn Future<Output = Vec<String>> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait { ... }
    fn terminate_inference<'life0, 'life1, 'async_trait>(
        &'life0 self,
        _inference_id: &'life1 str,
    ) -> Pin<Box<dyn Future<Output = InferenceTerminationAck> + Send + 'async_trait>>
       where Self: 'async_trait,
             'life0: 'async_trait,
             'life1: 'async_trait { ... }
}
Expand description

Handler that runs a fully-resolved on-device GenerateRequest somewhere isolated from the calling process (the daemon’s car-server --mlx-worker child). Implementations must run the request against a real in-process engine and return the same InferenceResult / StreamEvent stream the caller would have produced itself — the boundary is transparent except that a native abort surfaces as an Err/dropped stream instead of a crash.

Required Methods§

Source

fn generate<'life0, 'async_trait>( &'life0 self, request: GenerateRequest, ) -> Pin<Box<dyn Future<Output = Result<InferenceResult, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Run a non-streaming generation to completion in the worker.

Source

fn stream<'life0, 'async_trait>( &'life0 self, request: GenerateRequest, ) -> Pin<Box<dyn Future<Output = Result<Receiver<StreamEvent>, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Run a streaming generation in the worker, returning a receiver of the same StreamEvents the in-process path emits. The channel closes when the worker finishes (or dies); a mid-stream worker death is observed as the sender dropping, exactly like any other stream end.

Provided Methods§

Source

fn generate_admitted<'life0, 'async_trait>( &'life0 self, request: GenerateRequest, _admission: LocalWorkerAdmission, ) -> Pin<Box<dyn Future<Output = Result<LocalOffloadResult, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Admission-aware extension. The default preserves source compatibility for external legacy implementers and treats their result as transient; only implementations that explicitly override this method may report retained worker/process weights.

Source

fn stream_admitted<'life0, 'async_trait>( &'life0 self, request: GenerateRequest, _admission: LocalWorkerAdmission, ) -> Pin<Box<dyn Future<Output = Result<LocalOffloadStream, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Source

fn refresh_resource_policy(&self, _generation: u64)

Source

fn resident_models<'life0, 'async_trait>( &'life0 self, ) -> Pin<Box<dyn Future<Output = Vec<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Source

fn resident_allocation_id(&self, _model_id: &str) -> Option<String>

Exact process-generation allocation owner for parent-side residency publication. Legacy implementations may omit it and remain transient.

Source

fn release_model<'life0, 'life1, 'async_trait>( &'life0 self, _model_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = Result<bool, InferenceError>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Release worker-owned residency for one model.

Returning Ok(true) MUST mean both that the owning worker exited and that its resident allocation was retired from the admission coordinator before this future completed. The real WorkerOffload provides that guarantee through WorkerProcessGuard::confirm_exited → finish_accounting → finish_teardown_allocation. An implementation that cannot provide both parts of the guarantee must return Ok(false).

Source

fn release_idle_models<'life0, 'async_trait>( &'life0 self, _idle_for: Duration, ) -> Pin<Box<dyn Future<Output = Vec<String>> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait,

Release worker-owned model residents whose last completed use is at least idle_for old. Legacy offloads do not track residency age.

Source

fn terminate_inference<'life0, 'life1, 'async_trait>( &'life0 self, _inference_id: &'life1 str, ) -> Pin<Box<dyn Future<Output = InferenceTerminationAck> + Send + 'async_trait>>
where Self: 'async_trait, 'life0: 'async_trait, 'life1: 'async_trait,

Terminate the exact isolated process currently executing this ID. The compatibility default is intentionally conservative: only an exact kill-and-wait implementation may report confirmed termination.

Dyn Compatibility§

This trait is dyn compatible.

In older versions of Rust, dyn compatibility was called "object safety".

Implementors§