ml-runtime-provider 0.2.3

Provider contracts for rust-ml-runtime
Documentation
use async_trait::async_trait;
use ml_runtime_common::{BoxStream, CancellationToken, RuntimeResult};
use ml_runtime_inference::{InferenceChunk, InferenceRequest, InferenceResult};
use serde::{Deserialize, Serialize};

#[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Eq)]
pub struct ProviderCapabilities {
    pub available: bool,
    pub local: bool,
    pub remote: bool,
    pub streaming: bool,
    pub cancellation: bool,
    pub structured_output: bool,
    pub batching: bool,
    pub endpoint: Option<String>,
    pub notes: Vec<String>,
}

#[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Eq)]
pub struct ProviderCapability {
    pub name: String,
    pub available: bool,
    pub local: bool,
    pub remote: bool,
    pub streaming: bool,
    pub cancellation: bool,
    pub structured_output: bool,
    pub batching: bool,
    pub endpoint: Option<String>,
    pub notes: Vec<String>,
}

impl ProviderCapability {
    pub fn from_parts(name: impl Into<String>, capabilities: ProviderCapabilities) -> Self {
        Self {
            name: name.into(),
            available: capabilities.available,
            local: capabilities.local,
            remote: capabilities.remote,
            streaming: capabilities.streaming,
            cancellation: capabilities.cancellation,
            structured_output: capabilities.structured_output,
            batching: capabilities.batching,
            endpoint: capabilities.endpoint,
            notes: capabilities.notes,
        }
    }
}

#[async_trait]
pub trait Provider: Send + Sync {
    fn name(&self) -> &str;
    fn capabilities(&self) -> ProviderCapabilities;
    async fn infer(
        &self,
        request: InferenceRequest,
        cancellation: Option<CancellationToken>,
    ) -> RuntimeResult<InferenceResult>;
    async fn infer_stream(
        &self,
        request: InferenceRequest,
        cancellation: Option<CancellationToken>,
    ) -> RuntimeResult<BoxStream<RuntimeResult<InferenceChunk>>>;
}