rskit_inference/
inference.rs

1use async_trait::async_trait;
2use tokio_stream::Stream;
3
4use crate::types::{
5    InferenceDescriptor, InferenceError, PredictRequest, PredictResponse, StreamEventRef,
6};
7
8/// Inference is the model-serving runtime trait. Adapters cover text generation (vLLM, TGI),
9/// classification/regression (Triton KServe v2), embeddings (BentoML, custom), image/audio inference,
10/// and arbitrary tensor protocols. NOT chat completion — that lives in `rskit-llm`.
11///
12/// This trait extends `rskit_provider::RequestResponse<PredictRequest, PredictResponse>`
13/// so inference adapters natively expose the canonical request/response shape.
14#[async_trait]
15pub trait Inference:
16    rskit_provider::RequestResponse<PredictRequest, PredictResponse> + Send + Sync
17{
18    /// Execute one prediction request against the serving runtime.
19    async fn predict(&self, request: PredictRequest) -> Result<PredictResponse, InferenceError>;
20
21    /// Return the adapter descriptor and executable permission envelope.
22    fn descriptor(&self) -> InferenceDescriptor;
23}
24
25/// Implemented by serving runtimes that emit incremental output (vLLM, TGI).
26#[async_trait]
27pub trait StreamingInference: Inference {
28    /// Execute a prediction request and stream incremental output events.
29    async fn predict_stream(
30        &self,
31        request: PredictRequest,
32    ) -> Result<Box<dyn Stream<Item = StreamEventRef> + Send + Unpin>, InferenceError>;
33}
rskit_inference/inference.rs

rskit_inference/
inference.rs
