model-gateway-rs 0.1.7

A Rust library for model gateway services, providing traits and SDKs for various AI models.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16

use async_trait::async_trait;
use toolcraft_request::ByteStream;

use crate::error::Result;

/// Trait for normal model inference
#[async_trait]
pub trait ModelClient<I, O> {
    async fn infer(&self, input: I) -> Result<O>;
}

/// Trait for stream-based model inference
#[async_trait]
pub trait StreamModelClient<I> {
    async fn infer_stream(&self, input: I) -> Result<ByteStream>;
}