Trait GatewayTrait 

pub trait GatewayTrait: Send + Sync {
    // Required methods
    fn infer(
        &self,
        request: InferenceRequest,
    ) -> BoxFuture<'_, FederationResult<InferenceResponse>>;
    fn infer_stream(
        &self,
        request: InferenceRequest,
    ) -> BoxFuture<'_, FederationResult<Box<dyn TokenStream>>>;
    fn stats(&self) -> GatewayStats;
}

Gateway - the main entry point for federation requests

Required Methods

fn infer( &self, request: InferenceRequest, ) -> BoxFuture<'_, FederationResult<InferenceResponse>>

Execute an inference request through the federation

fn infer_stream( &self, request: InferenceRequest, ) -> BoxFuture<'_, FederationResult<Box<dyn TokenStream>>>

Execute with streaming response

fn stats(&self) -> GatewayStats

Get gateway statistics

Implementors

impl GatewayTrait for FederationGateway