Struct RouterEngine 

pub struct RouterEngine { /* private fields */ }

Implementations

impl RouterEngine

pub fn new(global_limits: GlobalLimits) -> Self

pub async fn add_deployment(&self, deployment: Deployment)

pub async fn remove_deployment(&self, deployment_id: &str)

pub async fn rebuild_pool(&self, deployments: Vec<Deployment>)

pub async fn register_strategy(&self, strategy: Box<dyn RoutingStrategy>)

pub async fn set_default_strategy(&self, name: &str)

pub async fn set_fallback_config(&self, config: FallbackConfig)

pub async fn set_alias(&self, alias: &str, target: &str)

pub async fn set_routing_group(&self, model: &str, strategy_name: &str)

pub async fn record_success( &self, deployment_id: &str, latency_ms: f64, tokens: u64, state: &dyn RoutingState, )

pub async fn record_failure( &self, deployment_id: &str, state: &dyn RoutingState, )

pub fn resolve_alias(model: &str, aliases: &HashMap<String, String>) -> String

pub async fn select_deployment( &self, model: &str, state: &dyn RoutingState, ctx: &RoutingContext, ) -> Result<RoutingResult, RoutingError>

pub async fn route_with_fallback<T: Send + 'static>( &self, model: &str, state: &dyn RoutingState, ctx: &RoutingContext, executor: impl Fn(Arc<Deployment>) -> Pin<Box<dyn Future<Output = Result<T, HyperInferError>> + Send>> + Send, ) -> Result<(RoutingResult, T), RoutingError>