pub struct ModelRouter { /* private fields */ }Expand description
Routes inference requests to the best available model based on prompt analysis and hardware.
Implementations§
Source§impl ModelRouter
impl ModelRouter
pub fn new(hw: HardwareInfo) -> Self
Sourcepub fn route_generate(
&self,
prompt: &str,
registry: &ModelRegistry,
) -> RoutingDecision
pub fn route_generate( &self, prompt: &str, registry: &ModelRegistry, ) -> RoutingDecision
Route a generation request to the best model.
Sourcepub fn route_small(&self, registry: &ModelRegistry) -> String
pub fn route_small(&self, registry: &ModelRegistry) -> String
Route classify to the smallest available generative model.
Sourcepub fn route_embedding(&self, _registry: &ModelRegistry) -> String
pub fn route_embedding(&self, _registry: &ModelRegistry) -> String
Route embedding to the dedicated embedding model.
Sourcepub fn assess_complexity(prompt: &str) -> TaskComplexity
pub fn assess_complexity(prompt: &str) -> TaskComplexity
Assess prompt complexity based on content analysis.
Auto Trait Implementations§
impl Freeze for ModelRouter
impl RefUnwindSafe for ModelRouter
impl Send for ModelRouter
impl Sync for ModelRouter
impl Unpin for ModelRouter
impl UnsafeUnpin for ModelRouter
impl UnwindSafe for ModelRouter
Blanket Implementations§
Source§impl<T> BorrowMut<T> for Twhere
T: ?Sized,
impl<T> BorrowMut<T> for Twhere
T: ?Sized,
Source§fn borrow_mut(&mut self) -> &mut T
fn borrow_mut(&mut self) -> &mut T
Mutably borrows from an owned value. Read more
Source§impl<T> Instrument for T
impl<T> Instrument for T
Source§fn instrument(self, span: Span) -> Instrumented<Self>
fn instrument(self, span: Span) -> Instrumented<Self>
Source§fn in_current_span(self) -> Instrumented<Self>
fn in_current_span(self) -> Instrumented<Self>
Source§impl<T> IntoEither for T
impl<T> IntoEither for T
Source§fn into_either(self, into_left: bool) -> Either<Self, Self>
fn into_either(self, into_left: bool) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left is true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read moreSource§fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
fn into_either_with<F>(self, into_left: F) -> Either<Self, Self>
Converts
self into a Left variant of Either<Self, Self>
if into_left(&self) returns true.
Converts self into a Right variant of Either<Self, Self>
otherwise. Read more