Struct RouterLLM 

pub struct RouterLLM { /* private fields */ }

A chat model pool with routing and fallback.

See the module docs for design rationale.

Implementations

impl RouterLLM

pub fn new(strategy: RoutingStrategy) -> Self

Create an empty router with the given strategy. Add models via the with_* builder methods.

pub fn with_name(self, name: impl Into<String>) -> Self

Set a human-readable name returned by model_name.

pub fn with_model<M>(self, model: M) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Register a model.

pub fn with_cost<M>(self, model: M, cost: f64) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Register a model with a relative cost used by RoutingStrategy::LowestCost.

pub fn with_fallbacks<M>(primary: M, fallbacks: Vec<M>) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Convenience constructor for primary-first fallback over models of the same type. The primary is tried first; each fallback is tried in order until one succeeds.

pub fn len(&self) -> usize

Number of registered models.

pub fn is_empty(&self) -> bool

Whether no models are registered.

Trait Implementations

impl BaseChatModel for RouterLLM

fn chat<'life0, 'async_trait>( &'life0 self, messages: Vec<Message>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<LLMResult, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Chat with the model.

fn stream_chat<'life0, 'async_trait>( &'life0 self, messages: Vec<Message>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<String, Self::Error>> + Send>>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Stream chat with the model.

fn chat_with_system<'life0, 'async_trait>( &'life0 self, system: String, messages: Vec<Message>, ) -> Pin<Box<dyn Future<Output = Result<LLMResult, Self::Error>> + Send + 'async_trait>>

where Self: Sync + 'async_trait, 'life0: 'async_trait,

Chat with system prompt.

fn bind_tools( &self, _tools: Vec<ToolDefinition>, ) -> Option<Box<dyn BaseChatModel<Error = Self::Error> + Send + Sync>>

Bind tool definitions for function calling.

impl BaseLanguageModel<Vec<Message>, LLMResult> for RouterLLM

fn model_name(&self) -> &str

Returns the model name.

fn get_num_tokens(&self, text: &str) -> usize

Calculates token count for text.

fn with_temperature(self, _temp: f32) -> Self

Sets the temperature parameter.

fn with_max_tokens(self, _max: usize) -> Self

Sets the max tokens limit.

fn temperature(&self) -> Option<f32>

Returns the temperature parameter.

fn max_tokens(&self) -> Option<usize>

Returns the max tokens limit.

impl Runnable<Vec<Message>, LLMResult> for RouterLLM

type Error = RouterError

Error type.

fn invoke<'life0, 'async_trait>( &'life0 self, input: Vec<Message>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<LLMResult, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Transforms single input to output.

fn batch<'life0, 'async_trait>( &'life0 self, inputs: Vec<Input>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Vec<Output>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Batch processing - transforms multiple inputs to outputs.

fn batch_as_completed<'life0, 'async_trait>( &'life0 self, inputs: Vec<Input>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Vec<(usize, Output)>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Batch processing that returns results in completion order.

fn stream<'life0, 'async_trait>( &'life0 self, input: Input, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<Output, Self::Error>> + Send>>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Streaming output - for real-time responses (LLM, etc).

fn transform<'life0, 'async_trait>( &'life0 self, input: Pin<Box<dyn Stream<Item = Result<Input, Self::Error>> + Send>>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<Output, Self::Error>> + Send>>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Stream-to-stream transformation - the core of LCEL streaming.