Struct RouterLLM 

pub struct RouterLLM { /* private fields */ }

A chat model pool with routing and fallback.

See the module docs for design rationale.

Implementations

impl RouterLLM

pub fn new(strategy: RoutingStrategy) -> Self

Create an empty router with the given strategy. Add models via the with_* builder methods.

pub fn with_budget(self, budget: Arc<RouterBudget>) -> Self

Attaches a shared RouterBudget. Every subsequent call projects its prompt spend against the remaining budget, skips paid slots once the breaker is tripped (free slots stay reachable as fallback), and records the model-reported token usage after a successful call.

pub fn with_registry(self, registry: Arc<ModelRegistry>) -> Self

Attaches a model registry. Keyed slots (RouterLLM::with_model_as) then take their RoutingStrategy::LowestCost weight from the registry’s blended per-1K price. An explicit cost from RouterLLM::with_cost always wins; a key the registry cannot resolve sorts last (treated as infinitely expensive).

pub fn with_name(self, name: impl Into<String>) -> Self

Set a human-readable name returned by model_name.

pub fn with_model<M>(self, model: M) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Register a model.

pub fn with_cost<M>(self, model: M, cost: f64) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Register a model with a relative cost used by RoutingStrategy::LowestCost.

pub fn with_model_as<M, K>(self, model: M, registry_key: K) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static, K: Into<String>,

Register a model keyed as "<provider>/<model-id>" in the attached ModelRegistry (see RouterLLM::with_registry). Under RoutingStrategy::LowestCost the slot is weighted by the registry’s blended per-1K price; other strategies ignore the key.

pub fn with_priced_model<M>(self, model: M, price: ModelPrice) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Register a model with an explicit per-token ModelPrice. The price both participates in RoutingStrategy::LowestCost (blended) and prices projected/actual spend for an attached RouterBudget.

pub fn with_model_rate_limited<M>(self, model: M, limit: ModelRateLimit) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Register a model behind a per-model ModelRateLimit (B13). Saturated slots queue callers FIFO; when the queue is full or the configured wait timeout elapses, the call skips to the next candidate model.

pub fn with_last_price(self, price: ModelPrice) -> Self

Attaches a ModelPrice to the most recently registered slot.

Use to combine registration styles, e.g. a keyed slot whose registry entry lacks pricing. A no-op with a warning when no slot exists.

pub fn with_last_rate_limit(self, limit: ModelRateLimit) -> Self

Attaches a ModelRateLimit to the most recently registered slot.

A no-op with a warning when no slot exists.

pub fn with_fallbacks<M>(primary: M, fallbacks: Vec<M>) -> Self

where M: BaseChatModel + 'static, M::Error: Error + Send + Sync + 'static,

Convenience constructor for primary-first fallback over models of the same type. The primary is tried first; each fallback is tried in order until one succeeds.

pub fn len(&self) -> usize

Number of registered models.

pub fn is_empty(&self) -> bool

Whether no models are registered.

Trait Implementations

impl BaseChatModel for RouterLLM

fn chat<'life0, 'async_trait>( &'life0 self, messages: Vec<Message>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<LLMResult, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Chat with the model.

fn stream_chat<'life0, 'async_trait>( &'life0 self, messages: Vec<Message>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<StreamChunk, Self::Error>> + Send>>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Stream chat with the model.

fn chat_with_system<'life0, 'async_trait>( &'life0 self, system: String, messages: Vec<Message>, ) -> Pin<Box<dyn Future<Output = Result<LLMResult, Self::Error>> + Send + 'async_trait>>

where Self: Sync + 'async_trait, 'life0: 'async_trait,

Chat with system prompt.

fn bind_tools( &self, _tools: Vec<ToolDefinition>, ) -> Option<Box<dyn BaseChatModel<Error = Self::Error> + Send + Sync>>

Bind tool definitions for function calling.

impl BaseLanguageModel<Vec<Message>, LLMResult> for RouterLLM

fn model_name(&self) -> &str

Returns the model name.

fn get_num_tokens(&self, text: &str) -> usize

Calculates token count for text.

fn with_temperature(self, _temp: f32) -> Self

Sets the temperature parameter.

fn with_max_tokens(self, _max: usize) -> Self

Sets the max tokens limit.

fn temperature(&self) -> Option<f32>

Returns the temperature parameter.

fn max_tokens(&self) -> Option<usize>

Returns the max tokens limit.

impl Runnable<Vec<Message>, LLMResult> for RouterLLM

type Error = RouterError

Error type.

fn invoke<'life0, 'async_trait>( &'life0 self, input: Vec<Message>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<LLMResult, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Transforms single input to output.

fn batch<'life0, 'async_trait>( &'life0 self, inputs: Vec<Input>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Vec<Output>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Batch processing - transforms multiple inputs to outputs.

fn batch_as_completed<'life0, 'async_trait>( &'life0 self, inputs: Vec<Input>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Vec<(usize, Output)>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Batch processing that returns results in completion order.

fn stream<'life0, 'async_trait>( &'life0 self, input: Input, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<Output, Self::Error>> + Send>>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Streaming output - for real-time responses (LLM, etc).

fn transform<'life0, 'async_trait>( &'life0 self, input: Pin<Box<dyn Stream<Item = Result<Input, Self::Error>> + Send>>, config: Option<RunnableConfig>, ) -> Pin<Box<dyn Future<Output = Result<Pin<Box<dyn Stream<Item = Result<Output, Self::Error>> + Send + '_>>, Self::Error>> + Send + 'async_trait>>

where Self: 'async_trait, 'life0: 'async_trait,

Stream-to-stream transformation - the core of LCEL streaming.