opensourcellmrouter 0.5.1

A fast, local-first LLM router — proxy any OpenAI/Anthropic/Ollama client to your own provider pipeline with classifiers, cost/latency/random routing rules, plugins, a live dashboard, and a TUI.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72

//! `pareto-router`: picks a provider from a named "coding quality tier"
//! (e.g. low/medium/high), letting a request opt into a cheaper or
//! higher-quality model without hardcoding a provider name.
//!
//! Config:
//!
//! ```toml
//! [plugins.pareto-router]
//! enabled = true
//! default_tier = "medium"
//!
//! [plugins.pareto-router.tiers]
//! low = ["local-llama"]
//! medium = ["openai"]
//! high = ["anthropic"]
//! ```
//!
//! A request can override the tier per-call:
//!
//! ```json
//! {"plugins": [{"id": "pareto-router", "tier": "high"}]}
//! ```

use async_trait::async_trait;
use serde_json::Value;

use super::{Flow, Plugin, PluginContext};
use crate::canonical::{ChatRequest, ChatResponse};

pub struct ParetoRouterPlugin;

#[async_trait]
impl Plugin for ParetoRouterPlugin {
    fn id(&self) -> &'static str {
        "pareto-router"
    }

    async fn pre_request(
        &self,
        ctx: &PluginContext,
        req: &mut ChatRequest,
        _resp: &mut Option<ChatResponse>,
    ) -> anyhow::Result<Flow> {
        let tier = ctx
            .get_str("tier")
            .or_else(|| ctx.get_str("default_tier"))
            .unwrap_or("medium");

        let tiers = ctx
            .settings
            .get("tiers")
            .and_then(Value::as_object)
            .ok_or_else(|| {
                anyhow::anyhow!(
                    "pareto-router: no `tiers` configured (e.g. [plugins.pareto-router.tiers])"
                )
            })?;

        let providers = tiers.get(tier).and_then(Value::as_array).ok_or_else(|| {
            let known: Vec<&str> = tiers.keys().map(String::as_str).collect();
            anyhow::anyhow!("pareto-router: unknown tier '{tier}' (configured tiers: {known:?})")
        })?;

        let provider = providers
            .first()
            .and_then(Value::as_str)
            .ok_or_else(|| anyhow::anyhow!("pareto-router: tier '{tier}' has no providers configured"))?;

        req.forced_provider = Some(provider.to_string());
        Ok(Flow::Modified)
    }
}