llmr 0.2.0

Reach language models across providers, with capabilities you can read before you ask and usage you can trust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86

//! Two providers, one router, and the floor that is not a preference.
//!
//! ```sh
//! cargo run --example routing
//! ```
//!
//! Needs no key and reaches nothing. Both providers are unreachable on purpose, which is
//! what makes the last case worth watching.

use llmr::providers::openai;
use llmr::{
    transport::Reqwest, Message, Provider, Reach, Registry, Requirements, Route, Router, Secret,
};
use std::sync::Arc;
use std::time::Duration;

#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error>> {
    let transport = Arc::new(Reqwest::new(Duration::from_secs(5))?);

    // A model on this machine, and a hosted one. Same protocol, completely different places
    // for your data to go, and only you can say which is which.
    let local = Arc::new(openai::api::at(
        "ollama",
        "http://localhost:11434/v1",
        Arc::clone(&transport) as Arc<dyn llmr::transport::HttpTransport>,
        Secret::new("ollama", "not-needed"),
        Reach::SelfHosted,
        Arc::new(shelf("llama3", Reach::SelfHosted)),
    ));

    let hosted = Arc::new(openai::api::at(
        "vendor",
        "https://api.example.invalid/v1",
        transport,
        Secret::new("vendor", "sk-not-real"),
        Reach::FirstPartyApi,
        Arc::new(shelf("some-hosted-model", Reach::FirstPartyApi)),
    ));

    let router = Router::new(vec![
        Route::new(local as Arc<dyn Provider>, "llama3"),
        Route::new(hosted as Arc<dyn Provider>, "some-hosted-model"),
    ]);

    for (name, capabilities) in router.routes() {
        println!("route {name:<28} {:?}", capabilities.map(|c| c.reach));
    }
    println!("unusable routes: {:?}\n", router.unusable());

    let request = llmr::ChatRequest::new("", vec![Message::user("Summarise this.")]);

    // Ordinary work. The local one is tried first and, being unreachable here, the hosted
    // one answers. `fell_through` is where that shows up.
    println!("--- no privacy floor");
    match router.chat(request.clone(), Requirements::default()).await {
        Ok(routed) => println!(
            "answered by {} after {:?}",
            routed.route, routed.fell_through
        ),
        Err(e) => println!("both were unreachable: {e}"),
    }

    // The same call, with a floor. The hosted provider is not tried at all, however
    // unreachable the local one is. That refusal is the feature.
    println!("\n--- data must stay on this machine");
    match router
        .chat(request, Requirements::default().on_device())
        .await
    {
        Ok(routed) => println!("answered by {}", routed.route),
        Err(e) => println!("refused rather than sent away: {e}"),
    }

    Ok(())
}

/// A one model table, so the router has capabilities to read.
fn shelf(model: &str, reach: Reach) -> Registry {
    let toml = format!(
        "provider = \"example\"\nreach = \"{reach:?}\"\n\n\
         [[model]]\nid = \"{model}\"\ncontext_window = 128000\nmax_output = 4096\n\
         source = \"an example\"\nverified_at = \"2026-08-28\"\n"
    );
    Registry::parse(&toml).unwrap_or_else(|_| Registry::empty("example", reach))
}