rust_web_server/proxy_config/
llm.rs

1//! `[route.action] type = "llm"`: the [LLM proxy](crate::llm) from the
2//! config file (18.9.0).
3//!
4//! ```toml
5//! [[route]]
6//!   [route.match]
7//!   path = "/v1/*"
8//!
9//!   [route.action]
10//!   type = "llm"
11//!
12//!   [route.action.llm]
13//!   audit_log          = true
14//!   redis              = "127.0.0.1:6379"   # share budgets between replicas
15//!   redis_password_env = "REDIS_PASSWORD"
16//!   redis_prefix       = "rws:llm:"
17//!   connect_timeout_ms = 10000
18//!   read_timeout_ms    = 300000
19//!
20//!   [[route.action.llm.provider]]         # tried in this order, per API
21//!   name    = "openai"
22//!   api     = "openai"                    # or "anthropic"
23//!   url     = "https://api.openai.com"
24//!   key_env = "OPENAI_API_KEY"
25//!
26//!   [[route.action.llm.key]]              # a virtual key clients send
27//!   identity      = "alice"
28//!   key_env       = "ALICE_LLM_KEY"
29//!   budget_tokens = 2000000               # optional, with window_secs
30//!   window_secs   = 86400
31//! ```
32//!
33//! Secrets never sit in the file: provider keys, virtual keys and the Redis
34//! password are read from the environment variables named.
35
36use std::time::Duration;
37
38use crate::config_check::ConfigReport;
39use crate::llm::{Api, LlmProxy, Provider};
40
41use super::parser::{get_str, get_u64, section_exists, SectionMap};
42
43#[derive(Debug, Clone, Default, PartialEq)]
44pub struct LlmConfig {
45    pub audit_log: bool,
46    pub allow_anonymous: bool,
47    pub redis: String,
48    pub redis_password_env: String,
49    pub redis_prefix: String,
50    pub connect_timeout_ms: u64,
51    pub read_timeout_ms: u64,
52    pub providers: Vec<LlmProviderConfig>,
53    pub keys: Vec<LlmKeyConfig>,
54}
55
56#[derive(Debug, Clone, Default, PartialEq)]
57pub struct LlmProviderConfig {
58    pub name: String,
59    pub api: String,
60    pub url: String,
61    pub key_env: String,
62}
63
64#[derive(Debug, Clone, Default, PartialEq)]
65pub struct LlmKeyConfig {
66    pub identity: String,
67    pub key_env: String,
68    pub budget_tokens: u64,
69    pub window_secs: u64,
70}
71
72/// The keys `check_unknown_keys` accepts in each part.
73pub(crate) const LLM_KEYS: &[&str] =
74    &["audit_log", "allow_anonymous", "redis", "redis_password_env", "redis_prefix", "connect_timeout_ms", "read_timeout_ms"];
75pub(crate) const PROVIDER_KEYS: &[&str] = &["name", "api", "url", "key_env"];
76pub(crate) const KEY_KEYS: &[&str] = &["identity", "key_env", "budget_tokens", "window_secs"];
77
78/// `section` is the action's, e.g. `route[0].action`.
79pub(crate) fn parse(map: &SectionMap, section: &str) -> LlmConfig {
80    let base = format!("{section}.llm");
81    let indexed = |kind: &str| -> Vec<String> {
82        (0..).map(|i| format!("{base}.{kind}[{i}]")).take_while(|s| section_exists(map, s)).collect()
83    };
84    let prefix = get_str(map, &base, "redis_prefix");
85    LlmConfig {
86        audit_log: get_str(map, &base, "audit_log").eq_ignore_ascii_case("true"),
87        allow_anonymous: get_str(map, &base, "allow_anonymous").eq_ignore_ascii_case("true"),
88        redis: get_str(map, &base, "redis"),
89        redis_password_env: get_str(map, &base, "redis_password_env"),
90        redis_prefix: if prefix.is_empty() { "rws:llm:".to_string() } else { prefix },
91        connect_timeout_ms: get_u64(map, &base, "connect_timeout_ms", 10_000),
92        read_timeout_ms: get_u64(map, &base, "read_timeout_ms", 300_000),
93        providers: indexed("provider")
94            .into_iter()
95            .map(|s| LlmProviderConfig {
96                name: get_str(map, &s, "name"),
97                api: get_str(map, &s, "api"),
98                url: get_str(map, &s, "url"),
99                key_env: get_str(map, &s, "key_env"),
100            })
101            .collect(),
102        keys: indexed("key")
103            .into_iter()
104            .map(|s| LlmKeyConfig {
105                identity: get_str(map, &s, "identity"),
106                key_env: get_str(map, &s, "key_env"),
107                budget_tokens: get_u64(map, &s, "budget_tokens", 0),
108                window_secs: get_u64(map, &s, "window_secs", 0),
109            })
110            .collect(),
111    }
112}
113
114fn env_set(name: &str) -> bool {
115    !std::env::var(name).unwrap_or_default().trim().is_empty()
116}
117
118/// Semantic checks for `rws --check` and startup.
119pub(crate) fn validate(config: &LlmConfig, label: &str, report: &mut ConfigReport) {
120    let offline = crate::config_check::offline();
121    if config.providers.is_empty() {
122        report.error(format!("{label}: [route.action.llm] has no [[route.action.llm.provider]]"));
123    }
124    let mut names = Vec::new();
125    for (i, p) in config.providers.iter().enumerate() {
126        let what = format!("{label}: provider #{}", i + 1);
127        if p.name.is_empty() {
128            report.error(format!("{what}: needs a name"));
129        } else if names.contains(&p.name) {
130            report.error(format!("{what}: name {:?} is used twice", p.name));
131        } else {
132            names.push(p.name.clone());
133        }
134        let Some(api) = Api::parse(&p.api) else {
135            report.error(format!("{what}: api {:?} must be \"openai\" or \"anthropic\"", p.api));
136            continue;
137        };
138        if let Err(e) = Provider::new(p.name.clone(), api, &p.url) {
139            report.error(format!("{what}: {e}"));
140        }
141        if !p.key_env.is_empty() && !offline && !env_set(&p.key_env) {
142            report.error(format!("{what}: key_env {} is unset or empty", p.key_env));
143        }
144    }
145    if config.keys.is_empty() && !config.allow_anonymous {
146        report.error(format!("{label}: no [[route.action.llm.key]] and allow_anonymous is off: nobody can call the proxy"));
147    }
148    for k in &config.keys {
149        if k.identity.is_empty() || k.key_env.is_empty() {
150            report.error(format!("{label}: a key needs identity and key_env"));
151        } else if !offline && !env_set(&k.key_env) {
152            report.error(format!("{label}: key for {} reads {}, which is unset or empty", k.identity, k.key_env));
153        }
154        if (k.budget_tokens > 0) != (k.window_secs > 0) {
155            report.error(format!("{label}: key for {}: budget_tokens and window_secs go together", k.identity));
156        }
157    }
158    if !config.redis_password_env.is_empty() && config.redis.is_empty() {
159        report.warn(format!("{label}: redis_password_env without redis: budgets stay in memory"));
160    }
161    if !config.redis.is_empty() && !config.keys.iter().any(|k| k.budget_tokens > 0) {
162        report.warn(format!("{label}: redis is set but no key has a budget"));
163    }
164}
165
166/// The proxy `config` describes, secrets read from the environment.
167pub(crate) fn build(config: &LlmConfig) -> LlmProxy {
168    let mut proxy = LlmProxy::new()
169        .audit_log(config.audit_log)
170        .timeouts(Duration::from_millis(config.connect_timeout_ms), Duration::from_millis(config.read_timeout_ms));
171    for p in &config.providers {
172        let Some(api) = Api::parse(&p.api) else { continue };
173        match Provider::new(p.name.clone(), api, &p.url) {
174            Ok(provider) => {
175                let key = if p.key_env.is_empty() { String::new() } else { std::env::var(&p.key_env).unwrap_or_default() };
176                proxy = proxy.provider(provider.api_key(key.trim()));
177            }
178            Err(e) => eprintln!("llm proxy: provider {} skipped: {e}", p.name),
179        }
180    }
181    for k in &config.keys {
182        let Ok(key) = std::env::var(&k.key_env) else { continue };
183        let key = key.trim();
184        if key.is_empty() {
185            continue;
186        }
187        proxy = proxy.virtual_key(key, k.identity.clone());
188        if k.budget_tokens > 0 && k.window_secs > 0 {
189            proxy = proxy.budget(k.identity.clone(), k.budget_tokens, k.window_secs);
190        }
191    }
192    if config.allow_anonymous {
193        proxy = proxy.allow_anonymous();
194    }
195    if !config.redis.is_empty() {
196        let password = (!config.redis_password_env.is_empty()).then(|| std::env::var(&config.redis_password_env).ok()).flatten();
197        proxy = proxy.redis(config.redis.clone(), password, config.redis_prefix.clone());
198    }
199    proxy
200}
201
202#[cfg(test)]
203mod tests;
rust_web_server/proxy_config/llm.rs

rust_web_server/proxy_config/
llm.rs
