rws_proxy/proxy_config/
llm.rs

1//! `[route.action] type = "llm"`: the [LLM proxy](crate::llm) from the
2//! config file (18.9.0).
3//!
4//! ```toml
5//! [[route]]
6//!   [route.match]
7//!   path = "/v1/*"
8//!
9//!   [route.action]
10//!   type = "llm"
11//!
12//!   [route.action.llm]
13//!   audit_log          = true
14//!   redis              = "127.0.0.1:6379"   # share budgets between replicas
15//!   redis_password_env = "REDIS_PASSWORD"
16//!   redis_prefix       = "rws:llm:"
17//!   connect_timeout_ms = 10000
18//!   read_timeout_ms    = 300000
19//!
20//!   [[route.action.llm.provider]]         # tried in this order, per API
21//!   name    = "openai"
22//!   api     = "openai"                    # or "anthropic"
23//!   url     = "https://api.openai.com"
24//!   key_env = "OPENAI_API_KEY"
25//!
26//!   [[route.action.llm.key]]              # a virtual key clients send
27//!   identity      = "alice"
28//!   key_env       = "ALICE_LLM_KEY"
29//!   budget_tokens = 2000000               # optional, with window_secs
30//!   window_secs   = 86400
31//! ```
32//!
33//! Secrets never sit in the file: provider keys, virtual keys and the Redis
34//! password are read from the environment variables named.
35
36use std::time::Duration;
37
38use crate::config_check::ConfigReport;
39use crate::llm::{Api, LlmProxy, Provider};
40
41use super::parser::{get_str, get_u64, section_exists, SectionMap};
42
43#[derive(Debug, Clone, Default, PartialEq)]
44pub struct LlmConfig {
45    pub audit_log: bool,
46    pub allow_anonymous: bool,
47    pub redis: String,
48    pub redis_password_env: String,
49    pub redis_prefix: String,
50    pub connect_timeout_ms: u64,
51    pub read_timeout_ms: u64,
52    pub providers: Vec<LlmProviderConfig>,
53    pub keys: Vec<LlmKeyConfig>,
54}
55
56#[derive(Debug, Clone, Default, PartialEq)]
57pub struct LlmProviderConfig {
58    pub name: String,
59    pub api: String,
60    pub url: String,
61    pub key_env: String,
62}
63
64#[derive(Debug, Clone, Default, PartialEq)]
65pub struct LlmKeyConfig {
66    pub identity: String,
67    pub key_env: String,
68    pub budget_tokens: u64,
69    pub window_secs: u64,
70}
71
72/// The keys `check_unknown_keys` accepts in each part.
73#[doc(hidden)]
74pub const LLM_KEYS: &[&str] =
75    &["audit_log", "allow_anonymous", "redis", "redis_password_env", "redis_prefix", "connect_timeout_ms", "read_timeout_ms"];
76#[doc(hidden)]
77pub const PROVIDER_KEYS: &[&str] = &["name", "api", "url", "key_env"];
78#[doc(hidden)]
79pub const KEY_KEYS: &[&str] = &["identity", "key_env", "budget_tokens", "window_secs"];
80
81/// `section` is the action's, e.g. `route[0].action`.
82#[doc(hidden)]
83pub fn parse(map: &SectionMap, section: &str) -> LlmConfig {
84    let base = format!("{section}.llm");
85    let indexed = |kind: &str| -> Vec<String> {
86        (0..).map(|i| format!("{base}.{kind}[{i}]")).take_while(|s| section_exists(map, s)).collect()
87    };
88    let prefix = get_str(map, &base, "redis_prefix");
89    LlmConfig {
90        audit_log: get_str(map, &base, "audit_log").eq_ignore_ascii_case("true"),
91        allow_anonymous: get_str(map, &base, "allow_anonymous").eq_ignore_ascii_case("true"),
92        redis: get_str(map, &base, "redis"),
93        redis_password_env: get_str(map, &base, "redis_password_env"),
94        redis_prefix: if prefix.is_empty() { "rws:llm:".to_string() } else { prefix },
95        connect_timeout_ms: get_u64(map, &base, "connect_timeout_ms", 10_000),
96        read_timeout_ms: get_u64(map, &base, "read_timeout_ms", 300_000),
97        providers: indexed("provider")
98            .into_iter()
99            .map(|s| LlmProviderConfig {
100                name: get_str(map, &s, "name"),
101                api: get_str(map, &s, "api"),
102                url: get_str(map, &s, "url"),
103                key_env: get_str(map, &s, "key_env"),
104            })
105            .collect(),
106        keys: indexed("key")
107            .into_iter()
108            .map(|s| LlmKeyConfig {
109                identity: get_str(map, &s, "identity"),
110                key_env: get_str(map, &s, "key_env"),
111                budget_tokens: get_u64(map, &s, "budget_tokens", 0),
112                window_secs: get_u64(map, &s, "window_secs", 0),
113            })
114            .collect(),
115    }
116}
117
118fn env_set(name: &str) -> bool {
119    !std::env::var(name).unwrap_or_default().trim().is_empty()
120}
121
122/// Semantic checks for `rws --check` and startup.
123#[doc(hidden)]
124pub fn validate(config: &LlmConfig, label: &str, report: &mut ConfigReport) {
125    let offline = crate::config_check::offline();
126    if config.providers.is_empty() {
127        report.error(format!("{label}: [route.action.llm] has no [[route.action.llm.provider]]"));
128    }
129    let mut names = Vec::new();
130    for (i, p) in config.providers.iter().enumerate() {
131        let what = format!("{label}: provider #{}", i + 1);
132        if p.name.is_empty() {
133            report.error(format!("{what}: needs a name"));
134        } else if names.contains(&p.name) {
135            report.error(format!("{what}: name {:?} is used twice", p.name));
136        } else {
137            names.push(p.name.clone());
138        }
139        let Some(api) = Api::parse(&p.api) else {
140            report.error(format!("{what}: api {:?} must be \"openai\" or \"anthropic\"", p.api));
141            continue;
142        };
143        if let Err(e) = Provider::new(p.name.clone(), api, &p.url) {
144            report.error(format!("{what}: {e}"));
145        }
146        if !p.key_env.is_empty() && !offline && !env_set(&p.key_env) {
147            report.error(format!("{what}: key_env {} is unset or empty", p.key_env));
148        }
149    }
150    if config.keys.is_empty() && !config.allow_anonymous {
151        report.error(format!("{label}: no [[route.action.llm.key]] and allow_anonymous is off: nobody can call the proxy"));
152    }
153    for k in &config.keys {
154        if k.identity.is_empty() || k.key_env.is_empty() {
155            report.error(format!("{label}: a key needs identity and key_env"));
156        } else if !offline && !env_set(&k.key_env) {
157            report.error(format!("{label}: key for {} reads {}, which is unset or empty", k.identity, k.key_env));
158        }
159        if (k.budget_tokens > 0) != (k.window_secs > 0) {
160            report.error(format!("{label}: key for {}: budget_tokens and window_secs go together", k.identity));
161        }
162    }
163    if !config.redis_password_env.is_empty() && config.redis.is_empty() {
164        report.warn(format!("{label}: redis_password_env without redis: budgets stay in memory"));
165    }
166    if !config.redis.is_empty() && !config.keys.iter().any(|k| k.budget_tokens > 0) {
167        report.warn(format!("{label}: redis is set but no key has a budget"));
168    }
169}
170
171/// The proxy `config` describes, secrets read from the environment.
172#[doc(hidden)]
173pub fn build(config: &LlmConfig) -> LlmProxy {
174    let mut proxy = LlmProxy::new()
175        .audit_log(config.audit_log)
176        .timeouts(Duration::from_millis(config.connect_timeout_ms), Duration::from_millis(config.read_timeout_ms));
177    for p in &config.providers {
178        let Some(api) = Api::parse(&p.api) else { continue };
179        match Provider::new(p.name.clone(), api, &p.url) {
180            Ok(provider) => {
181                let key = if p.key_env.is_empty() { String::new() } else { std::env::var(&p.key_env).unwrap_or_default() };
182                proxy = proxy.provider(provider.api_key(key.trim()));
183            }
184            Err(e) => eprintln!("llm proxy: provider {} skipped: {e}", p.name),
185        }
186    }
187    for k in &config.keys {
188        let Ok(key) = std::env::var(&k.key_env) else { continue };
189        let key = key.trim();
190        if key.is_empty() {
191            continue;
192        }
193        proxy = proxy.virtual_key(key, k.identity.clone());
194        if k.budget_tokens > 0 && k.window_secs > 0 {
195            proxy = proxy.budget(k.identity.clone(), k.budget_tokens, k.window_secs);
196        }
197    }
198    if config.allow_anonymous {
199        proxy = proxy.allow_anonymous();
200    }
201    if !config.redis.is_empty() {
202        let password = (!config.redis_password_env.is_empty()).then(|| std::env::var(&config.redis_password_env).ok()).flatten();
203        proxy = proxy.redis(config.redis.clone(), password, config.redis_prefix.clone());
204    }
205    proxy
206}
207
208#[cfg(test)]
209mod tests;
rws_proxy/proxy_config/llm.rs

rws_proxy/proxy_config/
llm.rs
