Skip to main content

graphrt_llm/
registry.rs

1//! Named model providers, registered as a host service.
2
3use std::fmt;
4use std::sync::Arc;
5
6use indexmap::IndexMap;
7
8use crate::error::LlmError;
9use crate::types::ChatModel;
10
11/// Maps provider names (as used in node configs: `model.provider`) to models.
12///
13/// Register it with `graphrt_core::Services::insert(registry)`.
14///
15/// Name resolution (see [`ModelRegistry::get`]) is exact, with two explicit
16/// exceptions: aliases added with [`ModelRegistry::alias`], and plugin-style
17/// identifiers `<org>/<provider>/<provider>` (for example
18/// `acme/openai/openai`), which resolve to `<provider>`. Other names, such as
19/// `acme/openai`, are not guessed.
20#[derive(Clone, Default)]
21pub struct ModelRegistry {
22    providers: IndexMap<String, Arc<dyn ChatModel>>,
23    aliases: IndexMap<String, String>,
24    /// Why a well-known provider was not registered (shown in "unknown provider" errors).
25    hints: IndexMap<String, String>,
26}
27
28impl fmt::Debug for ModelRegistry {
29    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
30        f.debug_struct("ModelRegistry")
31            .field("providers", &self.providers.keys().collect::<Vec<_>>())
32            .field("aliases", &self.aliases)
33            .finish()
34    }
35}
36
37impl ModelRegistry {
38    pub fn new() -> Self {
39        Self::default()
40    }
41
42    /// Register a provider, replacing any provider already registered under
43    /// `name`. Use [`ModelRegistry::try_add`] to reject duplicates instead.
44    pub fn add(&mut self, name: impl Into<String>, model: Arc<dyn ChatModel>) -> &mut Self {
45        self.providers.insert(name.into(), model);
46        self
47    }
48
49    /// Register a provider, failing if `name` is already registered.
50    pub fn try_add(&mut self, name: impl Into<String>, model: Arc<dyn ChatModel>) -> Result<&mut Self, LlmError> {
51        let name = name.into();
52        if self.providers.contains_key(&name) {
53            return Err(LlmError::Config(format!("model provider `{name}` is already registered")));
54        }
55        self.providers.insert(name, model);
56        Ok(self)
57    }
58
59    /// Make `alias` resolve to the provider registered as `target`.
60    pub fn alias(&mut self, alias: impl Into<String>, target: impl Into<String>) -> &mut Self {
61        self.aliases.insert(alias.into(), target.into());
62        self
63    }
64
65    /// Builder-style [`ModelRegistry::add`].
66    pub fn with(mut self, name: impl Into<String>, model: Arc<dyn ChatModel>) -> Self {
67        self.add(name, model);
68        self
69    }
70
71    /// Remove a provider (and any alias with that name). Returns whether something was removed.
72    pub fn remove(&mut self, name: &str) -> bool {
73        let provider = self.providers.shift_remove(name).is_some();
74        let alias = self.aliases.shift_remove(name).is_some();
75        provider || alias
76    }
77
78    /// Decompose into providers and aliases.
79    pub fn into_parts(self) -> (IndexMap<String, Arc<dyn ChatModel>>, IndexMap<String, String>) {
80        (self.providers, self.aliases)
81    }
82
83    pub fn names(&self) -> impl Iterator<Item = &str> {
84        self.providers.keys().map(String::as_str)
85    }
86
87    pub fn is_empty(&self) -> bool {
88        self.providers.is_empty()
89    }
90
91    /// Resolve a provider name: an exact name, then an alias, then a
92    /// plugin-style `<org>/<provider>/<provider>` identifier.
93    pub fn get(&self, name: &str) -> Result<Arc<dyn ChatModel>, LlmError> {
94        if let Some(m) = self.providers.get(name) {
95            return Ok(Arc::clone(m));
96        }
97        if let Some(m) = self.aliases.get(name).and_then(|target| self.providers.get(target)) {
98            return Ok(Arc::clone(m));
99        }
100        if let Some(m) = plugin_provider(name).and_then(|short| self.providers.get(short)) {
101            return Ok(Arc::clone(m));
102        }
103        let known: Vec<&str> = self.names().collect();
104        let hint = self.hints.get(name).map(|h| format!("; {h}")).unwrap_or_default();
105        Err(LlmError::Config(format!("unknown model provider `{name}` (configured: {known:?}){hint}")))
106    }
107
108    /// Build a registry from environment variables:
109    ///
110    /// | Variables | Provider name |
111    /// |---|---|
112    /// | `OPENAI_API_KEY`, optional `OPENAI_BASE_URL` | `openai` |
113    /// | `ANTHROPIC_API_KEY`, optional `ANTHROPIC_BASE_URL` | `anthropic` |
114    /// | `OLLAMA_HOST` (default `http://localhost:11434`; always registered) | `ollama` |
115    /// | `GRAPHRT_OPENAI_COMPAT_<NAME>_BASE_URL`, optional `..._API_KEY` | `<name>` (lowercase) |
116    /// | `GEMINI_API_KEY` or `GOOGLE_API_KEY`, optional `GEMINI_BASE_URL` | `gemini` |
117    /// | `AWS_REGION` (or `AWS_DEFAULT_REGION`) plus `AWS_BEARER_TOKEN_BEDROCK` or `AWS_ACCESS_KEY_ID` + `AWS_SECRET_ACCESS_KEY` (+ `AWS_SESSION_TOKEN`), optional `BEDROCK_ENDPOINT_URL` | `bedrock` |
118    #[cfg(feature = "http")]
119    pub fn from_env() -> Result<Self, LlmError> {
120        let vars: Vec<(String, String)> = std::env::vars().collect();
121        Self::from_vars(&vars)
122    }
123
124    /// Like [`ModelRegistry::from_env`] but reads from the given variables.
125    #[cfg(feature = "http")]
126    pub fn from_vars(vars: &[(String, String)]) -> Result<Self, LlmError> {
127        let get = |k: &str| vars.iter().find(|(name, v)| name == k && !v.is_empty()).map(|(_, v)| v.clone());
128        #[allow(unused_mut)]
129        let mut registry = Self::new();
130
131        #[cfg(feature = "openai")]
132        {
133            use crate::openai::{OpenAiConfig, OpenAiModel};
134            let key = get("OPENAI_API_KEY");
135            let base = get("OPENAI_BASE_URL");
136            if key.is_some() || base.is_some() {
137                let mut cfg = OpenAiConfig::new(base.unwrap_or_else(|| OpenAiConfig::default().base_url), key);
138                cfg.use_max_completion_tokens = cfg.base_url.contains("api.openai.com");
139                registry.try_add("openai", Arc::new(OpenAiModel::new(cfg)?))?;
140            }
141            {
142                // `ollama` is always available: it defaults to the standard local port, and
143                // only connects when a node actually uses it.
144                let host = get("OLLAMA_HOST").unwrap_or_else(|| "http://localhost:11434".to_owned());
145                let host = if host.starts_with("http") { host } else { format!("http://{host}") };
146                let cfg = OpenAiConfig::new(format!("{}/v1", host.trim_end_matches('/')), None);
147                registry.try_add("ollama", Arc::new(OpenAiModel::new(cfg)?))?;
148            }
149            const PREFIX: &str = "GRAPHRT_OPENAI_COMPAT_";
150            for (name, value) in vars {
151                let Some(rest) = name.strip_prefix(PREFIX) else { continue };
152                let Some(provider) = rest.strip_suffix("_BASE_URL") else { continue };
153                if value.is_empty() || provider.is_empty() {
154                    continue;
155                }
156                let key = get(&format!("{PREFIX}{provider}_API_KEY"));
157                let cfg = OpenAiConfig::new(value.clone(), key);
158                // A duplicate (e.g. `..._OPENAI_BASE_URL` next to `OPENAI_API_KEY`) is an error, not an override.
159                registry.try_add(provider.to_ascii_lowercase(), Arc::new(OpenAiModel::new(cfg)?))?;
160            }
161        }
162
163        #[cfg(feature = "anthropic")]
164        {
165            use crate::anthropic::{AnthropicConfig, AnthropicModel};
166            if let Some(key) = get("ANTHROPIC_API_KEY") {
167                let mut cfg = AnthropicConfig::new(key);
168                if let Some(base) = get("ANTHROPIC_BASE_URL") {
169                    cfg.base_url = base;
170                }
171                registry.try_add("anthropic", Arc::new(AnthropicModel::new(cfg)?))?;
172            }
173        }
174
175        #[cfg(feature = "gemini")]
176        {
177            use crate::gemini::{GeminiConfig, GeminiModel};
178            if let Some(key) = get("GEMINI_API_KEY").or_else(|| get("GOOGLE_API_KEY")) {
179                let mut cfg = GeminiConfig::new(key);
180                if let Some(base) = get("GEMINI_BASE_URL") {
181                    cfg.base_url = base;
182                }
183                registry.try_add("gemini", Arc::new(GeminiModel::new(cfg)?))?;
184            }
185        }
186
187        #[cfg(feature = "bedrock")]
188        {
189            use crate::bedrock::{AwsCredentials, BedrockAuth, BedrockConfig, BedrockModel};
190            // Without a region nothing is registered: AWS keys in the environment are
191            // common for unrelated tools and must not break startup.
192            if let Some(region) = get("AWS_REGION").or_else(|| get("AWS_DEFAULT_REGION")) {
193                let auth =
194                    match (get("AWS_BEARER_TOKEN_BEDROCK"), get("AWS_ACCESS_KEY_ID"), get("AWS_SECRET_ACCESS_KEY")) {
195                        (Some(token), _, _) => Some(BedrockAuth::ApiKey(token)),
196                        (None, Some(access_key_id), Some(secret_access_key)) => {
197                            Some(BedrockAuth::SigV4(AwsCredentials {
198                                access_key_id,
199                                secret_access_key,
200                                session_token: get("AWS_SESSION_TOKEN"),
201                            }))
202                        }
203                        (None, access, secret) => {
204                            let missing = match (access.is_some(), secret.is_some()) {
205                                (true, false) => "AWS_SECRET_ACCESS_KEY is not set",
206                                (false, true) => "AWS_ACCESS_KEY_ID is not set",
207                                _ => "neither AWS_BEARER_TOKEN_BEDROCK nor AWS access keys are set",
208                            };
209                            registry.hints.insert(
210                                "bedrock".into(),
211                                format!("bedrock was not registered: AWS region is set but {missing}"),
212                            );
213                            None
214                        }
215                    };
216                if let Some(auth) = auth {
217                    let mut cfg = BedrockConfig::new(region, auth);
218                    cfg.endpoint = get("BEDROCK_ENDPOINT_URL");
219                    registry.try_add("bedrock", Arc::new(BedrockModel::new(cfg)?))?;
220                }
221            }
222        }
223
224        let _ = &get;
225        Ok(registry)
226    }
227}
228
229/// `<org>/<provider>/<provider>` -> `<provider>`.
230fn plugin_provider(name: &str) -> Option<&str> {
231    let mut parts = name.split('/');
232    let (org, plugin, provider) = (parts.next()?, parts.next()?, parts.next()?);
233    if parts.next().is_some() || org.is_empty() || provider.is_empty() || plugin != provider {
234        return None;
235    }
236    Some(provider)
237}
238
239#[cfg(test)]
240mod tests {
241    use super::*;
242    use crate::mock::MockModel;
243
244    #[test]
245    fn resolves_names_explicitly() {
246        let mut reg = ModelRegistry::new().with("openai", Arc::new(MockModel::new(vec![])));
247        assert!(reg.get("openai").is_ok());
248        assert!(reg.get("vendor/openai/openai").is_ok());
249        assert!(matches!(reg.get("acme/openai"), Err(LlmError::Config(_))), "no guessing");
250        assert!(matches!(reg.get("a/b/openai"), Err(LlmError::Config(_))));
251        assert!(matches!(reg.get("nope"), Err(LlmError::Config(_))));
252        reg.alias("acme/openai", "openai");
253        assert!(reg.get("acme/openai").is_ok());
254    }
255
256    #[test]
257    fn try_add_rejects_duplicates() {
258        let mut reg = ModelRegistry::new();
259        assert!(reg.try_add("m", Arc::new(MockModel::new(vec![]))).is_ok());
260        assert!(matches!(reg.try_add("m", Arc::new(MockModel::new(vec![]))), Err(LlmError::Config(_))));
261    }
262
263    #[cfg(feature = "openai")]
264    #[test]
265    fn duplicate_env_providers_are_rejected() {
266        let vars = vec![
267            ("OPENAI_API_KEY".to_owned(), "sk-abc".to_owned()),
268            ("GRAPHRT_OPENAI_COMPAT_OPENAI_BASE_URL".to_owned(), "https://other/v1".to_owned()),
269        ];
270        assert!(matches!(ModelRegistry::from_vars(&vars), Err(LlmError::Config(_))));
271    }
272
273    #[cfg(all(feature = "openai", feature = "anthropic"))]
274    #[test]
275    fn from_vars_builds_providers_without_leaking_keys() {
276        let vars = vec![
277            ("OPENAI_API_KEY".to_owned(), "sk-abc".to_owned()),
278            ("ANTHROPIC_API_KEY".to_owned(), "ak-xyz".to_owned()),
279            ("OLLAMA_HOST".to_owned(), "localhost:11434".to_owned()),
280            ("GRAPHRT_OPENAI_COMPAT_GROQ_BASE_URL".to_owned(), "https://api.groq.com/openai/v1".to_owned()),
281        ];
282        let reg = ModelRegistry::from_vars(&vars).unwrap();
283        let names: Vec<&str> = reg.names().collect();
284        assert_eq!(names, vec!["openai", "ollama", "groq", "anthropic"]);
285        let debug = format!("{:?}", reg.get("openai").unwrap());
286        assert!(!debug.contains("sk-abc"));
287        let debug = format!("{:?}", reg.get("anthropic").unwrap());
288        assert!(!debug.contains("ak-xyz"));
289    }
290
291    #[cfg(all(feature = "gemini", feature = "bedrock"))]
292    #[test]
293    fn from_vars_registers_gemini_and_bedrock() {
294        let vars = |pairs: &[(&str, &str)]| -> Vec<(String, String)> {
295            pairs.iter().map(|(k, v)| ((*k).to_owned(), (*v).to_owned())).collect()
296        };
297        let reg = ModelRegistry::from_vars(&vars(&[
298            ("GOOGLE_API_KEY", "AIza-secret"),
299            ("AWS_REGION", "us-west-2"),
300            ("AWS_ACCESS_KEY_ID", "AKID"),
301            ("AWS_SECRET_ACCESS_KEY", "aws-secret"),
302            ("AWS_SESSION_TOKEN", "aws-token"),
303        ]))
304        .unwrap();
305        let debug = format!("{:?} {:?}", reg.get("gemini").unwrap(), reg.get("bedrock").unwrap());
306        assert!(
307            !debug.contains("AIza-secret") && !debug.contains("aws-secret") && !debug.contains("aws-token"),
308            "{debug}"
309        );
310        assert!(debug.contains("us-west-2"));
311
312        let bearer = ModelRegistry::from_vars(&vars(&[
313            ("AWS_DEFAULT_REGION", "eu-central-1"),
314            ("AWS_BEARER_TOKEN_BEDROCK", "bedrock-key"),
315        ]))
316        .unwrap();
317        assert!(format!("{:?}", bearer.get("bedrock").unwrap()).contains("ApiKey(<redacted>)"));
318
319        // Keys without a region register nothing.
320        let none =
321            ModelRegistry::from_vars(&vars(&[("AWS_ACCESS_KEY_ID", "A"), ("AWS_SECRET_ACCESS_KEY", "S")])).unwrap();
322        assert!(none.get("bedrock").is_err());
323
324        // A region with half the keys explains itself when the provider is used.
325        let partial =
326            ModelRegistry::from_vars(&vars(&[("AWS_REGION", "us-east-1"), ("AWS_ACCESS_KEY_ID", "A")])).unwrap();
327        let err = partial.get("bedrock").unwrap_err().to_string();
328        assert!(err.contains("AWS_SECRET_ACCESS_KEY is not set"), "{err}");
329    }
330}