oxicode_catalog/catalog/
materialize.rs

1//! models.dev → oxicode catalog materialization.
2//!
3//! Converts a [`MdCatalog`] (from models.dev `api.json`) into
4//! [`BuiltinProviderEntry`] and [`BuiltinModelEntry`] vectors.
5//!
6//! This is the shared entry point for both **SNAP** (compile-time embedded
7//! snapshot) and **LIVE** (runtime cache) sources.
8//!
9//! # Protocol resolution
10//!
11//! The [`protocol_for`] function maps a models.dev `npm` string to oxicode's
12//! [`crate::Api`] enum + [`crate::catalog::provider::AuthMethod`]. This is the **only** protocol knowledge
13//! oxicode has — 7 match arms covering all known API protocols. Unknown npm
14//! values default to OpenAI-compatible (`OpenAiCompletions` + `Bearer`).
15//!
16//! # Model-level overrides
17//!
18//! Per-model [`crate::catalog::models_dev::MdModelProvider`] can override three things:
19//! - `npm` → protocol + auth (via `protocol_for`)
20//! - `api` → base URL (55+ models as of 2026-06-17)
21//!
22//! # Attribution
23//!
24//! Model data © [models.dev](https://models.dev) (MIT).
25
26use crate::catalog::BuiltinModelEntry;
27use crate::catalog::models_dev::{MdCatalog, protocol_for};
28use crate::catalog::override_::{OverrideFile, apply_model_overrides, apply_provider_overrides};
29use crate::catalog::provider::BuiltinProviderEntry;
30use std::collections::BTreeMap;
31
32/// Convert a [`MdCatalog`] into oxicode's internal provider + model entries.
33///
34/// Applies product-meta extra headers and user overrides (Layer 2) after
35/// the base conversion.
36///
37/// Returns `(providers, models)` sorted by provider id.
38pub fn materialize(
39    catalog: &MdCatalog,
40    product_meta: &ProductMeta,
41    user_overrides: &OverrideFile,
42) -> (
43    Vec<BuiltinProviderEntry>,
44    BTreeMap<String, Vec<BuiltinModelEntry>>,
45) {
46    let mut providers = Vec::new();
47    let mut models: BTreeMap<String, Vec<BuiltinModelEntry>> = BTreeMap::new();
48
49    for (pid, mdprov) in &catalog.0 {
50        let (api, auth) = protocol_for(mdprov.npm.as_deref().unwrap_or(""));
51        let extra = product_meta
52            .extra_headers
53            .get(pid)
54            .cloned()
55            .unwrap_or_default();
56
57        providers.push(BuiltinProviderEntry {
58            id: pid.clone(),
59            display_name: mdprov.name.clone(),
60            aliases: vec![], // models.dev id만 사용, 호환성 버림
61            api: api.to_string(),
62            env_key: mdprov.env.first().cloned().unwrap_or_default(),
63            extra_env_keys: mdprov.env[1..].to_vec(),
64            base_url: mdprov.api.clone().unwrap_or_default(),
65            auth_method: auth,
66            extra_headers: extra,
67            category: String::new(),    // 제거됨 (Phase 2에서 최종 제거)
68            description: String::new(), // 제품 메타 불필요
69            default_enabled: true,
70        });
71
72        for (mid, mdmodel) in &mdprov.models {
73            // Model-level override (v3): protocol + auth + base_url
74            let model_prov = mdmodel.provider.as_ref();
75            let model_npm = model_prov
76                .and_then(|p| p.npm.as_deref())
77                .unwrap_or_else(|| mdprov.npm.as_deref().unwrap_or(""));
78            let (model_api, model_auth) = protocol_for(model_npm);
79            let model_base_url = model_prov.and_then(|p| p.api.clone());
80
81            models
82                .entry(pid.clone())
83                .or_default()
84                .push(BuiltinModelEntry {
85                    id: mid.clone(),
86                    name: mdmodel.name.clone(),
87                    api: model_api.to_string(),
88                    provider: pid.clone(),
89                    reasoning: mdmodel.reasoning,
90                    auth_method: model_auth,
91                    base_url: model_base_url,
92                    input: normalize_modalities(&mdmodel.modalities),
93                    cost_input: mdmodel.cost.as_ref().map(|c| c.input).unwrap_or(0.0),
94                    cost_output: mdmodel.cost.as_ref().map(|c| c.output).unwrap_or(0.0),
95                    cost_cache_read: mdmodel
96                        .cost
97                        .as_ref()
98                        .and_then(|c| c.cache_read)
99                        .unwrap_or(0.0),
100                    cost_cache_write: mdmodel
101                        .cost
102                        .as_ref()
103                        .and_then(|c| c.cache_write)
104                        .unwrap_or(0.0),
105                    context_window: mdmodel.limit.context as u32,
106                    max_tokens: mdmodel.limit.output as u32,
107                });
108        }
109    }
110
111    // Apply Layer 2 user overrides (highest precedence)
112    apply_provider_overrides(&mut providers, &user_overrides.provider);
113    apply_model_overrides(&mut models, &user_overrides.model);
114
115    (providers, models)
116}
117
118/// Raw gzip bytes of the embedded models.dev snapshot.
119///
120/// Single source of truth for the compiled-in catalog snapshot. Sibling
121/// crates (notably `oxicode-sdk`) reference these bytes through this accessor
122/// instead of their own `include_bytes!`, so the snapshot file is packaged
123/// exactly once — here, inside `oxicode-ai` — and never escapes the crate root
124/// in a published artifact. (A cross-crate `include_bytes!` in `oxicode-sdk`
125/// pointed at `oxicode-ai/data/...` and made the published `oxicode-sdk` crate
126/// uncompilable for downstream consumers — see 0.37.1.)
127///
128/// The path is relative to this source file and stays within `oxicode-ai`'s
129/// own package, so it resolves correctly both in-tree and when `oxicode-ai` is
130/// downloaded from crates.io.
131pub fn snapshot_gzip_bytes() -> &'static [u8] {
132    include_bytes!("../../data/catalog/_snapshot.json.gz")
133}
134
135/// Load the embedded SNAP snapshot, decompress it, and parse as `MdCatalog`.
136///
137/// This is the single source for both the provider registry and the model
138/// database. No network access — the snapshot is embedded at compile time.
139pub fn load_snapshot_catalog() -> Option<MdCatalog> {
140    use std::io::Read;
141    let compressed: &[u8] = snapshot_gzip_bytes();
142    let mut decoder = flate2::read::GzDecoder::new(compressed);
143    let mut json = String::new();
144    decoder.read_to_string(&mut json).ok()?;
145    serde_json::from_str::<MdCatalog>(&json).ok()
146}
147
148/// Materialize providers from the embedded snapshot.
149///
150/// Returns the full provider list (145+ providers from models.dev) after
151/// applying product-meta overrides and Layer 2 user overrides.
152pub fn materialize_providers() -> Vec<BuiltinProviderEntry> {
153    let Some(catalog) = load_snapshot_catalog() else {
154        return Vec::new();
155    };
156    let product_meta = ProductMeta::builtin();
157    let overrides = crate::catalog::load_overrides().unwrap_or_default();
158    let (providers, _models) = materialize(&catalog, &product_meta, &overrides);
159    providers
160}
161
162/// The raw `data/catalog/product-meta.toml` source embedded at compile time.
163///
164/// Exposed so sibling crates (e.g. `oxicode-cli`'s OAuth spec loader) can read
165/// this catalog's own data file **through the crate dependency** rather than
166/// reaching into `data/` via a relative filesystem path — which breaks
167/// standalone `cargo package` / crates.io publishing (the file is not included
168/// in a downstream consumer's tarball).
169pub fn product_meta_toml() -> &'static str {
170    include_str!("../../data/catalog/product-meta.toml")
171}
172
173/// Product-specific metadata: extra HTTP headers for a subset of providers.
174///
175/// Models.dev does not know oxicode-specific HTTP headers (e.g. `HTTP-Referer`
176/// for OpenRouter). These are maintained in `data/catalog/product-meta.toml`.
177#[derive(Default)]
178pub struct ProductMeta {
179    /// Extra HTTP headers keyed by models.dev provider id.
180    pub extra_headers: std::collections::HashMap<String, Vec<(String, String)>>,
181}
182
183impl ProductMeta {
184    /// Parse from the built-in product-meta.toml.
185    pub fn builtin() -> Self {
186        include_str!("../../data/catalog/product-meta.toml")
187            .parse()
188            .unwrap_or_default()
189    }
190}
191
192impl std::str::FromStr for ProductMeta {
193    type Err = String;
194    fn from_str(s: &str) -> Result<Self, Self::Err> {
195        #[derive(serde::Deserialize)]
196        struct Raw {
197            #[serde(default)]
198            provider: Vec<RawProvider>,
199        }
200        #[derive(serde::Deserialize)]
201        struct RawProvider {
202            id: String,
203            #[serde(default)]
204            extra_headers: Vec<(String, String)>,
205        }
206        let raw: Raw = toml::from_str(s).map_err(|e| e.to_string())?;
207        let mut extra_headers = std::collections::HashMap::new();
208        for p in raw.provider {
209            if !p.extra_headers.is_empty() {
210                extra_headers.insert(p.id, p.extra_headers);
211            }
212        }
213        Ok(Self { extra_headers })
214    }
215}
216
217/// Normalize models.dev modalities to oxicode input string list.
218fn normalize_modalities(md: &Option<crate::catalog::models_dev::MdModalities>) -> Vec<String> {
219    match md {
220        Some(m) => match &m.input {
221            Some(input) if !input.is_empty() => input.clone(),
222            _ => vec!["text".to_string()],
223        },
224        _ => vec!["text".to_string()],
225    }
226}
227
228#[cfg(test)]
229mod tests {
230    use super::*;
231    use crate::Api;
232
233    #[test]
234    fn protocol_for_anthropic() {
235        let (api, auth) = protocol_for("@ai-sdk/anthropic");
236        assert_eq!(api, Api::AnthropicMessages);
237        assert_eq!(auth, crate::catalog::provider::AuthMethod::XApiKey);
238    }
239
240    #[test]
241    fn protocol_for_google() {
242        let (api, auth) = protocol_for("@ai-sdk/google");
243        assert_eq!(api, Api::GoogleGenerativeAi);
244        assert_eq!(auth, crate::catalog::provider::AuthMethod::None);
245    }
246
247    #[test]
248    fn protocol_for_openai_compatible() {
249        let (api, auth) = protocol_for("@ai-sdk/openai-compatible");
250        assert_eq!(api, Api::OpenAiCompletions);
251        assert_eq!(auth, crate::catalog::provider::AuthMethod::Bearer);
252    }
253
254    #[test]
255    fn protocol_for_unknown() {
256        let (api, auth) = protocol_for("some-new-sdk");
257        assert_eq!(api, Api::OpenAiCompletions);
258        assert_eq!(auth, crate::catalog::provider::AuthMethod::Bearer);
259    }
260
261    #[test]
262    fn protocol_for_empty() {
263        let (api, auth) = protocol_for("");
264        assert_eq!(api, Api::OpenAiCompletions);
265        assert_eq!(auth, crate::catalog::provider::AuthMethod::Bearer);
266    }
267
268    #[test]
269    fn protocol_for_mistral() {
270        // omp treats Mistral as openai-completions-compatible (no separate
271        // dialect); it falls through to the default Bearer/openai-completions.
272        let (api, auth) = protocol_for("@ai-sdk/mistral");
273        assert_eq!(api, Api::OpenAiCompletions);
274        assert_eq!(auth, crate::catalog::provider::AuthMethod::Bearer);
275    }
276
277    #[test]
278    fn protocol_for_azure() {
279        let (api, auth) = protocol_for("@ai-sdk/azure");
280        assert_eq!(api, Api::AzureOpenAiResponses);
281        assert_eq!(auth, crate::catalog::provider::AuthMethod::ApiKey);
282    }
283
284    #[test]
285    fn protocol_for_amazon_bedrock() {
286        let (api, auth) = protocol_for("@ai-sdk/amazon-bedrock");
287        assert_eq!(api, Api::BedrockConverseStream);
288        assert_eq!(auth, crate::catalog::provider::AuthMethod::None);
289    }
290
291    #[test]
292    fn product_meta_parses() {
293        let meta = ProductMeta::builtin();
294        // OpenRouter must have its HTTP-Referer header
295        let openrouter_headers = meta.extra_headers.get("openrouter");
296        assert!(
297            openrouter_headers.is_some(),
298            "product-meta.toml should include openrouter"
299        );
300        if let Some(headers) = openrouter_headers {
301            assert!(headers.iter().any(|(k, _)| k == "HTTP-Referer"));
302        }
303    }
304
305    #[test]
306    fn materialize_snapshot_counts() {
307        use std::io::Read;
308
309        // Decompress the embedded SNAP snapshot
310        let compressed = include_bytes!("../../data/catalog/_snapshot.json.gz");
311        let mut decoder = flate2::read::GzDecoder::new(&compressed[..]);
312        let mut json = String::new();
313        decoder.read_to_string(&mut json).unwrap();
314
315        let catalog: crate::catalog::MdCatalog = serde_json::from_str(&json).unwrap();
316        let (providers, models) =
317            super::materialize(&catalog, &ProductMeta::default(), &Default::default());
318
319        assert!(!providers.is_empty(), "providers should not be empty");
320        assert_eq!(providers.len(), 145, "expected 145 providers");
321        let model_count: usize = models.values().map(|v| v.len()).sum();
322        assert_eq!(model_count, 5277, "expected 5277 models");
323    }
324}
oxicode_catalog/catalog/materialize.rs

oxicode_catalog/catalog/
materialize.rs
