llm_optimizer_decision/
model_registry.rs

1//! LLM Model Registry and Catalog
2//!
3//! This module provides a comprehensive registry of LLM models with their
4//! pricing, performance characteristics, and capabilities.
5
6use serde::{Deserialize, Serialize};
7use std::collections::HashMap;
8use uuid::Uuid;
9
10use crate::errors::Result;
11use crate::pareto::{ModelCandidate, Objectives};
12
13/// LLM Provider
14#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
15#[serde(rename_all = "snake_case")]
16pub enum Provider {
17    OpenAI,
18    Anthropic,
19    Google,
20    Meta,
21    Mistral,
22    XAI,
23    DeepSeek,
24    Alibaba,
25}
26
27impl Provider {
28    /// Get all supported providers
29    pub fn all() -> Vec<Provider> {
30        vec![
31            Provider::OpenAI,
32            Provider::Anthropic,
33            Provider::Google,
34            Provider::Meta,
35            Provider::Mistral,
36            Provider::XAI,
37            Provider::DeepSeek,
38            Provider::Alibaba,
39        ]
40    }
41
42    /// Get provider name
43    pub fn name(&self) -> &'static str {
44        match self {
45            Provider::OpenAI => "OpenAI",
46            Provider::Anthropic => "Anthropic",
47            Provider::Google => "Google",
48            Provider::Meta => "Meta",
49            Provider::Mistral => "Mistral AI",
50            Provider::XAI => "xAI",
51            Provider::DeepSeek => "DeepSeek",
52            Provider::Alibaba => "Alibaba Cloud",
53        }
54    }
55}
56
57/// Model tier for capability classification
58#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
59#[serde(rename_all = "snake_case")]
60pub enum ModelTier {
61    /// Flagship models (highest capability, highest cost)
62    Flagship,
63    /// Advanced models (high capability, moderate cost)
64    Advanced,
65    /// Standard models (good capability, low cost)
66    Standard,
67    /// Efficient models (basic capability, very low cost)
68    Efficient,
69}
70
71/// Model pricing information
72#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
73pub struct ModelPricing {
74    /// Input tokens price per 1,000 tokens (USD)
75    pub input_price_per_1k: f64,
76    /// Output tokens price per 1,000 tokens (USD)
77    pub output_price_per_1k: f64,
78    /// Cached input price per 1,000 tokens (USD), if supported
79    pub cached_input_price_per_1k: Option<f64>,
80}
81
82impl ModelPricing {
83    /// Create new pricing
84    pub fn new(input: f64, output: f64) -> Self {
85        Self {
86            input_price_per_1k: input,
87            output_price_per_1k: output,
88            cached_input_price_per_1k: None,
89        }
90    }
91
92    /// Create pricing with cache support
93    pub fn with_cache(input: f64, output: f64, cached: f64) -> Self {
94        Self {
95            input_price_per_1k: input,
96            output_price_per_1k: output,
97            cached_input_price_per_1k: Some(cached),
98        }
99    }
100
101    /// Calculate cost for given token counts
102    pub fn calculate_cost(&self, input_tokens: usize, output_tokens: usize) -> f64 {
103        let input_cost = (input_tokens as f64 / 1000.0) * self.input_price_per_1k;
104        let output_cost = (output_tokens as f64 / 1000.0) * self.output_price_per_1k;
105        input_cost + output_cost
106    }
107}
108
109/// Model performance characteristics
110#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
111pub struct ModelPerformance {
112    /// Average latency p50 in milliseconds
113    pub latency_p50_ms: f64,
114    /// P95 latency in milliseconds
115    pub latency_p95_ms: f64,
116    /// P99 latency in milliseconds
117    pub latency_p99_ms: f64,
118    /// Tokens per second throughput
119    pub tokens_per_second: f64,
120    /// Maximum context window (tokens)
121    pub max_context_tokens: usize,
122    /// Maximum output tokens
123    pub max_output_tokens: usize,
124}
125
126impl ModelPerformance {
127    /// Create new performance profile
128    pub fn new(
129        p50: f64,
130        p95: f64,
131        p99: f64,
132        throughput: f64,
133        context: usize,
134        max_output: usize,
135    ) -> Self {
136        Self {
137            latency_p50_ms: p50,
138            latency_p95_ms: p95,
139            latency_p99_ms: p99,
140            tokens_per_second: throughput,
141            max_context_tokens: context,
142            max_output_tokens: max_output,
143        }
144    }
145}
146
147/// Model capabilities
148#[derive(Debug, Clone, Serialize, Deserialize)]
149pub struct ModelCapabilities {
150    /// Supports function calling
151    pub function_calling: bool,
152    /// Supports vision/image input
153    pub vision: bool,
154    /// Supports streaming responses
155    pub streaming: bool,
156    /// Supports JSON mode
157    pub json_mode: bool,
158    /// Supports prompt caching
159    pub prompt_caching: bool,
160    /// Multimodal capabilities
161    pub multimodal: bool,
162    /// Supported languages (ISO codes)
163    pub languages: Vec<String>,
164}
165
166impl ModelCapabilities {
167    /// Standard capabilities (text-only, basic features)
168    pub fn standard() -> Self {
169        Self {
170            function_calling: true,
171            vision: false,
172            streaming: true,
173            json_mode: true,
174            prompt_caching: false,
175            multimodal: false,
176            languages: vec!["en".to_string()],
177        }
178    }
179
180    /// Advanced capabilities (multimodal, all features)
181    pub fn advanced() -> Self {
182        Self {
183            function_calling: true,
184            vision: true,
185            streaming: true,
186            json_mode: true,
187            prompt_caching: true,
188            multimodal: true,
189            languages: vec![
190                "en", "es", "fr", "de", "it", "pt", "ja", "ko", "zh",
191            ]
192            .iter()
193            .map(|s| s.to_string())
194            .collect(),
195        }
196    }
197}
198
199/// Complete model definition
200#[derive(Debug, Clone, Serialize, Deserialize)]
201pub struct ModelDefinition {
202    /// Model identifier
203    pub id: String,
204    /// Human-readable name
205    pub name: String,
206    /// Provider
207    pub provider: Provider,
208    /// Model tier
209    pub tier: ModelTier,
210    /// Pricing information
211    pub pricing: ModelPricing,
212    /// Performance characteristics
213    pub performance: ModelPerformance,
214    /// Capabilities
215    pub capabilities: ModelCapabilities,
216    /// Estimated quality score [0, 1]
217    pub quality_score: f64,
218    /// Release date (for versioning)
219    pub release_date: Option<String>,
220    /// Whether model is currently available
221    pub available: bool,
222}
223
224impl ModelDefinition {
225    /// Create a Pareto candidate from this model definition
226    pub fn to_pareto_candidate(
227        &self,
228        input_tokens: usize,
229        output_tokens: usize,
230    ) -> Result<ModelCandidate> {
231        let cost = self.pricing.calculate_cost(input_tokens, output_tokens);
232
233        ModelCandidate::new(
234            Uuid::new_v4(),
235            &self.id,
236            Objectives::new(self.quality_score, cost, self.performance.latency_p95_ms),
237        )
238    }
239}
240
241/// Model registry
242pub struct ModelRegistry {
243    models: HashMap<String, ModelDefinition>,
244}
245
246impl ModelRegistry {
247    /// Create a new model registry with all supported models
248    pub fn new() -> Self {
249        let mut registry = Self {
250            models: HashMap::new(),
251        };
252
253        // OpenAI Models
254        registry.register_openai_models();
255
256        // Anthropic Models
257        registry.register_anthropic_models();
258
259        // Google Models
260        registry.register_google_models();
261
262        // Meta Models
263        registry.register_meta_models();
264
265        // Mistral Models
266        registry.register_mistral_models();
267
268        // xAI Models
269        registry.register_xai_models();
270
271        // DeepSeek Models
272        registry.register_deepseek_models();
273
274        // Alibaba Models
275        registry.register_alibaba_models();
276
277        registry
278    }
279
280    /// Register a model
281    fn register(&mut self, model: ModelDefinition) {
282        self.models.insert(model.id.clone(), model);
283    }
284
285    /// Get model by ID
286    pub fn get(&self, id: &str) -> Option<&ModelDefinition> {
287        self.models.get(id)
288    }
289
290    /// Get all models
291    pub fn all_models(&self) -> Vec<&ModelDefinition> {
292        self.models.values().collect()
293    }
294
295    /// Get models by provider
296    pub fn models_by_provider(&self, provider: Provider) -> Vec<&ModelDefinition> {
297        self.models
298            .values()
299            .filter(|m| m.provider == provider)
300            .collect()
301    }
302
303    /// Get models by tier
304    pub fn models_by_tier(&self, tier: ModelTier) -> Vec<&ModelDefinition> {
305        self.models.values().filter(|m| m.tier == tier).collect()
306    }
307
308    /// Get available models
309    pub fn available_models(&self) -> Vec<&ModelDefinition> {
310        self.models.values().filter(|m| m.available).collect()
311    }
312
313    /// Compare models and find Pareto optimal set
314    pub fn find_pareto_optimal(
315        &self,
316        model_ids: &[String],
317        input_tokens: usize,
318        output_tokens: usize,
319    ) -> Result<Vec<ModelCandidate>> {
320        let candidates: Result<Vec<ModelCandidate>> = model_ids
321            .iter()
322            .filter_map(|id| self.get(id))
323            .map(|model| model.to_pareto_candidate(input_tokens, output_tokens))
324            .collect();
325
326        let candidates = candidates?;
327        Ok(crate::pareto::ParetoFrontier::find_pareto_optimal(
328            &candidates,
329        ))
330    }
331
332    // Provider-specific registration methods
333
334    fn register_openai_models(&mut self) {
335        // GPT-5 (Future flagship)
336        self.register(ModelDefinition {
337            id: "gpt-5".to_string(),
338            name: "GPT-5".to_string(),
339            provider: Provider::OpenAI,
340            tier: ModelTier::Flagship,
341            pricing: ModelPricing::with_cache(0.010, 0.030, 0.005),
342            performance: ModelPerformance::new(800.0, 1500.0, 2500.0, 120.0, 200000, 16384),
343            capabilities: ModelCapabilities::advanced(),
344            quality_score: 0.98,
345            release_date: Some("2025-Q3".to_string()),
346            available: false,
347        });
348
349        // GPT-4.5 (Future advanced)
350        self.register(ModelDefinition {
351            id: "gpt-4.5".to_string(),
352            name: "GPT-4.5".to_string(),
353            provider: Provider::OpenAI,
354            tier: ModelTier::Advanced,
355            pricing: ModelPricing::with_cache(0.005, 0.015, 0.0025),
356            performance: ModelPerformance::new(600.0, 1200.0, 2000.0, 150.0, 128000, 8192),
357            capabilities: ModelCapabilities::advanced(),
358            quality_score: 0.95,
359            release_date: Some("2025-Q2".to_string()),
360            available: false,
361        });
362
363        // GPT-4.1 (Projected improvement)
364        self.register(ModelDefinition {
365            id: "gpt-4.1".to_string(),
366            name: "GPT-4.1 Turbo".to_string(),
367            provider: Provider::OpenAI,
368            tier: ModelTier::Advanced,
369            pricing: ModelPricing::with_cache(0.003, 0.012, 0.0015),
370            performance: ModelPerformance::new(500.0, 1000.0, 1800.0, 180.0, 128000, 4096),
371            capabilities: ModelCapabilities::advanced(),
372            quality_score: 0.93,
373            release_date: Some("2025-Q1".to_string()),
374            available: false,
375        });
376
377        // Current GPT-4 Turbo
378        self.register(ModelDefinition {
379            id: "gpt-4-turbo".to_string(),
380            name: "GPT-4 Turbo".to_string(),
381            provider: Provider::OpenAI,
382            tier: ModelTier::Advanced,
383            pricing: ModelPricing::new(0.01, 0.03),
384            performance: ModelPerformance::new(800.0, 1600.0, 2800.0, 100.0, 128000, 4096),
385            capabilities: ModelCapabilities::advanced(),
386            quality_score: 0.92,
387            release_date: Some("2024-04".to_string()),
388            available: true,
389        });
390
391        // GPT-3.5 Turbo
392        self.register(ModelDefinition {
393            id: "gpt-3.5-turbo".to_string(),
394            name: "GPT-3.5 Turbo".to_string(),
395            provider: Provider::OpenAI,
396            tier: ModelTier::Efficient,
397            pricing: ModelPricing::new(0.0005, 0.0015),
398            performance: ModelPerformance::new(300.0, 600.0, 1000.0, 250.0, 16385, 4096),
399            capabilities: ModelCapabilities::standard(),
400            quality_score: 0.78,
401            release_date: Some("2023-11".to_string()),
402            available: true,
403        });
404    }
405
406    fn register_anthropic_models(&mut self) {
407        // Claude 4 Opus (Future flagship)
408        self.register(ModelDefinition {
409            id: "claude-4-opus".to_string(),
410            name: "Claude 4 Opus".to_string(),
411            provider: Provider::Anthropic,
412            tier: ModelTier::Flagship,
413            pricing: ModelPricing::with_cache(0.012, 0.060, 0.006),
414            performance: ModelPerformance::new(1000.0, 2000.0, 3500.0, 100.0, 200000, 8192),
415            capabilities: ModelCapabilities::advanced(),
416            quality_score: 0.97,
417            release_date: Some("2025-Q4".to_string()),
418            available: false,
419        });
420
421        // Claude Sonnet 4.5 (Future advanced) - Based on actual release
422        self.register(ModelDefinition {
423            id: "claude-sonnet-4.5".to_string(),
424            name: "Claude Sonnet 4.5".to_string(),
425            provider: Provider::Anthropic,
426            tier: ModelTier::Advanced,
427            pricing: ModelPricing::with_cache(0.003, 0.015, 0.0015),
428            performance: ModelPerformance::new(600.0, 1200.0, 2200.0, 140.0, 200000, 8192),
429            capabilities: ModelCapabilities::advanced(),
430            quality_score: 0.94,
431            release_date: Some("2024-10".to_string()),
432            available: true,
433        });
434
435        // Current Claude 3.5 Sonnet
436        self.register(ModelDefinition {
437            id: "claude-3.5-sonnet".to_string(),
438            name: "Claude 3.5 Sonnet".to_string(),
439            provider: Provider::Anthropic,
440            tier: ModelTier::Advanced,
441            pricing: ModelPricing::with_cache(0.003, 0.015, 0.0015),
442            performance: ModelPerformance::new(700.0, 1400.0, 2400.0, 120.0, 200000, 8192),
443            capabilities: ModelCapabilities::advanced(),
444            quality_score: 0.93,
445            release_date: Some("2024-06".to_string()),
446            available: true,
447        });
448
449        // Claude 3 Opus
450        self.register(ModelDefinition {
451            id: "claude-3-opus".to_string(),
452            name: "Claude 3 Opus".to_string(),
453            provider: Provider::Anthropic,
454            tier: ModelTier::Flagship,
455            pricing: ModelPricing::new(0.015, 0.075),
456            performance: ModelPerformance::new(1200.0, 2400.0, 4000.0, 80.0, 200000, 4096),
457            capabilities: ModelCapabilities::advanced(),
458            quality_score: 0.95,
459            release_date: Some("2024-03".to_string()),
460            available: true,
461        });
462
463        // Claude 3 Haiku
464        self.register(ModelDefinition {
465            id: "claude-3-haiku".to_string(),
466            name: "Claude 3 Haiku".to_string(),
467            provider: Provider::Anthropic,
468            tier: ModelTier::Efficient,
469            pricing: ModelPricing::new(0.00025, 0.00125),
470            performance: ModelPerformance::new(300.0, 600.0, 1000.0, 200.0, 200000, 4096),
471            capabilities: ModelCapabilities::standard(),
472            quality_score: 0.82,
473            release_date: Some("2024-03".to_string()),
474            available: true,
475        });
476    }
477
478    fn register_google_models(&mut self) {
479        // Gemini 2.5 Pro (Future)
480        self.register(ModelDefinition {
481            id: "gemini-2.5-pro".to_string(),
482            name: "Gemini 2.5 Pro".to_string(),
483            provider: Provider::Google,
484            tier: ModelTier::Flagship,
485            pricing: ModelPricing::new(0.00125, 0.005),
486            performance: ModelPerformance::new(700.0, 1400.0, 2400.0, 130.0, 1000000, 8192),
487            capabilities: ModelCapabilities::advanced(),
488            quality_score: 0.96,
489            release_date: Some("2025-Q2".to_string()),
490            available: false,
491        });
492
493        // Gemini 2.0 Flash
494        self.register(ModelDefinition {
495            id: "gemini-2.0-flash".to_string(),
496            name: "Gemini 2.0 Flash".to_string(),
497            provider: Provider::Google,
498            tier: ModelTier::Standard,
499            pricing: ModelPricing::new(0.0, 0.0), // Free tier available
500            performance: ModelPerformance::new(400.0, 800.0, 1400.0, 180.0, 1000000, 8192),
501            capabilities: ModelCapabilities::advanced(),
502            quality_score: 0.88,
503            release_date: Some("2024-12".to_string()),
504            available: true,
505        });
506
507        // Gemini 1.5 Pro
508        self.register(ModelDefinition {
509            id: "gemini-1.5-pro".to_string(),
510            name: "Gemini 1.5 Pro".to_string(),
511            provider: Provider::Google,
512            tier: ModelTier::Advanced,
513            pricing: ModelPricing::new(0.00125, 0.005),
514            performance: ModelPerformance::new(800.0, 1600.0, 2800.0, 110.0, 2000000, 8192),
515            capabilities: ModelCapabilities::advanced(),
516            quality_score: 0.91,
517            release_date: Some("2024-05".to_string()),
518            available: true,
519        });
520    }
521
522    fn register_meta_models(&mut self) {
523        // Llama 4 Scout (Future - reasoning-focused)
524        self.register(ModelDefinition {
525            id: "llama-4-scout".to_string(),
526            name: "Llama 4 Scout".to_string(),
527            provider: Provider::Meta,
528            tier: ModelTier::Advanced,
529            pricing: ModelPricing::new(0.0, 0.0), // Open source, self-hosted
530            performance: ModelPerformance::new(900.0, 1800.0, 3000.0, 90.0, 128000, 8192),
531            capabilities: ModelCapabilities::advanced(),
532            quality_score: 0.90,
533            release_date: Some("2025-Q3".to_string()),
534            available: false,
535        });
536
537        // Llama 4 Maverick (Future - efficient variant)
538        self.register(ModelDefinition {
539            id: "llama-4-maverick".to_string(),
540            name: "Llama 4 Maverick".to_string(),
541            provider: Provider::Meta,
542            tier: ModelTier::Standard,
543            pricing: ModelPricing::new(0.0, 0.0), // Open source
544            performance: ModelPerformance::new(500.0, 1000.0, 1800.0, 150.0, 128000, 8192),
545            capabilities: ModelCapabilities::standard(),
546            quality_score: 0.85,
547            release_date: Some("2025-Q3".to_string()),
548            available: false,
549        });
550
551        // Current Llama 3.3 70B
552        self.register(ModelDefinition {
553            id: "llama-3.3-70b".to_string(),
554            name: "Llama 3.3 70B".to_string(),
555            provider: Provider::Meta,
556            tier: ModelTier::Advanced,
557            pricing: ModelPricing::new(0.0, 0.0), // Open source
558            performance: ModelPerformance::new(800.0, 1600.0, 2800.0, 100.0, 128000, 8192),
559            capabilities: ModelCapabilities::standard(),
560            quality_score: 0.87,
561            release_date: Some("2024-12".to_string()),
562            available: true,
563        });
564    }
565
566    fn register_mistral_models(&mut self) {
567        // Mistral Large 2
568        self.register(ModelDefinition {
569            id: "mistral-large-2".to_string(),
570            name: "Mistral Large 2".to_string(),
571            provider: Provider::Mistral,
572            tier: ModelTier::Advanced,
573            pricing: ModelPricing::new(0.002, 0.006),
574            performance: ModelPerformance::new(600.0, 1200.0, 2000.0, 130.0, 128000, 8192),
575            capabilities: ModelCapabilities::advanced(),
576            quality_score: 0.89,
577            release_date: Some("2024-07".to_string()),
578            available: true,
579        });
580
581        // Mixtral 8x22B
582        self.register(ModelDefinition {
583            id: "mixtral-8x22b".to_string(),
584            name: "Mixtral 8x22B".to_string(),
585            provider: Provider::Mistral,
586            tier: ModelTier::Advanced,
587            pricing: ModelPricing::new(0.0, 0.0), // Open source
588            performance: ModelPerformance::new(700.0, 1400.0, 2400.0, 110.0, 65536, 8192),
589            capabilities: ModelCapabilities::standard(),
590            quality_score: 0.86,
591            release_date: Some("2024-04".to_string()),
592            available: true,
593        });
594
595        // Mistral Nemo
596        self.register(ModelDefinition {
597            id: "mistral-nemo".to_string(),
598            name: "Mistral Nemo".to_string(),
599            provider: Provider::Mistral,
600            tier: ModelTier::Efficient,
601            pricing: ModelPricing::new(0.0003, 0.0003),
602            performance: ModelPerformance::new(300.0, 600.0, 1000.0, 200.0, 128000, 8192),
603            capabilities: ModelCapabilities::standard(),
604            quality_score: 0.80,
605            release_date: Some("2024-07".to_string()),
606            available: true,
607        });
608    }
609
610    fn register_xai_models(&mut self) {
611        // Grok 4 (Future)
612        self.register(ModelDefinition {
613            id: "grok-4".to_string(),
614            name: "Grok 4".to_string(),
615            provider: Provider::XAI,
616            tier: ModelTier::Flagship,
617            pricing: ModelPricing::new(0.005, 0.015),
618            performance: ModelPerformance::new(800.0, 1600.0, 2800.0, 120.0, 128000, 8192),
619            capabilities: ModelCapabilities::advanced(),
620            quality_score: 0.92,
621            release_date: Some("2025-Q4".to_string()),
622            available: false,
623        });
624
625        // Grok 2 (Current)
626        self.register(ModelDefinition {
627            id: "grok-2".to_string(),
628            name: "Grok 2".to_string(),
629            provider: Provider::XAI,
630            tier: ModelTier::Advanced,
631            pricing: ModelPricing::new(0.002, 0.010),
632            performance: ModelPerformance::new(900.0, 1800.0, 3200.0, 100.0, 128000, 8192),
633            capabilities: ModelCapabilities::standard(),
634            quality_score: 0.88,
635            release_date: Some("2024-08".to_string()),
636            available: true,
637        });
638    }
639
640    fn register_deepseek_models(&mut self) {
641        // Gemma 3 (Future - Google/DeepMind)
642        self.register(ModelDefinition {
643            id: "gemma-3".to_string(),
644            name: "Gemma 3".to_string(),
645            provider: Provider::Google, // Gemma is from Google
646            tier: ModelTier::Standard,
647            pricing: ModelPricing::new(0.0, 0.0), // Open source
648            performance: ModelPerformance::new(400.0, 800.0, 1400.0, 170.0, 128000, 8192),
649            capabilities: ModelCapabilities::standard(),
650            quality_score: 0.83,
651            release_date: Some("2025-Q2".to_string()),
652            available: false,
653        });
654
655        // DeepSeek V3
656        self.register(ModelDefinition {
657            id: "deepseek-v3".to_string(),
658            name: "DeepSeek V3".to_string(),
659            provider: Provider::DeepSeek,
660            tier: ModelTier::Advanced,
661            pricing: ModelPricing::new(0.00027, 0.0011),
662            performance: ModelPerformance::new(600.0, 1200.0, 2000.0, 140.0, 128000, 8192),
663            capabilities: ModelCapabilities::standard(),
664            quality_score: 0.89,
665            release_date: Some("2024-12".to_string()),
666            available: true,
667        });
668    }
669
670    fn register_alibaba_models(&mut self) {
671        // Qwen 3 (Future)
672        self.register(ModelDefinition {
673            id: "qwen-3".to_string(),
674            name: "Qwen 3".to_string(),
675            provider: Provider::Alibaba,
676            tier: ModelTier::Advanced,
677            pricing: ModelPricing::new(0.0, 0.0), // Open source
678            performance: ModelPerformance::new(500.0, 1000.0, 1800.0, 150.0, 128000, 8192),
679            capabilities: ModelCapabilities::advanced(),
680            quality_score: 0.87,
681            release_date: Some("2025-Q2".to_string()),
682            available: false,
683        });
684
685        // Qwen 2.5 72B (Current)
686        self.register(ModelDefinition {
687            id: "qwen-2.5-72b".to_string(),
688            name: "Qwen 2.5 72B".to_string(),
689            provider: Provider::Alibaba,
690            tier: ModelTier::Advanced,
691            pricing: ModelPricing::new(0.0, 0.0), // Open source
692            performance: ModelPerformance::new(700.0, 1400.0, 2400.0, 120.0, 128000, 8192),
693            capabilities: ModelCapabilities::standard(),
694            quality_score: 0.85,
695            release_date: Some("2024-09".to_string()),
696            available: true,
697        });
698    }
699}
700
701impl Default for ModelRegistry {
702    fn default() -> Self {
703        Self::new()
704    }
705}
706
707#[cfg(test)]
708mod tests {
709    use super::*;
710
711    #[test]
712    fn test_model_registry_creation() {
713        let registry = ModelRegistry::new();
714        assert!(!registry.all_models().is_empty());
715    }
716
717    #[test]
718    fn test_all_specified_models_registered() {
719        let registry = ModelRegistry::new();
720
721        // Test that all requested models are registered
722        let required_models = vec![
723            "gpt-5",
724            "gpt-4.5",
725            "gpt-4.1",
726            "gemini-2.5-pro",
727            "claude-4-opus",
728            "claude-sonnet-4.5",
729            "llama-4-scout",
730            "llama-4-maverick",
731            "gemma-3",
732            "mistral-large-2",
733            "mixtral-8x22b",
734            "qwen-3",
735            "grok-4",
736        ];
737
738        for model_id in required_models {
739            assert!(
740                registry.get(model_id).is_some(),
741                "Model {} not found in registry",
742                model_id
743            );
744        }
745    }
746
747    #[test]
748    fn test_provider_filtering() {
749        let registry = ModelRegistry::new();
750
751        let openai_models = registry.models_by_provider(Provider::OpenAI);
752        assert!(!openai_models.is_empty());
753
754        let anthropic_models = registry.models_by_provider(Provider::Anthropic);
755        assert!(!anthropic_models.is_empty());
756    }
757
758    #[test]
759    fn test_tier_filtering() {
760        let registry = ModelRegistry::new();
761
762        let flagship = registry.models_by_tier(ModelTier::Flagship);
763        assert!(!flagship.is_empty());
764
765        let efficient = registry.models_by_tier(ModelTier::Efficient);
766        assert!(!efficient.is_empty());
767    }
768
769    #[test]
770    fn test_available_models() {
771        let registry = ModelRegistry::new();
772
773        let available = registry.available_models();
774        assert!(!available.is_empty());
775
776        // Check that some future models are not available
777        assert!(registry.get("gpt-5").unwrap().available == false);
778        assert!(registry.get("claude-4-opus").unwrap().available == false);
779    }
780
781    #[test]
782    fn test_pricing_calculation() {
783        let pricing = ModelPricing::new(0.01, 0.03);
784        let cost = pricing.calculate_cost(1000, 500);
785
786        // 1000 input tokens = 1.0 * 0.01 = 0.01
787        // 500 output tokens = 0.5 * 0.03 = 0.015
788        // Total = 0.025
789        assert!((cost - 0.025).abs() < 0.0001);
790    }
791
792    #[test]
793    fn test_pareto_candidate_creation() {
794        let registry = ModelRegistry::new();
795        let model = registry.get("gpt-4-turbo").unwrap();
796
797        let candidate = model.to_pareto_candidate(1000, 500).unwrap();
798        assert_eq!(candidate.name, "gpt-4-turbo");
799        assert!(candidate.objectives.quality > 0.9);
800        assert!(candidate.objectives.cost > 0.0);
801    }
802
803    #[test]
804    fn test_find_pareto_optimal_across_providers() {
805        let registry = ModelRegistry::new();
806
807        let model_ids = vec![
808            "gpt-4-turbo".to_string(),
809            "claude-3.5-sonnet".to_string(),
810            "gemini-1.5-pro".to_string(),
811            "mistral-large-2".to_string(),
812            "grok-2".to_string(),
813        ];
814
815        let pareto_set = registry
816            .find_pareto_optimal(&model_ids, 1000, 500)
817            .unwrap();
818
819        assert!(!pareto_set.is_empty());
820        assert!(pareto_set.len() <= model_ids.len());
821    }
822
823    #[test]
824    fn test_future_models_pareto_analysis() {
825        let registry = ModelRegistry::new();
826
827        // Test with future models
828        let model_ids = vec![
829            "gpt-5".to_string(),
830            "claude-4-opus".to_string(),
831            "gemini-2.5-pro".to_string(),
832            "llama-4-scout".to_string(),
833            "grok-4".to_string(),
834        ];
835
836        let pareto_set = registry
837            .find_pareto_optimal(&model_ids, 1000, 500)
838            .unwrap();
839
840        assert!(!pareto_set.is_empty());
841    }
842
843    #[test]
844    fn test_all_providers_represented() {
845        let registry = ModelRegistry::new();
846
847        for provider in Provider::all() {
848            let models = registry.models_by_provider(provider);
849            assert!(
850                !models.is_empty(),
851                "No models for provider: {:?}",
852                provider
853            );
854        }
855    }
856
857    #[test]
858    fn test_model_capabilities() {
859        let registry = ModelRegistry::new();
860
861        // Flagship models should have advanced capabilities
862        let flagship = registry.get("claude-3-opus").unwrap();
863        assert!(flagship.capabilities.vision);
864        assert!(flagship.capabilities.function_calling);
865
866        // Efficient models may have standard capabilities
867        let efficient = registry.get("gpt-3.5-turbo").unwrap();
868        assert!(efficient.tier == ModelTier::Efficient);
869    }
870
871    #[test]
872    fn test_performance_characteristics() {
873        let registry = ModelRegistry::new();
874
875        for model in registry.all_models() {
876            // Validate performance metrics
877            assert!(model.performance.latency_p50_ms > 0.0);
878            assert!(model.performance.latency_p95_ms >= model.performance.latency_p50_ms);
879            assert!(model.performance.latency_p99_ms >= model.performance.latency_p95_ms);
880            assert!(model.performance.tokens_per_second > 0.0);
881            assert!(model.performance.max_context_tokens > 0);
882            assert!(model.performance.max_output_tokens > 0);
883        }
884    }
885
886    #[test]
887    fn test_quality_scores_valid_range() {
888        let registry = ModelRegistry::new();
889
890        for model in registry.all_models() {
891            assert!(
892                model.quality_score >= 0.0 && model.quality_score <= 1.0,
893                "Invalid quality score for {}: {}",
894                model.name,
895                model.quality_score
896            );
897        }
898    }
899
900    #[test]
901    fn test_cost_comparison() {
902        let registry = ModelRegistry::new();
903
904        let gpt5 = registry.get("gpt-5").unwrap();
905        let claude_haiku = registry.get("claude-3-haiku").unwrap();
906
907        let gpt5_cost = gpt5.pricing.calculate_cost(1000, 1000);
908        let haiku_cost = claude_haiku.pricing.calculate_cost(1000, 1000);
909
910        // Haiku should be much cheaper
911        assert!(haiku_cost < gpt5_cost);
912    }
913}
llm_optimizer_decision/model_registry.rs

llm_optimizer_decision/
model_registry.rs
