1use rand::Rng;
13use serde::{Deserialize, Serialize};
14
15use std::sync::{Arc, Mutex};
16
17use crate::hardware::HardwareInfo;
18use crate::outcome::{InferenceTask, OutcomeTracker};
19use crate::registry::UnifiedRegistry;
20use crate::routing_ext::CircuitBreakerRegistry;
21use crate::schema::{ModelCapability, ModelSchema};
22use crate::tasks::RoutingWorkload;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
26#[serde(rename_all = "snake_case")]
27pub enum TaskComplexity {
28 Simple,
29 Medium,
30 Code,
31 Complex,
32}
33
34impl TaskComplexity {
35 pub fn assess(prompt: &str) -> Self {
42 let lower = prompt.to_lowercase();
43 let word_count = prompt.split_whitespace().count();
44 let estimated_tokens = (word_count as f64 * 1.3) as usize;
45
46 let has_code = Self::detect_code(prompt);
47
48 let repair_markers = [
49 "fix", "repair", "debug", "refactor", "broken", "failing", "error", "bug",
50 ];
51 let has_repair = repair_markers.iter().any(|m| lower.contains(m));
52
53 let reasoning_markers = [
54 "analyze",
55 "compare",
56 "explain why",
57 "step by step",
58 "think through",
59 "evaluate",
60 "trade-off",
61 "tradeoff",
62 "pros and cons",
63 "architecture",
64 "design",
65 "strategy",
66 "optimize",
67 "comprehensive",
68 ];
69 let has_reasoning = reasoning_markers.iter().any(|m| lower.contains(m));
70
71 let simple_patterns = [
72 "what is",
73 "who is",
74 "when did",
75 "where is",
76 "how many",
77 "yes or no",
78 "true or false",
79 "name the",
80 "list the",
81 "define ",
82 ];
83 let is_simple = simple_patterns.iter().any(|p| lower.contains(p));
84
85 if has_code || has_repair {
86 TaskComplexity::Code
87 } else if has_reasoning || estimated_tokens > 500 {
88 TaskComplexity::Complex
89 } else if is_simple || estimated_tokens < 30 {
90 TaskComplexity::Simple
91 } else {
92 TaskComplexity::Medium
93 }
94 }
95
96 fn detect_code(prompt: &str) -> bool {
102 #[cfg(feature = "ast")]
104 {
105 if let Some(is_code) = Self::detect_code_ast(prompt) {
106 return is_code;
107 }
108 }
109
110 let code_markers = [
112 "```",
113 "fn ",
114 "def ",
115 "class ",
116 "import ",
117 "require(",
118 "async fn",
119 "pub fn",
120 "function ",
121 "const ",
122 "let ",
123 "var ",
124 "#include",
125 "package ",
126 "impl ",
127 ];
128 code_markers.iter().any(|m| prompt.contains(m))
129 }
130
131 #[cfg(feature = "ast")]
135 fn detect_code_ast(prompt: &str) -> Option<bool> {
136 let mut blocks = Vec::new();
138 let mut rest = prompt;
139 while let Some(start) = rest.find("```") {
140 let after_fence = &rest[start + 3..];
141 let code_start = after_fence.find('\n').map(|i| i + 1).unwrap_or(0);
143 if let Some(end) = after_fence[code_start..].find("```") {
144 blocks.push(&after_fence[code_start..code_start + end]);
145 rest = &after_fence[code_start + end + 3..];
146 } else {
147 break;
148 }
149 }
150
151 if blocks.is_empty() {
152 return None; }
154
155 let languages = [
157 car_ast::Language::Rust,
158 car_ast::Language::Python,
159 car_ast::Language::TypeScript,
160 car_ast::Language::JavaScript,
161 car_ast::Language::Go,
162 ];
163
164 for block in &blocks {
165 let trimmed = block.trim();
166 if trimmed.is_empty() {
167 continue;
168 }
169
170 for lang in &languages {
171 if let Some(parsed) = car_ast::parse(trimmed, *lang) {
172 if !parsed.symbols.is_empty() {
174 return Some(true);
175 }
176 }
177 }
178 }
179
180 Some(false)
183 }
184
185 pub fn required_capabilities(&self) -> Vec<ModelCapability> {
187 match self {
188 TaskComplexity::Simple => vec![ModelCapability::Generate],
189 TaskComplexity::Medium => vec![ModelCapability::Generate],
190 TaskComplexity::Code => vec![ModelCapability::Code],
191 TaskComplexity::Complex => vec![ModelCapability::Reasoning],
192 }
193 }
194
195 pub fn inference_task(&self) -> InferenceTask {
197 match self {
198 TaskComplexity::Simple | TaskComplexity::Medium => InferenceTask::Generate,
199 TaskComplexity::Code => InferenceTask::Code,
200 TaskComplexity::Complex => InferenceTask::Reasoning,
201 }
202 }
203}
204
205#[derive(Debug, Clone, Serialize, Deserialize)]
207pub struct RoutingConfig {
208 pub min_observations: u64,
210 pub quality_weight: f64,
212 pub latency_weight: f64,
213 pub cost_weight: f64,
214 pub max_latency_ms: Option<u64>,
216 pub max_cost_usd: Option<f64>,
218 pub prefer_local: bool,
220 pub prior_strength: f64,
224 pub quality_first_cold_start: bool,
227 pub bootstrap_min_task_observations: u64,
230 pub bootstrap_quality_floor: f64,
233}
234
235impl Default for RoutingConfig {
236 fn default() -> Self {
237 Self {
238 min_observations: 2,
239 quality_weight: 0.45,
240 latency_weight: 0.4,
241 cost_weight: 0.15,
242 max_latency_ms: None,
243 max_cost_usd: None,
244 prefer_local: true,
245 prior_strength: 2.0,
246 quality_first_cold_start: true,
247 bootstrap_min_task_observations: 8,
248 bootstrap_quality_floor: 0.8,
249 }
250 }
251}
252
253#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
255#[serde(rename_all = "snake_case")]
256pub enum RoutingStrategy {
257 SchemaBased,
259 ProfileBased,
261 Exploration,
263 Explicit,
265}
266
267#[derive(Debug, Clone, Serialize, Deserialize)]
276pub struct RouteCandidate {
277 pub model_id: String,
279 pub reliability: f64,
281 pub score: f64,
283 pub selected: bool,
285 pub in_band: bool,
287}
288
289#[derive(Debug, Clone, Serialize, Deserialize)]
291pub struct AdaptiveRoutingDecision {
292 pub model_id: String,
294 pub model_name: String,
296 pub task: InferenceTask,
298 pub complexity: TaskComplexity,
300 pub reason: String,
302 pub strategy: RoutingStrategy,
304 pub predicted_quality: f64,
306 pub fallbacks: Vec<String>,
308 pub context_length: usize,
310 pub needs_compaction: bool,
312 #[serde(default)]
315 pub candidates: Vec<RouteCandidate>,
316}
317
318pub struct AdaptiveRouter {
320 hw: HardwareInfo,
321 config: RoutingConfig,
322 pub circuit_breakers: Arc<Mutex<CircuitBreakerRegistry>>,
324}
325
326pub struct RouteRequest<'a> {
342 pub prompt: &'a str,
343 pub registry: &'a UnifiedRegistry,
344 pub tracker: &'a OutcomeTracker,
345 pub estimated_total_tokens: usize,
348 pub estimated_input_tokens: usize,
351 pub estimated_output_tokens: usize,
353 pub estimated_cache_read_tokens: usize,
355 pub estimated_cache_write_tokens: usize,
357 pub has_tools: bool,
358 pub has_vision: bool,
359 pub workload: RoutingWorkload,
360 pub intent: Option<&'a crate::intent::IntentHint>,
363}
364
365impl<'a> RouteRequest<'a> {
366 pub fn new(
370 prompt: &'a str,
371 registry: &'a UnifiedRegistry,
372 tracker: &'a OutcomeTracker,
373 ) -> Self {
374 Self {
375 prompt,
376 registry,
377 tracker,
378 estimated_total_tokens: 0,
379 estimated_input_tokens: 0,
380 estimated_output_tokens: 0,
381 estimated_cache_read_tokens: 0,
382 estimated_cache_write_tokens: 0,
383 has_tools: false,
384 has_vision: false,
385 workload: RoutingWorkload::Interactive,
386 intent: None,
387 }
388 }
389}
390
391#[derive(Clone, Copy)]
395struct AvailabilitySnapshot {
396 openrouter: Option<crate::openrouter::CredentialSource>,
397}
398
399impl AvailabilitySnapshot {
400 fn capture(registry: &UnifiedRegistry) -> Self {
401 let has_openrouter = registry.list().into_iter().any(|schema| {
402 matches!(
403 schema.source,
404 crate::schema::ModelSource::RemoteApi {
405 protocol: crate::schema::ApiProtocol::OpenRouter,
406 ..
407 }
408 )
409 });
410 Self {
411 openrouter: has_openrouter
412 .then(crate::openrouter::credential_source)
413 .flatten(),
414 }
415 }
416
417 fn is_available(self, schema: &ModelSchema) -> bool {
418 match schema.source {
419 crate::schema::ModelSource::RemoteApi {
420 protocol: crate::schema::ApiProtocol::OpenRouter,
421 ..
422 } => self.openrouter.is_some(),
423 _ => schema.available,
424 }
425 }
426}
427
428static BENCHMARK_FRONTIER_REFS: std::sync::LazyLock<std::collections::HashMap<String, f64>> =
455 std::sync::LazyLock::new(|| compute_frontier_refs(crate::registry::builtin_catalog().iter()));
456
457fn compute_frontier_refs<'a>(
462 models: impl Iterator<Item = &'a ModelSchema>,
463) -> std::collections::HashMap<String, f64> {
464 let mut max_score: std::collections::HashMap<String, f64> = std::collections::HashMap::new();
468 let mut has_frontier_carrier: std::collections::HashMap<String, bool> =
469 std::collections::HashMap::new();
470 for model in models {
471 let frontier = model_is_frontier_class(model);
472 for b in &model.public_benchmarks {
473 let entry = max_score.entry(b.name.clone()).or_insert(0.0);
474 *entry = entry.max(b.score);
475 let seen = has_frontier_carrier.entry(b.name.clone()).or_insert(false);
476 *seen = *seen || frontier;
477 }
478 }
479 max_score
483 .into_iter()
484 .filter(|(name, _)| {
485 has_frontier_carrier.get(name).copied().unwrap_or(false) || {
486 tracing::warn!(
487 benchmark = %name,
488 "benchmark has no frontier-class carrier in the catalog — \
489 refusing to self-anchor (uncalibrated clamp). Add a \
490 frontier-tagged model that carries it, or an explicit \
491 reference, before it can calibrate the quality tier (#371)."
492 );
493 false
494 }
495 })
496 .collect()
497}
498
499fn model_is_frontier_class(model: &ModelSchema) -> bool {
505 model.tags.iter().any(|t| t == "frontier")
506}
507
508impl AdaptiveRouter {
509 pub fn new(hw: HardwareInfo, config: RoutingConfig) -> Self {
510 let circuit_breakers = Arc::new(Mutex::new(
511 CircuitBreakerRegistry::new(3, 300), ));
513 Self {
514 hw,
515 config,
516 circuit_breakers,
517 }
518 }
519
520 pub fn with_default_config(hw: HardwareInfo) -> Self {
521 Self::new(hw, RoutingConfig::default())
522 }
523
524 pub fn config(&self) -> &RoutingConfig {
525 &self.config
526 }
527
528 pub fn set_config(&mut self, config: RoutingConfig) {
529 self.config = config;
530 }
531
532 pub fn route_with(&self, req: RouteRequest<'_>) -> AdaptiveRoutingDecision {
538 let workload = Self::resolve_workload(req.intent, req.workload);
544 self.route_inner_with_intent(
545 req.prompt,
546 req.registry,
547 req.tracker,
548 req.has_tools,
549 req.has_vision,
550 req.estimated_total_tokens,
551 req.estimated_input_tokens,
552 req.estimated_output_tokens,
553 req.estimated_cache_read_tokens,
554 req.estimated_cache_write_tokens,
555 workload,
556 req.intent,
557 )
558 }
559
560 pub fn route(
563 &self,
564 prompt: &str,
565 registry: &UnifiedRegistry,
566 tracker: &OutcomeTracker,
567 ) -> AdaptiveRoutingDecision {
568 self.route_with(RouteRequest::new(prompt, registry, tracker))
569 }
570
571 pub fn route_editor(
577 &self,
578 prompt: &str,
579 registry: &UnifiedRegistry,
580 tracker: &OutcomeTracker,
581 ) -> AdaptiveRoutingDecision {
582 self.route_with(RouteRequest {
583 workload: RoutingWorkload::Background,
584 ..RouteRequest::new(prompt, registry, tracker)
585 })
586 }
587
588 pub fn route_with_tools(
590 &self,
591 prompt: &str,
592 registry: &UnifiedRegistry,
593 tracker: &OutcomeTracker,
594 ) -> AdaptiveRoutingDecision {
595 self.route_with(RouteRequest {
596 has_tools: true,
597 ..RouteRequest::new(prompt, registry, tracker)
598 })
599 }
600
601 pub fn route_with_vision(
603 &self,
604 prompt: &str,
605 registry: &UnifiedRegistry,
606 tracker: &OutcomeTracker,
607 has_tools: bool,
608 ) -> AdaptiveRoutingDecision {
609 self.route_with(RouteRequest {
610 has_tools,
611 has_vision: true,
612 ..RouteRequest::new(prompt, registry, tracker)
613 })
614 }
615
616 pub fn route_with_intent<'a>(
622 &self,
623 prompt: &'a str,
624 registry: &'a UnifiedRegistry,
625 tracker: &'a OutcomeTracker,
626 intent: &'a crate::intent::IntentHint,
627 ) -> AdaptiveRoutingDecision {
628 self.route_with(RouteRequest {
629 intent: Some(intent),
630 ..RouteRequest::new(prompt, registry, tracker)
631 })
632 }
633
634 pub fn route_context_aware(
637 &self,
638 prompt: &str,
639 estimated_total_tokens: usize,
640 registry: &UnifiedRegistry,
641 tracker: &OutcomeTracker,
642 has_tools: bool,
643 has_vision: bool,
644 workload: RoutingWorkload,
645 ) -> AdaptiveRoutingDecision {
646 self.route_with(RouteRequest {
647 estimated_total_tokens,
648 has_tools,
649 has_vision,
650 workload,
651 ..RouteRequest::new(prompt, registry, tracker)
652 })
653 }
654
655 pub fn route_context_aware_with_intent<'a>(
660 &self,
661 prompt: &'a str,
662 estimated_total_tokens: usize,
663 registry: &'a UnifiedRegistry,
664 tracker: &'a OutcomeTracker,
665 has_tools: bool,
666 has_vision: bool,
667 workload: RoutingWorkload,
668 intent: &'a crate::intent::IntentHint,
669 ) -> AdaptiveRoutingDecision {
670 self.route_with(RouteRequest {
671 estimated_total_tokens,
672 has_tools,
673 has_vision,
674 workload,
675 intent: Some(intent),
676 ..RouteRequest::new(prompt, registry, tracker)
677 })
678 }
679
680 fn route_inner_with_intent(
681 &self,
682 prompt: &str,
683 registry: &UnifiedRegistry,
684 tracker: &OutcomeTracker,
685 has_tools: bool,
686 has_vision: bool,
687 estimated_total_tokens: usize,
688 estimated_input_tokens: usize,
689 estimated_output_tokens: usize,
690 estimated_cache_read_tokens: usize,
691 estimated_cache_write_tokens: usize,
692 workload: RoutingWorkload,
693 intent: Option<&crate::intent::IntentHint>,
694 ) -> AdaptiveRoutingDecision {
695 let complexity = TaskComplexity::assess(prompt);
696 let task = intent
698 .and_then(|h| h.task)
699 .map(task_hint_to_inference_task)
700 .unwrap_or_else(|| complexity.inference_task());
701 let mut required_caps = complexity.required_capabilities();
702 let task_cap = inference_task_required_capability(task);
710 if !required_caps.contains(&task_cap) {
711 required_caps.push(task_cap);
712 }
713 if let Some(hint) = intent {
714 for cap in &hint.require {
715 if !required_caps.contains(cap) {
716 required_caps.push(*cap);
717 }
718 }
719 }
720 if has_vision {
721 required_caps.push(ModelCapability::Vision);
722 }
723 if has_tools {
724 required_caps.push(ModelCapability::ToolUse);
725 if Self::needs_multi_tool_call(prompt) {
728 required_caps.push(ModelCapability::MultiToolCall);
729 }
730 }
731
732 let exclude_set = self.build_exclude_set(intent, registry);
740 let strict_exclusions = intent.is_some_and(|hint| hint.strict_exclusions);
741 let availability = AvailabilitySnapshot::capture(registry);
742
743 let want_ready = intent.is_some_and(|h| h.require_ready);
748 let mut candidates = self.filter_candidates(
749 &required_caps,
750 registry,
751 tracker,
752 has_vision,
753 availability,
754 want_ready,
755 );
756 if candidates.is_empty() && want_ready {
757 candidates = self.filter_candidates(
758 &required_caps,
759 registry,
760 tracker,
761 has_vision,
762 availability,
763 false,
764 );
765 }
766
767 if candidates.is_empty() && required_caps.contains(&ModelCapability::MultiToolCall) {
770 required_caps.retain(|c| *c != ModelCapability::MultiToolCall);
771 candidates = self.filter_candidates(
772 &required_caps,
773 registry,
774 tracker,
775 has_vision,
776 availability,
777 want_ready,
778 );
779 if candidates.is_empty() && want_ready {
780 candidates = self.filter_candidates(
781 &required_caps,
782 registry,
783 tracker,
784 has_vision,
785 availability,
786 false,
787 );
788 }
789 }
790
791 if candidates.is_empty() {
792 return self.cold_start_decision(
796 complexity,
797 task,
798 &required_caps,
799 registry,
800 has_vision,
801 &exclude_set,
802 strict_exclusions,
803 availability,
804 );
805 }
806
807 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
820 {
821 let mut seen = std::collections::HashSet::new();
822 candidates = candidates
823 .into_iter()
824 .map(|m| registry.resolve_mlx_equivalent(&m).cloned().unwrap_or(m))
825 .filter(|m| seen.insert(m.id.clone()))
826 .collect();
827 }
828
829 if !exclude_set.is_empty() {
838 let filtered: Vec<ModelSchema> = candidates
839 .iter()
840 .filter(|m| !exclude_set.contains(&m.id))
841 .cloned()
842 .collect();
843 if filtered.is_empty() {
844 if strict_exclusions {
845 return Self::no_eligible_model_decision(complexity, task, exclude_set.len());
846 }
847 tracing::debug!(
848 ?exclude_set,
849 "model exclusion left no candidates; keeping full set"
850 );
851 } else {
852 candidates = filtered;
853 }
854 }
855
856 candidates = self.apply_quality_first_bootstrap_policy(
857 candidates, task, tracker, has_vision, has_tools, workload,
858 );
859
860 let (fits, needs_compaction_candidates) = if estimated_total_tokens > 0 {
863 let mut fits = Vec::new();
864 let mut tight = Vec::new();
865 for m in &candidates {
866 if m.context_length == 0 || m.context_length >= estimated_total_tokens {
867 fits.push(m.clone());
868 } else {
869 tight.push(m.clone());
870 }
871 }
872 (fits, tight)
873 } else {
874 (candidates.clone(), Vec::new())
875 };
876
877 let (scoring_candidates, compaction_needed) = if !fits.is_empty() {
879 (fits, false)
880 } else if !needs_compaction_candidates.is_empty() {
881 tracing::info!(
882 prompt_tokens = estimated_total_tokens,
883 candidates = needs_compaction_candidates.len(),
884 "no model fits full prompt — compaction will be needed"
885 );
886 (needs_compaction_candidates.clone(), true)
887 } else {
888 (candidates.clone(), false)
889 };
890
891 let pricing_input_tokens = estimated_input_tokens
897 .max(estimated_total_tokens.saturating_sub(estimated_output_tokens));
898 let scored = self.score_candidates_context_aware(
899 &scoring_candidates,
900 task,
901 tracker,
902 estimated_total_tokens,
903 pricing_input_tokens,
904 estimated_output_tokens,
905 estimated_cache_read_tokens,
906 estimated_cache_write_tokens,
907 workload,
908 );
909 tracing::debug!(
915 ?workload,
916 ?task,
917 candidates = ?scoring_candidates.iter().map(|m| m.id.as_str()).collect::<Vec<_>>(),
918 ?scored,
919 "route scoring"
920 );
921
922 let band_ids: Option<std::collections::HashSet<String>> =
933 if Self::applies_reliability_band(task, workload) {
934 Some(
935 self.outcome_first_band(scoring_candidates.clone(), task, tracker)
936 .into_iter()
937 .map(|m| m.id)
938 .collect(),
939 )
940 } else {
941 None
942 };
943 if let Some(ref ids) = band_ids {
944 tracing::debug!(
949 ?task,
950 band = Self::OUTCOME_FIRST_BAND,
951 band_size = ids.len(),
952 "outcome-first: selection restricted to reliability band"
953 );
954 }
955 let selection_pool: Vec<(String, f64)> = if let Some(ref ids) = band_ids {
956 let pool: Vec<(String, f64)> = scored
957 .iter()
958 .filter(|(id, _)| ids.contains(id))
959 .cloned()
960 .collect();
961 if pool.is_empty() {
962 debug_assert!(
968 false,
969 "reliability band produced no in-`scored` ids (workload {workload:?})"
970 );
971 scored.clone()
972 } else {
973 pool
974 }
975 } else {
976 scored.clone()
977 };
978
979 let (selected_id, strategy) = if workload == RoutingWorkload::Quality {
992 selection_pool
993 .iter()
994 .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
995 .map(|(id, _)| (id.clone(), RoutingStrategy::SchemaBased))
996 .unwrap_or_else(|| self.select_with_thompson_sampling(&selection_pool, tracker))
997 } else {
998 self.select_with_thompson_sampling(&selection_pool, tracker)
999 };
1000
1001 let mut fallbacks: Vec<String> = if let Some(ref ids) = band_ids {
1008 let (in_band, out_band): (Vec<String>, Vec<String>) = scored
1009 .iter()
1010 .filter(|(id, _)| *id != selected_id)
1011 .map(|(id, _)| id.clone())
1012 .partition(|id| ids.contains(id));
1013 in_band.into_iter().chain(out_band).collect()
1014 } else {
1015 scored
1016 .iter()
1017 .filter(|(id, _)| *id != selected_id)
1018 .map(|(id, _)| id.clone())
1019 .collect()
1020 };
1021 if !compaction_needed {
1023 for m in &needs_compaction_candidates {
1024 if m.id != selected_id && !fallbacks.contains(&m.id) {
1025 fallbacks.push(m.id.clone());
1026 }
1027 }
1028 }
1029
1030 let predicted_quality = scored
1031 .iter()
1032 .find(|(id, _)| *id == selected_id)
1033 .map(|(_, score)| *score)
1034 .unwrap_or(0.5);
1035
1036 let selected_schema = registry
1037 .get(&selected_id)
1038 .or_else(|| registry.find_by_name(&selected_id));
1039 let model_name = selected_schema
1040 .map(|m| m.name.clone())
1041 .unwrap_or_else(|| selected_id.clone());
1042 let context_length = selected_schema.map(|m| m.context_length).unwrap_or(0);
1043
1044 let needs_compact = compaction_needed
1045 || (estimated_total_tokens > 0
1046 && context_length > 0
1047 && estimated_total_tokens > context_length);
1048
1049 let compaction_note = if needs_compact {
1050 format!(
1051 " [compaction needed: {}→{}tok]",
1052 estimated_total_tokens, context_length
1053 )
1054 } else {
1055 String::new()
1056 };
1057
1058 let cost_quality_note = selected_schema
1059 .map(|schema| {
1060 let is_openrouter = schema.tags.iter().any(|tag| tag == "openrouter");
1061 if is_openrouter && schema.tags.iter().any(|tag| tag == "cheap") {
1062 let prices = schema.cost.prices_for(pricing_input_tokens);
1063 format!(
1064 ", low-cost tier (${:.3}/MTok input, ${:.3}/MTok output at {} prompt tokens)",
1065 prices.input_per_mtok.unwrap_or(0.0),
1066 prices.output_per_mtok.unwrap_or(0.0),
1067 pricing_input_tokens,
1068 )
1069 } else if is_openrouter && model_is_frontier_class(schema) {
1070 ", frontier quality tier".to_string()
1071 } else {
1072 String::new()
1073 }
1074 })
1075 .unwrap_or_default();
1076
1077 let reason = format!(
1078 "{:?} task → {} via {:?} (quality: {:.2}, {} candidates{}){}",
1079 complexity,
1080 model_name,
1081 strategy,
1082 predicted_quality,
1083 scoring_candidates.len(),
1084 cost_quality_note,
1085 compaction_note,
1086 );
1087
1088 let candidates: Vec<RouteCandidate> = scored
1095 .iter()
1096 .map(|(id, score)| {
1097 let reliability = match scoring_candidates.iter().find(|m| m.id == *id) {
1103 Some(m) => self.reliability(m, task, tracker),
1104 None => {
1105 debug_assert!(false, "scored id {id} absent from scoring_candidates");
1106 0.0
1107 }
1108 };
1109 RouteCandidate {
1110 model_id: id.clone(),
1111 reliability,
1112 score: *score,
1113 selected: *id == selected_id,
1114 in_band: band_ids.as_ref().is_none_or(|b| b.contains(id)),
1115 }
1116 })
1117 .collect();
1118
1119 AdaptiveRoutingDecision {
1120 model_id: selected_id,
1121 model_name,
1122 task,
1123 complexity,
1124 reason,
1125 strategy,
1126 predicted_quality,
1127 fallbacks,
1128 context_length,
1129 needs_compaction: needs_compact,
1130 candidates,
1131 }
1132 }
1133
1134 pub fn route_embedding(&self, registry: &UnifiedRegistry) -> String {
1136 let embed_models = registry.query_by_capability(ModelCapability::Embed);
1137 embed_models
1138 .first()
1139 .map(|m| m.name.clone())
1140 .unwrap_or_else(|| "Qwen3-Embedding-0.6B".to_string())
1141 }
1142
1143 pub fn route_small(&self, registry: &UnifiedRegistry) -> String {
1145 let gen_models = registry.query_by_capability(ModelCapability::Generate);
1146 gen_models
1148 .iter()
1149 .filter(|m| m.is_local())
1150 .min_by_key(|m| m.size_mb())
1151 .map(|m| m.name.clone())
1152 .unwrap_or_else(|| "Qwen3-0.6B".to_string())
1153 }
1154
1155 const LATENCY_CEILING_MS: f64 = 10000.0;
1161 const _TPS_CEILING: f64 = 150.0;
1163 const MOE_TPS_MULTIPLIER: f64 = 0.10;
1167 const MLX_MOE_TPS_MULTIPLIER: f64 = 0.50;
1170 const COST_CEILING_PER_1K: f64 = 0.1;
1172 const LOCAL_BONUS: f64 = 0.15;
1174 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1182 const HAS_GPU_BACKEND: bool = true;
1183 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
1184 const HAS_GPU_BACKEND: bool = false;
1185 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1188 const MLX_BONUS: f64 = 0.10;
1189 const SYSTEM_LLM_BONUS: f64 = 0.12;
1200
1201 pub(crate) fn build_exclude_set(
1220 &self,
1221 intent: Option<&crate::intent::IntentHint>,
1222 registry: &UnifiedRegistry,
1223 ) -> std::collections::HashSet<String> {
1224 let mut set = std::collections::HashSet::new();
1225 let Some(hint) = intent else {
1226 return set;
1227 };
1228 for id in &hint.exclude_models {
1229 set.insert(id.clone());
1230 let Some(schema) = registry.get(id).or_else(|| registry.find_by_name(id)) else {
1231 continue;
1232 };
1233 set.insert(schema.id.clone());
1234 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1235 {
1236 if let Some(mlx) = registry.resolve_mlx_equivalent(schema) {
1237 set.insert(mlx.id.clone());
1238 }
1239 }
1240 }
1241 set
1242 }
1243
1244 fn filter_candidates(
1245 &self,
1246 required_caps: &[ModelCapability],
1247 registry: &UnifiedRegistry,
1248 tracker: &OutcomeTracker,
1249 has_vision: bool,
1250 availability: AvailabilitySnapshot,
1251 require_ready: bool,
1252 ) -> Vec<ModelSchema> {
1253 registry
1254 .list()
1255 .into_iter()
1256 .filter(|m| {
1257 if !required_caps.iter().all(|c| m.has_capability(*c)) {
1259 return false;
1260 }
1261 if !has_vision && m.tags.iter().any(|t| t == "mlx-vlm-cli") {
1271 return false;
1272 }
1273 if !availability.is_available(m) {
1275 return false;
1276 }
1277 if require_ready && !m.weights_ready {
1284 return false;
1285 }
1286 if m.is_local() && m.size_mb() >= self.hw.max_model_mb {
1288 return false;
1289 }
1290 if let Some(max) = self.config.max_latency_ms {
1292 if let Some(p50) = m.performance.latency_p50_ms {
1293 if p50 > max {
1294 return false;
1295 }
1296 }
1297 }
1298 if let Some(max) = self.config.max_cost_usd {
1300 if m.cost_per_1k_output() > max {
1301 return false;
1302 }
1303 }
1304 if !self.config.prefer_local && m.is_local() {
1306 return false;
1307 }
1308 if tracker.is_excluded(&m.id) {
1310 return false;
1311 }
1312 if let Ok(mut cb) = self.circuit_breakers.lock() {
1314 if !cb.allow_request(&m.id) {
1315 tracing::debug!(model = %m.id, "skipped by circuit breaker");
1316 return false;
1317 }
1318 }
1319 true
1320 })
1321 .cloned()
1322 .collect()
1323 }
1324
1325 fn apply_quality_first_bootstrap_policy(
1326 &self,
1327 candidates: Vec<ModelSchema>,
1328 task: InferenceTask,
1329 tracker: &OutcomeTracker,
1330 has_vision: bool,
1331 has_tools: bool,
1332 workload: RoutingWorkload,
1333 ) -> Vec<ModelSchema> {
1334 if !self.config.quality_first_cold_start
1335 || !workload.is_latency_sensitive()
1336 || !self.is_quality_critical_bootstrap_task(task, has_vision, has_tools)
1337 {
1338 return candidates;
1339 }
1340
1341 let trusted_remote: Vec<ModelSchema> = candidates
1342 .iter()
1343 .filter(|model| self.is_trusted_quality_remote(model))
1344 .cloned()
1345 .collect();
1346
1347 if trusted_remote.is_empty() {
1348 return candidates;
1349 }
1350
1351 let proven_local: Vec<ModelSchema> = candidates
1352 .iter()
1353 .filter(|model| {
1354 model.is_local() && self.is_local_model_proven_for_task(model, task, tracker)
1355 })
1356 .cloned()
1357 .collect();
1358
1359 if !proven_local.is_empty() {
1360 return proven_local;
1361 }
1362
1363 trusted_remote
1364 }
1365
1366 fn score_candidates_context_aware(
1370 &self,
1371 candidates: &[ModelSchema],
1372 task: InferenceTask,
1373 tracker: &OutcomeTracker,
1374 estimated_total_tokens: usize,
1375 estimated_input_tokens: usize,
1376 estimated_output_tokens: usize,
1377 estimated_cache_read_tokens: usize,
1378 estimated_cache_write_tokens: usize,
1379 workload: RoutingWorkload,
1380 ) -> Vec<(String, f64)> {
1381 let mut scored: Vec<(String, f64)> = candidates
1382 .iter()
1383 .map(|m| {
1384 let base_score = self.score_model(
1385 m,
1386 task,
1387 tracker,
1388 workload,
1389 estimated_input_tokens,
1390 estimated_output_tokens,
1391 estimated_cache_read_tokens,
1392 estimated_cache_write_tokens,
1393 );
1394 let headroom_bonus = if estimated_total_tokens > 0 && m.context_length > 0 {
1397 let ratio = m.context_length as f64 / estimated_total_tokens as f64;
1398 if ratio >= 1.0 {
1399 (ratio.min(4.0) - 1.0) / 3.0 * 0.10 } else {
1401 -0.15 }
1403 } else {
1404 0.0
1405 };
1406 (m.id.clone(), base_score + headroom_bonus)
1407 })
1408 .collect();
1409
1410 scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
1411 scored
1412 }
1413
1414 fn task_aware_weights(task: InferenceTask, workload: RoutingWorkload) -> (f64, f64, f64) {
1430 match (task, workload) {
1431 (
1432 InferenceTask::Code,
1433 RoutingWorkload::Interactive | RoutingWorkload::LocalPreferred,
1434 ) => (0.62, 0.28, 0.10),
1435 _ => workload.weights(),
1436 }
1437 }
1438
1439 fn reliability(
1446 &self,
1447 model: &ModelSchema,
1448 task: InferenceTask,
1449 tracker: &OutcomeTracker,
1450 ) -> f64 {
1451 let schema_quality = self.schema_quality_estimate(model);
1452 let Some(p) = tracker.profile(&model.id) else {
1453 return schema_quality;
1455 };
1456
1457 let (ema, n, g) = p
1465 .task_stats(task)
1466 .map(|ts| {
1467 (
1468 ts.ema_quality,
1469 ts.prior_sample_size,
1470 ts.quality_observations,
1471 )
1472 })
1473 .unwrap_or((p.ema_quality, p.prior_sample_size, p.quality_observations));
1474
1475 let evidence = n as f64 + Self::LIVE_GRADE_WEIGHT * g as f64;
1501 let confidence = evidence / (evidence + Self::PRIOR_SHRINK_PSEUDOCOUNT);
1502 (schema_quality + confidence * (ema - schema_quality)).clamp(0.0, 1.0)
1503 }
1504
1505 const BENCH_TIER_FLOOR: f64 = 0.40;
1513 const BENCH_TIER_CEIL: f64 = 0.92;
1514
1515 const LOCAL_BENCH_TIER_FLOOR: f64 = 0.30;
1529 const LOCAL_BENCH_TIER_CEIL: f64 = 0.60;
1530
1531 const UNMEASURED_REMOTE_CURATED: f64 = 0.60;
1536 const UNMEASURED_REMOTE_COMMUNITY: f64 = 0.48;
1537 const CURATED_FRONTIER_TAG_PRIOR: f64 = 0.85;
1542 const CURATED_BALANCED_TAG_PRIOR: f64 = 0.72;
1543
1544 const LIVE_GRADE_WEIGHT: f64 = 2.0;
1555
1556 const PRIOR_SHRINK_PSEUDOCOUNT: f64 = 4.0;
1566
1567 const OUTCOME_FIRST_BAND: f64 = 0.02;
1583
1584 fn resolve_workload(
1590 intent: Option<&crate::intent::IntentHint>,
1591 base: RoutingWorkload,
1592 ) -> RoutingWorkload {
1593 match intent {
1594 Some(h) if h.high_stakes => RoutingWorkload::Quality,
1595 Some(h) if h.prefer_fast => RoutingWorkload::Fastest,
1596 Some(h) if h.prefer_quality => RoutingWorkload::Quality,
1597 Some(h) if h.prefer_local => RoutingWorkload::LocalPreferred,
1598 _ => base,
1599 }
1600 }
1601
1602 fn applies_reliability_band(task: InferenceTask, workload: RoutingWorkload) -> bool {
1618 match workload {
1619 RoutingWorkload::Quality => true,
1620 RoutingWorkload::Fastest
1621 | RoutingWorkload::Background
1622 | RoutingWorkload::LocalPreferred => false,
1623 _ => matches!(task, InferenceTask::Code | InferenceTask::Reasoning),
1624 }
1625 }
1626
1627 fn outcome_first_band(
1632 &self,
1633 candidates: Vec<ModelSchema>,
1634 task: InferenceTask,
1635 tracker: &OutcomeTracker,
1636 ) -> Vec<ModelSchema> {
1637 if candidates.len() <= 1 {
1638 return candidates;
1639 }
1640 let best = candidates
1641 .iter()
1642 .map(|m| self.reliability(m, task, tracker))
1643 .fold(f64::MIN, f64::max);
1644 candidates
1645 .into_iter()
1646 .filter(|m| {
1649 best - self.reliability(m, task, tracker) <= Self::OUTCOME_FIRST_BAND + 1e-9
1650 })
1651 .collect()
1652 }
1653
1654 fn score_model(
1655 &self,
1656 model: &ModelSchema,
1657 task: InferenceTask,
1658 tracker: &OutcomeTracker,
1659 workload: RoutingWorkload,
1660 estimated_input_tokens: usize,
1661 estimated_output_tokens: usize,
1662 estimated_cache_read_tokens: usize,
1663 estimated_cache_write_tokens: usize,
1664 ) -> f64 {
1665 let profile = tracker.profile(&model.id);
1666 let schema_latency = self.schema_latency_estimate(model);
1667 let (quality_weight, latency_weight, cost_weight) =
1668 Self::task_aware_weights(task, workload);
1669
1670 let quality = self.reliability(model, task, tracker);
1674
1675 let latency = match profile {
1678 Some(p) if p.total_calls >= self.config.min_observations => {
1679 let avg = p
1680 .task_stats(task)
1681 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1682 .map(|ts| ts.avg_latency_ms)
1683 .unwrap_or_else(|| p.avg_latency_ms());
1684 self.latency_ms_to_score(avg)
1685 }
1686 Some(p) if p.total_calls == 0 => p
1687 .task_stats(task)
1688 .filter(|ts| ts.avg_latency_ms > 0.0)
1689 .map(|ts| self.latency_ms_to_score(ts.avg_latency_ms))
1690 .unwrap_or(schema_latency),
1691 Some(p) if p.total_calls > 0 => {
1692 let observed = self.latency_ms_to_score(
1693 p.task_stats(task)
1694 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1695 .map(|ts| ts.avg_latency_ms)
1696 .unwrap_or_else(|| p.avg_latency_ms()),
1697 );
1698 let w = p.total_calls as f64 / self.config.min_observations as f64;
1699 schema_latency * (1.0 - w) + observed * w
1700 }
1701 _ => schema_latency,
1702 };
1703
1704 let cost = if model.is_local() {
1706 1.0
1707 } else if model.cost.output_per_mtok.is_none() {
1708 0.5
1715 } else {
1716 let normalized_cost_per_1k = if estimated_input_tokens
1717 + estimated_output_tokens
1718 + estimated_cache_read_tokens
1719 + estimated_cache_write_tokens
1720 > 0
1721 {
1722 let estimated_cost = model.cost.estimated_usd(
1723 estimated_input_tokens,
1724 estimated_output_tokens,
1725 estimated_cache_read_tokens,
1726 estimated_cache_write_tokens,
1727 );
1728 estimated_cost * 1000.0
1729 / (estimated_input_tokens + estimated_output_tokens).max(1) as f64
1730 } else {
1731 model.cost_per_1k_output()
1732 };
1733 (1.0 - (normalized_cost_per_1k / Self::COST_CEILING_PER_1K)).clamp(0.0, 1.0)
1734 };
1735
1736 let gpu_backend_or_latency_tolerant =
1755 Self::HAS_GPU_BACKEND || !workload.is_latency_sensitive();
1756 let local_bonus = if self.config.prefer_local
1757 && model.is_local()
1758 && gpu_backend_or_latency_tolerant
1759 {
1760 Self::LOCAL_BONUS
1761 } else {
1762 if self.config.prefer_local && model.is_local() && !Self::HAS_GPU_BACKEND {
1763 tracing::debug!(
1764 model = %model.id,
1765 "LOCAL_BONUS suppressed: no GPU backend on this host (Intel Mac or car_skip_mlx) for a latency-sensitive workload; cloud models will rank higher"
1766 );
1767 }
1768 0.0
1769 };
1770 let workload_local_bonus = if model.is_local() {
1771 workload.local_bonus()
1772 } else {
1773 0.0
1774 };
1775
1776 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1778 let mlx_bonus = if model.is_mlx() { Self::MLX_BONUS } else { 0.0 };
1779 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
1780 let mlx_bonus = 0.0;
1781
1782 let vllm_mlx_bonus = if model.is_vllm_mlx() {
1784 Self::LOCAL_BONUS + 0.05
1785 } else {
1786 0.0
1787 };
1788
1789 let system_llm_bonus = if model.tags.iter().any(|t| t == "low_latency")
1797 && model.tags.iter().any(|t| t == "private")
1798 {
1799 Self::SYSTEM_LLM_BONUS
1800 } else {
1801 0.0
1802 };
1803
1804 let (local_bonus, workload_local_bonus, mlx_bonus, vllm_mlx_bonus, system_llm_bonus) =
1815 if workload == RoutingWorkload::Quality {
1816 (0.0, 0.0, 0.0, 0.0, 0.0)
1817 } else {
1818 (
1819 local_bonus,
1820 workload_local_bonus,
1821 mlx_bonus,
1822 vllm_mlx_bonus,
1823 system_llm_bonus,
1824 )
1825 };
1826
1827 quality_weight * quality
1828 + latency_weight * latency
1829 + cost_weight * cost
1830 + local_bonus
1831 + workload_local_bonus
1832 + mlx_bonus
1833 + vllm_mlx_bonus
1834 + system_llm_bonus
1835 }
1836
1837 fn latency_ms_to_score(&self, ms: f64) -> f64 {
1840 (1.0 - (ms / Self::LATENCY_CEILING_MS)).clamp(0.0, 1.0)
1841 }
1842
1843 fn tps_to_latency_ms(tps: f64) -> f64 {
1845 if tps <= 0.0 {
1846 return Self::LATENCY_CEILING_MS;
1847 }
1848 (200.0 / tps) * 1000.0
1850 }
1851
1852 fn needs_multi_tool_call(prompt: &str) -> bool {
1855 let lower = prompt.to_lowercase();
1856
1857 let has_numbered_list = {
1859 let mut count = 0u32;
1860 for i in 1..=5u32 {
1861 if lower.contains(&format!("{}) ", i)) || lower.contains(&format!("{}. ", i)) {
1862 count += 1;
1863 }
1864 }
1865 count >= 2
1866 };
1867
1868 let multi_keywords = [
1870 "multiple edits",
1871 "several changes",
1872 "three changes",
1873 "two changes",
1874 "all of the following",
1875 "each of these",
1876 "do both",
1877 "do all",
1878 "and also",
1879 "additionally",
1880 "as well as",
1881 "then also",
1882 ];
1883 let has_multi_keywords = multi_keywords.iter().any(|kw| lower.contains(kw));
1884
1885 let bullet_actions = lower.matches("- add ").count()
1887 + lower.matches("- update ").count()
1888 + lower.matches("- change ").count()
1889 + lower.matches("- remove ").count()
1890 + lower.matches("- fix ").count()
1891 + lower.matches("- edit ").count()
1892 + lower.matches("- implement ").count()
1893 + lower.matches("- create ").count();
1894 let has_bullet_list = bullet_actions >= 2;
1895
1896 has_numbered_list || has_multi_keywords || has_bullet_list
1897 }
1898
1899 fn schema_quality_estimate(&self, model: &ModelSchema) -> f64 {
1921 if !model.public_benchmarks.is_empty() {
1922 return self.normalized_benchmark_quality(model).clamp(0.0, 1.0);
1923 }
1924 if model.is_remote() {
1925 if model.trust_tier == crate::schema::TrustTier::Curated
1931 && model.tags.iter().any(|tag| tag == "openrouter")
1932 {
1933 if model_is_frontier_class(model) {
1934 return Self::CURATED_FRONTIER_TAG_PRIOR;
1935 }
1936 if model.tags.iter().any(|tag| tag == "balanced") {
1937 return Self::CURATED_BALANCED_TAG_PRIOR;
1938 }
1939 }
1940 return match model.trust_tier {
1941 crate::schema::TrustTier::Community => Self::UNMEASURED_REMOTE_COMMUNITY,
1942 _ => Self::UNMEASURED_REMOTE_CURATED,
1943 };
1944 }
1945 match model.size_mb() {
1949 0 => 0.35, s if s < 1000 => 0.30, s if s < 2000 => 0.38, s if s < 3000 => 0.45, s if s < 6000 => 0.52, _ => 0.60, }
1956 }
1957
1958 fn normalized_benchmark_quality(&self, model: &ModelSchema) -> f64 {
1969 let is_local = model.is_local();
1970 let n = model.public_benchmarks.len() as f64;
1971 let sum: f64 = model
1972 .public_benchmarks
1973 .iter()
1974 .map(|b| match Self::benchmark_frontier_ref(&b.name) {
1975 Some(frontier_ref) => {
1976 if b.score > frontier_ref {
1985 tracing::warn!(
1986 model = %model.id,
1987 benchmark = %b.name,
1988 score = b.score,
1989 frontier_ref,
1990 "model scores ABOVE the catalog frontier_ref — re-anchor \
1991 candidate: update builtin_catalog.json so this benchmark's \
1992 champion calibrates the quality tier (#370)"
1993 );
1994 }
1995 let frac = (b.score / frontier_ref).clamp(0.0, 1.0);
1996 Self::BENCH_TIER_FLOOR + frac * (Self::BENCH_TIER_CEIL - Self::BENCH_TIER_FLOOR)
1997 }
1998 None if is_local => {
2004 b.score
2011 .clamp(Self::LOCAL_BENCH_TIER_FLOOR, Self::LOCAL_BENCH_TIER_CEIL)
2012 }
2013 None => b.score.clamp(Self::BENCH_TIER_FLOOR, Self::BENCH_TIER_CEIL),
2018 })
2019 .sum();
2020 sum / n
2027 }
2028
2029 fn benchmark_frontier_ref(name: &str) -> Option<f64> {
2041 BENCHMARK_FRONTIER_REFS
2042 .get(name)
2043 .copied()
2044 .filter(|r| *r > 0.0)
2045 }
2046
2047 fn schema_latency_estimate(&self, model: &ModelSchema) -> f64 {
2052 let is_moe = model.tags.contains(&"moe".to_string());
2053
2054 if model.is_local() {
2055 if let Some(tps) = model.performance.tokens_per_second {
2056 let measured = model.performance.latency_p50_ms.is_some();
2069 let effective_tps = if is_moe && !measured {
2070 let multiplier = if model.is_mlx() {
2071 Self::MLX_MOE_TPS_MULTIPLIER
2072 } else {
2073 Self::MOE_TPS_MULTIPLIER
2074 };
2075 tps * multiplier
2076 } else {
2077 tps
2078 };
2079 let estimated_ms = Self::tps_to_latency_ms(effective_tps);
2080 return self.latency_ms_to_score(estimated_ms);
2081 }
2082 return 0.5; }
2084
2085 if let Some(p50) = model.performance.latency_p50_ms {
2087 return self.latency_ms_to_score(p50 as f64);
2088 }
2089 0.3 }
2091
2092 fn is_quality_critical_bootstrap_task(
2093 &self,
2094 task: InferenceTask,
2095 has_vision: bool,
2096 has_tools: bool,
2097 ) -> bool {
2098 has_vision
2099 || has_tools
2100 || matches!(
2101 task,
2102 InferenceTask::Generate | InferenceTask::Code | InferenceTask::Reasoning
2103 )
2104 }
2105
2106 fn is_trusted_quality_remote(&self, model: &ModelSchema) -> bool {
2107 let trusted_provider = matches!(
2108 model.provider.as_str(),
2109 "openai" | "anthropic" | "google" | "openrouter"
2110 );
2111 model.is_remote()
2112 && model.trust_tier == crate::schema::TrustTier::Curated
2113 && (trusted_provider || crate::openrouter::is_managed_gateway_schema(model))
2114 && !model.has_capability(ModelCapability::SpeechToText)
2115 && !model.has_capability(ModelCapability::TextToSpeech)
2116 }
2117
2118 fn is_local_model_proven_for_task(
2119 &self,
2120 model: &ModelSchema,
2121 task: InferenceTask,
2122 tracker: &OutcomeTracker,
2123 ) -> bool {
2124 let Some(profile) = tracker.profile(&model.id) else {
2125 return false;
2126 };
2127 if let Some(task_stats) = profile.task_stats(task) {
2128 if task_stats.calls >= self.config.bootstrap_min_task_observations
2129 && task_stats.ema_quality >= self.config.bootstrap_quality_floor
2130 {
2131 return true;
2132 }
2133 }
2134
2135 profile.total_calls >= self.config.bootstrap_min_task_observations
2136 && profile.ema_quality >= self.config.bootstrap_quality_floor
2137 }
2138
2139 fn select_with_thompson_sampling(
2149 &self,
2150 scored: &[(String, f64)],
2151 tracker: &OutcomeTracker,
2152 ) -> (String, RoutingStrategy) {
2153 if scored.is_empty() {
2154 return (String::new(), RoutingStrategy::SchemaBased);
2155 }
2156
2157 let mut rng = rand::rng();
2158 let mut best_sample = f64::NEG_INFINITY;
2159 let mut best_id = scored[0].0.clone();
2160 let mut best_strategy = RoutingStrategy::SchemaBased;
2161
2162 for (id, phase2_score) in scored {
2163 let profile = tracker.profile(id);
2164 let prior = self.config.prior_strength;
2165
2166 let prior_mean = phase2_score.clamp(0.0, 1.0);
2168
2169 let prior_alpha = prior * prior_mean;
2171 let prior_beta = prior * (1.0 - prior_mean);
2172
2173 let (obs_alpha, obs_beta) = match profile {
2175 Some(p) => (p.success_count as f64, p.fail_count as f64),
2176 None => (0.0, 0.0),
2177 };
2178
2179 let alpha = (prior_alpha + obs_alpha).max(0.01);
2181 let beta = (prior_beta + obs_beta).max(0.01);
2182
2183 let sample = sample_beta(&mut rng, alpha, beta);
2185
2186 if sample > best_sample {
2187 best_sample = sample;
2188 best_id = id.clone();
2189 best_strategy = match profile {
2190 Some(p) if p.total_calls >= self.config.min_observations => {
2191 RoutingStrategy::ProfileBased
2192 }
2193 Some(p) if p.total_calls > 0 => {
2194 RoutingStrategy::Exploration
2196 }
2197 _ => RoutingStrategy::SchemaBased,
2198 };
2199 }
2200 }
2201
2202 (best_id, best_strategy)
2203 }
2204
2205 fn cold_start_decision(
2222 &self,
2223 complexity: TaskComplexity,
2224 task: InferenceTask,
2225 required_caps: &[ModelCapability],
2226 registry: &UnifiedRegistry,
2227 has_vision: bool,
2228 exclude: &std::collections::HashSet<String>,
2229 strict_exclusions: bool,
2230 availability: AvailabilitySnapshot,
2231 ) -> AdaptiveRoutingDecision {
2232 if has_vision {
2237 if let Some(model) = registry
2238 .query_by_capability(ModelCapability::Vision)
2239 .into_iter()
2240 .filter(|model| !exclude.contains(&model.id))
2241 .find(|model| {
2242 availability.is_available(model) && self.is_trusted_quality_remote(model)
2243 })
2244 .or_else(|| {
2245 registry
2246 .query_by_capability(ModelCapability::Vision)
2247 .into_iter()
2248 .find(|model| !exclude.contains(&model.id))
2249 })
2250 {
2251 return AdaptiveRoutingDecision {
2252 model_id: model.id.clone(),
2253 model_name: model.name.clone(),
2254 task,
2255 complexity,
2256 reason: format!(
2257 "{:?} task → {} (cold start, vision fallback)",
2258 complexity, model.name
2259 ),
2260 strategy: RoutingStrategy::SchemaBased,
2261 predicted_quality: 0.5,
2262 fallbacks: vec![],
2263 context_length: model.context_length,
2264 needs_compaction: false,
2265 candidates: vec![],
2266 };
2267 }
2268 }
2269
2270 if self.config.quality_first_cold_start {
2271 if let Some(model) = registry
2272 .list()
2273 .into_iter()
2274 .filter(|model| {
2275 availability.is_available(model)
2276 && !exclude.contains(&model.id)
2277 && required_caps.iter().all(|cap| model.has_capability(*cap))
2278 && self.is_trusted_quality_remote(model)
2279 })
2280 .max_by(|a, b| {
2281 self.schema_quality_estimate(a)
2282 .partial_cmp(&self.schema_quality_estimate(b))
2283 .unwrap_or(std::cmp::Ordering::Equal)
2284 })
2285 {
2286 return AdaptiveRoutingDecision {
2287 model_id: model.id.clone(),
2288 model_name: model.name.clone(),
2289 task,
2290 complexity,
2291 reason: format!(
2292 "{:?} task → {} (quality-first cold start)",
2293 complexity, model.name
2294 ),
2295 strategy: RoutingStrategy::SchemaBased,
2296 predicted_quality: self.schema_quality_estimate(model),
2297 fallbacks: vec![],
2298 context_length: model.context_length,
2299 needs_compaction: false,
2300 candidates: vec![],
2301 };
2302 }
2303 }
2304
2305 if let Some(model) = registry
2313 .list()
2314 .into_iter()
2315 .filter(|model| {
2316 availability.is_available(model)
2317 && !exclude.contains(&model.id)
2318 && model.is_local()
2319 && model.size_mb() < self.hw.max_model_mb
2320 && required_caps.iter().all(|cap| model.has_capability(*cap))
2321 })
2322 .max_by(|a, b| {
2323 self.schema_quality_estimate(a)
2324 .partial_cmp(&self.schema_quality_estimate(b))
2325 .unwrap_or(std::cmp::Ordering::Equal)
2326 })
2327 {
2328 return AdaptiveRoutingDecision {
2329 model_id: model.id.clone(),
2330 model_name: model.name.clone(),
2331 task,
2332 complexity,
2333 reason: format!(
2334 "{:?} task → {} (capability-satisfying local cold start)",
2335 complexity, model.name
2336 ),
2337 strategy: RoutingStrategy::SchemaBased,
2338 predicted_quality: self.schema_quality_estimate(model),
2339 fallbacks: vec![],
2340 context_length: model.context_length,
2341 needs_compaction: false,
2342 candidates: vec![],
2343 };
2344 }
2345
2346 if let Some(model) = registry
2357 .list()
2358 .into_iter()
2359 .filter(|model| {
2360 availability.is_available(model)
2361 && !exclude.contains(&model.id)
2362 && required_caps.iter().all(|cap| model.has_capability(*cap))
2363 })
2364 .max_by(|a, b| {
2365 let qa = (
2366 self.is_trusted_quality_remote(a),
2367 self.schema_quality_estimate(a),
2368 );
2369 let qb = (
2370 self.is_trusted_quality_remote(b),
2371 self.schema_quality_estimate(b),
2372 );
2373 qa.partial_cmp(&qb).unwrap_or(std::cmp::Ordering::Equal)
2374 })
2375 {
2376 return AdaptiveRoutingDecision {
2377 model_id: model.id.clone(),
2378 model_name: model.name.clone(),
2379 task,
2380 complexity,
2381 reason: format!(
2382 "{:?} task → {} (capability-satisfying cold start)",
2383 complexity, model.name
2384 ),
2385 strategy: RoutingStrategy::SchemaBased,
2386 predicted_quality: self.schema_quality_estimate(model),
2387 fallbacks: vec![],
2388 context_length: model.context_length,
2389 needs_compaction: false,
2390 candidates: vec![],
2391 };
2392 }
2393
2394 if strict_exclusions && !exclude.is_empty() {
2398 return Self::no_eligible_model_decision(complexity, task, exclude.len());
2399 }
2400
2401 let model_name = match complexity {
2408 TaskComplexity::Simple => "Qwen3-0.6B",
2409 TaskComplexity::Medium => "Qwen3-1.7B",
2410 TaskComplexity::Code => "Qwen3-4B",
2411 TaskComplexity::Complex => &self.hw.recommended_model,
2412 };
2413
2414 let model_id = registry
2415 .find_by_name(model_name)
2416 .map(|m| m.id.clone())
2417 .unwrap_or_else(|| model_name.to_string());
2418
2419 let context_length = registry
2420 .find_by_name(model_name)
2421 .map(|m| m.context_length)
2422 .unwrap_or(0);
2423
2424 AdaptiveRoutingDecision {
2425 model_id,
2426 model_name: model_name.to_string(),
2427 task,
2428 complexity,
2429 reason: format!(
2430 "{:?} task → {} (cold start, no candidates)",
2431 complexity, model_name
2432 ),
2433 strategy: RoutingStrategy::SchemaBased,
2434 predicted_quality: 0.5,
2435 fallbacks: vec![],
2436 context_length,
2437 needs_compaction: false,
2438 candidates: vec![],
2439 }
2440 }
2441
2442 fn no_eligible_model_decision(
2443 complexity: TaskComplexity,
2444 task: InferenceTask,
2445 excluded_count: usize,
2446 ) -> AdaptiveRoutingDecision {
2447 AdaptiveRoutingDecision {
2448 model_id: String::new(),
2449 model_name: String::new(),
2450 task,
2451 complexity,
2452 reason: format!(
2453 "strict model exclusions left no eligible model ({excluded_count} catalog ids excluded)"
2454 ),
2455 strategy: RoutingStrategy::SchemaBased,
2456 predicted_quality: 0.0,
2457 fallbacks: vec![],
2458 context_length: 0,
2459 needs_compaction: false,
2460 candidates: vec![],
2461 }
2462 }
2463}
2464
2465fn task_hint_to_inference_task(hint: crate::intent::TaskHint) -> InferenceTask {
2471 use crate::intent::TaskHint;
2472 match hint {
2473 TaskHint::Chat => InferenceTask::Generate,
2474 TaskHint::Classify => InferenceTask::Classify,
2475 TaskHint::Reasoning => InferenceTask::Reasoning,
2476 TaskHint::Code => InferenceTask::Code,
2477 }
2478}
2479
2480fn inference_task_required_capability(task: InferenceTask) -> ModelCapability {
2487 match task {
2488 InferenceTask::Generate => ModelCapability::Generate,
2489 InferenceTask::Code => ModelCapability::Code,
2490 InferenceTask::Reasoning => ModelCapability::Reasoning,
2491 InferenceTask::Classify => ModelCapability::Classify,
2492 InferenceTask::Embed => ModelCapability::Embed,
2493 }
2494}
2495
2496fn sample_beta(rng: &mut impl Rng, alpha: f64, beta: f64) -> f64 {
2504 let x = sample_gamma(rng, alpha);
2505 let y = sample_gamma(rng, beta);
2506 if x + y == 0.0 {
2507 0.5 } else {
2509 x / (x + y)
2510 }
2511}
2512
2513fn sample_gamma(rng: &mut impl Rng, shape: f64) -> f64 {
2516 if shape < 1.0 {
2517 let u: f64 = rng.random();
2519 return sample_gamma(rng, shape + 1.0) * u.powf(1.0 / shape);
2520 }
2521
2522 let d = shape - 1.0 / 3.0;
2524 let c = 1.0 / (9.0 * d).sqrt();
2525
2526 loop {
2527 let x: f64 = loop {
2528 let n = sample_standard_normal(rng);
2529 if 1.0 + c * n > 0.0 {
2530 break n;
2531 }
2532 };
2533
2534 let v = (1.0 + c * x).powi(3);
2535 let u: f64 = rng.random();
2536
2537 if u < 1.0 - 0.0331 * x.powi(4) {
2538 return d * v;
2539 }
2540 if u.ln() < 0.5 * x * x + d * (1.0 - v + v.ln()) {
2541 return d * v;
2542 }
2543 }
2544}
2545
2546fn sample_standard_normal(rng: &mut impl Rng) -> f64 {
2548 let u1: f64 = rng.random();
2549 let u2: f64 = rng.random();
2550 (-2.0 * u1.max(1e-300).ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
2551}
2552
2553#[cfg(test)]
2554mod tests {
2555 use super::*;
2556 use crate::outcome::InferredOutcome;
2557 use std::ffi::OsString;
2558
2559 fn route_model_ids(decision: &AdaptiveRoutingDecision) -> impl Iterator<Item = &str> {
2560 std::iter::once(decision.model_id.as_str())
2561 .chain(
2562 decision
2563 .candidates
2564 .iter()
2565 .map(|candidate| candidate.model_id.as_str()),
2566 )
2567 .chain(decision.fallbacks.iter().map(String::as_str))
2568 }
2569
2570 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
2571
2572 impl RestoredEnvironment {
2573 fn capture(names: &[&'static str]) -> Self {
2574 Self(
2575 names
2576 .iter()
2577 .map(|name| (*name, std::env::var_os(name)))
2578 .collect(),
2579 )
2580 }
2581 }
2582
2583 impl Drop for RestoredEnvironment {
2584 fn drop(&mut self) {
2585 for (name, value) in &self.0 {
2586 unsafe {
2587 match value {
2588 Some(value) => std::env::set_var(name, value),
2589 None => std::env::remove_var(name),
2590 }
2591 }
2592 }
2593 }
2594 }
2595
2596 struct TestRegistry {
2597 registry: UnifiedRegistry,
2598 _restore: RestoredEnvironment,
2601 _environment: tokio::sync::MutexGuard<'static, ()>,
2602 }
2603
2604 impl std::ops::Deref for TestRegistry {
2605 type Target = UnifiedRegistry;
2606
2607 fn deref(&self) -> &Self::Target {
2608 &self.registry
2609 }
2610 }
2611
2612 impl std::ops::DerefMut for TestRegistry {
2613 fn deref_mut(&mut self) -> &mut Self::Target {
2614 &mut self.registry
2615 }
2616 }
2617
2618 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2624 const QWEN3_8B_ROUTED_ID: &str = "mlx/qwen3-8b:4bit";
2625 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2626 const QWEN3_8B_ROUTED_ID: &str = "qwen/qwen3-8b:q4_k_m";
2627 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2628 const QWEN3_4B_ROUTED_ID: &str = "mlx/qwen3-4b:4bit";
2629 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2630 const QWEN3_4B_ROUTED_ID: &str = "qwen/qwen3-4b:q4_k_m";
2631
2632 fn test_hw() -> HardwareInfo {
2633 HardwareInfo {
2634 os: "macos".into(),
2635 arch: "aarch64".into(),
2636 cpu_cores: 10,
2637 total_ram_mb: 32768,
2638 gpu_backend: crate::hardware::GpuBackend::Metal,
2639 gpu_memory_mb: Some(28672),
2640 gpu_devices: Vec::new(),
2641 recommended_model: "Qwen3-8B".into(),
2642 recommended_context: 8192,
2643 max_model_mb: 18000, }
2645 }
2646
2647 fn test_registry() -> TestRegistry {
2648 let environment = crate::openrouter::test_environment_scope();
2649 let restore = RestoredEnvironment::capture(&["OPENAI_API_KEY", "CAR_SECRETS_FILE_DIR"]);
2650 let tmp = std::path::PathBuf::from("/tmp/car-test-adaptive-router");
2651 unsafe {
2652 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
2653 std::env::set_var(
2659 "CAR_SECRETS_FILE_DIR",
2660 std::env::temp_dir().join("car-test-adaptive-router-empty-secrets"),
2661 );
2662 }
2663 for name in &[
2665 "Qwen3-0.6B",
2666 "Qwen3-1.7B",
2667 "Qwen3-4B",
2668 "Qwen3-8B",
2669 "Qwen3-Embedding-0.6B",
2670 ] {
2671 let dir = tmp.join(name);
2672 let _ = std::fs::create_dir_all(&dir);
2673 let _ = std::fs::write(dir.join("model.gguf"), b"fake");
2674 let _ = std::fs::write(dir.join("tokenizer.json"), b"{}");
2675 }
2676 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
2677 reg.register_project_model(ModelSchema {
2678 id: "openai/gpt-5.4-mini:latest".into(),
2679 name: "gpt-5.4-mini".into(),
2680 provider: "openai".into(),
2681 family: "gpt-5.4".into(),
2682 version: "latest".into(),
2683 capabilities: vec![
2684 ModelCapability::Generate,
2685 ModelCapability::Code,
2686 ModelCapability::Reasoning,
2687 ModelCapability::ToolUse,
2688 ModelCapability::MultiToolCall,
2689 ModelCapability::Vision,
2690 ],
2691 context_length: 128_000,
2692 max_output_tokens: None,
2693 param_count: "api".into(),
2694 quantization: None,
2695 performance: Default::default(),
2696 cost: Default::default(),
2697 source: crate::schema::ModelSource::RemoteApi {
2698 endpoint: "https://api.openai.com/v1".into(),
2699 api_key_env: "OPENAI_API_KEY".into(),
2700 api_key_envs: vec![],
2701 api_version: None,
2702 protocol: crate::schema::ApiProtocol::OpenAiCompat,
2703 },
2704 tags: vec!["trusted-remote".into()],
2705 supported_params: vec![],
2706 public_benchmarks: vec![],
2707 trust_tier: crate::schema::TrustTier::Curated,
2708 deprecated: false,
2709 available: true,
2710 weights_ready: true,
2711 });
2712 TestRegistry {
2713 registry: reg,
2714 _restore: restore,
2715 _environment: environment,
2716 }
2717 }
2718
2719 #[test]
2720 fn openrouter_tags_drive_cost_quality_routing_and_live_availability() {
2721 let tmp = tempfile::tempdir().unwrap();
2722 let _credential_scope = crate::openrouter::test_credential_scope();
2723 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
2724 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
2725 for schema in crate::openrouter::curated_schemas()
2726 .into_iter()
2727 .filter(|schema| schema.provider == "openrouter")
2728 {
2729 registry.register_project_model(schema);
2730 }
2731 let router = AdaptiveRouter::new(
2732 test_hw(),
2733 RoutingConfig {
2734 prefer_local: false,
2735 prior_strength: 1_000_000.0,
2739 ..RoutingConfig::default()
2740 },
2741 );
2742 let tracker = OutcomeTracker::new();
2743 let mut simple_tracker = OutcomeTracker::new();
2744 let evidenced_cheap = "openrouter/qwen/qwen3-coder-next";
2745 for _ in 0..8 {
2746 let trace = simple_tracker.record_start(
2747 evidenced_cheap,
2748 InferenceTask::Generate,
2749 "OpenRouter cheap-model calibration",
2750 );
2751 simple_tracker.record_complete(&trace, 100, 10, 10);
2752 simple_tracker
2753 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
2754 }
2755
2756 crate::openrouter::reset_credential_source_call_count();
2757 let simple = router.route_with(RouteRequest {
2758 workload: RoutingWorkload::Background,
2759 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
2760 });
2761 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
2762 let simple_schema = registry.get(&simple.model_id).unwrap();
2763 assert!(simple_schema.tags.iter().any(|tag| tag == "cheap"));
2764 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
2765 assert!(simple
2766 .candidates
2767 .iter()
2768 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
2769
2770 let quality_intent = crate::intent::IntentHint {
2771 task: Some(crate::intent::TaskHint::Reasoning),
2772 prefer_quality: true,
2773 ..Default::default()
2774 };
2775 let complex = router.route_with(RouteRequest {
2776 intent: Some(&quality_intent),
2777 ..RouteRequest::new(
2778 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
2779 ®istry,
2780 &tracker,
2781 )
2782 });
2783 let complex_schema = registry.get(&complex.model_id).unwrap();
2784 assert!(complex_schema.tags.iter().any(|tag| tag == "frontier"));
2785 assert!(
2786 complex.reason.contains("frontier quality tier"),
2787 "{}",
2788 complex.reason
2789 );
2790
2791 let vision_intent = crate::intent::IntentHint {
2792 require: vec![ModelCapability::Vision],
2793 ..Default::default()
2794 };
2795 let vision = router.route_with(RouteRequest {
2796 intent: Some(&vision_intent),
2797 ..RouteRequest::new("Inspect this image", ®istry, &tracker)
2798 });
2799 assert!(vision.candidates.iter().all(|candidate| registry
2800 .get(&candidate.model_id)
2801 .is_some_and(|schema| schema.has_capability(ModelCapability::Vision))));
2802
2803 let failed_model = simple.model_id.clone();
2804 {
2805 let mut breakers = router.circuit_breakers.lock().unwrap();
2806 for _ in 0..3 {
2807 breakers.record_failure(&failed_model);
2808 }
2809 }
2810 let after_failures = router.route_with(RouteRequest {
2811 workload: RoutingWorkload::Background,
2812 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
2813 });
2814 assert_ne!(after_failures.model_id, failed_model);
2815 assert!(!after_failures
2816 .candidates
2817 .iter()
2818 .any(|candidate| candidate.model_id == failed_model));
2819
2820 crate::openrouter::set_test_credential(None);
2821 crate::openrouter::reset_credential_source_call_count();
2822 let without_key = router.route("Say hello.", ®istry, &tracker);
2823 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
2824 assert!(!without_key.model_id.starts_with("openrouter/"));
2825 assert!(!without_key
2826 .fallbacks
2827 .iter()
2828 .any(|model| model.starts_with("openrouter/")));
2829 assert!(!without_key
2830 .candidates
2831 .iter()
2832 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
2833 }
2834
2835 #[tokio::test(flavor = "current_thread")]
2836 async fn managed_openrouter_aliases_survive_quality_bootstrap_without_a_personal_key() {
2837 let tmp = tempfile::tempdir().unwrap();
2838 let _credential_scope = crate::openrouter::test_credential_scope();
2839 let _environment = crate::openrouter::test_environment_scope_async().await;
2840 let _restore = RestoredEnvironment::capture(&[
2841 "CAR_SECRETS_FILE_DIR",
2842 "CAR_MANAGED_ROUTING_TEST_KEY",
2843 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
2844 ]);
2845 let _home = crate::openrouter::StateRootScope::new();
2854 crate::openrouter::clear_gateway_unconfigured();
2855 crate::parslee_credential::clear_credential_rejected();
2856 let secrets_dir = tmp.path().join("secrets");
2857 unsafe {
2858 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
2859 std::env::set_var("CAR_MANAGED_ROUTING_TEST_KEY", "test-peer-key");
2860 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
2861 }
2862 crate::openrouter::set_test_credential(None);
2863
2864 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("models"));
2865 for schema in crate::openrouter::curated_schemas() {
2866 registry.register_project_model(schema);
2867 }
2868 let mut bootstrap_peer = remote_model(
2869 "openai/bootstrap-peer",
2870 "bootstrap-peer",
2871 vec![
2872 ModelCapability::Generate,
2873 ModelCapability::Code,
2874 ModelCapability::Reasoning,
2875 ],
2876 );
2877 if let crate::schema::ModelSource::RemoteApi {
2878 ref mut api_key_env,
2879 ..
2880 } = bootstrap_peer.source
2881 {
2882 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
2883 }
2884 registry.register_project_model(bootstrap_peer);
2885 let mut unrelated = remote_model(
2886 "other/unrelated-curated",
2887 "unrelated-curated",
2888 vec![
2889 ModelCapability::Generate,
2890 ModelCapability::Code,
2891 ModelCapability::Reasoning,
2892 ],
2893 );
2894 unrelated.provider = "other".into();
2895 if let crate::schema::ModelSource::RemoteApi {
2896 ref mut api_key_env,
2897 ..
2898 } = unrelated.source
2899 {
2900 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
2901 }
2902 registry.register_project_model(unrelated);
2903 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
2908
2909 let managed: Vec<_> = registry
2910 .list()
2911 .into_iter()
2912 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
2913 .collect();
2914 assert_eq!(managed.len(), crate::openrouter::curated_model_count());
2915 assert!(managed.iter().all(|schema| {
2916 schema.available_now()
2917 && schema.tags.iter().any(|tag| tag == "openrouter")
2918 && schema.tags.iter().any(|tag| tag == "managed")
2919 && schema.cost.input_per_mtok.is_some_and(|price| price > 0.0)
2920 && schema.cost.output_per_mtok.is_some_and(|price| price > 0.0)
2921 && schema.has_capability(ModelCapability::Generate)
2922 }));
2923 assert!(registry
2924 .list()
2925 .into_iter()
2926 .filter(|schema| schema.id.starts_with("openrouter/"))
2927 .all(|schema| !schema.available_now()));
2928
2929 let router = AdaptiveRouter::new(
2930 test_hw(),
2931 RoutingConfig {
2932 prefer_local: false,
2933 prior_strength: 1_000_000.0,
2934 ..RoutingConfig::default()
2935 },
2936 );
2937 let tracker = OutcomeTracker::new();
2938
2939 let interactive = router.route("Say hello.", ®istry, &tracker);
2940 assert!(
2941 interactive
2942 .candidates
2943 .iter()
2944 .any(|candidate| candidate.model_id.starts_with("parslee/openrouter/")),
2945 "signed-in managed aliases must survive quality-first bootstrap: {interactive:?}"
2946 );
2947 assert!(
2948 interactive
2949 .fallbacks
2950 .iter()
2951 .any(|id| id.starts_with("parslee/openrouter/")),
2952 "signed-in managed aliases must remain fallback-eligible: {interactive:?}"
2953 );
2954 assert!(
2955 interactive
2956 .candidates
2957 .iter()
2958 .any(|candidate| candidate.model_id == "openai/bootstrap-peer"),
2959 "adding managed aliases must not remove existing trusted providers: {interactive:?}"
2960 );
2961 assert!(
2962 !interactive
2963 .candidates
2964 .iter()
2965 .any(|candidate| candidate.model_id == "other/unrelated-curated"),
2966 "managed-alias trust must not broaden the bootstrap to unrelated providers"
2967 );
2968
2969 let mut simple_tracker = OutcomeTracker::new();
2970 let evidenced_cheap = "parslee/openrouter/open-fast";
2971 for _ in 0..8 {
2972 let trace = simple_tracker.record_start(
2973 evidenced_cheap,
2974 InferenceTask::Generate,
2975 "managed OpenRouter cheap-model calibration",
2976 );
2977 simple_tracker.record_complete(&trace, 100, 10, 10);
2978 simple_tracker
2979 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
2980 }
2981 let simple = router.route_with(RouteRequest {
2982 workload: RoutingWorkload::Background,
2983 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
2984 });
2985 let simple_schema = registry.get(&simple.model_id).unwrap();
2986 assert!(
2987 simple.model_id.starts_with("parslee/openrouter/")
2988 && simple_schema.tags.iter().any(|tag| tag == "cheap"),
2989 "simple work should prefer an evidenced cheap managed alias: {simple:?}"
2990 );
2991 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
2992
2993 let quality_intent = crate::intent::IntentHint {
2994 task: Some(crate::intent::TaskHint::Reasoning),
2995 prefer_quality: true,
2996 ..Default::default()
2997 };
2998 let complex = router.route_with(RouteRequest {
2999 intent: Some(&quality_intent),
3000 ..RouteRequest::new(
3001 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
3002 ®istry,
3003 &tracker,
3004 )
3005 });
3006 let complex_schema = registry.get(&complex.model_id).unwrap();
3007 assert!(
3008 complex.model_id.starts_with("parslee/openrouter/")
3009 && complex_schema.tags.iter().any(|tag| tag == "frontier"),
3010 "complex quality work should escalate to a managed frontier alias: {complex:?}"
3011 );
3012 assert!(
3013 complex.reason.contains("frontier quality tier"),
3014 "{}",
3015 complex.reason
3016 );
3017
3018 for decision in [&interactive, &simple, &complex] {
3019 assert!(
3020 !route_model_ids(decision).any(|id| id.starts_with("openrouter/")),
3021 "personal OpenRouter rows must stay excluded without a personal key: {decision:?}"
3022 );
3023 }
3024
3025 let spoofed_id = "parslee/openrouter/frontier-general";
3026 let spoofed_endpoint = "https://attacker.invalid/v1/chat/completions";
3027 let spoofed_value = serde_json::json!({
3028 "id": spoofed_id,
3029 "name": spoofed_id,
3030 "provider": "parslee",
3031 "family": "spoof",
3032 "capabilities": ["generate", "code", "reasoning"],
3033 "context_length": 1_000_000,
3034 "performance": {"latency_p50_ms": 1},
3035 "cost": {"input_per_mtok": 0.0, "output_per_mtok": 0.0},
3036 "source": {
3037 "type": "remote_api",
3038 "endpoint": spoofed_endpoint,
3039 "api_key_env": "CAR_MANAGED_ROUTING_TEST_KEY",
3040 "protocol": "open_ai_compat"
3041 },
3042 "tags": ["openrouter", "managed", "frontier"],
3043 "public_benchmarks": [{"name": "attacker-unanchored", "score": 1.0}]
3044 });
3045 let omitted_tier: ModelSchema = serde_json::from_value(spoofed_value.clone()).unwrap();
3046 assert_eq!(
3047 omitted_tier.trust_tier,
3048 crate::schema::TrustTier::Curated,
3049 "the regression must exercise the legacy omitted-tier default"
3050 );
3051 let mut direct_spoof_registry =
3052 UnifiedRegistry::new_empty(tmp.path().join("direct-spoof-models"));
3053 direct_spoof_registry.register(omitted_tier);
3054 assert!(
3055 direct_spoof_registry.get(spoofed_id).is_none(),
3056 "public registration must reject the reserved managed-alias namespace"
3057 );
3058 direct_spoof_registry
3059 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3060 let direct_spoof_route = router.route(
3061 "Analyze this architecture and prove the failure modes.",
3062 &direct_spoof_registry,
3063 &OutcomeTracker::new(),
3064 );
3065 assert!(
3066 !route_model_ids(&direct_spoof_route).any(|id| id == spoofed_id),
3067 "a rejected in-memory lookalike must not be routable: {direct_spoof_route:?}"
3068 );
3069 std::fs::write(
3070 tmp.path().join("models.json"),
3071 serde_json::to_vec_pretty(&vec![spoofed_value]).unwrap(),
3072 )
3073 .unwrap();
3074
3075 let mut reloaded = UnifiedRegistry::new_empty(tmp.path().join("reloaded-models"));
3076 reloaded.load_user_config().unwrap();
3077 assert!(
3078 reloaded.get(spoofed_id).is_none(),
3079 "persisted user config must reject the reserved managed-alias namespace"
3080 );
3081 reloaded.register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3082 let spoofed_route = router.route(
3083 "Analyze this architecture and prove the failure modes.",
3084 &reloaded,
3085 &OutcomeTracker::new(),
3086 );
3087 assert!(
3088 !route_model_ids(&spoofed_route).any(|id| id == spoofed_id),
3089 "a rejected persisted lookalike must not be routable: {spoofed_route:?}"
3090 );
3091
3092 let discovery_state_root = tmp.path().join("discovery-spoof-root");
3093 let discovery_models_dir = discovery_state_root.join("models");
3094 std::fs::create_dir_all(&discovery_models_dir).unwrap();
3095 let discovery_spoof_id = "openrouter/forged-discovery-cache";
3096 let mut discovery_spoof = remote_model(
3097 discovery_spoof_id,
3098 "forged-discovery-cache",
3099 vec![
3100 ModelCapability::Generate,
3101 ModelCapability::Code,
3102 ModelCapability::Reasoning,
3103 ],
3104 );
3105 discovery_spoof.provider = "openrouter".into();
3106 discovery_spoof.tags = vec!["openrouter".into(), "frontier".into()];
3107 discovery_spoof.trust_tier = crate::schema::TrustTier::Curated;
3108 discovery_spoof.performance.latency_p50_ms = Some(1);
3109 discovery_spoof.cost = crate::schema::CostModel::default();
3110 if let crate::schema::ModelSource::RemoteApi {
3111 ref mut api_key_env,
3112 ..
3113 } = discovery_spoof.source
3114 {
3115 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3116 }
3117 crate::discovery::save_cache(
3118 &crate::discovery::cache_path(&discovery_models_dir),
3119 &[discovery_spoof],
3120 )
3121 .unwrap();
3122
3123 let mut discovery_registry =
3124 UnifiedRegistry::new_with_state_root(discovery_state_root, discovery_models_dir);
3125 discovery_registry
3126 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3127 assert_eq!(
3128 discovery_registry
3129 .get(discovery_spoof_id)
3130 .unwrap()
3131 .trust_tier,
3132 crate::schema::TrustTier::Community,
3133 "unsigned discovery cache rows must be demoted before registration"
3134 );
3135 let discovery_spoof_route = router.route(
3136 "Analyze this architecture and prove the failure modes.",
3137 &discovery_registry,
3138 &OutcomeTracker::new(),
3139 );
3140 assert!(
3141 !discovery_spoof_route
3142 .candidates
3143 .iter()
3144 .any(|candidate| candidate.model_id == discovery_spoof_id),
3145 "a forged direct-provider discovery row must not enter trusted candidates: \
3146 {discovery_spoof_route:?}"
3147 );
3148 assert!(
3149 !discovery_spoof_route
3150 .fallbacks
3151 .iter()
3152 .any(|id| id == discovery_spoof_id),
3153 "a forged direct-provider discovery row must not enter trusted fallbacks: \
3154 {discovery_spoof_route:?}"
3155 );
3156 }
3157
3158 #[test]
3159 fn openrouter_becomes_require_ready_eligible_after_live_key_activation() {
3160 let tmp = tempfile::tempdir().unwrap();
3161 let _credential_scope = crate::openrouter::test_credential_scope();
3162 let _provider_env = crate::openrouter::test_environment_scope();
3163 crate::openrouter::set_test_credential(None);
3164 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
3165 let personal = crate::openrouter::curated_schemas()
3166 .into_iter()
3167 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3168 .unwrap();
3169 registry.register_project_model(personal);
3170
3171 unsafe {
3172 std::env::set_var("OPENAI_API_KEY", "ready-peer-key");
3173 }
3174 registry.register_project_model(remote_model(
3175 "ready-peer",
3176 "ready-peer",
3177 vec![ModelCapability::Generate, ModelCapability::Reasoning],
3178 ));
3179
3180 crate::openrouter::set_test_credential(Some("activated-after-register"));
3181 let hint = crate::intent::IntentHint {
3182 require_ready: true,
3183 prefer_quality: true,
3184 ..Default::default()
3185 };
3186 let router = AdaptiveRouter::new(
3187 test_hw(),
3188 RoutingConfig {
3189 prefer_local: false,
3190 prior_strength: 1_000_000.0,
3191 ..RoutingConfig::default()
3192 },
3193 );
3194 let tracker = OutcomeTracker::new();
3195 let decision = router.route_with(RouteRequest {
3196 intent: Some(&hint),
3197 ..RouteRequest::new("Prove the architecture tradeoff.", ®istry, &tracker)
3198 });
3199 assert!(
3200 decision
3201 .candidates
3202 .iter()
3203 .any(|candidate| candidate.model_id == "openrouter/openai/gpt-5.4"),
3204 "post-login OpenRouter model must remain require_ready-eligible: {:?}",
3205 decision.candidates
3206 );
3207 unsafe {
3208 std::env::remove_var("OPENAI_API_KEY");
3209 }
3210 }
3211
3212 #[test]
3213 fn route_cost_score_uses_prompt_tiers_and_model_cache_prices() {
3214 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
3215 let tracker = OutcomeTracker::new();
3216 let mut tiered = crate::openrouter::curated_schemas()
3217 .into_iter()
3218 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3219 .unwrap();
3220 let mut flat = tiered.clone();
3221 tiered.id = "tiered".into();
3222 flat.id = "flat".into();
3223 flat.cost = crate::schema::CostModel {
3224 input_per_mtok: Some(4.0),
3225 output_per_mtok: Some(22.5),
3226 cache_read_input_per_mtok: Some(4.0),
3227 ..Default::default()
3228 };
3229
3230 let score = |model: &ModelSchema, prompt, cache_read| {
3231 router.score_model(
3232 model,
3233 InferenceTask::Generate,
3234 &tracker,
3235 RoutingWorkload::Background,
3236 prompt,
3237 0,
3238 cache_read,
3239 0,
3240 )
3241 };
3242 assert!(score(&tiered, 100_000, 0) > score(&flat, 100_000, 0));
3243 assert!(score(&tiered, 300_000, 0) < score(&flat, 300_000, 0));
3244 assert!(score(&tiered, 300_000, 300_000) > score(&flat, 300_000, 300_000));
3245 }
3246
3247 #[test]
3248 fn routes_simple_to_trusted_remote_during_cold_start() {
3249 let router = AdaptiveRouter::new(
3250 test_hw(),
3251 RoutingConfig {
3252 prior_strength: 100.0, ..Default::default()
3254 },
3255 );
3256 let reg = test_registry();
3257 let tracker = OutcomeTracker::new();
3258
3259 let decision = router.route("What is 2+2?", ®, &tracker);
3260 assert_eq!(decision.complexity, TaskComplexity::Simple);
3261 assert_eq!(decision.strategy, RoutingStrategy::SchemaBased);
3262 let schema = reg
3264 .find_by_name(&decision.model_name)
3265 .expect("selected model should exist in registry");
3266 assert!(
3267 !schema.is_local(),
3268 "simple task should route to trusted remote model during cold start"
3269 );
3270 assert!(matches!(
3271 schema.provider.as_str(),
3272 "openai" | "anthropic" | "google"
3273 ));
3274 }
3275
3276 #[test]
3277 fn routes_code_to_code_capable_remote_during_cold_start() {
3278 let router = AdaptiveRouter::new(
3279 test_hw(),
3280 RoutingConfig {
3281 prior_strength: 100.0, ..Default::default()
3283 },
3284 );
3285 let reg = test_registry();
3286 let tracker = OutcomeTracker::new();
3287
3288 let decision = router.route(
3289 "Fix this function:\n```rust\nfn main() {}\n```",
3290 ®,
3291 &tracker,
3292 );
3293 assert_eq!(decision.complexity, TaskComplexity::Code);
3294 assert_eq!(decision.task, InferenceTask::Code);
3295 let schema = reg
3297 .find_by_name(&decision.model_name)
3298 .expect("model should exist");
3299 assert!(
3300 schema.has_capability(ModelCapability::Code),
3301 "selected model must support Code"
3302 );
3303 assert!(!schema.is_local(), "should route to trusted remote model");
3304 }
3305
3306 #[test]
3307 fn routes_images_to_vision_capable_model() {
3308 let router = AdaptiveRouter::new(
3309 test_hw(),
3310 RoutingConfig {
3311 prior_strength: 1_000_000.0,
3314 ..Default::default()
3315 },
3316 );
3317 let mut reg = test_registry();
3318 let tracker = OutcomeTracker::new();
3319
3320 reg.register_project_model(ModelSchema {
3321 id: "mlx-vlm/qwen3-vl-2b:bf16".into(),
3322 name: "Qwen3-VL-2B-mlx-vlm".into(),
3323 provider: "qwen".into(),
3324 family: "qwen3-vl".into(),
3325 version: "bf16".into(),
3326 capabilities: vec![
3327 ModelCapability::Generate,
3328 ModelCapability::Vision,
3329 ModelCapability::Grounding,
3330 ],
3331 context_length: 262_144,
3332 max_output_tokens: None,
3333 param_count: "2B".into(),
3334 quantization: None,
3335 performance: Default::default(),
3336 cost: Default::default(),
3337 source: crate::schema::ModelSource::Mlx {
3338 hf_repo: "Qwen/Qwen3-VL-2B-Instruct".into(),
3339 hf_weight_file: None,
3340 },
3341 tags: vec!["vision".into(), "mlx-vlm-cli".into()],
3342 supported_params: vec![],
3343 public_benchmarks: vec![],
3344 trust_tier: crate::schema::TrustTier::Curated,
3345 deprecated: false,
3346 available: true,
3347 weights_ready: true,
3348 });
3349
3350 let decision = router.route_with_vision("What is in this image?", ®, &tracker, false);
3351 let schema = reg
3352 .find_by_name(&decision.model_name)
3353 .expect("model should exist");
3354 assert!(
3355 schema.has_capability(ModelCapability::Vision),
3356 "selected model must support Vision"
3357 );
3358 }
3359
3360 #[test]
3361 fn profile_based_routing_favors_proven_model() {
3362 let router = AdaptiveRouter::new(
3363 test_hw(),
3364 RoutingConfig {
3365 prior_strength: 0.5, min_observations: 3,
3367 ..Default::default()
3368 },
3369 );
3370 let reg = test_registry();
3371 let mut tracker = OutcomeTracker::new();
3372
3373 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3375 for _ in 0..20 {
3376 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Code, "test");
3377 tracker.record_complete(&trace, 500, 100, 50);
3378 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3379 }
3380
3381 let mut wins = 0;
3383 for _ in 0..20 {
3384 let decision = router.route("Fix this bug in the parser", ®, &tracker);
3385 assert_eq!(decision.complexity, TaskComplexity::Code);
3386 if decision.model_id == qwen_8b_id {
3387 wins += 1;
3388 }
3389 }
3390 assert!(
3391 wins >= 12,
3392 "proven model won only {wins}/20 times (expected >= 12)"
3393 );
3394 }
3395
3396 #[test]
3397 fn proven_local_model_can_displace_bootstrap_remote() {
3398 let router = AdaptiveRouter::new(
3399 test_hw(),
3400 RoutingConfig {
3401 prior_strength: 100.0,
3402 bootstrap_min_task_observations: 6,
3403 bootstrap_quality_floor: 0.8,
3404 ..Default::default()
3405 },
3406 );
3407 let reg = test_registry();
3408 let mut tracker = OutcomeTracker::new();
3409
3410 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3411 for _ in 0..12 {
3412 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Generate, "test");
3413 tracker.record_complete(&trace, 300, 50, 20);
3414 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3415 }
3416
3417 let mut local_wins = 0;
3418 for _ in 0..20 {
3419 let decision = router.route("Summarize this design decision.", ®, &tracker);
3420 let schema = reg
3421 .get(&decision.model_id)
3422 .expect("selected model should exist");
3423 if schema.is_local() {
3424 local_wins += 1;
3425 }
3426 }
3427
3428 assert!(
3429 local_wins >= 12,
3430 "proven local model won only {local_wins}/20 times (expected >= 12)"
3431 );
3432 }
3433
3434 #[test]
3459 fn background_routing_prefers_local_models() {
3460 let router = AdaptiveRouter::new(
3461 test_hw(),
3462 RoutingConfig {
3463 prior_strength: 1_000_000.0,
3464 ..Default::default()
3465 },
3466 );
3467 let reg = test_registry();
3468 let tracker = OutcomeTracker::new();
3469
3470 let draws = 200;
3485 let mut local_wins = 0;
3486 for _ in 0..draws {
3487 let decision = router.route_context_aware(
3488 "Write a Python fibonacci function.",
3489 128,
3490 ®,
3491 &tracker,
3492 false,
3493 false,
3494 RoutingWorkload::Background,
3495 );
3496 let schema = reg
3497 .get(&decision.model_id)
3498 .expect("selected model should exist");
3499 if schema.is_local() {
3500 local_wins += 1;
3501 }
3502 }
3503
3504 assert!(
3505 local_wins >= 170,
3506 "background routing should keep work local; local won only \
3507 {local_wins}/{draws} (measured baseline: 96.2% off Apple Silicon, \
3508 100% on it)"
3509 );
3510 }
3511
3512 #[test]
3539 fn benchmark_prior_with_evidence_steers_background_routing() {
3540 fn target_score(profile: Option<crate::outcome::ModelProfile>) -> f64 {
3541 let router = AdaptiveRouter::new(
3542 test_hw(),
3543 RoutingConfig {
3544 prior_strength: 100.0,
3545 ..Default::default()
3546 },
3547 );
3548 let reg = test_registry();
3549 let mut tracker = OutcomeTracker::new();
3550 if let Some(p) = profile {
3551 tracker.import_profiles(vec![p]);
3552 }
3553 let decision = router.route_context_aware(
3554 "Write a Python fibonacci function.",
3555 128,
3556 ®,
3557 &tracker,
3558 false,
3559 false,
3560 RoutingWorkload::Background,
3561 );
3562 decision
3563 .candidates
3564 .iter()
3565 .find(|c| c.model_id == QWEN3_8B_ROUTED_ID)
3566 .unwrap_or_else(|| panic!("{QWEN3_8B_ROUTED_ID} should be a candidate"))
3567 .score
3568 }
3569
3570 fn profile_with(ema: f64) -> crate::outcome::ModelProfile {
3571 let mut p = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3572 p.ema_quality = ema;
3573 p.prior_sample_size = 30;
3577 p.quality_observations = 30;
3578 p
3579 }
3580
3581 let control = target_score(None);
3582 let good = target_score(Some(profile_with(0.99)));
3583 let bad = target_score(Some(profile_with(0.05)));
3584
3585 assert!(
3586 good > control,
3587 "an evidence-backed good prior must raise the score it vouches for \
3588 ({good} vs control {control}); equal means this test would pass with \
3589 the prior wiring removed"
3590 );
3591 assert!(
3592 bad < control,
3593 "an evidence-backed poor prior must lower the score ({bad} vs \
3594 control {control})"
3595 );
3596
3597 let mut unsupported = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3601 unsupported.ema_quality = 0.99;
3602 assert_eq!(
3603 target_score(Some(unsupported)),
3604 control,
3605 "an EMA with no observations behind it must not move the score"
3606 );
3607 }
3608
3609 #[test]
3610 fn task_specific_benchmark_prior_informs_cold_start_routing() {
3611 let router = AdaptiveRouter::new(
3612 test_hw(),
3613 RoutingConfig {
3614 prior_strength: 100.0,
3615 ..Default::default()
3616 },
3617 );
3618 let reg = test_registry();
3619 let mut tracker = OutcomeTracker::new();
3620 let mut profile = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3625 profile.task_stats.insert(
3626 crate::outcome::InferenceTask::Code.to_string(),
3627 crate::outcome::TaskStats {
3628 ema_quality: 0.95,
3629 prior_sample_size: 30,
3630 quality_observations: 30,
3631 ..Default::default()
3632 },
3633 );
3634 tracker.import_profiles(vec![profile]);
3635
3636 let decision = router.route_context_aware(
3637 "Write a Python fibonacci function.",
3638 128,
3639 ®,
3640 &tracker,
3641 false,
3642 false,
3643 RoutingWorkload::Background,
3644 );
3645
3646 let top_scored = decision
3650 .candidates
3651 .first()
3652 .expect("routing should expose ranked candidates");
3653 let schema = reg
3654 .get(&top_scored.model_id)
3655 .expect("top-scored model should exist");
3656 assert!(
3657 schema.is_local(),
3658 "background routing should score the task-specific local code prior highest"
3659 );
3660 }
3661
3662 #[test]
3663 fn task_specific_latency_prior_affects_cold_start_score() {
3664 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
3665 let reg = test_registry();
3666 let model = reg
3667 .get("qwen/qwen3-8b:q4_k_m")
3668 .expect("local test model should exist");
3669
3670 let mut fast_tracker = OutcomeTracker::new();
3671 let mut fast_profile = crate::outcome::ModelProfile::new(model.id.clone());
3672 fast_profile.task_stats.insert(
3673 crate::outcome::InferenceTask::Generate.to_string(),
3674 crate::outcome::TaskStats {
3675 ema_quality: 0.95,
3676 avg_latency_ms: 1200.0,
3677 ..Default::default()
3678 },
3679 );
3680 fast_tracker.import_profiles(vec![fast_profile]);
3681
3682 let mut slow_tracker = OutcomeTracker::new();
3683 let mut slow_profile = crate::outcome::ModelProfile::new(model.id.clone());
3684 slow_profile.task_stats.insert(
3685 crate::outcome::InferenceTask::Generate.to_string(),
3686 crate::outcome::TaskStats {
3687 ema_quality: 0.95,
3688 avg_latency_ms: 120_000.0,
3689 ..Default::default()
3690 },
3691 );
3692 slow_tracker.import_profiles(vec![slow_profile]);
3693
3694 let fast_score = router.score_model(
3695 model,
3696 InferenceTask::Generate,
3697 &fast_tracker,
3698 RoutingWorkload::Interactive,
3699 0,
3700 0,
3701 0,
3702 0,
3703 );
3704 let slow_score = router.score_model(
3705 model,
3706 InferenceTask::Generate,
3707 &slow_tracker,
3708 RoutingWorkload::Interactive,
3709 0,
3710 0,
3711 0,
3712 0,
3713 );
3714
3715 assert!(
3716 fast_score > slow_score,
3717 "faster task latency prior should improve cold-start score ({fast_score} <= {slow_score})"
3718 );
3719 }
3720
3721 #[test]
3722 fn interactive_workload_keeps_remote_bootstrap_bias() {
3723 let router = AdaptiveRouter::new(
3724 test_hw(),
3725 RoutingConfig {
3726 prior_strength: 100.0,
3727 ..Default::default()
3728 },
3729 );
3730 let reg = test_registry();
3731 let mut tracker = OutcomeTracker::new();
3732 let mut profile = crate::outcome::ModelProfile::new("qwen/qwen3-8b:q4_k_m".into());
3733 profile.ema_quality = 0.95;
3734 tracker.import_profiles(vec![profile]);
3735
3736 let decision = router.route_context_aware(
3737 "Write a Python fibonacci function.",
3738 128,
3739 ®,
3740 &tracker,
3741 false,
3742 false,
3743 RoutingWorkload::Interactive,
3744 );
3745
3746 let schema = reg
3747 .get(&decision.model_id)
3748 .expect("selected model should exist");
3749 assert!(
3750 !schema.is_local(),
3751 "interactive routing should still prefer trusted remote models during cold start"
3752 );
3753 }
3754
3755 #[test]
3756 fn code_interactive_weights_prioritise_quality_over_speed_over_cost() {
3757 let (q, lat, cost) =
3763 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Interactive);
3764 assert!(
3765 q > lat && lat > cost,
3766 "expected quality > speed > cost, got ({q}, {lat}, {cost})"
3767 );
3768 assert!(
3769 (q + lat + cost - 1.0).abs() < 1e-9,
3770 "weights must sum to 1.0"
3771 );
3772 let (default_q, default_lat, default_cost) = RoutingWorkload::Interactive.weights();
3773 assert!(
3774 q > default_q && lat < default_lat && cost < default_cost,
3775 "code weighting must lift quality and cut latency/cost vs the generic interactive profile"
3776 );
3777
3778 assert_eq!(
3780 AdaptiveRouter::task_aware_weights(
3781 InferenceTask::Code,
3782 RoutingWorkload::LocalPreferred,
3783 ),
3784 (q, lat, cost),
3785 );
3786
3787 assert_eq!(
3790 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Fastest),
3791 RoutingWorkload::Fastest.weights(),
3792 );
3793 assert_eq!(
3795 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Background),
3796 RoutingWorkload::Background.weights(),
3797 );
3798 assert_eq!(
3800 AdaptiveRouter::task_aware_weights(
3801 InferenceTask::Generate,
3802 RoutingWorkload::Interactive,
3803 ),
3804 RoutingWorkload::Interactive.weights(),
3805 );
3806 }
3807
3808 #[test]
3809 fn fallback_chain_has_alternatives() {
3810 let router = AdaptiveRouter::new(
3811 test_hw(),
3812 RoutingConfig {
3813 prior_strength: 100.0, ..Default::default()
3815 },
3816 );
3817 let reg = test_registry();
3818 let tracker = OutcomeTracker::new();
3819
3820 let decision = router.route("Analyze the architecture trade-offs", ®, &tracker);
3821 assert!(!decision.fallbacks.is_empty());
3822 assert!(!decision.fallbacks.contains(&decision.model_id));
3824 }
3825
3826 #[test]
3827 fn latency_scoring_is_consistent() {
3828 let router = AdaptiveRouter::with_default_config(test_hw());
3830
3831 let schema_score = router.latency_ms_to_score(AdaptiveRouter::tps_to_latency_ms(25.0));
3833 let observed_score = router.latency_ms_to_score(8000.0);
3835 assert!(
3836 (schema_score - observed_score).abs() < 0.01,
3837 "schema ({schema_score}) and observed ({observed_score}) should match"
3838 );
3839 }
3840
3841 #[test]
3842 fn complexity_assessment() {
3843 assert_eq!(
3844 TaskComplexity::assess("What is the capital of France?"),
3845 TaskComplexity::Simple
3846 );
3847 assert_eq!(
3848 TaskComplexity::assess("Fix this broken test"),
3849 TaskComplexity::Code
3850 );
3851 assert_eq!(
3852 TaskComplexity::assess("Analyze the trade-offs between A and B"),
3853 TaskComplexity::Complex
3854 );
3855 }
3856
3857 #[test]
3858 fn beta_sampling_produces_valid_values() {
3859 let mut rng = rand::rng();
3860 for _ in 0..100 {
3862 let s = sample_beta(&mut rng, 2.0, 5.0);
3863 assert!((0.0..=1.0).contains(&s), "sample {s} out of [0,1] range");
3864 }
3865 let samples: Vec<f64> = (0..1000).map(|_| sample_beta(&mut rng, 1.0, 1.0)).collect();
3867 let mean = samples.iter().sum::<f64>() / samples.len() as f64;
3868 assert!(
3869 (mean - 0.5).abs() < 0.05,
3870 "Beta(1,1) mean {mean} should be ~0.5"
3871 );
3872 }
3873
3874 #[test]
3875 fn thompson_sampling_converges_to_best() {
3876 let router = AdaptiveRouter::new(
3878 test_hw(),
3879 RoutingConfig {
3880 prior_strength: 1.0, ..Default::default()
3882 },
3883 );
3884 let reg = test_registry();
3885 let mut tracker = OutcomeTracker::new();
3886
3887 let qwen_4b_id = QWEN3_4B_ROUTED_ID;
3889 for _ in 0..20 {
3890 let trace = tracker.record_start(qwen_4b_id, InferenceTask::Code, "test");
3891 tracker.record_complete(&trace, 500, 100, 50);
3892 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3893 }
3894
3895 let mut wins = 0;
3897 for _ in 0..20 {
3898 let decision = router.route("Fix this parser bug", ®, &tracker);
3899 if decision.model_id == qwen_4b_id {
3900 wins += 1;
3901 }
3902 }
3903 assert!(
3904 wins >= 14,
3905 "strong model won only {wins}/20 times (expected >= 14)"
3906 );
3907 }
3908
3909 #[test]
3912 fn cold_start_never_returns_a_model_lacking_a_required_capability() {
3913 let router = AdaptiveRouter::new(
3921 test_hw(),
3922 RoutingConfig {
3923 quality_first_cold_start: false,
3924 ..Default::default()
3925 },
3926 );
3927 let reg = test_registry();
3928
3929 for caps in [
3930 vec![ModelCapability::Code],
3931 vec![ModelCapability::Code, ModelCapability::ToolUse],
3932 ] {
3933 let decision = router.cold_start_decision(
3936 TaskComplexity::Simple,
3937 InferenceTask::Code,
3938 &caps,
3939 ®,
3940 false,
3941 &std::collections::HashSet::new(),
3942 false,
3943 AvailabilitySnapshot::capture(®),
3944 );
3945 let chosen = reg
3946 .find_by_name(&decision.model_name)
3947 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
3948 .unwrap_or_else(|| panic!("cold start named an unknown model: {decision:?}"));
3949 for cap in &caps {
3950 assert!(
3951 chosen.has_capability(*cap),
3952 "cold start chose {} which lacks required {:?} (caps {:?})",
3953 chosen.name,
3954 cap,
3955 caps
3956 );
3957 }
3958 assert_eq!(
3965 decision.strategy,
3966 RoutingStrategy::SchemaBased,
3967 "expected schema-based cold start"
3968 );
3969 assert!(
3970 decision.reason.contains("capability-satisfying"),
3971 "expected a capability-satisfying cold-start branch, got: {}",
3972 decision.reason
3973 );
3974 }
3975 }
3976
3977 #[test]
3978 fn cold_start_skips_a_capable_local_that_exceeds_the_memory_budget() {
3979 let mut hw = test_hw();
3984 hw.max_model_mb = 300; let router = AdaptiveRouter::new(
3986 hw,
3987 RoutingConfig {
3988 quality_first_cold_start: false,
3989 ..Default::default()
3990 },
3991 );
3992 let reg = test_registry();
3993 let decision = router.cold_start_decision(
3994 TaskComplexity::Code,
3995 InferenceTask::Code,
3996 &[ModelCapability::Code],
3997 ®,
3998 false,
3999 &std::collections::HashSet::new(),
4000 false,
4001 AvailabilitySnapshot::capture(®),
4002 );
4003 assert!(
4004 !decision.reason.contains("local cold start"),
4005 "no local fits the 300MB budget, so the local branch must not fire: {}",
4006 decision.reason
4007 );
4008 }
4009
4010 #[test]
4011 fn intent_require_filters_out_models_lacking_capability() {
4012 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4017 let reg = test_registry();
4018 let tracker = OutcomeTracker::new();
4019
4020 let intent = crate::intent::IntentHint {
4021 require: vec![ModelCapability::Vision],
4022 ..Default::default()
4023 };
4024 let decision = router.route_with_intent("hello", ®, &tracker, &intent);
4025
4026 assert_eq!(
4031 decision.strategy,
4032 RoutingStrategy::SchemaBased,
4033 "require=[vision] with no vision-capable candidates must drop to schema cold-start"
4034 );
4035 }
4036
4037 #[test]
4038 fn unbenchmarked_remote_is_conservative_not_assumed_frontier() {
4039 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4048 let reg = test_registry();
4049 let remote = reg
4050 .list()
4051 .into_iter()
4052 .find(|m| m.is_remote() && m.public_benchmarks.is_empty())
4053 .expect("a benchmark-less remote in the fixture")
4054 .clone();
4055 let q = router.schema_quality_estimate(&remote);
4056 assert!(
4057 (q - 0.60).abs() < 1e-9,
4058 "unmeasured curated remote should be a conservative 0.60, got {q}"
4059 );
4060
4061 let mut benched = remote.clone();
4065 benched.public_benchmarks = vec![
4066 crate::schema::BenchmarkScore {
4067 name: "A".into(),
4068 score: 0.9,
4069 harness: None,
4070 source_url: None,
4071 measured_at: None,
4072 },
4073 crate::schema::BenchmarkScore {
4074 name: "B".into(),
4075 score: 0.7,
4076 harness: None,
4077 source_url: None,
4078 measured_at: None,
4079 },
4080 ];
4081 let qb = router.schema_quality_estimate(&benched);
4082 assert!((qb - 0.8).abs() < 1e-9);
4083 assert!(
4084 qb > q,
4085 "real benchmarks must lift a model above the unmeasured default"
4086 );
4087 }
4088
4089 #[test]
4090 fn calibrated_benchmark_is_normalized_onto_the_tier_scale() {
4091 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4095 let bench = |raw: f64| {
4096 let mut m = outcome_model("m", None);
4097 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4098 name: "tau-bench-airline".into(),
4099 score: raw,
4100 harness: None,
4101 source_url: None,
4102 measured_at: None,
4103 }];
4104 router.schema_quality_estimate(&m)
4105 };
4106 let frontier = bench(0.65); let strong = bench(0.35); assert!(
4110 (frontier - 0.92).abs() < 0.01,
4111 "best-in-class → ~CEIL, got {frontier}"
4112 );
4113 assert!(
4114 (strong - 0.68).abs() < 0.02,
4115 "0.35 raw → ~0.68 tier, got {strong}"
4116 );
4117 assert!(
4118 strong > router.schema_quality_estimate(&local_model("local-30b", 18_000)),
4119 "a measured frontier model must out-rank an unmeasured 30B local"
4120 );
4121 }
4122
4123 #[test]
4124 fn uncalibrated_benchmark_is_clamped_into_the_tier_band() {
4125 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4129 let q = |raw: f64| {
4130 let mut m = outcome_model("m", None);
4131 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4132 name: "some-new-unknown-bench".into(),
4133 score: raw,
4134 harness: None,
4135 source_url: None,
4136 measured_at: None,
4137 }];
4138 router.schema_quality_estimate(&m)
4139 };
4140 assert!((q(0.73) - 0.73).abs() < 1e-9);
4143 assert!((q(0.10) - AdaptiveRouter::BENCH_TIER_FLOOR).abs() < 1e-9);
4144 assert!((q(0.99) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9);
4145 }
4146
4147 #[test]
4148 fn local_uncalibrated_benchmark_maps_into_the_local_tier_band() {
4149 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4155 let q = |raw: f64| {
4156 let mut m = local_model("local-bench", 2500); m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4158 name: "car-agentic-suite".into(), score: raw,
4160 harness: None,
4161 source_url: None,
4162 measured_at: None,
4163 }];
4164 router.schema_quality_estimate(&m)
4165 };
4166 assert!(
4170 (q(0.45) - 0.45).abs() < 1e-9,
4171 "in-band local score passes through"
4172 );
4173 assert!(
4174 (q(0.05) - AdaptiveRouter::LOCAL_BENCH_TIER_FLOOR).abs() < 1e-9,
4175 "sub-floor clamps to LOCAL floor 0.30, not frontier FLOOR 0.40"
4176 );
4177 assert!(
4178 (q(0.95) - AdaptiveRouter::LOCAL_BENCH_TIER_CEIL).abs() < 1e-9,
4179 "a perfect local-suite score tops at LOCAL ceil 0.60, not frontier CEIL 0.92"
4180 );
4181 assert!(q(0.55) > q(0.35));
4183 let mut frontier = outcome_model("frontier", None);
4186 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
4187 name: "tau-bench-airline".into(),
4188 score: 0.65,
4189 harness: None,
4190 source_url: None,
4191 measured_at: None,
4192 }];
4193 assert!(
4194 router.schema_quality_estimate(&frontier) > q(0.95),
4195 "a benchmarked frontier model out-ranks even a perfect-scoring local"
4196 );
4197 }
4198
4199 #[test]
4200 fn frontier_ref_is_derived_from_catalog_best_in_class() {
4201 use std::collections::HashMap;
4206 let mut max_by_bench: HashMap<String, f64> = HashMap::new();
4207 for model in crate::registry::builtin_catalog() {
4208 for b in &model.public_benchmarks {
4209 let e = max_by_bench.entry(b.name.clone()).or_insert(0.0);
4210 *e = e.max(b.score);
4211 }
4212 }
4213 assert!(!max_by_bench.is_empty(), "catalog should carry benchmarks");
4214 for (name, max) in max_by_bench {
4215 let derived = AdaptiveRouter::benchmark_frontier_ref(&name)
4216 .unwrap_or_else(|| panic!("benchmark '{name}' must derive a frontier_ref"));
4217 assert!(
4218 (derived - max).abs() < 1e-9,
4219 "frontier_ref for '{name}' should be the catalog max {max}, got {derived}"
4220 );
4221 }
4222 assert_eq!(
4224 AdaptiveRouter::benchmark_frontier_ref("tau-bench-airline"),
4225 Some(0.65)
4226 );
4227 assert_eq!(
4229 AdaptiveRouter::benchmark_frontier_ref("nonexistent-bench"),
4230 None
4231 );
4232 }
4233
4234 fn benchmarked_model(id: &str, bench: &str, score: f64, frontier: bool) -> ModelSchema {
4238 let mut m = outcome_model(id, None);
4239 if frontier {
4240 m.tags.push("frontier".into());
4241 }
4242 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4243 name: bench.into(),
4244 score,
4245 harness: None,
4246 source_url: None,
4247 measured_at: None,
4248 }];
4249 m
4250 }
4251
4252 #[test]
4253 fn frontier_ref_guard_drops_frontier_less_benchmark() {
4254 let models = [
4259 benchmarked_model("mid-a", "midtier-only-bench", 0.50, false),
4260 benchmarked_model("mid-b", "midtier-only-bench", 0.40, false),
4261 ];
4262 let refs = compute_frontier_refs(models.iter());
4263 assert!(
4264 !refs.contains_key("midtier-only-bench"),
4265 "a benchmark with no frontier-class carrier must not self-anchor, got {refs:?}"
4266 );
4267 }
4268
4269 #[test]
4270 fn frontier_ref_guard_anchors_when_a_frontier_model_carries_the_benchmark() {
4271 let models = [
4276 benchmarked_model("frontier-x", "shared-bench", 0.60, true),
4277 benchmarked_model("mid-y", "shared-bench", 0.70, false),
4278 ];
4279 let refs = compute_frontier_refs(models.iter());
4280 assert_eq!(
4281 refs.get("shared-bench").copied(),
4282 Some(0.70),
4283 "anchor is the catalog max across all carriers once a frontier carrier exists"
4284 );
4285 }
4286
4287 #[test]
4288 fn frontier_ref_guard_is_noop_on_todays_catalog() {
4289 use std::collections::HashMap;
4294 let catalog = crate::registry::builtin_catalog();
4295 let mut raw_max: HashMap<String, f64> = HashMap::new();
4296 for m in &catalog {
4297 for b in &m.public_benchmarks {
4298 let e = raw_max.entry(b.name.clone()).or_insert(0.0);
4299 *e = e.max(b.score);
4300 }
4301 }
4302 let guarded = compute_frontier_refs(catalog.iter());
4303 assert_eq!(
4304 raw_max, guarded,
4305 "guard dropped a benchmark — a catalog benchmark lost its frontier-class carrier"
4306 );
4307 assert!(
4308 model_is_frontier_class(
4309 catalog
4310 .iter()
4311 .find(|m| m.id == "anthropic/claude-opus-4-7:latest")
4312 .expect("opus-4-7 in catalog")
4313 ),
4314 "the tau-bench-airline champion must be frontier-tagged for the anchor to be valid"
4315 );
4316 }
4317
4318 #[test]
4319 fn score_above_the_frontier_clamps_to_ceil() {
4320 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4324 let mut m = outcome_model("future-sota", None);
4325 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4326 name: "tau-bench-airline".into(),
4327 score: 0.80, harness: None,
4329 source_url: None,
4330 measured_at: None,
4331 }];
4332 assert!(
4333 (router.schema_quality_estimate(&m) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9,
4334 "above-frontier score should clamp to CEIL"
4335 );
4336 }
4337
4338 #[test]
4339 fn local_size_heuristic_stays_below_measured_frontier_floor() {
4340 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4344 for size_mb in [500u64, 1500, 2500, 4500, 18_000] {
4345 let q = router.schema_quality_estimate(&local_model("local", size_mb));
4346 assert!(
4347 q <= AdaptiveRouter::BENCH_TIER_FLOOR + 0.21, "local {size_mb}MB quality {q} must stay near/below the measured floor"
4349 );
4350 }
4351 }
4352
4353 #[test]
4354 fn community_remote_quality_is_damped_below_curated() {
4355 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4360 let reg = test_registry();
4361 let mut remote = reg
4362 .list()
4363 .into_iter()
4364 .find(|m| m.is_remote() && m.public_benchmarks.is_empty())
4365 .expect("a benchmark-less remote in the fixture")
4366 .clone();
4367
4368 remote.trust_tier = crate::schema::TrustTier::Curated;
4370 let qc = router.schema_quality_estimate(&remote);
4371 assert!(
4372 (qc - 0.60).abs() < 1e-9,
4373 "curated unmeasured remote should be 0.60, got {qc}"
4374 );
4375
4376 remote.trust_tier = crate::schema::TrustTier::Community;
4378 let q = router.schema_quality_estimate(&remote);
4379 assert!(
4380 (q - 0.48).abs() < 1e-9,
4381 "community remote should be damped to 0.48, got {q}"
4382 );
4383 assert!(
4384 q < qc,
4385 "community remote must rank below an unmeasured curated remote"
4386 );
4387 }
4388
4389 #[test]
4390 fn quality_workload_is_quality_dominant() {
4391 let (q, lat, cost) = RoutingWorkload::Quality.weights();
4394 assert!(
4395 (q + lat + cost - 1.0).abs() < 1e-9,
4396 "weights must sum to 1.0"
4397 );
4398 assert!(q >= 0.8, "quality must dominate, got {q}");
4399 assert!(
4400 lat <= 0.1 && cost <= 0.15,
4401 "latency/cost near floor: {lat}/{cost}"
4402 );
4403 for w in [
4405 RoutingWorkload::Interactive,
4406 RoutingWorkload::Batch,
4407 RoutingWorkload::Background,
4408 RoutingWorkload::LocalPreferred,
4409 RoutingWorkload::Fastest,
4410 ] {
4411 assert!(
4412 q > w.weights().0,
4413 "Quality must out-weight {w:?} on quality"
4414 );
4415 }
4416 }
4417
4418 #[test]
4419 fn prefer_quality_scores_the_more_capable_model_higher() {
4420 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4425 let reg = test_registry();
4426 let tracker = OutcomeTracker::new();
4427 let small = reg
4428 .find_by_name("Qwen3-1.7B")
4429 .expect("1.7B in fixture")
4430 .clone();
4431 let big = reg.find_by_name("Qwen3-4B").expect("4B in fixture").clone();
4432 assert!(small.has_capability(ModelCapability::Code));
4433 assert!(big.has_capability(ModelCapability::Code));
4434
4435 let score = |m: &ModelSchema, w: RoutingWorkload| {
4436 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
4437 };
4438 assert!(
4440 score(&big, RoutingWorkload::Quality) > score(&small, RoutingWorkload::Quality),
4441 "prefer_quality must rank the capable 4B above the cheap 1.7B"
4442 );
4443 }
4444
4445 fn local_mlx_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
4448 let mut m = outcome_model(id, benchmark);
4449 m.provider = "local".into();
4450 m.source = crate::schema::ModelSource::Mlx {
4451 hf_repo: "mlx-community/test".into(),
4452 hf_weight_file: None,
4453 };
4454 m
4455 }
4456
4457 #[test]
4458 fn quality_workload_suppresses_local_mlx_bonus_vs_capable_remote() {
4459 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4464 let tracker = OutcomeTracker::new();
4465 let score = |m: &ModelSchema, w: RoutingWorkload| {
4466 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
4467 };
4468
4469 let remote_capable = outcome_model("remote/frontier", Some(0.85));
4470 let mlx_tiny = local_mlx_model("mlx/tiny", Some(0.55));
4471 let remote_peer = outcome_model("remote/peer", Some(0.55));
4473 assert!(mlx_tiny.is_mlx() && mlx_tiny.is_local());
4474
4475 assert!(
4479 score(&mlx_tiny, RoutingWorkload::Background)
4480 > score(&remote_peer, RoutingWorkload::Background),
4481 "the local/MLX bonus should still apply on non-Quality workloads"
4482 );
4483
4484 assert!(
4487 score(&remote_capable, RoutingWorkload::Quality)
4488 > score(&mlx_tiny, RoutingWorkload::Quality),
4489 "prefer_quality must suppress the locality bonus so the capable remote wins"
4490 );
4491 }
4492
4493 fn local_model(id: &str, size_mb: u64) -> ModelSchema {
4498 let mut m = outcome_model(id, None);
4499 m.provider = "local".into();
4500 m.source = crate::schema::ModelSource::Local {
4501 hf_repo: "test/repo".into(),
4502 hf_filename: "model.gguf".into(),
4503 tokenizer_repo: "test/repo".into(),
4504 };
4505 m.cost.size_mb = Some(size_mb);
4506 m
4507 }
4508
4509 #[test]
4520 fn a_measured_moe_rate_is_not_discounted_again() {
4521 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4522
4523 let mut measured = local_model("local/moe-measured", 17_000);
4524 measured.tags.push("moe".into());
4525 measured.performance.tokens_per_second = Some(35.0);
4526 measured.performance.latency_p50_ms = Some(1_200);
4527
4528 let mut declared = local_model("local/moe-declared", 17_000);
4530 declared.tags.push("moe".into());
4531 declared.performance.tokens_per_second = Some(35.0);
4532
4533 let mut dense = local_model("local/dense", 17_000);
4535 dense.performance.tokens_per_second = Some(35.0);
4536 dense.performance.latency_p50_ms = Some(1_200);
4537
4538 let measured_score = router.schema_latency_estimate(&measured);
4539 let declared_score = router.schema_latency_estimate(&declared);
4540 let dense_score = router.schema_latency_estimate(&dense);
4541
4542 assert_eq!(
4543 measured_score, dense_score,
4544 "a measured MoE rate must score as the rate it measured"
4545 );
4546 assert!(
4547 declared_score < measured_score,
4548 "a declared MoE rate still gets the discount: {declared_score} vs {measured_score}"
4549 );
4550 }
4551
4552 fn outcome_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
4556 ModelSchema {
4557 id: id.into(),
4558 name: id.into(),
4559 provider: "openai".into(),
4560 family: "test".into(),
4561 version: "latest".into(),
4562 capabilities: vec![
4563 ModelCapability::Generate,
4564 ModelCapability::Code,
4565 ModelCapability::Reasoning,
4566 ],
4567 context_length: 128_000,
4568 max_output_tokens: None,
4569 param_count: "api".into(),
4570 quantization: None,
4571 performance: Default::default(),
4572 cost: Default::default(),
4573 source: crate::schema::ModelSource::RemoteApi {
4574 endpoint: "https://api.openai.com/v1".into(),
4575 api_key_env: "OPENAI_API_KEY".into(),
4576 api_key_envs: vec![],
4577 api_version: None,
4578 protocol: crate::schema::ApiProtocol::OpenAiCompat,
4579 },
4580 tags: vec![],
4581 supported_params: vec![],
4582 public_benchmarks: benchmark
4583 .map(|s| {
4584 vec![crate::schema::BenchmarkScore {
4585 name: "t".into(),
4586 score: s,
4587 harness: None,
4588 source_url: None,
4589 measured_at: None,
4590 }]
4591 })
4592 .unwrap_or_default(),
4593 trust_tier: crate::schema::TrustTier::Curated,
4594 deprecated: false,
4595 available: true,
4596 weights_ready: true,
4597 }
4598 }
4599
4600 #[test]
4601 fn reliability_band_applies_to_quality_and_substantive_normal_lanes() {
4602 use RoutingWorkload as W;
4603 assert!(AdaptiveRouter::applies_reliability_band(
4605 InferenceTask::Code,
4606 W::Interactive
4607 ));
4608 assert!(AdaptiveRouter::applies_reliability_band(
4609 InferenceTask::Reasoning,
4610 W::Batch
4611 ));
4612 assert!(AdaptiveRouter::applies_reliability_band(
4615 InferenceTask::Code,
4616 W::Quality
4617 ));
4618 assert!(AdaptiveRouter::applies_reliability_band(
4619 InferenceTask::Generate,
4620 W::Quality
4621 ));
4622 assert!(!AdaptiveRouter::applies_reliability_band(
4624 InferenceTask::Generate,
4625 W::Interactive
4626 ));
4627 assert!(!AdaptiveRouter::applies_reliability_band(
4629 InferenceTask::Code,
4630 W::Background
4631 ));
4632 assert!(!AdaptiveRouter::applies_reliability_band(
4633 InferenceTask::Code,
4634 W::Fastest
4635 ));
4636 assert!(!AdaptiveRouter::applies_reliability_band(
4638 InferenceTask::Code,
4639 W::LocalPreferred
4640 ));
4641 }
4642
4643 #[test]
4644 fn high_stakes_intent_forces_quality_over_everything() {
4645 use crate::intent::IntentHint;
4646 let hs = IntentHint {
4649 high_stakes: true,
4650 prefer_fast: true,
4651 prefer_local: true,
4652 ..Default::default()
4653 };
4654 assert_eq!(
4655 AdaptiveRouter::resolve_workload(Some(&hs), RoutingWorkload::Interactive),
4656 RoutingWorkload::Quality
4657 );
4658 let f = IntentHint {
4660 prefer_fast: true,
4661 ..Default::default()
4662 };
4663 assert_eq!(
4664 AdaptiveRouter::resolve_workload(Some(&f), RoutingWorkload::Interactive),
4665 RoutingWorkload::Fastest
4666 );
4667 assert_eq!(
4669 AdaptiveRouter::resolve_workload(None, RoutingWorkload::Batch),
4670 RoutingWorkload::Batch
4671 );
4672 }
4673
4674 #[test]
4675 fn band_excludes_a_real_quality_gap_even_if_cheaper() {
4676 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4678 let tracker = OutcomeTracker::new();
4679 let strong = outcome_model("strong", Some(0.90));
4680 let weak_cheap = outcome_model("weak-cheap", Some(0.83));
4681 let band =
4682 router.outcome_first_band(vec![strong, weak_cheap], InferenceTask::Code, &tracker);
4683 assert_eq!(
4684 band.len(),
4685 1,
4686 "0.07 gap (> band 0.02) must drop the weaker model"
4687 );
4688 assert_eq!(band[0].id, "strong");
4689 }
4690
4691 #[test]
4692 fn band_keeps_genuinely_near_equal_models() {
4693 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4695 let tracker = OutcomeTracker::new();
4696 let band = router.outcome_first_band(
4697 vec![
4698 outcome_model("a", Some(0.90)),
4699 outcome_model("a2", Some(0.89)),
4700 ],
4701 InferenceTask::Code,
4702 &tracker,
4703 );
4704 assert_eq!(
4705 band.len(),
4706 2,
4707 "0.01 gap (<= band 0.02) keeps both for tie-break"
4708 );
4709 }
4710
4711 #[test]
4712 fn band_cold_start_frontier_beats_cheap_unknown() {
4713 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4716 let tracker = OutcomeTracker::new();
4717 let band = router.outcome_first_band(
4718 vec![
4719 outcome_model("frontier", Some(0.85)),
4720 outcome_model("unknown", None),
4721 ],
4722 InferenceTask::Code,
4723 &tracker,
4724 );
4725 assert_eq!(band.len(), 1);
4726 assert_eq!(
4727 band[0].id, "frontier",
4728 "0.85 vs 0.60 unknown → unknown excluded"
4729 );
4730 }
4731
4732 #[test]
4733 fn quality_lane_bands_out_a_weak_local_but_local_preferred_keeps_it() {
4734 let mut reg = test_registry();
4741 let mut frontier = outcome_model("frontier-remote", None);
4742 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
4743 name: "tau-bench-airline".into(),
4744 score: 0.65, harness: None,
4746 source_url: None,
4747 measured_at: None,
4748 }];
4749 reg.register_project_model(frontier);
4750
4751 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4752 let tracker = OutcomeTracker::new();
4753 let prompt = "Implement a balanced binary search tree with deletion.";
4754
4755 let q = router.route_context_aware(
4758 prompt,
4759 256,
4760 ®,
4761 &tracker,
4762 false,
4763 false,
4764 RoutingWorkload::Quality,
4765 );
4766 assert_eq!(
4767 q.model_id, "frontier-remote",
4768 "prefer_quality must pick the benchmarked frontier over locals-on-bonuses"
4769 );
4770 assert_eq!(
4771 q.strategy,
4772 RoutingStrategy::SchemaBased,
4773 "Quality lane must use deterministic argmax, not Thompson exploration"
4774 );
4775 assert!(
4776 q.candidates.iter().any(|c| !c.in_band),
4777 "Quality lane must apply a reliability band (some candidates excluded)"
4778 );
4779
4780 let lp = router.route_context_aware(
4783 prompt,
4784 256,
4785 ®,
4786 &tracker,
4787 false,
4788 false,
4789 RoutingWorkload::LocalPreferred,
4790 );
4791 assert!(
4792 lp.candidates.iter().all(|c| c.in_band),
4793 "LocalPreferred must NOT band — all candidates eligible"
4794 );
4795 }
4796
4797 #[test]
4798 fn band_never_empties() {
4799 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4800 let tracker = OutcomeTracker::new();
4801 assert_eq!(
4803 router
4804 .outcome_first_band(
4805 vec![outcome_model("only", Some(0.5))],
4806 InferenceTask::Code,
4807 &tracker
4808 )
4809 .len(),
4810 1
4811 );
4812 let band = router.outcome_first_band(
4814 vec![
4815 outcome_model("x", Some(0.40)),
4816 outcome_model("y", Some(0.41)),
4817 ],
4818 InferenceTask::Code,
4819 &tracker,
4820 );
4821 assert!(!band.is_empty() && band.iter().any(|m| m.id == "y"));
4822 }
4823
4824 #[test]
4825 fn outcome_first_band_is_wired_into_selection_and_keeps_fallbacks() {
4826 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-route");
4833 unsafe {
4834 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
4835 }
4836 let mut reg = UnifiedRegistry::new_empty(tmp);
4837 let mut strong = outcome_model("strong-expensive", Some(0.90));
4838 strong.cost = crate::schema::CostModel {
4839 input_per_mtok: Some(20.0),
4840 output_per_mtok: Some(20.0),
4841 ..Default::default()
4842 };
4843 let mut weak = outcome_model("weak-cheap", Some(0.80));
4844 weak.cost = crate::schema::CostModel {
4845 input_per_mtok: Some(0.1),
4846 output_per_mtok: Some(0.1),
4847 ..Default::default()
4848 };
4849 reg.register_project_model(strong);
4850 reg.register_project_model(weak);
4851
4852 let router = AdaptiveRouter::new(
4853 test_hw(),
4854 RoutingConfig {
4855 prior_strength: 100.0, ..Default::default()
4857 },
4858 );
4859 let tracker = OutcomeTracker::new();
4860 let decision = router.route(
4862 "Analyze the architecture trade-offs in depth",
4863 ®,
4864 &tracker,
4865 );
4866
4867 assert_eq!(
4868 decision.model_id, "strong-expensive",
4869 "band must select the higher-reliability model even though it's pricier"
4870 );
4871 assert!(
4872 decision.fallbacks.contains(&"weak-cheap".to_string()),
4873 "the out-of-band weaker model must remain as a fallback, got {:?}",
4874 decision.fallbacks
4875 );
4876 }
4877
4878 #[test]
4879 fn outcome_first_fallbacks_rank_in_band_before_out_of_band() {
4880 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-fb-order");
4884 unsafe {
4885 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
4886 }
4887 let mut reg = UnifiedRegistry::new_empty(tmp);
4888 reg.register_project_model(outcome_model("strong", Some(0.90)));
4889 let mut peer = outcome_model("peer-in-band", Some(0.89)); peer.cost = crate::schema::CostModel {
4891 input_per_mtok: Some(30.0),
4892 output_per_mtok: Some(30.0),
4893 ..Default::default()
4894 };
4895 reg.register_project_model(peer);
4896 let mut weak = outcome_model("weak-out-of-band", Some(0.70)); weak.cost = crate::schema::CostModel {
4898 input_per_mtok: Some(0.01),
4899 output_per_mtok: Some(0.01),
4900 ..Default::default()
4901 };
4902 reg.register_project_model(weak);
4903
4904 let router = AdaptiveRouter::new(
4905 test_hw(),
4906 RoutingConfig {
4907 prior_strength: 100.0,
4908 ..Default::default()
4909 },
4910 );
4911 let tracker = OutcomeTracker::new();
4912 let decision = router.route(
4913 "Analyze the architecture trade-offs in depth",
4914 ®,
4915 &tracker,
4916 );
4917
4918 assert!(
4920 decision.model_id == "strong" || decision.model_id == "peer-in-band",
4921 "primary must be from the band, got {}",
4922 decision.model_id
4923 );
4924 let pos = |id: &str| decision.fallbacks.iter().position(|x| x == id);
4925 let in_band_id = if decision.model_id == "strong" {
4926 "peer-in-band"
4927 } else {
4928 "strong"
4929 };
4930 let in_band_pos = pos(in_band_id).expect("in-band peer must be a fallback");
4931 let weak_pos = pos("weak-out-of-band").expect("weak must remain a fallback (availability)");
4932 assert!(
4933 in_band_pos < weak_pos,
4934 "in-band fallback ({in_band_id}@{in_band_pos}) must precede out-of-band weak (@{weak_pos})"
4935 );
4936 }
4937
4938 #[test]
4939 fn reliability_is_cost_free() {
4940 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4943 let tracker = OutcomeTracker::new();
4944 let mut cheap = outcome_model("cheap", Some(0.8));
4945 cheap.cost = crate::schema::CostModel {
4946 input_per_mtok: Some(0.1),
4947 output_per_mtok: Some(0.1),
4948 ..Default::default()
4949 };
4950 let mut pricey = outcome_model("pricey", Some(0.8));
4951 pricey.cost = crate::schema::CostModel {
4952 input_per_mtok: Some(50.0),
4953 output_per_mtok: Some(50.0),
4954 ..Default::default()
4955 };
4956 let rc = router.reliability(&cheap, InferenceTask::Code, &tracker);
4957 let rp = router.reliability(&pricey, InferenceTask::Code, &tracker);
4958 assert!(
4959 (rc - rp).abs() < 1e-9,
4960 "reliability must not depend on cost"
4961 );
4962 }
4963
4964 #[test]
4965 fn cold_start_with_no_sample_size_sits_at_schema_estimate() {
4966 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4971 let unmeasured = outcome_model("unmeasured", None); let schema = router.schema_quality_estimate(&unmeasured);
4973
4974 let mut tracker = OutcomeTracker::new();
4975 let mut p = crate::outcome::ModelProfile::new("unmeasured".into());
4976 p.ema_quality = 0.5; tracker.import_profiles(vec![p]);
4978
4979 let r = router.reliability(&unmeasured, InferenceTask::Code, &tracker);
4980 assert!(
4981 (r - schema).abs() < 1e-9,
4982 "no sample size ⇒ reliability == schema estimate ({schema}), got {r}"
4983 );
4984 }
4985
4986 #[test]
4987 fn sparse_seeded_prior_barely_perturbs_schema_anchor() {
4988 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4991 let benchmarked = outcome_model("frontier", Some(0.55)); let mut tracker = OutcomeTracker::new();
4994 let mut p = crate::outcome::ModelProfile::new("frontier".into());
4995 p.ema_quality = 1.0;
4996 p.prior_sample_size = 1;
4997 p.task_stats.insert(
4998 InferenceTask::Code.to_string(),
4999 crate::outcome::TaskStats {
5000 ema_quality: 1.0,
5001 prior_sample_size: 1,
5002 ..Default::default()
5003 },
5004 );
5005 tracker.import_profiles(vec![p]);
5006
5007 let r = router.reliability(&benchmarked, InferenceTask::Code, &tracker);
5008 assert!((r - 0.64).abs() < 0.01, "expected ~0.64, got {r}");
5010 assert!(
5011 r < 0.7,
5012 "a 1-case prior must not pull reliability near its raw 1.0, got {r}"
5013 );
5014 }
5015
5016 #[test]
5017 fn dense_seeded_prior_is_trusted_near_raw() {
5018 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5021 let m = outcome_model("dense", Some(0.55)); let mut tracker = OutcomeTracker::new();
5024 let mut p = crate::outcome::ModelProfile::new("dense".into());
5025 p.ema_quality = 0.95;
5026 p.prior_sample_size = 40;
5027 tracker.import_profiles(vec![p]);
5028
5029 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5030 assert!(
5032 r > 0.88,
5033 "dense prior should be trusted near raw 0.95, got {r}"
5034 );
5035 }
5036
5037 #[test]
5038 fn warm_ungraded_successes_sit_at_schema_not_neutral_ema() {
5039 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5044 let frontier = outcome_model("frontier", Some(0.55)); let schema = router.schema_quality_estimate(&frontier);
5046
5047 let mut tracker = OutcomeTracker::new();
5048 let mut p = crate::outcome::ModelProfile::new("frontier".into());
5049 p.total_calls = 9;
5050 p.success_count = 9; p.ema_quality = 0.5; p.quality_observations = 0;
5053 tracker.import_profiles(vec![p]);
5054
5055 let r = router.reliability(&frontier, InferenceTask::Code, &tracker);
5056 assert!(
5057 (r - schema).abs() < 1e-9,
5058 "ungraded successes ⇒ reliability == schema ({schema}), not neutral EMA; got {r}"
5059 );
5060 }
5061
5062 #[test]
5063 fn ungraded_task_stats_with_zero_ema_still_resolve_to_schema() {
5064 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5071 let m = outcome_model("mech-only", Some(0.55)); let schema = router.schema_quality_estimate(&m);
5073
5074 let mut tracker = OutcomeTracker::new();
5075 let mut p = crate::outcome::ModelProfile::new("mech-only".into());
5076 p.total_calls = 7;
5077 p.success_count = 7;
5078 p.task_stats.insert(
5081 InferenceTask::Code.to_string(),
5082 crate::outcome::TaskStats {
5083 calls: 7,
5084 successes: 7,
5085 ..Default::default()
5086 },
5087 );
5088 tracker.import_profiles(vec![p]);
5089
5090 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5091 assert!(
5092 (r - schema).abs() < 1e-9,
5093 "ungraded task stats (ema 0.0, e=0) must resolve to schema {schema}, got {r}"
5094 );
5095 }
5096
5097 #[test]
5098 fn graded_outcomes_earn_trust_in_the_live_ema() {
5099 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5102 let m = outcome_model("graded", Some(0.55)); let mut tracker = OutcomeTracker::new();
5105 let mut p = crate::outcome::ModelProfile::new("graded".into());
5106 p.total_calls = 30;
5107 p.ema_quality = 0.9;
5108 p.quality_observations = 30; tracker.import_profiles(vec![p]);
5110
5111 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5112 assert!(
5114 r > 0.83,
5115 "dense graded signal should trust the live EMA, got {r}"
5116 );
5117 }
5118
5119 #[test]
5120 fn live_grades_outweigh_benchmark_cases_of_equal_count() {
5121 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5127 let benched = outcome_model("benched", Some(0.7));
5128 let graded = outcome_model("graded", Some(0.7));
5129
5130 let mut tracker = OutcomeTracker::new();
5131 let mut pb = crate::outcome::ModelProfile::new("benched".into());
5132 pb.ema_quality = 0.2;
5133 pb.prior_sample_size = 4; let mut pg = crate::outcome::ModelProfile::new("graded".into());
5135 pg.ema_quality = 0.2;
5136 pg.total_calls = 4;
5137 pg.quality_observations = 4; tracker.import_profiles(vec![pb, pg]);
5139
5140 let r_bench = router.reliability(&benched, InferenceTask::Code, &tracker);
5141 let r_graded = router.reliability(&graded, InferenceTask::Code, &tracker);
5142 assert!(
5143 r_graded < r_bench,
5144 "4 live grades must distrust more than 4 benchmark cases: \
5145 graded={r_graded} bench={r_bench}"
5146 );
5147 }
5148
5149 #[test]
5150 fn graded_failures_count_as_evidence_and_pull_reliability_down() {
5151 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5155 let m = outcome_model("flaky", Some(0.8)); let schema = router.schema_quality_estimate(&m);
5157
5158 let mut tracker = OutcomeTracker::new();
5159 let mut p = crate::outcome::ModelProfile::new("flaky".into());
5160 p.total_calls = 12;
5161 p.fail_count = 12;
5162 p.ema_quality = 0.1; p.quality_observations = 12;
5164 tracker.import_profiles(vec![p]);
5165
5166 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5167 assert!(
5168 r < schema - 0.2,
5169 "12 graded failures must pull reliability well below schema {schema}, got {r}"
5170 );
5171 }
5172
5173 #[test]
5174 fn route_exposes_ranked_advisory_candidates() {
5175 let _environment = crate::openrouter::test_environment_scope();
5180 let tmp = std::path::PathBuf::from("/tmp/car-test-route-candidates");
5186 unsafe {
5187 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5188 }
5189 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
5190 reg.register_project_model(outcome_model("strong", Some(0.90)));
5191 let mut weak = outcome_model("weak-cheap", Some(0.70)); weak.cost = crate::schema::CostModel {
5193 input_per_mtok: Some(0.01),
5194 output_per_mtok: Some(0.01),
5195 ..Default::default()
5196 };
5197 reg.register_project_model(weak);
5198
5199 let router = AdaptiveRouter::new(
5200 test_hw(),
5201 RoutingConfig {
5203 prior_strength: 100.0,
5204 ..Default::default()
5205 },
5206 );
5207 let tracker = OutcomeTracker::new();
5208 let decision = router.route(
5209 "Analyze the architecture trade-offs in depth",
5210 ®,
5211 &tracker,
5212 );
5213
5214 assert!(
5216 decision.candidates.iter().any(|c| c.model_id == "strong"),
5217 "strong must appear as a candidate, got {:?}",
5218 decision.candidates
5219 );
5220 assert!(
5221 decision
5222 .candidates
5223 .iter()
5224 .any(|c| c.model_id == "weak-cheap"),
5225 "weak-cheap must appear as a candidate even though out of band"
5226 );
5227
5228 let selected: Vec<&RouteCandidate> =
5230 decision.candidates.iter().filter(|c| c.selected).collect();
5231 assert_eq!(selected.len(), 1, "exactly one candidate may be selected");
5232 assert_eq!(
5233 selected[0].model_id, decision.model_id,
5234 "selected flag must match model_id"
5235 );
5236
5237 let strong = decision
5239 .candidates
5240 .iter()
5241 .find(|c| c.model_id == "strong")
5242 .unwrap();
5243 let weak_c = decision
5244 .candidates
5245 .iter()
5246 .find(|c| c.model_id == "weak-cheap")
5247 .unwrap();
5248 assert!(strong.in_band, "strong must be in the reliability band");
5249 assert!(
5250 !weak_c.in_band,
5251 "weak-cheap (0.20 below) must be out of band"
5252 );
5253 assert!(
5255 strong.reliability > weak_c.reliability,
5256 "reliability must reflect quality, not the weak model's lower cost"
5257 );
5258 }
5259
5260 #[test]
5261 fn prefer_quality_selection_is_deterministic_no_exploration() {
5262 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5267 let reg = test_registry();
5268 let tracker = OutcomeTracker::new();
5269 let intent = crate::intent::IntentHint {
5270 task: Some(crate::intent::TaskHint::Code),
5271 require: vec![ModelCapability::Code],
5272 prefer_quality: true,
5273 ..Default::default()
5274 };
5275 let first = router.route_with_intent("write a function", ®, &tracker, &intent);
5276 for _ in 0..8 {
5277 let again = router.route_with_intent("write a function", ®, &tracker, &intent);
5278 assert_eq!(
5279 again.model_id, first.model_id,
5280 "prefer_quality must be deterministic (no Thompson exploration)"
5281 );
5282 }
5283 let chosen = reg
5285 .find_by_name(&first.model_name)
5286 .or_else(|| reg.list().into_iter().find(|m| m.id == first.model_id));
5287 if let Some(m) = chosen {
5288 assert!(m.has_capability(ModelCapability::Code));
5289 }
5290 }
5291
5292 fn remote_model(id: &str, name: &str, caps: Vec<ModelCapability>) -> ModelSchema {
5294 ModelSchema {
5295 id: id.into(),
5296 name: name.into(),
5297 provider: "openai".into(),
5298 family: name.into(),
5299 version: "latest".into(),
5300 capabilities: caps,
5301 context_length: 128_000,
5302 max_output_tokens: None,
5303 param_count: "api".into(),
5304 quantization: None,
5305 performance: Default::default(),
5306 cost: Default::default(),
5307 source: crate::schema::ModelSource::RemoteApi {
5308 endpoint: "https://api.openai.com/v1".into(),
5309 api_key_env: "OPENAI_API_KEY".into(),
5310 api_key_envs: vec![],
5311 api_version: None,
5312 protocol: crate::schema::ApiProtocol::OpenAiCompat,
5313 },
5314 tags: vec!["trusted-remote".into()],
5315 supported_params: vec![],
5316 public_benchmarks: vec![],
5317 trust_tier: crate::schema::TrustTier::Curated,
5318 deprecated: false,
5319 available: true,
5320 weights_ready: true,
5321 }
5322 }
5323
5324 #[test]
5325 fn code_intent_filters_generate_only_models_on_simple_prompt() {
5326 let _environment = crate::openrouter::test_environment_scope();
5331 unsafe {
5338 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5339 }
5340 let intent_filter_dir = std::path::PathBuf::from("/tmp/car-test-code-intent-filter");
5341 let mut reg =
5342 UnifiedRegistry::new_with_state_root(intent_filter_dir.clone(), intent_filter_dir);
5343 reg.register_project_model(remote_model(
5344 "openai/gen-only:latest",
5345 "gen-only",
5346 vec![ModelCapability::Generate],
5347 ));
5348 reg.register_project_model(remote_model(
5349 "openai/coder:latest",
5350 "coder",
5351 vec![
5352 ModelCapability::Generate,
5353 ModelCapability::Code,
5354 ModelCapability::Reasoning,
5355 ],
5356 ));
5357
5358 let router = AdaptiveRouter::new(
5359 test_hw(),
5360 RoutingConfig {
5361 prior_strength: 100.0,
5362 ..Default::default()
5363 },
5364 );
5365 let tracker = OutcomeTracker::new();
5366 let intent = crate::intent::IntentHint {
5367 task: Some(crate::intent::TaskHint::Code),
5368 prefer_quality: true,
5369 ..Default::default() };
5371
5372 let decision = router.route_with_intent(
5374 "Give me a short two-sentence overview.",
5375 ®,
5376 &tracker,
5377 &intent,
5378 );
5379
5380 assert_eq!(
5381 decision.task,
5382 InferenceTask::Code,
5383 "explicit task=code must be honored"
5384 );
5385 assert!(
5388 !decision
5389 .candidates
5390 .iter()
5391 .any(|c| c.model_id == "openai/gen-only:latest"),
5392 "generate-only model must be filtered out for task=code; candidates: {:?}",
5393 decision
5394 .candidates
5395 .iter()
5396 .map(|c| &c.model_id)
5397 .collect::<Vec<_>>()
5398 );
5399 let schema = reg
5400 .find_by_name(&decision.model_name)
5401 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
5402 .expect("selected model exists");
5403 assert!(
5404 schema.has_capability(ModelCapability::Code),
5405 "task=code on a simple prompt must route to a code-capable model, got {}",
5406 decision.model_name,
5407 );
5408 }
5409
5410 #[test]
5411 fn intent_default_does_not_override_task_or_caps() {
5412 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5419 let reg = test_registry();
5420 let tracker = OutcomeTracker::new();
5421
5422 let baseline = router.route("write a haiku", ®, &tracker);
5423 let with_default = router.route_with_intent(
5424 "write a haiku",
5425 ®,
5426 &tracker,
5427 &crate::intent::IntentHint::default(),
5428 );
5429
5430 assert_eq!(
5431 baseline.task, with_default.task,
5432 "default IntentHint must not change the prompt-derived task"
5433 );
5434 assert_eq!(
5435 baseline.complexity, with_default.complexity,
5436 "default IntentHint must not change the prompt-derived complexity"
5437 );
5438 }
5439
5440 #[test]
5451 fn require_ready_skips_models_that_are_not_on_disk() {
5452 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5453 let reg = test_registry();
5454 let tracker = OutcomeTracker::new();
5455
5456 let ready: std::collections::HashSet<String> = reg
5458 .list()
5459 .into_iter()
5460 .filter(|m| m.weights_ready)
5461 .map(|m| m.id.clone())
5462 .collect();
5463 assert!(!ready.is_empty(), "fixture should have ready models");
5464
5465 let hint = crate::intent::IntentHint {
5466 require_ready: true,
5467 ..Default::default()
5468 };
5469 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
5470 assert!(
5471 ready.contains(&decision.model_id),
5472 "require_ready picked {} which is not on disk; ready set: {:?}",
5473 decision.model_id,
5474 ready
5475 );
5476 }
5477
5478 #[test]
5481 fn require_ready_falls_back_when_nothing_is_ready() {
5482 let _environment = crate::openrouter::test_environment_scope();
5487 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5488 let tracker = OutcomeTracker::new();
5489 let empty_dir = std::env::temp_dir().join("car-test-require-ready-empty");
5491 let _ = std::fs::create_dir_all(&empty_dir);
5492 let reg = UnifiedRegistry::new_with_state_root(empty_dir.clone(), empty_dir);
5493
5494 let hint = crate::intent::IntentHint {
5495 require_ready: true,
5496 ..Default::default()
5497 };
5498 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
5499 assert!(
5500 !decision.model_id.is_empty(),
5501 "soft constraint must still yield a model when nothing is ready"
5502 );
5503 }
5504
5505 #[test]
5506 fn exclude_models_picks_a_different_model() {
5507 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5512 let reg = test_registry();
5513 let tracker = OutcomeTracker::new();
5514 let base = crate::intent::IntentHint {
5515 prefer_quality: true,
5516 ..Default::default()
5517 };
5518 let author = router.route_with_intent("review this", ®, &tracker, &base);
5519
5520 let with_exclude = crate::intent::IntentHint {
5521 prefer_quality: true,
5522 exclude_models: vec![author.model_id.clone()],
5523 ..Default::default()
5524 };
5525 let reviewer = router.route_with_intent("review this", ®, &tracker, &with_exclude);
5526 assert_ne!(
5527 reviewer.model_id, author.model_id,
5528 "an excluded model must never be the routed pick"
5529 );
5530 assert!(
5533 !reviewer.fallbacks.contains(&author.model_id),
5534 "excluded model leaked into fallbacks"
5535 );
5536 assert!(
5537 !reviewer
5538 .candidates
5539 .iter()
5540 .any(|c| c.model_id == author.model_id),
5541 "excluded model leaked into candidates[]"
5542 );
5543 }
5544
5545 #[test]
5546 fn strict_exclusions_refuse_when_every_candidate_is_a_panel_seat() {
5547 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5548 let reg = test_registry();
5549 let tracker = OutcomeTracker::new();
5550 let panel: Vec<String> = reg
5551 .list()
5552 .into_iter()
5553 .map(|model| model.name.clone())
5554 .collect();
5555 assert!(!panel.is_empty(), "fixture must contain candidate models");
5556
5557 let decision = router.route_with_intent(
5558 "repair this code",
5559 ®,
5560 &tracker,
5561 &crate::intent::IntentHint {
5562 task: Some(crate::intent::TaskHint::Code),
5563 exclude_models: panel,
5564 strict_exclusions: true,
5565 ..Default::default()
5566 },
5567 );
5568
5569 assert!(decision.model_id.is_empty(), "{decision:?}");
5570 assert!(decision.model_name.is_empty(), "{decision:?}");
5571 assert!(decision.fallbacks.is_empty(), "{decision:?}");
5572 assert!(
5573 decision.reason.contains("no eligible model"),
5574 "the refusal must tell the operator what is missing: {decision:?}"
5575 );
5576 }
5577
5578 #[test]
5579 fn exclude_models_accepts_the_name_a_result_reports() {
5580 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5588 let reg = test_registry();
5589 let tracker = OutcomeTracker::new();
5590 let base = crate::intent::IntentHint {
5591 prefer_quality: true,
5592 ..Default::default()
5593 };
5594 let author = router.route_with_intent("write a python function", ®, &tracker, &base);
5595 let author_name = reg
5596 .get(&author.model_id)
5597 .expect("the routed pick is in the registry")
5598 .name
5599 .clone();
5600 assert_ne!(
5601 author_name, author.model_id,
5602 "this test is only meaningful on a model whose name and id differ"
5603 );
5604
5605 let by_name = crate::intent::IntentHint {
5606 prefer_quality: true,
5607 exclude_models: vec![author_name.clone()],
5608 ..Default::default()
5609 };
5610 let rerouted =
5611 router.route_with_intent("write a python function", ®, &tracker, &by_name);
5612 assert_ne!(
5613 rerouted.model_id, author.model_id,
5614 "excluding by the name a result reports ({author_name}) must route elsewhere"
5615 );
5616 assert!(
5617 !rerouted.fallbacks.contains(&author.model_id),
5618 "name-excluded model leaked into fallbacks"
5619 );
5620 }
5621
5622 #[test]
5623 fn cold_start_honors_exclusion() {
5624 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5629 let reg = test_registry();
5630 let caps = [ModelCapability::Generate];
5631 let base = router.cold_start_decision(
5632 TaskComplexity::Simple,
5633 InferenceTask::Generate,
5634 &caps,
5635 ®,
5636 false,
5637 &std::collections::HashSet::new(),
5638 false,
5639 AvailabilitySnapshot::capture(®),
5640 );
5641 let mut exclude = std::collections::HashSet::new();
5642 exclude.insert(base.model_id.clone());
5643 let excluded = router.cold_start_decision(
5644 TaskComplexity::Simple,
5645 InferenceTask::Generate,
5646 &caps,
5647 ®,
5648 false,
5649 &exclude,
5650 false,
5651 AvailabilitySnapshot::capture(®),
5652 );
5653 let other_generate_exists = reg
5657 .list()
5658 .into_iter()
5659 .filter(|m| m.available && m.has_capability(ModelCapability::Generate))
5660 .any(|m| m.id != base.model_id);
5661 if other_generate_exists {
5662 assert_ne!(
5663 excluded.model_id, base.model_id,
5664 "cold start re-picked the excluded model despite an alternative"
5665 );
5666 }
5667 }
5668
5669 #[test]
5670 fn exclude_all_models_falls_back_to_full_set() {
5671 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5674 let reg = test_registry();
5675 let tracker = OutcomeTracker::new();
5676 let exclude_everything: Vec<String> =
5677 reg.list().into_iter().map(|m| m.id.clone()).collect();
5678 let hint = crate::intent::IntentHint {
5679 exclude_models: exclude_everything,
5680 ..Default::default()
5681 };
5682 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
5683 assert!(
5684 !decision.model_id.is_empty(),
5685 "excluding everything must still yield a routed model (fallback)"
5686 );
5687 }
5688
5689 #[test]
5690 fn intent_task_hint_overrides_prompt_complexity() {
5691 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5694 let reg = test_registry();
5695 let tracker = OutcomeTracker::new();
5696
5697 let hint = crate::intent::IntentHint {
5698 task: Some(crate::intent::TaskHint::Reasoning),
5699 ..Default::default()
5700 };
5701 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
5702
5703 assert_eq!(
5704 decision.task,
5705 InferenceTask::Reasoning,
5706 "TaskHint::Reasoning should override the prompt-derived task"
5707 );
5708 }
5709
5710 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5719 #[test]
5720 fn router_proposes_mlx_twin_not_pre_redirect_gguf() {
5721 let gguf_id = "qwen/qwen3-8b:q4_k_m";
5722 let mlx_id = "mlx/qwen3-8b:4bit";
5723
5724 let reg = test_registry();
5725 let gguf = reg.get(gguf_id).expect("gguf in builtin catalog").clone();
5728 assert_eq!(
5729 reg.resolve_mlx_equivalent(&gguf).map(|m| m.id.as_str()),
5730 Some(mlx_id),
5731 "GGUF must resolve to its SAME-SIZE MLX twin"
5732 );
5733
5734 let router = AdaptiveRouter::new(
5735 test_hw(),
5736 RoutingConfig {
5737 prior_strength: 0.5,
5738 min_observations: 3,
5739 ..Default::default()
5740 },
5741 );
5742 let mut tracker = OutcomeTracker::new();
5743 for _ in 0..20 {
5745 let t = tracker.record_start(mlx_id, InferenceTask::Code, "test");
5746 tracker.record_complete(&t, 500, 100, 50);
5747 tracker.record_inferred_outcome(&t, InferredOutcome::Accepted { confidence: 0.95 });
5748 }
5749
5750 let mut mlx_wins = 0;
5751 for _ in 0..20 {
5752 let d = router.route("Fix this bug in the parser", ®, &tracker);
5753 assert_ne!(
5756 d.model_id, gguf_id,
5757 "router proposed the pre-redirect GGUF id"
5758 );
5759 assert!(
5760 !d.fallbacks.iter().any(|f| f == gguf_id),
5761 "pre-redirect GGUF id leaked into the fallback chain"
5762 );
5763 if d.model_id == mlx_id {
5764 mlx_wins += 1;
5765 }
5766 }
5767 assert!(
5768 mlx_wins >= 12,
5769 "proven MLX twin should win the majority (the router reads the id it \
5770 learns/runs); won only {mlx_wins}/20"
5771 );
5772 }
5773}