1use rand::Rng;
13use serde::{Deserialize, Serialize};
14
15use std::sync::{Arc, Mutex};
16
17use crate::hardware::HardwareInfo;
18use crate::outcome::{InferenceTask, OutcomeTracker};
19use crate::registry::UnifiedRegistry;
20use crate::routing_ext::CircuitBreakerRegistry;
21use crate::schema::{ModelCapability, ModelSchema};
22use crate::tasks::RoutingWorkload;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
26#[serde(rename_all = "snake_case")]
27pub enum TaskComplexity {
28 Simple,
29 Medium,
30 Code,
31 Complex,
32}
33
34impl TaskComplexity {
35 pub fn assess(prompt: &str) -> Self {
42 let lower = prompt.to_lowercase();
43 let word_count = prompt.split_whitespace().count();
44 let estimated_tokens = (word_count as f64 * 1.3) as usize;
45
46 let has_code = Self::detect_code(prompt);
47
48 let repair_markers = [
49 "fix", "repair", "debug", "refactor", "broken", "failing", "error", "bug",
50 ];
51 let has_repair = repair_markers.iter().any(|m| lower.contains(m));
52
53 let reasoning_markers = [
54 "analyze",
55 "compare",
56 "explain why",
57 "step by step",
58 "think through",
59 "evaluate",
60 "trade-off",
61 "tradeoff",
62 "pros and cons",
63 "architecture",
64 "design",
65 "strategy",
66 "optimize",
67 "comprehensive",
68 ];
69 let has_reasoning = reasoning_markers.iter().any(|m| lower.contains(m));
70
71 let simple_patterns = [
72 "what is",
73 "who is",
74 "when did",
75 "where is",
76 "how many",
77 "yes or no",
78 "true or false",
79 "name the",
80 "list the",
81 "define ",
82 ];
83 let is_simple = simple_patterns.iter().any(|p| lower.contains(p));
84
85 if has_code || has_repair {
86 TaskComplexity::Code
87 } else if has_reasoning || estimated_tokens > 500 {
88 TaskComplexity::Complex
89 } else if is_simple || estimated_tokens < 30 {
90 TaskComplexity::Simple
91 } else {
92 TaskComplexity::Medium
93 }
94 }
95
96 fn detect_code(prompt: &str) -> bool {
102 #[cfg(feature = "ast")]
104 {
105 if let Some(is_code) = Self::detect_code_ast(prompt) {
106 return is_code;
107 }
108 }
109
110 let code_markers = [
112 "```",
113 "fn ",
114 "def ",
115 "class ",
116 "import ",
117 "require(",
118 "async fn",
119 "pub fn",
120 "function ",
121 "const ",
122 "let ",
123 "var ",
124 "#include",
125 "package ",
126 "impl ",
127 ];
128 code_markers.iter().any(|m| prompt.contains(m))
129 }
130
131 #[cfg(feature = "ast")]
135 fn detect_code_ast(prompt: &str) -> Option<bool> {
136 let mut blocks = Vec::new();
138 let mut rest = prompt;
139 while let Some(start) = rest.find("```") {
140 let after_fence = &rest[start + 3..];
141 let code_start = after_fence.find('\n').map(|i| i + 1).unwrap_or(0);
143 if let Some(end) = after_fence[code_start..].find("```") {
144 blocks.push(&after_fence[code_start..code_start + end]);
145 rest = &after_fence[code_start + end + 3..];
146 } else {
147 break;
148 }
149 }
150
151 if blocks.is_empty() {
152 return None; }
154
155 let languages = [
157 car_ast::Language::Rust,
158 car_ast::Language::Python,
159 car_ast::Language::TypeScript,
160 car_ast::Language::JavaScript,
161 car_ast::Language::Go,
162 ];
163
164 for block in &blocks {
165 let trimmed = block.trim();
166 if trimmed.is_empty() {
167 continue;
168 }
169
170 for lang in &languages {
171 if let Some(parsed) = car_ast::parse(trimmed, *lang) {
172 if !parsed.symbols.is_empty() {
174 return Some(true);
175 }
176 }
177 }
178 }
179
180 Some(false)
183 }
184
185 pub fn required_capabilities(&self) -> Vec<ModelCapability> {
187 match self {
188 TaskComplexity::Simple => vec![ModelCapability::Generate],
189 TaskComplexity::Medium => vec![ModelCapability::Generate],
190 TaskComplexity::Code => vec![ModelCapability::Code],
191 TaskComplexity::Complex => vec![ModelCapability::Reasoning],
192 }
193 }
194
195 pub fn inference_task(&self) -> InferenceTask {
197 match self {
198 TaskComplexity::Simple | TaskComplexity::Medium => InferenceTask::Generate,
199 TaskComplexity::Code => InferenceTask::Code,
200 TaskComplexity::Complex => InferenceTask::Reasoning,
201 }
202 }
203}
204
205#[derive(Debug, Clone, Serialize, Deserialize)]
207pub struct RoutingConfig {
208 pub min_observations: u64,
210 pub quality_weight: f64,
212 pub latency_weight: f64,
213 pub cost_weight: f64,
214 pub max_latency_ms: Option<u64>,
216 pub max_cost_usd: Option<f64>,
218 pub prefer_local: bool,
220 pub prior_strength: f64,
224 pub quality_first_cold_start: bool,
227 pub bootstrap_min_task_observations: u64,
230 pub bootstrap_quality_floor: f64,
233}
234
235impl Default for RoutingConfig {
236 fn default() -> Self {
237 Self {
238 min_observations: 2,
239 quality_weight: 0.45,
240 latency_weight: 0.4,
241 cost_weight: 0.15,
242 max_latency_ms: None,
243 max_cost_usd: None,
244 prefer_local: true,
245 prior_strength: 2.0,
246 quality_first_cold_start: true,
247 bootstrap_min_task_observations: 8,
248 bootstrap_quality_floor: 0.8,
249 }
250 }
251}
252
253#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
255#[serde(rename_all = "snake_case")]
256pub enum RoutingStrategy {
257 SchemaBased,
259 ProfileBased,
261 Exploration,
263 Explicit,
265}
266
267#[derive(Debug, Clone, Serialize, Deserialize)]
276pub struct RouteCandidate {
277 pub model_id: String,
279 pub reliability: f64,
281 pub score: f64,
283 pub selected: bool,
285 pub in_band: bool,
287}
288
289#[derive(Debug, Clone, Copy, PartialEq, Eq)]
291pub enum RouterExclusionReason {
292 RateLimited,
294 CircuitOpen,
296}
297
298impl RouterExclusionReason {
299 pub fn as_str(self) -> &'static str {
301 match self {
302 Self::RateLimited => "rate_limited",
303 Self::CircuitOpen => "circuit_open",
304 }
305 }
306}
307
308#[derive(Debug, Clone, PartialEq, Eq)]
310pub struct RouterExclusion {
311 pub lane: String,
313 pub reason: RouterExclusionReason,
315 pub until: Option<u64>,
319}
320
321#[derive(Debug, Clone, Serialize, Deserialize)]
323pub struct AdaptiveRoutingDecision {
324 pub model_id: String,
326 pub model_name: String,
328 pub task: InferenceTask,
330 pub complexity: TaskComplexity,
332 pub reason: String,
334 pub strategy: RoutingStrategy,
336 pub predicted_quality: f64,
338 pub fallbacks: Vec<String>,
340 pub context_length: usize,
342 pub needs_compaction: bool,
344 #[serde(default)]
347 pub candidates: Vec<RouteCandidate>,
348}
349
350pub struct AdaptiveRouter {
352 hw: HardwareInfo,
353 config: RoutingConfig,
354 pub circuit_breakers: Arc<Mutex<CircuitBreakerRegistry>>,
356}
357
358pub struct RouteRequest<'a> {
374 pub prompt: &'a str,
375 pub registry: &'a UnifiedRegistry,
376 pub tracker: &'a OutcomeTracker,
377 pub estimated_total_tokens: usize,
380 pub estimated_input_tokens: usize,
383 pub estimated_output_tokens: usize,
385 pub estimated_cache_read_tokens: usize,
387 pub estimated_cache_write_tokens: usize,
389 pub has_tools: bool,
390 pub has_vision: bool,
391 pub workload: RoutingWorkload,
392 pub intent: Option<&'a crate::intent::IntentHint>,
395}
396
397impl<'a> RouteRequest<'a> {
398 pub fn new(
402 prompt: &'a str,
403 registry: &'a UnifiedRegistry,
404 tracker: &'a OutcomeTracker,
405 ) -> Self {
406 Self {
407 prompt,
408 registry,
409 tracker,
410 estimated_total_tokens: 0,
411 estimated_input_tokens: 0,
412 estimated_output_tokens: 0,
413 estimated_cache_read_tokens: 0,
414 estimated_cache_write_tokens: 0,
415 has_tools: false,
416 has_vision: false,
417 workload: RoutingWorkload::Interactive,
418 intent: None,
419 }
420 }
421}
422
423#[derive(Clone, Copy)]
427struct AvailabilitySnapshot {
428 openrouter: Option<crate::openrouter::CredentialSource>,
429}
430
431impl AvailabilitySnapshot {
432 fn capture(registry: &UnifiedRegistry) -> Self {
433 let has_openrouter = registry.list().into_iter().any(|schema| {
434 matches!(
435 schema.source,
436 crate::schema::ModelSource::RemoteApi {
437 protocol: crate::schema::ApiProtocol::OpenRouter,
438 ..
439 }
440 )
441 });
442 Self {
443 openrouter: has_openrouter
444 .then(crate::openrouter::credential_source)
445 .flatten(),
446 }
447 }
448
449 fn is_available(self, schema: &ModelSchema) -> bool {
450 match schema.source {
451 crate::schema::ModelSource::RemoteApi {
452 protocol: crate::schema::ApiProtocol::OpenRouter,
453 ..
454 } => self.openrouter.is_some(),
455 _ => schema.available,
456 }
457 }
458}
459
460static BENCHMARK_FRONTIER_REFS: std::sync::LazyLock<std::collections::HashMap<String, f64>> =
487 std::sync::LazyLock::new(|| compute_frontier_refs(crate::registry::builtin_catalog().iter()));
488
489fn compute_frontier_refs<'a>(
494 models: impl Iterator<Item = &'a ModelSchema>,
495) -> std::collections::HashMap<String, f64> {
496 let mut max_score: std::collections::HashMap<String, f64> = std::collections::HashMap::new();
500 let mut has_frontier_carrier: std::collections::HashMap<String, bool> =
501 std::collections::HashMap::new();
502 for model in models {
503 let frontier = model_is_frontier_class(model);
504 for b in &model.public_benchmarks {
505 let entry = max_score.entry(b.name.clone()).or_insert(0.0);
506 *entry = entry.max(b.score);
507 let seen = has_frontier_carrier.entry(b.name.clone()).or_insert(false);
508 *seen = *seen || frontier;
509 }
510 }
511 max_score
515 .into_iter()
516 .filter(|(name, _)| {
517 has_frontier_carrier.get(name).copied().unwrap_or(false) || {
518 tracing::warn!(
519 benchmark = %name,
520 "benchmark has no frontier-class carrier in the catalog — \
521 refusing to self-anchor (uncalibrated clamp). Add a \
522 frontier-tagged model that carries it, or an explicit \
523 reference, before it can calibrate the quality tier (#371)."
524 );
525 false
526 }
527 })
528 .collect()
529}
530
531fn model_is_frontier_class(model: &ModelSchema) -> bool {
537 model.tags.iter().any(|t| t == "frontier")
538}
539
540impl AdaptiveRouter {
541 pub fn new(hw: HardwareInfo, config: RoutingConfig) -> Self {
542 let circuit_breakers = Arc::new(Mutex::new(
543 CircuitBreakerRegistry::new(3, 300), ));
545 Self {
546 hw,
547 config,
548 circuit_breakers,
549 }
550 }
551
552 pub fn with_default_config(hw: HardwareInfo) -> Self {
553 Self::new(hw, RoutingConfig::default())
554 }
555
556 pub fn config(&self) -> &RoutingConfig {
557 &self.config
558 }
559
560 pub fn set_config(&mut self, config: RoutingConfig) {
561 self.config = config;
562 }
563
564 pub fn route_with(&self, req: RouteRequest<'_>) -> AdaptiveRoutingDecision {
570 let mut ignore_exclusion = |_: RouterExclusion| {};
571 self.route_with_exclusion_observer(req, &mut ignore_exclusion)
572 }
573
574 pub(crate) fn route_with_exclusion_observer(
578 &self,
579 req: RouteRequest<'_>,
580 exclusion_observer: &mut dyn FnMut(RouterExclusion),
581 ) -> AdaptiveRoutingDecision {
582 let workload = Self::resolve_workload(req.intent, req.workload);
588 self.route_inner_with_intent(
589 req.prompt,
590 req.registry,
591 req.tracker,
592 req.has_tools,
593 req.has_vision,
594 req.estimated_total_tokens,
595 req.estimated_input_tokens,
596 req.estimated_output_tokens,
597 req.estimated_cache_read_tokens,
598 req.estimated_cache_write_tokens,
599 workload,
600 req.intent,
601 exclusion_observer,
602 )
603 }
604
605 pub fn route(
608 &self,
609 prompt: &str,
610 registry: &UnifiedRegistry,
611 tracker: &OutcomeTracker,
612 ) -> AdaptiveRoutingDecision {
613 self.route_with(RouteRequest::new(prompt, registry, tracker))
614 }
615
616 pub fn route_editor(
622 &self,
623 prompt: &str,
624 registry: &UnifiedRegistry,
625 tracker: &OutcomeTracker,
626 ) -> AdaptiveRoutingDecision {
627 self.route_with(RouteRequest {
628 workload: RoutingWorkload::Background,
629 ..RouteRequest::new(prompt, registry, tracker)
630 })
631 }
632
633 pub fn route_with_tools(
635 &self,
636 prompt: &str,
637 registry: &UnifiedRegistry,
638 tracker: &OutcomeTracker,
639 ) -> AdaptiveRoutingDecision {
640 self.route_with(RouteRequest {
641 has_tools: true,
642 ..RouteRequest::new(prompt, registry, tracker)
643 })
644 }
645
646 pub fn route_with_vision(
648 &self,
649 prompt: &str,
650 registry: &UnifiedRegistry,
651 tracker: &OutcomeTracker,
652 has_tools: bool,
653 ) -> AdaptiveRoutingDecision {
654 self.route_with(RouteRequest {
655 has_tools,
656 has_vision: true,
657 ..RouteRequest::new(prompt, registry, tracker)
658 })
659 }
660
661 pub fn route_with_intent<'a>(
667 &self,
668 prompt: &'a str,
669 registry: &'a UnifiedRegistry,
670 tracker: &'a OutcomeTracker,
671 intent: &'a crate::intent::IntentHint,
672 ) -> AdaptiveRoutingDecision {
673 self.route_with(RouteRequest {
674 intent: Some(intent),
675 ..RouteRequest::new(prompt, registry, tracker)
676 })
677 }
678
679 pub fn route_context_aware(
682 &self,
683 prompt: &str,
684 estimated_total_tokens: usize,
685 registry: &UnifiedRegistry,
686 tracker: &OutcomeTracker,
687 has_tools: bool,
688 has_vision: bool,
689 workload: RoutingWorkload,
690 ) -> AdaptiveRoutingDecision {
691 self.route_with(RouteRequest {
692 estimated_total_tokens,
693 has_tools,
694 has_vision,
695 workload,
696 ..RouteRequest::new(prompt, registry, tracker)
697 })
698 }
699
700 pub fn route_context_aware_with_intent<'a>(
705 &self,
706 prompt: &'a str,
707 estimated_total_tokens: usize,
708 registry: &'a UnifiedRegistry,
709 tracker: &'a OutcomeTracker,
710 has_tools: bool,
711 has_vision: bool,
712 workload: RoutingWorkload,
713 intent: &'a crate::intent::IntentHint,
714 ) -> AdaptiveRoutingDecision {
715 self.route_with(RouteRequest {
716 estimated_total_tokens,
717 has_tools,
718 has_vision,
719 workload,
720 intent: Some(intent),
721 ..RouteRequest::new(prompt, registry, tracker)
722 })
723 }
724
725 fn route_inner_with_intent(
726 &self,
727 prompt: &str,
728 registry: &UnifiedRegistry,
729 tracker: &OutcomeTracker,
730 has_tools: bool,
731 has_vision: bool,
732 estimated_total_tokens: usize,
733 estimated_input_tokens: usize,
734 estimated_output_tokens: usize,
735 estimated_cache_read_tokens: usize,
736 estimated_cache_write_tokens: usize,
737 workload: RoutingWorkload,
738 intent: Option<&crate::intent::IntentHint>,
739 exclusion_observer: &mut dyn FnMut(RouterExclusion),
740 ) -> AdaptiveRoutingDecision {
741 let complexity = TaskComplexity::assess(prompt);
742 let task = intent
744 .and_then(|h| h.task)
745 .map(task_hint_to_inference_task)
746 .unwrap_or_else(|| complexity.inference_task());
747 let mut required_caps = complexity.required_capabilities();
748 let task_cap = inference_task_required_capability(task);
756 if !required_caps.contains(&task_cap) {
757 required_caps.push(task_cap);
758 }
759 if let Some(hint) = intent {
760 for cap in &hint.require {
761 if !required_caps.contains(cap) {
762 required_caps.push(*cap);
763 }
764 }
765 }
766 if has_vision {
767 required_caps.push(ModelCapability::Vision);
768 }
769 if has_tools {
770 required_caps.push(ModelCapability::ToolUse);
771 if Self::needs_multi_tool_call(prompt) {
774 required_caps.push(ModelCapability::MultiToolCall);
775 }
776 }
777
778 let exclude_set = self.build_exclude_set(intent, registry);
786 let strict_exclusions = intent.is_some_and(|hint| hint.strict_exclusions);
787 let availability = AvailabilitySnapshot::capture(registry);
788
789 let want_ready = intent.is_some_and(|h| h.require_ready);
794 let mut candidates = self.filter_candidates(
795 &required_caps,
796 registry,
797 tracker,
798 has_vision,
799 availability,
800 want_ready,
801 exclusion_observer,
802 );
803 if candidates.is_empty() && want_ready {
804 candidates = self.filter_candidates(
805 &required_caps,
806 registry,
807 tracker,
808 has_vision,
809 availability,
810 false,
811 exclusion_observer,
812 );
813 }
814
815 if candidates.is_empty() && required_caps.contains(&ModelCapability::MultiToolCall) {
818 required_caps.retain(|c| *c != ModelCapability::MultiToolCall);
819 candidates = self.filter_candidates(
820 &required_caps,
821 registry,
822 tracker,
823 has_vision,
824 availability,
825 want_ready,
826 exclusion_observer,
827 );
828 if candidates.is_empty() && want_ready {
829 candidates = self.filter_candidates(
830 &required_caps,
831 registry,
832 tracker,
833 has_vision,
834 availability,
835 false,
836 exclusion_observer,
837 );
838 }
839 }
840
841 if candidates.is_empty() {
842 return self.cold_start_decision(
846 complexity,
847 task,
848 &required_caps,
849 registry,
850 has_vision,
851 &exclude_set,
852 strict_exclusions,
853 availability,
854 );
855 }
856
857 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
870 {
871 let mut seen = std::collections::HashSet::new();
872 candidates = candidates
873 .into_iter()
874 .map(|m| registry.resolve_mlx_equivalent(&m).cloned().unwrap_or(m))
875 .filter(|m| seen.insert(m.id.clone()))
876 .collect();
877 }
878
879 if !exclude_set.is_empty() {
888 let filtered: Vec<ModelSchema> = candidates
889 .iter()
890 .filter(|m| !exclude_set.contains(&m.id))
891 .cloned()
892 .collect();
893 if filtered.is_empty() {
894 if strict_exclusions {
895 return Self::no_eligible_model_decision(complexity, task, exclude_set.len());
896 }
897 tracing::debug!(
898 ?exclude_set,
899 "model exclusion left no candidates; keeping full set"
900 );
901 } else {
902 candidates = filtered;
903 }
904 }
905
906 candidates = self.apply_quality_first_bootstrap_policy(
907 candidates, task, tracker, has_vision, has_tools, workload,
908 );
909
910 let (fits, needs_compaction_candidates) = if estimated_total_tokens > 0 {
913 let mut fits = Vec::new();
914 let mut tight = Vec::new();
915 for m in &candidates {
916 if m.context_length == 0 || m.context_length >= estimated_total_tokens {
917 fits.push(m.clone());
918 } else {
919 tight.push(m.clone());
920 }
921 }
922 (fits, tight)
923 } else {
924 (candidates.clone(), Vec::new())
925 };
926
927 let (scoring_candidates, compaction_needed) = if !fits.is_empty() {
929 (fits, false)
930 } else if !needs_compaction_candidates.is_empty() {
931 tracing::info!(
932 prompt_tokens = estimated_total_tokens,
933 candidates = needs_compaction_candidates.len(),
934 "no model fits full prompt — compaction will be needed"
935 );
936 (needs_compaction_candidates.clone(), true)
937 } else {
938 (candidates.clone(), false)
939 };
940
941 let pricing_input_tokens = estimated_input_tokens
947 .max(estimated_total_tokens.saturating_sub(estimated_output_tokens));
948 let scored = self.score_candidates_context_aware(
949 &scoring_candidates,
950 task,
951 tracker,
952 estimated_total_tokens,
953 pricing_input_tokens,
954 estimated_output_tokens,
955 estimated_cache_read_tokens,
956 estimated_cache_write_tokens,
957 workload,
958 );
959 tracing::debug!(
965 ?workload,
966 ?task,
967 candidates = ?scoring_candidates.iter().map(|m| m.id.as_str()).collect::<Vec<_>>(),
968 ?scored,
969 "route scoring"
970 );
971
972 let band_ids: Option<std::collections::HashSet<String>> =
983 if Self::applies_reliability_band(task, workload) {
984 Some(
985 self.outcome_first_band(scoring_candidates.clone(), task, tracker)
986 .into_iter()
987 .map(|m| m.id)
988 .collect(),
989 )
990 } else {
991 None
992 };
993 if let Some(ref ids) = band_ids {
994 tracing::debug!(
999 ?task,
1000 band = Self::OUTCOME_FIRST_BAND,
1001 band_size = ids.len(),
1002 "outcome-first: selection restricted to reliability band"
1003 );
1004 }
1005 let selection_pool: Vec<(String, f64)> = if let Some(ref ids) = band_ids {
1006 let pool: Vec<(String, f64)> = scored
1007 .iter()
1008 .filter(|(id, _)| ids.contains(id))
1009 .cloned()
1010 .collect();
1011 if pool.is_empty() {
1012 debug_assert!(
1018 false,
1019 "reliability band produced no in-`scored` ids (workload {workload:?})"
1020 );
1021 scored.clone()
1022 } else {
1023 pool
1024 }
1025 } else {
1026 scored.clone()
1027 };
1028
1029 let (selected_id, strategy) = if workload == RoutingWorkload::Quality {
1042 selection_pool
1043 .iter()
1044 .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
1045 .map(|(id, _)| (id.clone(), RoutingStrategy::SchemaBased))
1046 .unwrap_or_else(|| self.select_with_thompson_sampling(&selection_pool, tracker))
1047 } else {
1048 self.select_with_thompson_sampling(&selection_pool, tracker)
1049 };
1050
1051 let mut fallbacks: Vec<String> = if let Some(ref ids) = band_ids {
1058 let (in_band, out_band): (Vec<String>, Vec<String>) = scored
1059 .iter()
1060 .filter(|(id, _)| *id != selected_id)
1061 .map(|(id, _)| id.clone())
1062 .partition(|id| ids.contains(id));
1063 in_band.into_iter().chain(out_band).collect()
1064 } else {
1065 scored
1066 .iter()
1067 .filter(|(id, _)| *id != selected_id)
1068 .map(|(id, _)| id.clone())
1069 .collect()
1070 };
1071 if !compaction_needed {
1073 for m in &needs_compaction_candidates {
1074 if m.id != selected_id && !fallbacks.contains(&m.id) {
1075 fallbacks.push(m.id.clone());
1076 }
1077 }
1078 }
1079
1080 let predicted_quality = scored
1081 .iter()
1082 .find(|(id, _)| *id == selected_id)
1083 .map(|(_, score)| *score)
1084 .unwrap_or(0.5);
1085
1086 let selected_schema = registry
1087 .get(&selected_id)
1088 .or_else(|| registry.find_by_name(&selected_id));
1089 let model_name = selected_schema
1090 .map(|m| m.name.clone())
1091 .unwrap_or_else(|| selected_id.clone());
1092 let context_length = selected_schema.map(|m| m.context_length).unwrap_or(0);
1093
1094 let needs_compact = compaction_needed
1095 || (estimated_total_tokens > 0
1096 && context_length > 0
1097 && estimated_total_tokens > context_length);
1098
1099 let compaction_note = if needs_compact {
1100 format!(
1101 " [compaction needed: {}→{}tok]",
1102 estimated_total_tokens, context_length
1103 )
1104 } else {
1105 String::new()
1106 };
1107
1108 let cost_quality_note = selected_schema
1109 .map(|schema| {
1110 let is_openrouter = schema.tags.iter().any(|tag| tag == "openrouter");
1111 if is_openrouter && schema.tags.iter().any(|tag| tag == "cheap") {
1112 let prices = schema.cost.prices_for(pricing_input_tokens);
1113 format!(
1114 ", low-cost tier (${:.3}/MTok input, ${:.3}/MTok output at {} prompt tokens)",
1115 prices.input_per_mtok.unwrap_or(0.0),
1116 prices.output_per_mtok.unwrap_or(0.0),
1117 pricing_input_tokens,
1118 )
1119 } else if is_openrouter && model_is_frontier_class(schema) {
1120 ", frontier quality tier".to_string()
1121 } else {
1122 String::new()
1123 }
1124 })
1125 .unwrap_or_default();
1126
1127 let reason = format!(
1128 "{:?} task → {} via {:?} (quality: {:.2}, {} candidates{}){}",
1129 complexity,
1130 model_name,
1131 strategy,
1132 predicted_quality,
1133 scoring_candidates.len(),
1134 cost_quality_note,
1135 compaction_note,
1136 );
1137
1138 let candidates: Vec<RouteCandidate> = scored
1145 .iter()
1146 .map(|(id, score)| {
1147 let reliability = match scoring_candidates.iter().find(|m| m.id == *id) {
1153 Some(m) => self.reliability(m, task, tracker),
1154 None => {
1155 debug_assert!(false, "scored id {id} absent from scoring_candidates");
1156 0.0
1157 }
1158 };
1159 RouteCandidate {
1160 model_id: id.clone(),
1161 reliability,
1162 score: *score,
1163 selected: *id == selected_id,
1164 in_band: band_ids.as_ref().is_none_or(|b| b.contains(id)),
1165 }
1166 })
1167 .collect();
1168
1169 AdaptiveRoutingDecision {
1170 model_id: selected_id,
1171 model_name,
1172 task,
1173 complexity,
1174 reason,
1175 strategy,
1176 predicted_quality,
1177 fallbacks,
1178 context_length,
1179 needs_compaction: needs_compact,
1180 candidates,
1181 }
1182 }
1183
1184 pub fn route_embedding(&self, registry: &UnifiedRegistry) -> String {
1186 let embed_models = registry.query_by_capability(ModelCapability::Embed);
1187 embed_models
1188 .first()
1189 .map(|m| m.name.clone())
1190 .unwrap_or_else(|| "Qwen3-Embedding-0.6B".to_string())
1191 }
1192
1193 pub fn route_small(&self, registry: &UnifiedRegistry) -> String {
1195 let gen_models = registry.query_by_capability(ModelCapability::Generate);
1196 gen_models
1198 .iter()
1199 .filter(|m| m.is_local())
1200 .min_by_key(|m| m.size_mb())
1201 .map(|m| m.name.clone())
1202 .unwrap_or_else(|| "Qwen3-0.6B".to_string())
1203 }
1204
1205 const LATENCY_CEILING_MS: f64 = 10000.0;
1211 const _TPS_CEILING: f64 = 150.0;
1213 const MOE_TPS_MULTIPLIER: f64 = 0.10;
1217 const MLX_MOE_TPS_MULTIPLIER: f64 = 0.50;
1220 const COST_CEILING_PER_1K: f64 = 0.1;
1222 const LOCAL_BONUS: f64 = 0.15;
1224 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1232 const HAS_GPU_BACKEND: bool = true;
1233 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
1234 const HAS_GPU_BACKEND: bool = false;
1235 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1238 const MLX_BONUS: f64 = 0.10;
1239 const SYSTEM_LLM_BONUS: f64 = 0.12;
1250
1251 pub(crate) fn build_exclude_set(
1270 &self,
1271 intent: Option<&crate::intent::IntentHint>,
1272 registry: &UnifiedRegistry,
1273 ) -> std::collections::HashSet<String> {
1274 let mut set = std::collections::HashSet::new();
1275 let Some(hint) = intent else {
1276 return set;
1277 };
1278 for id in &hint.exclude_models {
1279 set.insert(id.clone());
1280 let Some(schema) = registry.get(id).or_else(|| registry.find_by_name(id)) else {
1281 continue;
1282 };
1283 set.insert(schema.id.clone());
1284 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1285 {
1286 if let Some(mlx) = registry.resolve_mlx_equivalent(schema) {
1287 set.insert(mlx.id.clone());
1288 }
1289 }
1290 }
1291 set
1292 }
1293
1294 fn filter_candidates(
1295 &self,
1296 required_caps: &[ModelCapability],
1297 registry: &UnifiedRegistry,
1298 tracker: &OutcomeTracker,
1299 has_vision: bool,
1300 availability: AvailabilitySnapshot,
1301 require_ready: bool,
1302 exclusion_observer: &mut dyn FnMut(RouterExclusion),
1303 ) -> Vec<ModelSchema> {
1304 registry
1305 .list()
1306 .into_iter()
1307 .filter(|m| {
1308 if !required_caps.iter().all(|c| m.has_capability(*c)) {
1310 return false;
1311 }
1312 if !has_vision && m.tags.iter().any(|t| t == "mlx-vlm-cli") {
1322 return false;
1323 }
1324 if !availability.is_available(m) {
1326 return false;
1327 }
1328 if require_ready && !m.weights_ready {
1335 return false;
1336 }
1337 if m.is_local() && m.size_mb() >= self.hw.max_model_mb {
1339 return false;
1340 }
1341 if let Some(max) = self.config.max_latency_ms {
1343 if let Some(p50) = m.performance.latency_p50_ms {
1344 if p50 > max {
1345 return false;
1346 }
1347 }
1348 }
1349 if let Some(max) = self.config.max_cost_usd {
1351 if m.cost_per_1k_output() > max {
1352 return false;
1353 }
1354 }
1355 if !self.config.prefer_local && m.is_local() {
1357 return false;
1358 }
1359 if tracker.is_excluded(&m.id) {
1361 exclusion_observer(RouterExclusion {
1362 lane: m.id.clone(),
1363 reason: RouterExclusionReason::RateLimited,
1364 until: None,
1365 });
1366 return false;
1367 }
1368 if let Ok(mut cb) = self.circuit_breakers.lock() {
1370 if !cb.allow_request(&m.id) {
1371 let until = cb.blocked_until(&m.id);
1372 tracing::debug!(model = %m.id, ?until, "skipped by circuit breaker");
1373 exclusion_observer(RouterExclusion {
1374 lane: m.id.clone(),
1375 reason: RouterExclusionReason::CircuitOpen,
1376 until,
1377 });
1378 return false;
1379 }
1380 }
1381 true
1382 })
1383 .cloned()
1384 .collect()
1385 }
1386
1387 fn apply_quality_first_bootstrap_policy(
1388 &self,
1389 candidates: Vec<ModelSchema>,
1390 task: InferenceTask,
1391 tracker: &OutcomeTracker,
1392 has_vision: bool,
1393 has_tools: bool,
1394 workload: RoutingWorkload,
1395 ) -> Vec<ModelSchema> {
1396 if !self.config.quality_first_cold_start
1397 || !workload.is_latency_sensitive()
1398 || !self.is_quality_critical_bootstrap_task(task, has_vision, has_tools)
1399 {
1400 return candidates;
1401 }
1402
1403 let trusted_remote: Vec<ModelSchema> = candidates
1404 .iter()
1405 .filter(|model| self.is_trusted_quality_remote(model))
1406 .cloned()
1407 .collect();
1408
1409 if trusted_remote.is_empty() {
1410 return candidates;
1411 }
1412
1413 let proven_local: Vec<ModelSchema> = candidates
1414 .iter()
1415 .filter(|model| {
1416 model.is_local() && self.is_local_model_proven_for_task(model, task, tracker)
1417 })
1418 .cloned()
1419 .collect();
1420
1421 if !proven_local.is_empty() {
1422 return proven_local;
1423 }
1424
1425 trusted_remote
1426 }
1427
1428 fn score_candidates_context_aware(
1432 &self,
1433 candidates: &[ModelSchema],
1434 task: InferenceTask,
1435 tracker: &OutcomeTracker,
1436 estimated_total_tokens: usize,
1437 estimated_input_tokens: usize,
1438 estimated_output_tokens: usize,
1439 estimated_cache_read_tokens: usize,
1440 estimated_cache_write_tokens: usize,
1441 workload: RoutingWorkload,
1442 ) -> Vec<(String, f64)> {
1443 let mut scored: Vec<(String, f64)> = candidates
1444 .iter()
1445 .map(|m| {
1446 let base_score = self.score_model(
1447 m,
1448 task,
1449 tracker,
1450 workload,
1451 estimated_input_tokens,
1452 estimated_output_tokens,
1453 estimated_cache_read_tokens,
1454 estimated_cache_write_tokens,
1455 );
1456 let headroom_bonus = if estimated_total_tokens > 0 && m.context_length > 0 {
1459 let ratio = m.context_length as f64 / estimated_total_tokens as f64;
1460 if ratio >= 1.0 {
1461 (ratio.min(4.0) - 1.0) / 3.0 * 0.10 } else {
1463 -0.15 }
1465 } else {
1466 0.0
1467 };
1468 (m.id.clone(), base_score + headroom_bonus)
1469 })
1470 .collect();
1471
1472 scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
1473 scored
1474 }
1475
1476 fn task_aware_weights(task: InferenceTask, workload: RoutingWorkload) -> (f64, f64, f64) {
1492 match (task, workload) {
1493 (
1494 InferenceTask::Code,
1495 RoutingWorkload::Interactive | RoutingWorkload::LocalPreferred,
1496 ) => (0.62, 0.28, 0.10),
1497 _ => workload.weights(),
1498 }
1499 }
1500
1501 fn reliability(
1508 &self,
1509 model: &ModelSchema,
1510 task: InferenceTask,
1511 tracker: &OutcomeTracker,
1512 ) -> f64 {
1513 let schema_quality = self.schema_quality_estimate(model);
1514 let Some(p) = tracker.profile(&model.id) else {
1515 return schema_quality;
1517 };
1518
1519 let (ema, n, g) = p
1527 .task_stats(task)
1528 .map(|ts| {
1529 (
1530 ts.ema_quality,
1531 ts.prior_sample_size,
1532 ts.quality_observations,
1533 )
1534 })
1535 .unwrap_or((p.ema_quality, p.prior_sample_size, p.quality_observations));
1536
1537 let evidence = n as f64 + Self::LIVE_GRADE_WEIGHT * g as f64;
1563 let confidence = evidence / (evidence + Self::PRIOR_SHRINK_PSEUDOCOUNT);
1564 (schema_quality + confidence * (ema - schema_quality)).clamp(0.0, 1.0)
1565 }
1566
1567 const BENCH_TIER_FLOOR: f64 = 0.40;
1575 const BENCH_TIER_CEIL: f64 = 0.92;
1576
1577 const LOCAL_BENCH_TIER_FLOOR: f64 = 0.30;
1591 const LOCAL_BENCH_TIER_CEIL: f64 = 0.60;
1592
1593 const UNMEASURED_REMOTE_CURATED: f64 = 0.60;
1598 const UNMEASURED_REMOTE_COMMUNITY: f64 = 0.48;
1599 const CURATED_FRONTIER_TAG_PRIOR: f64 = 0.85;
1604 const CURATED_BALANCED_TAG_PRIOR: f64 = 0.72;
1605
1606 const LIVE_GRADE_WEIGHT: f64 = 2.0;
1617
1618 const PRIOR_SHRINK_PSEUDOCOUNT: f64 = 4.0;
1628
1629 const OUTCOME_FIRST_BAND: f64 = 0.02;
1645
1646 fn resolve_workload(
1652 intent: Option<&crate::intent::IntentHint>,
1653 base: RoutingWorkload,
1654 ) -> RoutingWorkload {
1655 match intent {
1656 Some(h) if h.high_stakes => RoutingWorkload::Quality,
1657 Some(h) if h.prefer_fast => RoutingWorkload::Fastest,
1658 Some(h) if h.prefer_quality => RoutingWorkload::Quality,
1659 Some(h) if h.prefer_local => RoutingWorkload::LocalPreferred,
1660 _ => base,
1661 }
1662 }
1663
1664 fn applies_reliability_band(task: InferenceTask, workload: RoutingWorkload) -> bool {
1680 match workload {
1681 RoutingWorkload::Quality => true,
1682 RoutingWorkload::Fastest
1683 | RoutingWorkload::Background
1684 | RoutingWorkload::LocalPreferred => false,
1685 _ => matches!(task, InferenceTask::Code | InferenceTask::Reasoning),
1686 }
1687 }
1688
1689 fn outcome_first_band(
1694 &self,
1695 candidates: Vec<ModelSchema>,
1696 task: InferenceTask,
1697 tracker: &OutcomeTracker,
1698 ) -> Vec<ModelSchema> {
1699 if candidates.len() <= 1 {
1700 return candidates;
1701 }
1702 let best = candidates
1703 .iter()
1704 .map(|m| self.reliability(m, task, tracker))
1705 .fold(f64::MIN, f64::max);
1706 candidates
1707 .into_iter()
1708 .filter(|m| {
1711 best - self.reliability(m, task, tracker) <= Self::OUTCOME_FIRST_BAND + 1e-9
1712 })
1713 .collect()
1714 }
1715
1716 fn score_model(
1717 &self,
1718 model: &ModelSchema,
1719 task: InferenceTask,
1720 tracker: &OutcomeTracker,
1721 workload: RoutingWorkload,
1722 estimated_input_tokens: usize,
1723 estimated_output_tokens: usize,
1724 estimated_cache_read_tokens: usize,
1725 estimated_cache_write_tokens: usize,
1726 ) -> f64 {
1727 let profile = tracker.profile(&model.id);
1728 let schema_latency = self.schema_latency_estimate(model);
1729 let (quality_weight, latency_weight, cost_weight) =
1730 Self::task_aware_weights(task, workload);
1731
1732 let quality = self.reliability(model, task, tracker);
1736
1737 let latency = match profile {
1740 Some(p) if p.total_calls >= self.config.min_observations => {
1741 let avg = p
1742 .task_stats(task)
1743 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1744 .map(|ts| ts.avg_latency_ms)
1745 .unwrap_or_else(|| p.avg_latency_ms());
1746 self.latency_ms_to_score(avg)
1747 }
1748 Some(p) if p.total_calls == 0 => p
1749 .task_stats(task)
1750 .filter(|ts| ts.avg_latency_ms > 0.0)
1751 .map(|ts| self.latency_ms_to_score(ts.avg_latency_ms))
1752 .unwrap_or(schema_latency),
1753 Some(p) if p.total_calls > 0 => {
1754 let observed = self.latency_ms_to_score(
1755 p.task_stats(task)
1756 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1757 .map(|ts| ts.avg_latency_ms)
1758 .unwrap_or_else(|| p.avg_latency_ms()),
1759 );
1760 let w = p.total_calls as f64 / self.config.min_observations as f64;
1761 schema_latency * (1.0 - w) + observed * w
1762 }
1763 _ => schema_latency,
1764 };
1765
1766 let cost = if model.is_local() {
1768 1.0
1769 } else if model.cost.output_per_mtok.is_none() {
1770 0.5
1777 } else {
1778 let normalized_cost_per_1k = if estimated_input_tokens
1779 + estimated_output_tokens
1780 + estimated_cache_read_tokens
1781 + estimated_cache_write_tokens
1782 > 0
1783 {
1784 let estimated_cost = model.cost.estimated_usd(
1785 estimated_input_tokens,
1786 estimated_output_tokens,
1787 estimated_cache_read_tokens,
1788 estimated_cache_write_tokens,
1789 );
1790 estimated_cost * 1000.0
1791 / (estimated_input_tokens + estimated_output_tokens).max(1) as f64
1792 } else {
1793 model.cost_per_1k_output()
1794 };
1795 (1.0 - (normalized_cost_per_1k / Self::COST_CEILING_PER_1K)).clamp(0.0, 1.0)
1796 };
1797
1798 let gpu_backend_or_latency_tolerant =
1817 Self::HAS_GPU_BACKEND || !workload.is_latency_sensitive();
1818 let local_bonus = if self.config.prefer_local
1819 && model.is_local()
1820 && gpu_backend_or_latency_tolerant
1821 {
1822 Self::LOCAL_BONUS
1823 } else {
1824 if self.config.prefer_local && model.is_local() && !Self::HAS_GPU_BACKEND {
1825 tracing::debug!(
1826 model = %model.id,
1827 "LOCAL_BONUS suppressed: no GPU backend on this host (Intel Mac or car_skip_mlx) for a latency-sensitive workload; cloud models will rank higher"
1828 );
1829 }
1830 0.0
1831 };
1832 let workload_local_bonus = if model.is_local() {
1833 workload.local_bonus()
1834 } else {
1835 0.0
1836 };
1837
1838 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1840 let mlx_bonus = if model.is_mlx() { Self::MLX_BONUS } else { 0.0 };
1841 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
1842 let mlx_bonus = 0.0;
1843
1844 let vllm_mlx_bonus = if model.is_vllm_mlx() {
1846 Self::LOCAL_BONUS + 0.05
1847 } else {
1848 0.0
1849 };
1850
1851 let system_llm_bonus = if model.tags.iter().any(|t| t == "low_latency")
1859 && model.tags.iter().any(|t| t == "private")
1860 {
1861 Self::SYSTEM_LLM_BONUS
1862 } else {
1863 0.0
1864 };
1865
1866 let (local_bonus, workload_local_bonus, mlx_bonus, vllm_mlx_bonus, system_llm_bonus) =
1877 if workload == RoutingWorkload::Quality {
1878 (0.0, 0.0, 0.0, 0.0, 0.0)
1879 } else {
1880 (
1881 local_bonus,
1882 workload_local_bonus,
1883 mlx_bonus,
1884 vllm_mlx_bonus,
1885 system_llm_bonus,
1886 )
1887 };
1888
1889 quality_weight * quality
1890 + latency_weight * latency
1891 + cost_weight * cost
1892 + local_bonus
1893 + workload_local_bonus
1894 + mlx_bonus
1895 + vllm_mlx_bonus
1896 + system_llm_bonus
1897 }
1898
1899 fn latency_ms_to_score(&self, ms: f64) -> f64 {
1902 (1.0 - (ms / Self::LATENCY_CEILING_MS)).clamp(0.0, 1.0)
1903 }
1904
1905 fn tps_to_latency_ms(tps: f64) -> f64 {
1907 if tps <= 0.0 {
1908 return Self::LATENCY_CEILING_MS;
1909 }
1910 (200.0 / tps) * 1000.0
1912 }
1913
1914 fn needs_multi_tool_call(prompt: &str) -> bool {
1917 let lower = prompt.to_lowercase();
1918
1919 let has_numbered_list = {
1921 let mut count = 0u32;
1922 for i in 1..=5u32 {
1923 if lower.contains(&format!("{}) ", i)) || lower.contains(&format!("{}. ", i)) {
1924 count += 1;
1925 }
1926 }
1927 count >= 2
1928 };
1929
1930 let multi_keywords = [
1932 "multiple edits",
1933 "several changes",
1934 "three changes",
1935 "two changes",
1936 "all of the following",
1937 "each of these",
1938 "do both",
1939 "do all",
1940 "and also",
1941 "additionally",
1942 "as well as",
1943 "then also",
1944 ];
1945 let has_multi_keywords = multi_keywords.iter().any(|kw| lower.contains(kw));
1946
1947 let bullet_actions = lower.matches("- add ").count()
1949 + lower.matches("- update ").count()
1950 + lower.matches("- change ").count()
1951 + lower.matches("- remove ").count()
1952 + lower.matches("- fix ").count()
1953 + lower.matches("- edit ").count()
1954 + lower.matches("- implement ").count()
1955 + lower.matches("- create ").count();
1956 let has_bullet_list = bullet_actions >= 2;
1957
1958 has_numbered_list || has_multi_keywords || has_bullet_list
1959 }
1960
1961 fn schema_quality_estimate(&self, model: &ModelSchema) -> f64 {
1983 if !model.public_benchmarks.is_empty() {
1984 return self.normalized_benchmark_quality(model).clamp(0.0, 1.0);
1985 }
1986 if model.is_remote() {
1987 if model.trust_tier == crate::schema::TrustTier::Curated
1993 && model.tags.iter().any(|tag| tag == "openrouter")
1994 {
1995 if model_is_frontier_class(model) {
1996 return Self::CURATED_FRONTIER_TAG_PRIOR;
1997 }
1998 if model.tags.iter().any(|tag| tag == "balanced") {
1999 return Self::CURATED_BALANCED_TAG_PRIOR;
2000 }
2001 }
2002 return match model.trust_tier {
2003 crate::schema::TrustTier::Community => Self::UNMEASURED_REMOTE_COMMUNITY,
2004 _ => Self::UNMEASURED_REMOTE_CURATED,
2005 };
2006 }
2007 match model.size_mb() {
2011 0 => 0.35, s if s < 1000 => 0.30, s if s < 2000 => 0.38, s if s < 3000 => 0.45, s if s < 6000 => 0.52, _ => 0.60, }
2018 }
2019
2020 fn normalized_benchmark_quality(&self, model: &ModelSchema) -> f64 {
2031 let is_local = model.is_local();
2032 let n = model.public_benchmarks.len() as f64;
2033 let sum: f64 = model
2034 .public_benchmarks
2035 .iter()
2036 .map(|b| match Self::benchmark_frontier_ref(&b.name) {
2037 Some(frontier_ref) => {
2038 if b.score > frontier_ref {
2047 tracing::warn!(
2048 model = %model.id,
2049 benchmark = %b.name,
2050 score = b.score,
2051 frontier_ref,
2052 "model scores ABOVE the catalog frontier_ref — re-anchor \
2053 candidate: update builtin_catalog.json so this benchmark's \
2054 champion calibrates the quality tier (#370)"
2055 );
2056 }
2057 let frac = (b.score / frontier_ref).clamp(0.0, 1.0);
2058 Self::BENCH_TIER_FLOOR + frac * (Self::BENCH_TIER_CEIL - Self::BENCH_TIER_FLOOR)
2059 }
2060 None if is_local => {
2066 b.score
2073 .clamp(Self::LOCAL_BENCH_TIER_FLOOR, Self::LOCAL_BENCH_TIER_CEIL)
2074 }
2075 None => b.score.clamp(Self::BENCH_TIER_FLOOR, Self::BENCH_TIER_CEIL),
2080 })
2081 .sum();
2082 sum / n
2089 }
2090
2091 fn benchmark_frontier_ref(name: &str) -> Option<f64> {
2103 BENCHMARK_FRONTIER_REFS
2104 .get(name)
2105 .copied()
2106 .filter(|r| *r > 0.0)
2107 }
2108
2109 fn schema_latency_estimate(&self, model: &ModelSchema) -> f64 {
2114 let is_moe = model.tags.contains(&"moe".to_string());
2115
2116 if model.is_local() {
2117 if let Some(tps) = model.performance.tokens_per_second {
2118 let measured = model.performance.latency_p50_ms.is_some();
2131 let effective_tps = if is_moe && !measured {
2132 let multiplier = if model.is_mlx() {
2133 Self::MLX_MOE_TPS_MULTIPLIER
2134 } else {
2135 Self::MOE_TPS_MULTIPLIER
2136 };
2137 tps * multiplier
2138 } else {
2139 tps
2140 };
2141 let estimated_ms = Self::tps_to_latency_ms(effective_tps);
2142 return self.latency_ms_to_score(estimated_ms);
2143 }
2144 return 0.5; }
2146
2147 if let Some(p50) = model.performance.latency_p50_ms {
2149 return self.latency_ms_to_score(p50 as f64);
2150 }
2151 0.3 }
2153
2154 fn is_quality_critical_bootstrap_task(
2155 &self,
2156 task: InferenceTask,
2157 has_vision: bool,
2158 has_tools: bool,
2159 ) -> bool {
2160 has_vision
2161 || has_tools
2162 || matches!(
2163 task,
2164 InferenceTask::Generate | InferenceTask::Code | InferenceTask::Reasoning
2165 )
2166 }
2167
2168 fn is_trusted_quality_remote(&self, model: &ModelSchema) -> bool {
2169 let trusted_provider = matches!(
2170 model.provider.as_str(),
2171 "openai" | "anthropic" | "google" | "openrouter"
2172 );
2173 model.is_remote()
2174 && model.trust_tier == crate::schema::TrustTier::Curated
2175 && (trusted_provider || crate::openrouter::is_managed_gateway_schema(model))
2176 && !model.has_capability(ModelCapability::SpeechToText)
2177 && !model.has_capability(ModelCapability::TextToSpeech)
2178 }
2179
2180 fn is_local_model_proven_for_task(
2181 &self,
2182 model: &ModelSchema,
2183 task: InferenceTask,
2184 tracker: &OutcomeTracker,
2185 ) -> bool {
2186 let Some(profile) = tracker.profile(&model.id) else {
2187 return false;
2188 };
2189 if let Some(task_stats) = profile.task_stats(task) {
2190 if task_stats.calls >= self.config.bootstrap_min_task_observations
2191 && task_stats.ema_quality >= self.config.bootstrap_quality_floor
2192 {
2193 return true;
2194 }
2195 }
2196
2197 profile.total_calls >= self.config.bootstrap_min_task_observations
2198 && profile.ema_quality >= self.config.bootstrap_quality_floor
2199 }
2200
2201 fn select_with_thompson_sampling(
2211 &self,
2212 scored: &[(String, f64)],
2213 tracker: &OutcomeTracker,
2214 ) -> (String, RoutingStrategy) {
2215 if scored.is_empty() {
2216 return (String::new(), RoutingStrategy::SchemaBased);
2217 }
2218
2219 let mut rng = rand::rng();
2220 let mut best_sample = f64::NEG_INFINITY;
2221 let mut best_id = scored[0].0.clone();
2222 let mut best_strategy = RoutingStrategy::SchemaBased;
2223
2224 for (id, phase2_score) in scored {
2225 let profile = tracker.profile(id);
2226 let prior = self.config.prior_strength;
2227
2228 let prior_mean = phase2_score.clamp(0.0, 1.0);
2230
2231 let prior_alpha = prior * prior_mean;
2233 let prior_beta = prior * (1.0 - prior_mean);
2234
2235 let (obs_alpha, obs_beta) = match profile {
2237 Some(p) => (p.success_count as f64, p.fail_count as f64),
2238 None => (0.0, 0.0),
2239 };
2240
2241 let alpha = (prior_alpha + obs_alpha).max(0.01);
2243 let beta = (prior_beta + obs_beta).max(0.01);
2244
2245 let sample = sample_beta(&mut rng, alpha, beta);
2247
2248 if sample > best_sample {
2249 best_sample = sample;
2250 best_id = id.clone();
2251 best_strategy = match profile {
2252 Some(p) if p.total_calls >= self.config.min_observations => {
2253 RoutingStrategy::ProfileBased
2254 }
2255 Some(p) if p.total_calls > 0 => {
2256 RoutingStrategy::Exploration
2258 }
2259 _ => RoutingStrategy::SchemaBased,
2260 };
2261 }
2262 }
2263
2264 (best_id, best_strategy)
2265 }
2266
2267 fn cold_start_decision(
2284 &self,
2285 complexity: TaskComplexity,
2286 task: InferenceTask,
2287 required_caps: &[ModelCapability],
2288 registry: &UnifiedRegistry,
2289 has_vision: bool,
2290 exclude: &std::collections::HashSet<String>,
2291 strict_exclusions: bool,
2292 availability: AvailabilitySnapshot,
2293 ) -> AdaptiveRoutingDecision {
2294 if has_vision {
2299 if let Some(model) = registry
2300 .query_by_capability(ModelCapability::Vision)
2301 .into_iter()
2302 .filter(|model| !exclude.contains(&model.id))
2303 .find(|model| {
2304 availability.is_available(model) && self.is_trusted_quality_remote(model)
2305 })
2306 .or_else(|| {
2307 registry
2308 .query_by_capability(ModelCapability::Vision)
2309 .into_iter()
2310 .find(|model| !exclude.contains(&model.id))
2311 })
2312 {
2313 return AdaptiveRoutingDecision {
2314 model_id: model.id.clone(),
2315 model_name: model.name.clone(),
2316 task,
2317 complexity,
2318 reason: format!(
2319 "{:?} task → {} (cold start, vision fallback)",
2320 complexity, model.name
2321 ),
2322 strategy: RoutingStrategy::SchemaBased,
2323 predicted_quality: 0.5,
2324 fallbacks: vec![],
2325 context_length: model.context_length,
2326 needs_compaction: false,
2327 candidates: vec![],
2328 };
2329 }
2330 }
2331
2332 if self.config.quality_first_cold_start {
2333 if let Some(model) = registry
2334 .list()
2335 .into_iter()
2336 .filter(|model| {
2337 availability.is_available(model)
2338 && !exclude.contains(&model.id)
2339 && required_caps.iter().all(|cap| model.has_capability(*cap))
2340 && self.is_trusted_quality_remote(model)
2341 })
2342 .max_by(|a, b| {
2343 self.schema_quality_estimate(a)
2344 .partial_cmp(&self.schema_quality_estimate(b))
2345 .unwrap_or(std::cmp::Ordering::Equal)
2346 })
2347 {
2348 return AdaptiveRoutingDecision {
2349 model_id: model.id.clone(),
2350 model_name: model.name.clone(),
2351 task,
2352 complexity,
2353 reason: format!(
2354 "{:?} task → {} (quality-first cold start)",
2355 complexity, model.name
2356 ),
2357 strategy: RoutingStrategy::SchemaBased,
2358 predicted_quality: self.schema_quality_estimate(model),
2359 fallbacks: vec![],
2360 context_length: model.context_length,
2361 needs_compaction: false,
2362 candidates: vec![],
2363 };
2364 }
2365 }
2366
2367 if let Some(model) = registry
2375 .list()
2376 .into_iter()
2377 .filter(|model| {
2378 availability.is_available(model)
2379 && !exclude.contains(&model.id)
2380 && model.is_local()
2381 && model.size_mb() < self.hw.max_model_mb
2382 && required_caps.iter().all(|cap| model.has_capability(*cap))
2383 })
2384 .max_by(|a, b| {
2385 self.schema_quality_estimate(a)
2386 .partial_cmp(&self.schema_quality_estimate(b))
2387 .unwrap_or(std::cmp::Ordering::Equal)
2388 })
2389 {
2390 return AdaptiveRoutingDecision {
2391 model_id: model.id.clone(),
2392 model_name: model.name.clone(),
2393 task,
2394 complexity,
2395 reason: format!(
2396 "{:?} task → {} (capability-satisfying local cold start)",
2397 complexity, model.name
2398 ),
2399 strategy: RoutingStrategy::SchemaBased,
2400 predicted_quality: self.schema_quality_estimate(model),
2401 fallbacks: vec![],
2402 context_length: model.context_length,
2403 needs_compaction: false,
2404 candidates: vec![],
2405 };
2406 }
2407
2408 if let Some(model) = registry
2419 .list()
2420 .into_iter()
2421 .filter(|model| {
2422 availability.is_available(model)
2423 && !exclude.contains(&model.id)
2424 && required_caps.iter().all(|cap| model.has_capability(*cap))
2425 })
2426 .max_by(|a, b| {
2427 let qa = (
2428 self.is_trusted_quality_remote(a),
2429 self.schema_quality_estimate(a),
2430 );
2431 let qb = (
2432 self.is_trusted_quality_remote(b),
2433 self.schema_quality_estimate(b),
2434 );
2435 qa.partial_cmp(&qb).unwrap_or(std::cmp::Ordering::Equal)
2436 })
2437 {
2438 return AdaptiveRoutingDecision {
2439 model_id: model.id.clone(),
2440 model_name: model.name.clone(),
2441 task,
2442 complexity,
2443 reason: format!(
2444 "{:?} task → {} (capability-satisfying cold start)",
2445 complexity, model.name
2446 ),
2447 strategy: RoutingStrategy::SchemaBased,
2448 predicted_quality: self.schema_quality_estimate(model),
2449 fallbacks: vec![],
2450 context_length: model.context_length,
2451 needs_compaction: false,
2452 candidates: vec![],
2453 };
2454 }
2455
2456 if strict_exclusions && !exclude.is_empty() {
2460 return Self::no_eligible_model_decision(complexity, task, exclude.len());
2461 }
2462
2463 let model_name = match complexity {
2470 TaskComplexity::Simple => "Qwen3-0.6B",
2471 TaskComplexity::Medium => "Qwen3-1.7B",
2472 TaskComplexity::Code => "Qwen3-4B",
2473 TaskComplexity::Complex => &self.hw.recommended_model,
2474 };
2475
2476 let model_id = registry
2477 .find_by_name(model_name)
2478 .map(|m| m.id.clone())
2479 .unwrap_or_else(|| model_name.to_string());
2480
2481 let context_length = registry
2482 .find_by_name(model_name)
2483 .map(|m| m.context_length)
2484 .unwrap_or(0);
2485
2486 AdaptiveRoutingDecision {
2487 model_id,
2488 model_name: model_name.to_string(),
2489 task,
2490 complexity,
2491 reason: format!(
2492 "{:?} task → {} (cold start, no candidates)",
2493 complexity, model_name
2494 ),
2495 strategy: RoutingStrategy::SchemaBased,
2496 predicted_quality: 0.5,
2497 fallbacks: vec![],
2498 context_length,
2499 needs_compaction: false,
2500 candidates: vec![],
2501 }
2502 }
2503
2504 fn no_eligible_model_decision(
2505 complexity: TaskComplexity,
2506 task: InferenceTask,
2507 excluded_count: usize,
2508 ) -> AdaptiveRoutingDecision {
2509 AdaptiveRoutingDecision {
2510 model_id: String::new(),
2511 model_name: String::new(),
2512 task,
2513 complexity,
2514 reason: format!(
2515 "strict model exclusions left no eligible model ({excluded_count} catalog ids excluded)"
2516 ),
2517 strategy: RoutingStrategy::SchemaBased,
2518 predicted_quality: 0.0,
2519 fallbacks: vec![],
2520 context_length: 0,
2521 needs_compaction: false,
2522 candidates: vec![],
2523 }
2524 }
2525}
2526
2527fn task_hint_to_inference_task(hint: crate::intent::TaskHint) -> InferenceTask {
2533 use crate::intent::TaskHint;
2534 match hint {
2535 TaskHint::Chat => InferenceTask::Generate,
2536 TaskHint::Classify => InferenceTask::Classify,
2537 TaskHint::Reasoning => InferenceTask::Reasoning,
2538 TaskHint::Code => InferenceTask::Code,
2539 }
2540}
2541
2542fn inference_task_required_capability(task: InferenceTask) -> ModelCapability {
2549 match task {
2550 InferenceTask::Generate => ModelCapability::Generate,
2551 InferenceTask::Code => ModelCapability::Code,
2552 InferenceTask::Reasoning => ModelCapability::Reasoning,
2553 InferenceTask::Classify => ModelCapability::Classify,
2554 InferenceTask::Embed => ModelCapability::Embed,
2555 }
2556}
2557
2558fn sample_beta(rng: &mut impl Rng, alpha: f64, beta: f64) -> f64 {
2566 let x = sample_gamma(rng, alpha);
2567 let y = sample_gamma(rng, beta);
2568 if x + y == 0.0 {
2569 0.5 } else {
2571 x / (x + y)
2572 }
2573}
2574
2575fn sample_gamma(rng: &mut impl Rng, shape: f64) -> f64 {
2578 if shape < 1.0 {
2579 let u: f64 = rng.random();
2581 return sample_gamma(rng, shape + 1.0) * u.powf(1.0 / shape);
2582 }
2583
2584 let d = shape - 1.0 / 3.0;
2586 let c = 1.0 / (9.0 * d).sqrt();
2587
2588 loop {
2589 let x: f64 = loop {
2590 let n = sample_standard_normal(rng);
2591 if 1.0 + c * n > 0.0 {
2592 break n;
2593 }
2594 };
2595
2596 let v = (1.0 + c * x).powi(3);
2597 let u: f64 = rng.random();
2598
2599 if u < 1.0 - 0.0331 * x.powi(4) {
2600 return d * v;
2601 }
2602 if u.ln() < 0.5 * x * x + d * (1.0 - v + v.ln()) {
2603 return d * v;
2604 }
2605 }
2606}
2607
2608fn sample_standard_normal(rng: &mut impl Rng) -> f64 {
2610 let u1: f64 = rng.random();
2611 let u2: f64 = rng.random();
2612 (-2.0 * u1.max(1e-300).ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
2613}
2614
2615#[cfg(test)]
2616mod tests {
2617 use super::*;
2618 use crate::outcome::InferredOutcome;
2619 use std::ffi::OsString;
2620
2621 fn route_model_ids(decision: &AdaptiveRoutingDecision) -> impl Iterator<Item = &str> {
2622 std::iter::once(decision.model_id.as_str())
2623 .chain(
2624 decision
2625 .candidates
2626 .iter()
2627 .map(|candidate| candidate.model_id.as_str()),
2628 )
2629 .chain(decision.fallbacks.iter().map(String::as_str))
2630 }
2631
2632 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
2633
2634 impl RestoredEnvironment {
2635 fn capture(names: &[&'static str]) -> Self {
2636 Self(
2637 names
2638 .iter()
2639 .map(|name| (*name, std::env::var_os(name)))
2640 .collect(),
2641 )
2642 }
2643 }
2644
2645 impl Drop for RestoredEnvironment {
2646 fn drop(&mut self) {
2647 for (name, value) in &self.0 {
2648 unsafe {
2649 match value {
2650 Some(value) => std::env::set_var(name, value),
2651 None => std::env::remove_var(name),
2652 }
2653 }
2654 }
2655 }
2656 }
2657
2658 struct TestRegistry {
2659 registry: UnifiedRegistry,
2660 _restore: RestoredEnvironment,
2663 _environment: tokio::sync::MutexGuard<'static, ()>,
2664 }
2665
2666 impl std::ops::Deref for TestRegistry {
2667 type Target = UnifiedRegistry;
2668
2669 fn deref(&self) -> &Self::Target {
2670 &self.registry
2671 }
2672 }
2673
2674 impl std::ops::DerefMut for TestRegistry {
2675 fn deref_mut(&mut self) -> &mut Self::Target {
2676 &mut self.registry
2677 }
2678 }
2679
2680 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2686 const QWEN3_8B_ROUTED_ID: &str = "mlx/qwen3-8b:4bit";
2687 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2688 const QWEN3_8B_ROUTED_ID: &str = "qwen/qwen3-8b:q4_k_m";
2689 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2690 const QWEN3_4B_ROUTED_ID: &str = "mlx/qwen3-4b:4bit";
2691 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2692 const QWEN3_4B_ROUTED_ID: &str = "qwen/qwen3-4b:q4_k_m";
2693
2694 fn test_hw() -> HardwareInfo {
2695 HardwareInfo {
2696 os: "macos".into(),
2697 arch: "aarch64".into(),
2698 cpu_cores: 10,
2699 total_ram_mb: 32768,
2700 gpu_backend: crate::hardware::GpuBackend::Metal,
2701 gpu_memory_mb: Some(28672),
2702 gpu_devices: Vec::new(),
2703 recommended_model: "Qwen3-8B".into(),
2704 recommended_context: 8192,
2705 max_model_mb: 18000, }
2707 }
2708
2709 fn test_registry() -> TestRegistry {
2710 let environment = crate::openrouter::test_environment_scope();
2711 let restore = RestoredEnvironment::capture(&["OPENAI_API_KEY", "CAR_SECRETS_FILE_DIR"]);
2712 let tmp = std::path::PathBuf::from("/tmp/car-test-adaptive-router");
2713 unsafe {
2714 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
2715 std::env::set_var(
2721 "CAR_SECRETS_FILE_DIR",
2722 std::env::temp_dir().join("car-test-adaptive-router-empty-secrets"),
2723 );
2724 }
2725 for name in &[
2727 "Qwen3-0.6B",
2728 "Qwen3-1.7B",
2729 "Qwen3-4B",
2730 "Qwen3-8B",
2731 "Qwen3-Embedding-0.6B",
2732 ] {
2733 let dir = tmp.join(name);
2734 let _ = std::fs::create_dir_all(&dir);
2735 let _ = std::fs::write(dir.join("model.gguf"), b"fake");
2736 let _ = std::fs::write(dir.join("tokenizer.json"), b"{}");
2737 }
2738 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
2739 reg.register_project_model(ModelSchema {
2740 id: "openai/gpt-5.4-mini:latest".into(),
2741 name: "gpt-5.4-mini".into(),
2742 provider: "openai".into(),
2743 family: "gpt-5.4".into(),
2744 version: "latest".into(),
2745 capabilities: vec![
2746 ModelCapability::Generate,
2747 ModelCapability::Code,
2748 ModelCapability::Reasoning,
2749 ModelCapability::ToolUse,
2750 ModelCapability::MultiToolCall,
2751 ModelCapability::Vision,
2752 ],
2753 context_length: 128_000,
2754 max_output_tokens: None,
2755 param_count: "api".into(),
2756 quantization: None,
2757 performance: Default::default(),
2758 cost: Default::default(),
2759 source: crate::schema::ModelSource::RemoteApi {
2760 endpoint: "https://api.openai.com/v1".into(),
2761 api_key_env: "OPENAI_API_KEY".into(),
2762 api_key_envs: vec![],
2763 api_version: None,
2764 protocol: crate::schema::ApiProtocol::OpenAiCompat,
2765 },
2766 tags: vec!["trusted-remote".into()],
2767 supported_params: vec![],
2768 public_benchmarks: vec![],
2769 trust_tier: crate::schema::TrustTier::Curated,
2770 deprecated: false,
2771 available: true,
2772 weights_ready: true,
2773 });
2774 TestRegistry {
2775 registry: reg,
2776 _restore: restore,
2777 _environment: environment,
2778 }
2779 }
2780
2781 #[test]
2782 fn route_observer_reports_a_429_exclusion_before_dispatch() {
2783 let registry = test_registry();
2784 let router = AdaptiveRouter::with_default_config(test_hw());
2785 let mut tracker = OutcomeTracker::new();
2786 let lane = "openai/gpt-5.4-mini:latest";
2787 let trace = tracker.record_start(lane, InferenceTask::Generate, "test 429");
2788 tracker.record_failure(&trace, "API returned 429 Too Many Requests");
2789 assert!(
2790 tracker.is_excluded(lane),
2791 "429 must seed the router exclusion"
2792 );
2793
2794 let mut exclusions = Vec::new();
2795 let mut observe = |exclusion| exclusions.push(exclusion);
2796 let decision = router.route_with_exclusion_observer(
2797 RouteRequest::new("Say hello.", ®istry, &tracker),
2798 &mut observe,
2799 );
2800
2801 assert_ne!(decision.model_id, lane);
2802 assert!(exclusions.contains(&RouterExclusion {
2803 lane: lane.into(),
2804 reason: RouterExclusionReason::RateLimited,
2805 until: None,
2806 }));
2807 }
2808
2809 #[test]
2810 fn route_observer_reports_an_open_circuit_with_its_cooldown() {
2811 let registry = test_registry();
2812 let router = AdaptiveRouter::with_default_config(test_hw());
2813 let tracker = OutcomeTracker::new();
2814 let lane = "openai/gpt-5.4-mini:latest";
2815 {
2816 let mut breakers = router.circuit_breakers.lock().unwrap();
2817 for _ in 0..3 {
2818 breakers.record_failure(lane);
2819 }
2820 }
2821
2822 let mut exclusions = Vec::new();
2823 let mut observe = |exclusion| exclusions.push(exclusion);
2824 let decision = router.route_with_exclusion_observer(
2825 RouteRequest::new("Say hello.", ®istry, &tracker),
2826 &mut observe,
2827 );
2828
2829 assert_ne!(decision.model_id, lane);
2830 let exclusion = exclusions
2831 .iter()
2832 .find(|exclusion| exclusion.lane == lane)
2833 .expect("open lane must be reported");
2834 assert_eq!(exclusion.reason, RouterExclusionReason::CircuitOpen);
2835 assert!(exclusion.until.is_some(), "open circuit has a cooldown end");
2836 }
2837
2838 #[test]
2839 fn openrouter_tags_drive_cost_quality_routing_and_live_availability() {
2840 let tmp = tempfile::tempdir().unwrap();
2841 let _credential_scope = crate::openrouter::test_credential_scope();
2842 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
2843 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
2844 for schema in crate::openrouter::curated_schemas()
2845 .into_iter()
2846 .filter(|schema| schema.provider == "openrouter")
2847 {
2848 registry.register_project_model(schema);
2849 }
2850 let router = AdaptiveRouter::new(
2851 test_hw(),
2852 RoutingConfig {
2853 prefer_local: false,
2854 prior_strength: 1_000_000.0,
2858 ..RoutingConfig::default()
2859 },
2860 );
2861 let tracker = OutcomeTracker::new();
2862 let mut simple_tracker = OutcomeTracker::new();
2863 let evidenced_cheap = "openrouter/qwen/qwen3-coder-next";
2864 for _ in 0..8 {
2865 let trace = simple_tracker.record_start(
2866 evidenced_cheap,
2867 InferenceTask::Generate,
2868 "OpenRouter cheap-model calibration",
2869 );
2870 simple_tracker.record_complete(&trace, 100, 10, 10);
2871 simple_tracker
2872 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
2873 }
2874
2875 crate::openrouter::reset_credential_source_call_count();
2876 let simple = router.route_with(RouteRequest {
2877 workload: RoutingWorkload::Background,
2878 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
2879 });
2880 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
2881 let simple_schema = registry.get(&simple.model_id).unwrap();
2882 assert!(simple_schema.tags.iter().any(|tag| tag == "cheap"));
2883 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
2884 assert!(simple
2885 .candidates
2886 .iter()
2887 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
2888
2889 let quality_intent = crate::intent::IntentHint {
2890 task: Some(crate::intent::TaskHint::Reasoning),
2891 prefer_quality: true,
2892 ..Default::default()
2893 };
2894 let complex = router.route_with(RouteRequest {
2895 intent: Some(&quality_intent),
2896 ..RouteRequest::new(
2897 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
2898 ®istry,
2899 &tracker,
2900 )
2901 });
2902 let complex_schema = registry.get(&complex.model_id).unwrap();
2903 assert!(complex_schema.tags.iter().any(|tag| tag == "frontier"));
2904 assert!(
2905 complex.reason.contains("frontier quality tier"),
2906 "{}",
2907 complex.reason
2908 );
2909
2910 let vision_intent = crate::intent::IntentHint {
2911 require: vec![ModelCapability::Vision],
2912 ..Default::default()
2913 };
2914 let vision = router.route_with(RouteRequest {
2915 intent: Some(&vision_intent),
2916 ..RouteRequest::new("Inspect this image", ®istry, &tracker)
2917 });
2918 assert!(vision.candidates.iter().all(|candidate| registry
2919 .get(&candidate.model_id)
2920 .is_some_and(|schema| schema.has_capability(ModelCapability::Vision))));
2921
2922 let failed_model = simple.model_id.clone();
2923 {
2924 let mut breakers = router.circuit_breakers.lock().unwrap();
2925 for _ in 0..3 {
2926 breakers.record_failure(&failed_model);
2927 }
2928 }
2929 let after_failures = router.route_with(RouteRequest {
2930 workload: RoutingWorkload::Background,
2931 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
2932 });
2933 assert_ne!(after_failures.model_id, failed_model);
2934 assert!(!after_failures
2935 .candidates
2936 .iter()
2937 .any(|candidate| candidate.model_id == failed_model));
2938
2939 crate::openrouter::set_test_credential(None);
2940 crate::openrouter::reset_credential_source_call_count();
2941 let without_key = router.route("Say hello.", ®istry, &tracker);
2942 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
2943 assert!(!without_key.model_id.starts_with("openrouter/"));
2944 assert!(!without_key
2945 .fallbacks
2946 .iter()
2947 .any(|model| model.starts_with("openrouter/")));
2948 assert!(!without_key
2949 .candidates
2950 .iter()
2951 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
2952 }
2953
2954 #[tokio::test(flavor = "current_thread")]
2955 async fn managed_openrouter_aliases_survive_quality_bootstrap_without_a_personal_key() {
2956 let tmp = tempfile::tempdir().unwrap();
2957 let (_credential_scope, _environment) =
2958 crate::openrouter::test_credential_and_environment_scope_async().await;
2959 let _restore = RestoredEnvironment::capture(&[
2960 "CAR_SECRETS_FILE_DIR",
2961 "CAR_MANAGED_ROUTING_TEST_KEY",
2962 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
2963 ]);
2964 let _home = crate::openrouter::StateRootScope::new();
2973 crate::openrouter::clear_gateway_unconfigured();
2974 crate::parslee_credential::clear_credential_rejected();
2975 let secrets_dir = tmp.path().join("secrets");
2976 unsafe {
2977 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
2978 std::env::set_var("CAR_MANAGED_ROUTING_TEST_KEY", "test-peer-key");
2979 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
2980 }
2981 crate::openrouter::set_test_credential(None);
2982
2983 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("models"));
2984 for schema in crate::openrouter::curated_schemas() {
2985 registry.register_project_model(schema);
2986 }
2987 let mut bootstrap_peer = remote_model(
2988 "openai/bootstrap-peer",
2989 "bootstrap-peer",
2990 vec![
2991 ModelCapability::Generate,
2992 ModelCapability::Code,
2993 ModelCapability::Reasoning,
2994 ],
2995 );
2996 if let crate::schema::ModelSource::RemoteApi {
2997 ref mut api_key_env,
2998 ..
2999 } = bootstrap_peer.source
3000 {
3001 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3002 }
3003 registry.register_project_model(bootstrap_peer);
3004 let mut unrelated = remote_model(
3005 "other/unrelated-curated",
3006 "unrelated-curated",
3007 vec![
3008 ModelCapability::Generate,
3009 ModelCapability::Code,
3010 ModelCapability::Reasoning,
3011 ],
3012 );
3013 unrelated.provider = "other".into();
3014 if let crate::schema::ModelSource::RemoteApi {
3015 ref mut api_key_env,
3016 ..
3017 } = unrelated.source
3018 {
3019 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3020 }
3021 registry.register_project_model(unrelated);
3022 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
3027
3028 let managed: Vec<_> = registry
3029 .list()
3030 .into_iter()
3031 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
3032 .collect();
3033 assert_eq!(managed.len(), crate::openrouter::managed_alias_count());
3034 assert!(managed.iter().all(|schema| {
3035 schema.available_now()
3036 && schema.tags.iter().any(|tag| tag == "openrouter")
3037 && schema.tags.iter().any(|tag| tag == "managed")
3038 && schema.cost.input_per_mtok.is_some_and(|price| price > 0.0)
3039 && schema.cost.output_per_mtok.is_some_and(|price| price > 0.0)
3040 && schema.has_capability(ModelCapability::Generate)
3041 }));
3042 assert!(registry
3043 .list()
3044 .into_iter()
3045 .filter(|schema| schema.id.starts_with("openrouter/"))
3046 .all(|schema| !schema.available_now()));
3047
3048 let router = AdaptiveRouter::new(
3049 test_hw(),
3050 RoutingConfig {
3051 prefer_local: false,
3052 prior_strength: 1_000_000.0,
3053 ..RoutingConfig::default()
3054 },
3055 );
3056 let tracker = OutcomeTracker::new();
3057
3058 let interactive = router.route("Say hello.", ®istry, &tracker);
3059 assert!(
3060 interactive
3061 .candidates
3062 .iter()
3063 .any(|candidate| candidate.model_id.starts_with("parslee/openrouter/")),
3064 "signed-in managed aliases must survive quality-first bootstrap: {interactive:?}"
3065 );
3066 assert!(
3067 interactive
3068 .fallbacks
3069 .iter()
3070 .any(|id| id.starts_with("parslee/openrouter/")),
3071 "signed-in managed aliases must remain fallback-eligible: {interactive:?}"
3072 );
3073 assert!(
3074 interactive
3075 .candidates
3076 .iter()
3077 .any(|candidate| candidate.model_id == "openai/bootstrap-peer"),
3078 "adding managed aliases must not remove existing trusted providers: {interactive:?}"
3079 );
3080 assert!(
3081 !interactive
3082 .candidates
3083 .iter()
3084 .any(|candidate| candidate.model_id == "other/unrelated-curated"),
3085 "managed-alias trust must not broaden the bootstrap to unrelated providers"
3086 );
3087
3088 let mut simple_tracker = OutcomeTracker::new();
3089 let evidenced_cheap = "parslee/openrouter/open-fast";
3090 for _ in 0..8 {
3091 let trace = simple_tracker.record_start(
3092 evidenced_cheap,
3093 InferenceTask::Generate,
3094 "managed OpenRouter cheap-model calibration",
3095 );
3096 simple_tracker.record_complete(&trace, 100, 10, 10);
3097 simple_tracker
3098 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
3099 }
3100 let simple = router.route_with(RouteRequest {
3101 workload: RoutingWorkload::Background,
3102 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3103 });
3104 let simple_schema = registry.get(&simple.model_id).unwrap();
3105 assert!(
3106 simple.model_id.starts_with("parslee/openrouter/")
3107 && simple_schema.tags.iter().any(|tag| tag == "cheap"),
3108 "simple work should prefer an evidenced cheap managed alias: {simple:?}"
3109 );
3110 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
3111
3112 let quality_intent = crate::intent::IntentHint {
3113 task: Some(crate::intent::TaskHint::Reasoning),
3114 prefer_quality: true,
3115 ..Default::default()
3116 };
3117 let complex = router.route_with(RouteRequest {
3118 intent: Some(&quality_intent),
3119 ..RouteRequest::new(
3120 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
3121 ®istry,
3122 &tracker,
3123 )
3124 });
3125 let complex_schema = registry.get(&complex.model_id).unwrap();
3126 assert!(
3127 complex.model_id.starts_with("parslee/openrouter/")
3128 && complex_schema.tags.iter().any(|tag| tag == "frontier"),
3129 "complex quality work should escalate to a managed frontier alias: {complex:?}"
3130 );
3131 assert!(
3132 complex.reason.contains("frontier quality tier"),
3133 "{}",
3134 complex.reason
3135 );
3136
3137 for decision in [&interactive, &simple, &complex] {
3138 assert!(
3139 !route_model_ids(decision).any(|id| id.starts_with("openrouter/")),
3140 "personal OpenRouter rows must stay excluded without a personal key: {decision:?}"
3141 );
3142 }
3143
3144 let spoofed_id = "parslee/openrouter/frontier-general";
3145 let spoofed_endpoint = "https://attacker.invalid/v1/chat/completions";
3146 let spoofed_value = serde_json::json!({
3147 "id": spoofed_id,
3148 "name": spoofed_id,
3149 "provider": "parslee",
3150 "family": "spoof",
3151 "capabilities": ["generate", "code", "reasoning"],
3152 "context_length": 1_000_000,
3153 "performance": {"latency_p50_ms": 1},
3154 "cost": {"input_per_mtok": 0.0, "output_per_mtok": 0.0},
3155 "source": {
3156 "type": "remote_api",
3157 "endpoint": spoofed_endpoint,
3158 "api_key_env": "CAR_MANAGED_ROUTING_TEST_KEY",
3159 "protocol": "open_ai_compat"
3160 },
3161 "tags": ["openrouter", "managed", "frontier"],
3162 "public_benchmarks": [{"name": "attacker-unanchored", "score": 1.0}]
3163 });
3164 let omitted_tier: ModelSchema = serde_json::from_value(spoofed_value.clone()).unwrap();
3165 assert_eq!(
3166 omitted_tier.trust_tier,
3167 crate::schema::TrustTier::Curated,
3168 "the regression must exercise the legacy omitted-tier default"
3169 );
3170 let mut direct_spoof_registry =
3171 UnifiedRegistry::new_empty(tmp.path().join("direct-spoof-models"));
3172 direct_spoof_registry.register(omitted_tier);
3173 assert!(
3174 direct_spoof_registry.get(spoofed_id).is_none(),
3175 "public registration must reject the reserved managed-alias namespace"
3176 );
3177 direct_spoof_registry
3178 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3179 let direct_spoof_route = router.route(
3180 "Analyze this architecture and prove the failure modes.",
3181 &direct_spoof_registry,
3182 &OutcomeTracker::new(),
3183 );
3184 assert!(
3185 !route_model_ids(&direct_spoof_route).any(|id| id == spoofed_id),
3186 "a rejected in-memory lookalike must not be routable: {direct_spoof_route:?}"
3187 );
3188 std::fs::write(
3189 tmp.path().join("models.json"),
3190 serde_json::to_vec_pretty(&vec![spoofed_value]).unwrap(),
3191 )
3192 .unwrap();
3193
3194 let mut reloaded = UnifiedRegistry::new_empty(tmp.path().join("reloaded-models"));
3195 reloaded.load_user_config().unwrap();
3196 assert!(
3197 reloaded.get(spoofed_id).is_none(),
3198 "persisted user config must reject the reserved managed-alias namespace"
3199 );
3200 reloaded.register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3201 let spoofed_route = router.route(
3202 "Analyze this architecture and prove the failure modes.",
3203 &reloaded,
3204 &OutcomeTracker::new(),
3205 );
3206 assert!(
3207 !route_model_ids(&spoofed_route).any(|id| id == spoofed_id),
3208 "a rejected persisted lookalike must not be routable: {spoofed_route:?}"
3209 );
3210
3211 let discovery_state_root = tmp.path().join("discovery-spoof-root");
3212 let discovery_models_dir = discovery_state_root.join("models");
3213 std::fs::create_dir_all(&discovery_models_dir).unwrap();
3214 let discovery_spoof_id = "openrouter/forged-discovery-cache";
3215 let mut discovery_spoof = remote_model(
3216 discovery_spoof_id,
3217 "forged-discovery-cache",
3218 vec![
3219 ModelCapability::Generate,
3220 ModelCapability::Code,
3221 ModelCapability::Reasoning,
3222 ],
3223 );
3224 discovery_spoof.provider = "openrouter".into();
3225 discovery_spoof.tags = vec!["openrouter".into(), "frontier".into()];
3226 discovery_spoof.trust_tier = crate::schema::TrustTier::Curated;
3227 discovery_spoof.performance.latency_p50_ms = Some(1);
3228 discovery_spoof.cost = crate::schema::CostModel::default();
3229 if let crate::schema::ModelSource::RemoteApi {
3230 ref mut api_key_env,
3231 ..
3232 } = discovery_spoof.source
3233 {
3234 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3235 }
3236 crate::discovery::save_cache(
3237 &crate::discovery::cache_path(&discovery_models_dir),
3238 &[discovery_spoof],
3239 )
3240 .unwrap();
3241
3242 let mut discovery_registry =
3243 UnifiedRegistry::new_with_state_root(discovery_state_root, discovery_models_dir);
3244 discovery_registry
3245 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3246 assert_eq!(
3247 discovery_registry
3248 .get(discovery_spoof_id)
3249 .unwrap()
3250 .trust_tier,
3251 crate::schema::TrustTier::Community,
3252 "unsigned discovery cache rows must be demoted before registration"
3253 );
3254 let discovery_spoof_route = router.route(
3255 "Analyze this architecture and prove the failure modes.",
3256 &discovery_registry,
3257 &OutcomeTracker::new(),
3258 );
3259 assert!(
3260 !discovery_spoof_route
3261 .candidates
3262 .iter()
3263 .any(|candidate| candidate.model_id == discovery_spoof_id),
3264 "a forged direct-provider discovery row must not enter trusted candidates: \
3265 {discovery_spoof_route:?}"
3266 );
3267 assert!(
3268 !discovery_spoof_route
3269 .fallbacks
3270 .iter()
3271 .any(|id| id == discovery_spoof_id),
3272 "a forged direct-provider discovery row must not enter trusted fallbacks: \
3273 {discovery_spoof_route:?}"
3274 );
3275 }
3276
3277 #[test]
3278 fn openrouter_becomes_require_ready_eligible_after_live_key_activation() {
3279 let tmp = tempfile::tempdir().unwrap();
3280 let _credential_scope = crate::openrouter::test_credential_scope();
3281 let _provider_env = crate::openrouter::test_environment_scope();
3282 crate::openrouter::set_test_credential(None);
3283 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
3284 let personal = crate::openrouter::curated_schemas()
3285 .into_iter()
3286 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3287 .unwrap();
3288 registry.register_project_model(personal);
3289
3290 unsafe {
3291 std::env::set_var("OPENAI_API_KEY", "ready-peer-key");
3292 }
3293 registry.register_project_model(remote_model(
3294 "ready-peer",
3295 "ready-peer",
3296 vec![ModelCapability::Generate, ModelCapability::Reasoning],
3297 ));
3298
3299 crate::openrouter::set_test_credential(Some("activated-after-register"));
3300 let hint = crate::intent::IntentHint {
3301 require_ready: true,
3302 prefer_quality: true,
3303 ..Default::default()
3304 };
3305 let router = AdaptiveRouter::new(
3306 test_hw(),
3307 RoutingConfig {
3308 prefer_local: false,
3309 prior_strength: 1_000_000.0,
3310 ..RoutingConfig::default()
3311 },
3312 );
3313 let tracker = OutcomeTracker::new();
3314 let decision = router.route_with(RouteRequest {
3315 intent: Some(&hint),
3316 ..RouteRequest::new("Prove the architecture tradeoff.", ®istry, &tracker)
3317 });
3318 assert!(
3319 decision
3320 .candidates
3321 .iter()
3322 .any(|candidate| candidate.model_id == "openrouter/openai/gpt-5.4"),
3323 "post-login OpenRouter model must remain require_ready-eligible: {:?}",
3324 decision.candidates
3325 );
3326 unsafe {
3327 std::env::remove_var("OPENAI_API_KEY");
3328 }
3329 }
3330
3331 #[test]
3332 fn route_cost_score_uses_prompt_tiers_and_model_cache_prices() {
3333 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
3334 let tracker = OutcomeTracker::new();
3335 let mut tiered = crate::openrouter::curated_schemas()
3336 .into_iter()
3337 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3338 .unwrap();
3339 let mut flat = tiered.clone();
3340 tiered.id = "tiered".into();
3341 flat.id = "flat".into();
3342 flat.cost = crate::schema::CostModel {
3343 input_per_mtok: Some(4.0),
3344 output_per_mtok: Some(22.5),
3345 cache_read_input_per_mtok: Some(4.0),
3346 ..Default::default()
3347 };
3348
3349 let score = |model: &ModelSchema, prompt, cache_read| {
3350 router.score_model(
3351 model,
3352 InferenceTask::Generate,
3353 &tracker,
3354 RoutingWorkload::Background,
3355 prompt,
3356 0,
3357 cache_read,
3358 0,
3359 )
3360 };
3361 assert!(score(&tiered, 100_000, 0) > score(&flat, 100_000, 0));
3362 assert!(score(&tiered, 300_000, 0) < score(&flat, 300_000, 0));
3363 assert!(score(&tiered, 300_000, 300_000) > score(&flat, 300_000, 300_000));
3364 }
3365
3366 #[test]
3367 fn routes_simple_to_trusted_remote_during_cold_start() {
3368 let router = AdaptiveRouter::new(
3369 test_hw(),
3370 RoutingConfig {
3371 prior_strength: 100.0, ..Default::default()
3373 },
3374 );
3375 let reg = test_registry();
3376 let tracker = OutcomeTracker::new();
3377
3378 let decision = router.route("What is 2+2?", ®, &tracker);
3379 assert_eq!(decision.complexity, TaskComplexity::Simple);
3380 assert_eq!(decision.strategy, RoutingStrategy::SchemaBased);
3381 let schema = reg
3383 .find_by_name(&decision.model_name)
3384 .expect("selected model should exist in registry");
3385 assert!(
3386 !schema.is_local(),
3387 "simple task should route to trusted remote model during cold start"
3388 );
3389 assert!(matches!(
3390 schema.provider.as_str(),
3391 "openai" | "anthropic" | "google"
3392 ));
3393 }
3394
3395 #[test]
3396 fn routes_code_to_code_capable_remote_during_cold_start() {
3397 let router = AdaptiveRouter::new(
3398 test_hw(),
3399 RoutingConfig {
3400 prior_strength: 100.0, ..Default::default()
3402 },
3403 );
3404 let reg = test_registry();
3405 let tracker = OutcomeTracker::new();
3406
3407 let decision = router.route(
3408 "Fix this function:\n```rust\nfn main() {}\n```",
3409 ®,
3410 &tracker,
3411 );
3412 assert_eq!(decision.complexity, TaskComplexity::Code);
3413 assert_eq!(decision.task, InferenceTask::Code);
3414 let schema = reg
3416 .find_by_name(&decision.model_name)
3417 .expect("model should exist");
3418 assert!(
3419 schema.has_capability(ModelCapability::Code),
3420 "selected model must support Code"
3421 );
3422 assert!(!schema.is_local(), "should route to trusted remote model");
3423 }
3424
3425 #[test]
3426 fn routes_images_to_vision_capable_model() {
3427 let router = AdaptiveRouter::new(
3428 test_hw(),
3429 RoutingConfig {
3430 prior_strength: 1_000_000.0,
3433 ..Default::default()
3434 },
3435 );
3436 let mut reg = test_registry();
3437 let tracker = OutcomeTracker::new();
3438
3439 reg.register_project_model(ModelSchema {
3440 id: "mlx-vlm/qwen3-vl-2b:bf16".into(),
3441 name: "Qwen3-VL-2B-mlx-vlm".into(),
3442 provider: "qwen".into(),
3443 family: "qwen3-vl".into(),
3444 version: "bf16".into(),
3445 capabilities: vec![
3446 ModelCapability::Generate,
3447 ModelCapability::Vision,
3448 ModelCapability::Grounding,
3449 ],
3450 context_length: 262_144,
3451 max_output_tokens: None,
3452 param_count: "2B".into(),
3453 quantization: None,
3454 performance: Default::default(),
3455 cost: Default::default(),
3456 source: crate::schema::ModelSource::Mlx {
3457 hf_repo: "Qwen/Qwen3-VL-2B-Instruct".into(),
3458 hf_weight_file: None,
3459 },
3460 tags: vec!["vision".into(), "mlx-vlm-cli".into()],
3461 supported_params: vec![],
3462 public_benchmarks: vec![],
3463 trust_tier: crate::schema::TrustTier::Curated,
3464 deprecated: false,
3465 available: true,
3466 weights_ready: true,
3467 });
3468
3469 let decision = router.route_with_vision("What is in this image?", ®, &tracker, false);
3470 let schema = reg
3471 .find_by_name(&decision.model_name)
3472 .expect("model should exist");
3473 assert!(
3474 schema.has_capability(ModelCapability::Vision),
3475 "selected model must support Vision"
3476 );
3477 }
3478
3479 #[test]
3480 fn profile_based_routing_favors_proven_model() {
3481 let router = AdaptiveRouter::new(
3482 test_hw(),
3483 RoutingConfig {
3484 prior_strength: 0.5, min_observations: 3,
3486 ..Default::default()
3487 },
3488 );
3489 let reg = test_registry();
3490 let mut tracker = OutcomeTracker::new();
3491
3492 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3494 for _ in 0..20 {
3495 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Code, "test");
3496 tracker.record_complete(&trace, 500, 100, 50);
3497 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3498 }
3499
3500 let mut wins = 0;
3502 for _ in 0..20 {
3503 let decision = router.route("Fix this bug in the parser", ®, &tracker);
3504 assert_eq!(decision.complexity, TaskComplexity::Code);
3505 if decision.model_id == qwen_8b_id {
3506 wins += 1;
3507 }
3508 }
3509 assert!(
3510 wins >= 12,
3511 "proven model won only {wins}/20 times (expected >= 12)"
3512 );
3513 }
3514
3515 #[test]
3516 fn proven_local_model_can_displace_bootstrap_remote() {
3517 let router = AdaptiveRouter::new(
3518 test_hw(),
3519 RoutingConfig {
3520 prior_strength: 100.0,
3521 bootstrap_min_task_observations: 6,
3522 bootstrap_quality_floor: 0.8,
3523 ..Default::default()
3524 },
3525 );
3526 let reg = test_registry();
3527 let mut tracker = OutcomeTracker::new();
3528
3529 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3530 for _ in 0..12 {
3531 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Generate, "test");
3532 tracker.record_complete(&trace, 300, 50, 20);
3533 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3534 }
3535
3536 let mut local_wins = 0;
3537 for _ in 0..20 {
3538 let decision = router.route("Summarize this design decision.", ®, &tracker);
3539 let schema = reg
3540 .get(&decision.model_id)
3541 .expect("selected model should exist");
3542 if schema.is_local() {
3543 local_wins += 1;
3544 }
3545 }
3546
3547 assert!(
3548 local_wins >= 12,
3549 "proven local model won only {local_wins}/20 times (expected >= 12)"
3550 );
3551 }
3552
3553 #[test]
3578 fn background_routing_prefers_local_models() {
3579 let router = AdaptiveRouter::new(
3580 test_hw(),
3581 RoutingConfig {
3582 prior_strength: 1_000_000.0,
3583 ..Default::default()
3584 },
3585 );
3586 let reg = test_registry();
3587 let tracker = OutcomeTracker::new();
3588
3589 let draws = 200;
3604 let mut local_wins = 0;
3605 for _ in 0..draws {
3606 let decision = router.route_context_aware(
3607 "Write a Python fibonacci function.",
3608 128,
3609 ®,
3610 &tracker,
3611 false,
3612 false,
3613 RoutingWorkload::Background,
3614 );
3615 let schema = reg
3616 .get(&decision.model_id)
3617 .expect("selected model should exist");
3618 if schema.is_local() {
3619 local_wins += 1;
3620 }
3621 }
3622
3623 assert!(
3624 local_wins >= 170,
3625 "background routing should keep work local; local won only \
3626 {local_wins}/{draws} (measured baseline: 96.2% off Apple Silicon, \
3627 100% on it)"
3628 );
3629 }
3630
3631 #[test]
3658 fn benchmark_prior_with_evidence_steers_background_routing() {
3659 fn target_score(profile: Option<crate::outcome::ModelProfile>) -> f64 {
3660 let router = AdaptiveRouter::new(
3661 test_hw(),
3662 RoutingConfig {
3663 prior_strength: 100.0,
3664 ..Default::default()
3665 },
3666 );
3667 let reg = test_registry();
3668 let mut tracker = OutcomeTracker::new();
3669 if let Some(p) = profile {
3670 tracker.import_profiles(vec![p]);
3671 }
3672 let decision = router.route_context_aware(
3673 "Write a Python fibonacci function.",
3674 128,
3675 ®,
3676 &tracker,
3677 false,
3678 false,
3679 RoutingWorkload::Background,
3680 );
3681 decision
3682 .candidates
3683 .iter()
3684 .find(|c| c.model_id == QWEN3_8B_ROUTED_ID)
3685 .unwrap_or_else(|| panic!("{QWEN3_8B_ROUTED_ID} should be a candidate"))
3686 .score
3687 }
3688
3689 fn profile_with(ema: f64) -> crate::outcome::ModelProfile {
3690 let mut p = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3691 p.ema_quality = ema;
3692 p.prior_sample_size = 30;
3696 p.quality_observations = 30;
3697 p
3698 }
3699
3700 let control = target_score(None);
3701 let good = target_score(Some(profile_with(0.99)));
3702 let bad = target_score(Some(profile_with(0.05)));
3703
3704 assert!(
3705 good > control,
3706 "an evidence-backed good prior must raise the score it vouches for \
3707 ({good} vs control {control}); equal means this test would pass with \
3708 the prior wiring removed"
3709 );
3710 assert!(
3711 bad < control,
3712 "an evidence-backed poor prior must lower the score ({bad} vs \
3713 control {control})"
3714 );
3715
3716 let mut unsupported = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3720 unsupported.ema_quality = 0.99;
3721 assert_eq!(
3722 target_score(Some(unsupported)),
3723 control,
3724 "an EMA with no observations behind it must not move the score"
3725 );
3726 }
3727
3728 #[test]
3729 fn task_specific_benchmark_prior_informs_cold_start_routing() {
3730 let router = AdaptiveRouter::new(
3731 test_hw(),
3732 RoutingConfig {
3733 prior_strength: 100.0,
3734 ..Default::default()
3735 },
3736 );
3737 let reg = test_registry();
3738 let mut tracker = OutcomeTracker::new();
3739 let mut profile = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3744 profile.task_stats.insert(
3745 crate::outcome::InferenceTask::Code.to_string(),
3746 crate::outcome::TaskStats {
3747 ema_quality: 0.95,
3748 prior_sample_size: 30,
3749 quality_observations: 30,
3750 ..Default::default()
3751 },
3752 );
3753 tracker.import_profiles(vec![profile]);
3754
3755 let decision = router.route_context_aware(
3756 "Write a Python fibonacci function.",
3757 128,
3758 ®,
3759 &tracker,
3760 false,
3761 false,
3762 RoutingWorkload::Background,
3763 );
3764
3765 let top_scored = decision
3769 .candidates
3770 .first()
3771 .expect("routing should expose ranked candidates");
3772 let schema = reg
3773 .get(&top_scored.model_id)
3774 .expect("top-scored model should exist");
3775 assert!(
3776 schema.is_local(),
3777 "background routing should score the task-specific local code prior highest"
3778 );
3779 }
3780
3781 #[test]
3782 fn task_specific_latency_prior_affects_cold_start_score() {
3783 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
3784 let reg = test_registry();
3785 let model = reg
3786 .get("qwen/qwen3-8b:q4_k_m")
3787 .expect("local test model should exist");
3788
3789 let mut fast_tracker = OutcomeTracker::new();
3790 let mut fast_profile = crate::outcome::ModelProfile::new(model.id.clone());
3791 fast_profile.task_stats.insert(
3792 crate::outcome::InferenceTask::Generate.to_string(),
3793 crate::outcome::TaskStats {
3794 ema_quality: 0.95,
3795 avg_latency_ms: 1200.0,
3796 ..Default::default()
3797 },
3798 );
3799 fast_tracker.import_profiles(vec![fast_profile]);
3800
3801 let mut slow_tracker = OutcomeTracker::new();
3802 let mut slow_profile = crate::outcome::ModelProfile::new(model.id.clone());
3803 slow_profile.task_stats.insert(
3804 crate::outcome::InferenceTask::Generate.to_string(),
3805 crate::outcome::TaskStats {
3806 ema_quality: 0.95,
3807 avg_latency_ms: 120_000.0,
3808 ..Default::default()
3809 },
3810 );
3811 slow_tracker.import_profiles(vec![slow_profile]);
3812
3813 let fast_score = router.score_model(
3814 model,
3815 InferenceTask::Generate,
3816 &fast_tracker,
3817 RoutingWorkload::Interactive,
3818 0,
3819 0,
3820 0,
3821 0,
3822 );
3823 let slow_score = router.score_model(
3824 model,
3825 InferenceTask::Generate,
3826 &slow_tracker,
3827 RoutingWorkload::Interactive,
3828 0,
3829 0,
3830 0,
3831 0,
3832 );
3833
3834 assert!(
3835 fast_score > slow_score,
3836 "faster task latency prior should improve cold-start score ({fast_score} <= {slow_score})"
3837 );
3838 }
3839
3840 #[test]
3841 fn interactive_workload_keeps_remote_bootstrap_bias() {
3842 let router = AdaptiveRouter::new(
3843 test_hw(),
3844 RoutingConfig {
3845 prior_strength: 100.0,
3846 ..Default::default()
3847 },
3848 );
3849 let reg = test_registry();
3850 let mut tracker = OutcomeTracker::new();
3851 let mut profile = crate::outcome::ModelProfile::new("qwen/qwen3-8b:q4_k_m".into());
3852 profile.ema_quality = 0.95;
3853 tracker.import_profiles(vec![profile]);
3854
3855 let decision = router.route_context_aware(
3856 "Write a Python fibonacci function.",
3857 128,
3858 ®,
3859 &tracker,
3860 false,
3861 false,
3862 RoutingWorkload::Interactive,
3863 );
3864
3865 let schema = reg
3866 .get(&decision.model_id)
3867 .expect("selected model should exist");
3868 assert!(
3869 !schema.is_local(),
3870 "interactive routing should still prefer trusted remote models during cold start"
3871 );
3872 }
3873
3874 #[test]
3875 fn code_interactive_weights_prioritise_quality_over_speed_over_cost() {
3876 let (q, lat, cost) =
3882 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Interactive);
3883 assert!(
3884 q > lat && lat > cost,
3885 "expected quality > speed > cost, got ({q}, {lat}, {cost})"
3886 );
3887 assert!(
3888 (q + lat + cost - 1.0).abs() < 1e-9,
3889 "weights must sum to 1.0"
3890 );
3891 let (default_q, default_lat, default_cost) = RoutingWorkload::Interactive.weights();
3892 assert!(
3893 q > default_q && lat < default_lat && cost < default_cost,
3894 "code weighting must lift quality and cut latency/cost vs the generic interactive profile"
3895 );
3896
3897 assert_eq!(
3899 AdaptiveRouter::task_aware_weights(
3900 InferenceTask::Code,
3901 RoutingWorkload::LocalPreferred,
3902 ),
3903 (q, lat, cost),
3904 );
3905
3906 assert_eq!(
3909 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Fastest),
3910 RoutingWorkload::Fastest.weights(),
3911 );
3912 assert_eq!(
3914 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Background),
3915 RoutingWorkload::Background.weights(),
3916 );
3917 assert_eq!(
3919 AdaptiveRouter::task_aware_weights(
3920 InferenceTask::Generate,
3921 RoutingWorkload::Interactive,
3922 ),
3923 RoutingWorkload::Interactive.weights(),
3924 );
3925 }
3926
3927 #[test]
3928 fn fallback_chain_has_alternatives() {
3929 let router = AdaptiveRouter::new(
3930 test_hw(),
3931 RoutingConfig {
3932 prior_strength: 100.0, ..Default::default()
3934 },
3935 );
3936 let reg = test_registry();
3937 let tracker = OutcomeTracker::new();
3938
3939 let decision = router.route("Analyze the architecture trade-offs", ®, &tracker);
3940 assert!(!decision.fallbacks.is_empty());
3941 assert!(!decision.fallbacks.contains(&decision.model_id));
3943 }
3944
3945 #[test]
3946 fn latency_scoring_is_consistent() {
3947 let router = AdaptiveRouter::with_default_config(test_hw());
3949
3950 let schema_score = router.latency_ms_to_score(AdaptiveRouter::tps_to_latency_ms(25.0));
3952 let observed_score = router.latency_ms_to_score(8000.0);
3954 assert!(
3955 (schema_score - observed_score).abs() < 0.01,
3956 "schema ({schema_score}) and observed ({observed_score}) should match"
3957 );
3958 }
3959
3960 #[test]
3961 fn complexity_assessment() {
3962 assert_eq!(
3963 TaskComplexity::assess("What is the capital of France?"),
3964 TaskComplexity::Simple
3965 );
3966 assert_eq!(
3967 TaskComplexity::assess("Fix this broken test"),
3968 TaskComplexity::Code
3969 );
3970 assert_eq!(
3971 TaskComplexity::assess("Analyze the trade-offs between A and B"),
3972 TaskComplexity::Complex
3973 );
3974 }
3975
3976 #[test]
3977 fn beta_sampling_produces_valid_values() {
3978 let mut rng = rand::rng();
3979 for _ in 0..100 {
3981 let s = sample_beta(&mut rng, 2.0, 5.0);
3982 assert!((0.0..=1.0).contains(&s), "sample {s} out of [0,1] range");
3983 }
3984 let samples: Vec<f64> = (0..1000).map(|_| sample_beta(&mut rng, 1.0, 1.0)).collect();
3986 let mean = samples.iter().sum::<f64>() / samples.len() as f64;
3987 assert!(
3988 (mean - 0.5).abs() < 0.05,
3989 "Beta(1,1) mean {mean} should be ~0.5"
3990 );
3991 }
3992
3993 #[test]
3994 fn thompson_sampling_converges_to_best() {
3995 let router = AdaptiveRouter::new(
3997 test_hw(),
3998 RoutingConfig {
3999 prior_strength: 1.0, ..Default::default()
4001 },
4002 );
4003 let reg = test_registry();
4004 let mut tracker = OutcomeTracker::new();
4005
4006 let qwen_4b_id = QWEN3_4B_ROUTED_ID;
4008 for _ in 0..20 {
4009 let trace = tracker.record_start(qwen_4b_id, InferenceTask::Code, "test");
4010 tracker.record_complete(&trace, 500, 100, 50);
4011 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
4012 }
4013
4014 let mut wins = 0;
4016 for _ in 0..20 {
4017 let decision = router.route("Fix this parser bug", ®, &tracker);
4018 if decision.model_id == qwen_4b_id {
4019 wins += 1;
4020 }
4021 }
4022 assert!(
4023 wins >= 14,
4024 "strong model won only {wins}/20 times (expected >= 14)"
4025 );
4026 }
4027
4028 #[test]
4031 fn cold_start_never_returns_a_model_lacking_a_required_capability() {
4032 let router = AdaptiveRouter::new(
4040 test_hw(),
4041 RoutingConfig {
4042 quality_first_cold_start: false,
4043 ..Default::default()
4044 },
4045 );
4046 let reg = test_registry();
4047
4048 for caps in [
4049 vec![ModelCapability::Code],
4050 vec![ModelCapability::Code, ModelCapability::ToolUse],
4051 ] {
4052 let decision = router.cold_start_decision(
4055 TaskComplexity::Simple,
4056 InferenceTask::Code,
4057 &caps,
4058 ®,
4059 false,
4060 &std::collections::HashSet::new(),
4061 false,
4062 AvailabilitySnapshot::capture(®),
4063 );
4064 let chosen = reg
4065 .find_by_name(&decision.model_name)
4066 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
4067 .unwrap_or_else(|| panic!("cold start named an unknown model: {decision:?}"));
4068 for cap in &caps {
4069 assert!(
4070 chosen.has_capability(*cap),
4071 "cold start chose {} which lacks required {:?} (caps {:?})",
4072 chosen.name,
4073 cap,
4074 caps
4075 );
4076 }
4077 assert_eq!(
4084 decision.strategy,
4085 RoutingStrategy::SchemaBased,
4086 "expected schema-based cold start"
4087 );
4088 assert!(
4089 decision.reason.contains("capability-satisfying"),
4090 "expected a capability-satisfying cold-start branch, got: {}",
4091 decision.reason
4092 );
4093 }
4094 }
4095
4096 #[test]
4097 fn cold_start_skips_a_capable_local_that_exceeds_the_memory_budget() {
4098 let mut hw = test_hw();
4103 hw.max_model_mb = 300; let router = AdaptiveRouter::new(
4105 hw,
4106 RoutingConfig {
4107 quality_first_cold_start: false,
4108 ..Default::default()
4109 },
4110 );
4111 let reg = test_registry();
4112 let decision = router.cold_start_decision(
4113 TaskComplexity::Code,
4114 InferenceTask::Code,
4115 &[ModelCapability::Code],
4116 ®,
4117 false,
4118 &std::collections::HashSet::new(),
4119 false,
4120 AvailabilitySnapshot::capture(®),
4121 );
4122 assert!(
4123 !decision.reason.contains("local cold start"),
4124 "no local fits the 300MB budget, so the local branch must not fire: {}",
4125 decision.reason
4126 );
4127 }
4128
4129 #[test]
4130 fn intent_require_filters_out_models_lacking_capability() {
4131 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4136 let reg = test_registry();
4137 let tracker = OutcomeTracker::new();
4138
4139 let intent = crate::intent::IntentHint {
4140 require: vec![ModelCapability::Vision],
4141 ..Default::default()
4142 };
4143 let decision = router.route_with_intent("hello", ®, &tracker, &intent);
4144
4145 assert_eq!(
4150 decision.strategy,
4151 RoutingStrategy::SchemaBased,
4152 "require=[vision] with no vision-capable candidates must drop to schema cold-start"
4153 );
4154 }
4155
4156 #[test]
4157 fn unbenchmarked_remote_is_conservative_not_assumed_frontier() {
4158 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4167 let reg = test_registry();
4168 let remote = reg
4169 .list()
4170 .into_iter()
4171 .find(|m| m.is_remote() && m.public_benchmarks.is_empty())
4172 .expect("a benchmark-less remote in the fixture")
4173 .clone();
4174 let q = router.schema_quality_estimate(&remote);
4175 assert!(
4176 (q - 0.60).abs() < 1e-9,
4177 "unmeasured curated remote should be a conservative 0.60, got {q}"
4178 );
4179
4180 let mut benched = remote.clone();
4184 benched.public_benchmarks = vec![
4185 crate::schema::BenchmarkScore {
4186 name: "A".into(),
4187 score: 0.9,
4188 harness: None,
4189 source_url: None,
4190 measured_at: None,
4191 runs: None,
4192 spread: None,
4193 },
4194 crate::schema::BenchmarkScore {
4195 name: "B".into(),
4196 score: 0.7,
4197 harness: None,
4198 source_url: None,
4199 measured_at: None,
4200 runs: None,
4201 spread: None,
4202 },
4203 ];
4204 let qb = router.schema_quality_estimate(&benched);
4205 assert!((qb - 0.8).abs() < 1e-9);
4206 assert!(
4207 qb > q,
4208 "real benchmarks must lift a model above the unmeasured default"
4209 );
4210 }
4211
4212 #[test]
4213 fn calibrated_benchmark_is_normalized_onto_the_tier_scale() {
4214 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4218 let bench = |raw: f64| {
4219 let mut m = outcome_model("m", None);
4220 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4221 name: "tau-bench-airline".into(),
4222 score: raw,
4223 harness: None,
4224 source_url: None,
4225 measured_at: None,
4226 runs: None,
4227 spread: None,
4228 }];
4229 router.schema_quality_estimate(&m)
4230 };
4231 let frontier = bench(0.65); let strong = bench(0.35); assert!(
4235 (frontier - 0.92).abs() < 0.01,
4236 "best-in-class → ~CEIL, got {frontier}"
4237 );
4238 assert!(
4239 (strong - 0.68).abs() < 0.02,
4240 "0.35 raw → ~0.68 tier, got {strong}"
4241 );
4242 assert!(
4243 strong > router.schema_quality_estimate(&local_model("local-30b", 18_000)),
4244 "a measured frontier model must out-rank an unmeasured 30B local"
4245 );
4246 }
4247
4248 #[test]
4249 fn uncalibrated_benchmark_is_clamped_into_the_tier_band() {
4250 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4254 let q = |raw: f64| {
4255 let mut m = outcome_model("m", None);
4256 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4257 name: "some-new-unknown-bench".into(),
4258 score: raw,
4259 harness: None,
4260 source_url: None,
4261 measured_at: None,
4262 runs: None,
4263 spread: None,
4264 }];
4265 router.schema_quality_estimate(&m)
4266 };
4267 assert!((q(0.73) - 0.73).abs() < 1e-9);
4270 assert!((q(0.10) - AdaptiveRouter::BENCH_TIER_FLOOR).abs() < 1e-9);
4271 assert!((q(0.99) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9);
4272 }
4273
4274 #[test]
4275 fn local_uncalibrated_benchmark_maps_into_the_local_tier_band() {
4276 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4282 let q = |raw: f64| {
4283 let mut m = local_model("local-bench", 2500); m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4285 name: "car-agentic-suite".into(), score: raw,
4287 harness: None,
4288 source_url: None,
4289 measured_at: None,
4290 runs: None,
4291 spread: None,
4292 }];
4293 router.schema_quality_estimate(&m)
4294 };
4295 assert!(
4299 (q(0.45) - 0.45).abs() < 1e-9,
4300 "in-band local score passes through"
4301 );
4302 assert!(
4303 (q(0.05) - AdaptiveRouter::LOCAL_BENCH_TIER_FLOOR).abs() < 1e-9,
4304 "sub-floor clamps to LOCAL floor 0.30, not frontier FLOOR 0.40"
4305 );
4306 assert!(
4307 (q(0.95) - AdaptiveRouter::LOCAL_BENCH_TIER_CEIL).abs() < 1e-9,
4308 "a perfect local-suite score tops at LOCAL ceil 0.60, not frontier CEIL 0.92"
4309 );
4310 assert!(q(0.55) > q(0.35));
4312 let mut frontier = outcome_model("frontier", None);
4315 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
4316 name: "tau-bench-airline".into(),
4317 score: 0.65,
4318 harness: None,
4319 source_url: None,
4320 measured_at: None,
4321 runs: None,
4322 spread: None,
4323 }];
4324 assert!(
4325 router.schema_quality_estimate(&frontier) > q(0.95),
4326 "a benchmarked frontier model out-ranks even a perfect-scoring local"
4327 );
4328 }
4329
4330 #[test]
4331 fn frontier_ref_is_derived_from_catalog_best_in_class() {
4332 use std::collections::HashMap;
4337 let mut max_by_bench: HashMap<String, f64> = HashMap::new();
4338 for model in crate::registry::builtin_catalog() {
4339 for b in &model.public_benchmarks {
4340 let e = max_by_bench.entry(b.name.clone()).or_insert(0.0);
4341 *e = e.max(b.score);
4342 }
4343 }
4344 assert!(!max_by_bench.is_empty(), "catalog should carry benchmarks");
4345 for (name, max) in max_by_bench {
4346 let derived = AdaptiveRouter::benchmark_frontier_ref(&name)
4347 .unwrap_or_else(|| panic!("benchmark '{name}' must derive a frontier_ref"));
4348 assert!(
4349 (derived - max).abs() < 1e-9,
4350 "frontier_ref for '{name}' should be the catalog max {max}, got {derived}"
4351 );
4352 }
4353 assert_eq!(
4355 AdaptiveRouter::benchmark_frontier_ref("tau-bench-airline"),
4356 Some(0.65)
4357 );
4358 assert_eq!(
4360 AdaptiveRouter::benchmark_frontier_ref("nonexistent-bench"),
4361 None
4362 );
4363 }
4364
4365 fn benchmarked_model(id: &str, bench: &str, score: f64, frontier: bool) -> ModelSchema {
4369 let mut m = outcome_model(id, None);
4370 if frontier {
4371 m.tags.push("frontier".into());
4372 }
4373 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4374 name: bench.into(),
4375 score,
4376 harness: None,
4377 source_url: None,
4378 measured_at: None,
4379 runs: None,
4380 spread: None,
4381 }];
4382 m
4383 }
4384
4385 #[test]
4386 fn frontier_ref_guard_drops_frontier_less_benchmark() {
4387 let models = [
4392 benchmarked_model("mid-a", "midtier-only-bench", 0.50, false),
4393 benchmarked_model("mid-b", "midtier-only-bench", 0.40, false),
4394 ];
4395 let refs = compute_frontier_refs(models.iter());
4396 assert!(
4397 !refs.contains_key("midtier-only-bench"),
4398 "a benchmark with no frontier-class carrier must not self-anchor, got {refs:?}"
4399 );
4400 }
4401
4402 #[test]
4403 fn frontier_ref_guard_anchors_when_a_frontier_model_carries_the_benchmark() {
4404 let models = [
4409 benchmarked_model("frontier-x", "shared-bench", 0.60, true),
4410 benchmarked_model("mid-y", "shared-bench", 0.70, false),
4411 ];
4412 let refs = compute_frontier_refs(models.iter());
4413 assert_eq!(
4414 refs.get("shared-bench").copied(),
4415 Some(0.70),
4416 "anchor is the catalog max across all carriers once a frontier carrier exists"
4417 );
4418 }
4419
4420 #[test]
4421 fn frontier_ref_guard_is_noop_on_todays_catalog() {
4422 use std::collections::HashMap;
4427 let catalog = crate::registry::builtin_catalog();
4428 let mut raw_max: HashMap<String, f64> = HashMap::new();
4429 for m in &catalog {
4430 for b in &m.public_benchmarks {
4431 let e = raw_max.entry(b.name.clone()).or_insert(0.0);
4432 *e = e.max(b.score);
4433 }
4434 }
4435 let guarded = compute_frontier_refs(catalog.iter());
4436 assert_eq!(
4437 raw_max, guarded,
4438 "guard dropped a benchmark — a catalog benchmark lost its frontier-class carrier"
4439 );
4440 assert!(
4441 model_is_frontier_class(
4442 catalog
4443 .iter()
4444 .find(|m| m.id == "anthropic/claude-opus-4-7:latest")
4445 .expect("opus-4-7 in catalog")
4446 ),
4447 "the tau-bench-airline champion must be frontier-tagged for the anchor to be valid"
4448 );
4449 }
4450
4451 #[test]
4452 fn score_above_the_frontier_clamps_to_ceil() {
4453 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4457 let mut m = outcome_model("future-sota", None);
4458 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4459 name: "tau-bench-airline".into(),
4460 score: 0.80, harness: None,
4462 source_url: None,
4463 measured_at: None,
4464 runs: None,
4465 spread: None,
4466 }];
4467 assert!(
4468 (router.schema_quality_estimate(&m) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9,
4469 "above-frontier score should clamp to CEIL"
4470 );
4471 }
4472
4473 #[test]
4474 fn local_size_heuristic_stays_below_measured_frontier_floor() {
4475 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4479 for size_mb in [500u64, 1500, 2500, 4500, 18_000] {
4480 let q = router.schema_quality_estimate(&local_model("local", size_mb));
4481 assert!(
4482 q <= AdaptiveRouter::BENCH_TIER_FLOOR + 0.21, "local {size_mb}MB quality {q} must stay near/below the measured floor"
4484 );
4485 }
4486 }
4487
4488 #[test]
4489 fn community_remote_quality_is_damped_below_curated() {
4490 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4495 let reg = test_registry();
4496 let mut remote = reg
4497 .list()
4498 .into_iter()
4499 .find(|m| m.is_remote() && m.public_benchmarks.is_empty())
4500 .expect("a benchmark-less remote in the fixture")
4501 .clone();
4502
4503 remote.trust_tier = crate::schema::TrustTier::Curated;
4505 let qc = router.schema_quality_estimate(&remote);
4506 assert!(
4507 (qc - 0.60).abs() < 1e-9,
4508 "curated unmeasured remote should be 0.60, got {qc}"
4509 );
4510
4511 remote.trust_tier = crate::schema::TrustTier::Community;
4513 let q = router.schema_quality_estimate(&remote);
4514 assert!(
4515 (q - 0.48).abs() < 1e-9,
4516 "community remote should be damped to 0.48, got {q}"
4517 );
4518 assert!(
4519 q < qc,
4520 "community remote must rank below an unmeasured curated remote"
4521 );
4522 }
4523
4524 #[test]
4525 fn quality_workload_is_quality_dominant() {
4526 let (q, lat, cost) = RoutingWorkload::Quality.weights();
4529 assert!(
4530 (q + lat + cost - 1.0).abs() < 1e-9,
4531 "weights must sum to 1.0"
4532 );
4533 assert!(q >= 0.8, "quality must dominate, got {q}");
4534 assert!(
4535 lat <= 0.1 && cost <= 0.15,
4536 "latency/cost near floor: {lat}/{cost}"
4537 );
4538 for w in [
4540 RoutingWorkload::Interactive,
4541 RoutingWorkload::Batch,
4542 RoutingWorkload::Background,
4543 RoutingWorkload::LocalPreferred,
4544 RoutingWorkload::Fastest,
4545 ] {
4546 assert!(
4547 q > w.weights().0,
4548 "Quality must out-weight {w:?} on quality"
4549 );
4550 }
4551 }
4552
4553 #[test]
4554 fn prefer_quality_scores_the_more_capable_model_higher() {
4555 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4560 let reg = test_registry();
4561 let tracker = OutcomeTracker::new();
4562 let small = reg
4563 .find_by_name("Qwen3-1.7B")
4564 .expect("1.7B in fixture")
4565 .clone();
4566 let big = reg.find_by_name("Qwen3-4B").expect("4B in fixture").clone();
4567 assert!(small.has_capability(ModelCapability::Code));
4568 assert!(big.has_capability(ModelCapability::Code));
4569
4570 let score = |m: &ModelSchema, w: RoutingWorkload| {
4571 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
4572 };
4573 assert!(
4575 score(&big, RoutingWorkload::Quality) > score(&small, RoutingWorkload::Quality),
4576 "prefer_quality must rank the capable 4B above the cheap 1.7B"
4577 );
4578 }
4579
4580 fn local_mlx_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
4583 let mut m = outcome_model(id, benchmark);
4584 m.provider = "local".into();
4585 m.source = crate::schema::ModelSource::Mlx {
4586 hf_repo: "mlx-community/test".into(),
4587 hf_weight_file: None,
4588 };
4589 m
4590 }
4591
4592 #[test]
4593 fn quality_workload_suppresses_local_mlx_bonus_vs_capable_remote() {
4594 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4599 let tracker = OutcomeTracker::new();
4600 let score = |m: &ModelSchema, w: RoutingWorkload| {
4601 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
4602 };
4603
4604 let remote_capable = outcome_model("remote/frontier", Some(0.85));
4605 let mlx_tiny = local_mlx_model("mlx/tiny", Some(0.55));
4606 let remote_peer = outcome_model("remote/peer", Some(0.55));
4608 assert!(mlx_tiny.is_mlx() && mlx_tiny.is_local());
4609
4610 assert!(
4614 score(&mlx_tiny, RoutingWorkload::Background)
4615 > score(&remote_peer, RoutingWorkload::Background),
4616 "the local/MLX bonus should still apply on non-Quality workloads"
4617 );
4618
4619 assert!(
4622 score(&remote_capable, RoutingWorkload::Quality)
4623 > score(&mlx_tiny, RoutingWorkload::Quality),
4624 "prefer_quality must suppress the locality bonus so the capable remote wins"
4625 );
4626 }
4627
4628 fn local_model(id: &str, size_mb: u64) -> ModelSchema {
4633 let mut m = outcome_model(id, None);
4634 m.provider = "local".into();
4635 m.source = crate::schema::ModelSource::Local {
4636 hf_repo: "test/repo".into(),
4637 hf_filename: "model.gguf".into(),
4638 tokenizer_repo: "test/repo".into(),
4639 };
4640 m.cost.size_mb = Some(size_mb);
4641 m
4642 }
4643
4644 #[test]
4655 fn a_measured_moe_rate_is_not_discounted_again() {
4656 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4657
4658 let mut measured = local_model("local/moe-measured", 17_000);
4659 measured.tags.push("moe".into());
4660 measured.performance.tokens_per_second = Some(35.0);
4661 measured.performance.latency_p50_ms = Some(1_200);
4662
4663 let mut declared = local_model("local/moe-declared", 17_000);
4665 declared.tags.push("moe".into());
4666 declared.performance.tokens_per_second = Some(35.0);
4667
4668 let mut dense = local_model("local/dense", 17_000);
4670 dense.performance.tokens_per_second = Some(35.0);
4671 dense.performance.latency_p50_ms = Some(1_200);
4672
4673 let measured_score = router.schema_latency_estimate(&measured);
4674 let declared_score = router.schema_latency_estimate(&declared);
4675 let dense_score = router.schema_latency_estimate(&dense);
4676
4677 assert_eq!(
4678 measured_score, dense_score,
4679 "a measured MoE rate must score as the rate it measured"
4680 );
4681 assert!(
4682 declared_score < measured_score,
4683 "a declared MoE rate still gets the discount: {declared_score} vs {measured_score}"
4684 );
4685 }
4686
4687 fn outcome_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
4691 ModelSchema {
4692 id: id.into(),
4693 name: id.into(),
4694 provider: "openai".into(),
4695 family: "test".into(),
4696 version: "latest".into(),
4697 capabilities: vec![
4698 ModelCapability::Generate,
4699 ModelCapability::Code,
4700 ModelCapability::Reasoning,
4701 ],
4702 context_length: 128_000,
4703 max_output_tokens: None,
4704 param_count: "api".into(),
4705 quantization: None,
4706 performance: Default::default(),
4707 cost: Default::default(),
4708 source: crate::schema::ModelSource::RemoteApi {
4709 endpoint: "https://api.openai.com/v1".into(),
4710 api_key_env: "OPENAI_API_KEY".into(),
4711 api_key_envs: vec![],
4712 api_version: None,
4713 protocol: crate::schema::ApiProtocol::OpenAiCompat,
4714 },
4715 tags: vec![],
4716 supported_params: vec![],
4717 public_benchmarks: benchmark
4718 .map(|s| {
4719 vec![crate::schema::BenchmarkScore {
4720 name: "t".into(),
4721 score: s,
4722 harness: None,
4723 source_url: None,
4724 measured_at: None,
4725 runs: None,
4726 spread: None,
4727 }]
4728 })
4729 .unwrap_or_default(),
4730 trust_tier: crate::schema::TrustTier::Curated,
4731 deprecated: false,
4732 available: true,
4733 weights_ready: true,
4734 }
4735 }
4736
4737 #[test]
4738 fn reliability_band_applies_to_quality_and_substantive_normal_lanes() {
4739 use RoutingWorkload as W;
4740 assert!(AdaptiveRouter::applies_reliability_band(
4742 InferenceTask::Code,
4743 W::Interactive
4744 ));
4745 assert!(AdaptiveRouter::applies_reliability_band(
4746 InferenceTask::Reasoning,
4747 W::Batch
4748 ));
4749 assert!(AdaptiveRouter::applies_reliability_band(
4752 InferenceTask::Code,
4753 W::Quality
4754 ));
4755 assert!(AdaptiveRouter::applies_reliability_band(
4756 InferenceTask::Generate,
4757 W::Quality
4758 ));
4759 assert!(!AdaptiveRouter::applies_reliability_band(
4761 InferenceTask::Generate,
4762 W::Interactive
4763 ));
4764 assert!(!AdaptiveRouter::applies_reliability_band(
4766 InferenceTask::Code,
4767 W::Background
4768 ));
4769 assert!(!AdaptiveRouter::applies_reliability_band(
4770 InferenceTask::Code,
4771 W::Fastest
4772 ));
4773 assert!(!AdaptiveRouter::applies_reliability_band(
4775 InferenceTask::Code,
4776 W::LocalPreferred
4777 ));
4778 }
4779
4780 #[test]
4781 fn high_stakes_intent_forces_quality_over_everything() {
4782 use crate::intent::IntentHint;
4783 let hs = IntentHint {
4786 high_stakes: true,
4787 prefer_fast: true,
4788 prefer_local: true,
4789 ..Default::default()
4790 };
4791 assert_eq!(
4792 AdaptiveRouter::resolve_workload(Some(&hs), RoutingWorkload::Interactive),
4793 RoutingWorkload::Quality
4794 );
4795 let f = IntentHint {
4797 prefer_fast: true,
4798 ..Default::default()
4799 };
4800 assert_eq!(
4801 AdaptiveRouter::resolve_workload(Some(&f), RoutingWorkload::Interactive),
4802 RoutingWorkload::Fastest
4803 );
4804 assert_eq!(
4806 AdaptiveRouter::resolve_workload(None, RoutingWorkload::Batch),
4807 RoutingWorkload::Batch
4808 );
4809 }
4810
4811 #[test]
4812 fn band_excludes_a_real_quality_gap_even_if_cheaper() {
4813 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4815 let tracker = OutcomeTracker::new();
4816 let strong = outcome_model("strong", Some(0.90));
4817 let weak_cheap = outcome_model("weak-cheap", Some(0.83));
4818 let band =
4819 router.outcome_first_band(vec![strong, weak_cheap], InferenceTask::Code, &tracker);
4820 assert_eq!(
4821 band.len(),
4822 1,
4823 "0.07 gap (> band 0.02) must drop the weaker model"
4824 );
4825 assert_eq!(band[0].id, "strong");
4826 }
4827
4828 #[test]
4829 fn band_keeps_genuinely_near_equal_models() {
4830 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4832 let tracker = OutcomeTracker::new();
4833 let band = router.outcome_first_band(
4834 vec![
4835 outcome_model("a", Some(0.90)),
4836 outcome_model("a2", Some(0.89)),
4837 ],
4838 InferenceTask::Code,
4839 &tracker,
4840 );
4841 assert_eq!(
4842 band.len(),
4843 2,
4844 "0.01 gap (<= band 0.02) keeps both for tie-break"
4845 );
4846 }
4847
4848 #[test]
4849 fn band_cold_start_frontier_beats_cheap_unknown() {
4850 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4853 let tracker = OutcomeTracker::new();
4854 let band = router.outcome_first_band(
4855 vec![
4856 outcome_model("frontier", Some(0.85)),
4857 outcome_model("unknown", None),
4858 ],
4859 InferenceTask::Code,
4860 &tracker,
4861 );
4862 assert_eq!(band.len(), 1);
4863 assert_eq!(
4864 band[0].id, "frontier",
4865 "0.85 vs 0.60 unknown → unknown excluded"
4866 );
4867 }
4868
4869 #[test]
4870 fn quality_lane_bands_out_a_weak_local_but_local_preferred_keeps_it() {
4871 let mut reg = test_registry();
4878 let mut frontier = outcome_model("frontier-remote", None);
4879 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
4880 name: "tau-bench-airline".into(),
4881 score: 0.65, harness: None,
4883 source_url: None,
4884 measured_at: None,
4885 runs: None,
4886 spread: None,
4887 }];
4888 reg.register_project_model(frontier);
4889
4890 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4891 let tracker = OutcomeTracker::new();
4892 let prompt = "Implement a balanced binary search tree with deletion.";
4893
4894 let q = router.route_context_aware(
4897 prompt,
4898 256,
4899 ®,
4900 &tracker,
4901 false,
4902 false,
4903 RoutingWorkload::Quality,
4904 );
4905 assert_eq!(
4906 q.model_id, "frontier-remote",
4907 "prefer_quality must pick the benchmarked frontier over locals-on-bonuses"
4908 );
4909 assert_eq!(
4910 q.strategy,
4911 RoutingStrategy::SchemaBased,
4912 "Quality lane must use deterministic argmax, not Thompson exploration"
4913 );
4914 assert!(
4915 q.candidates.iter().any(|c| !c.in_band),
4916 "Quality lane must apply a reliability band (some candidates excluded)"
4917 );
4918
4919 let lp = router.route_context_aware(
4922 prompt,
4923 256,
4924 ®,
4925 &tracker,
4926 false,
4927 false,
4928 RoutingWorkload::LocalPreferred,
4929 );
4930 assert!(
4931 lp.candidates.iter().all(|c| c.in_band),
4932 "LocalPreferred must NOT band — all candidates eligible"
4933 );
4934 }
4935
4936 #[test]
4937 fn band_never_empties() {
4938 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4939 let tracker = OutcomeTracker::new();
4940 assert_eq!(
4942 router
4943 .outcome_first_band(
4944 vec![outcome_model("only", Some(0.5))],
4945 InferenceTask::Code,
4946 &tracker
4947 )
4948 .len(),
4949 1
4950 );
4951 let band = router.outcome_first_band(
4953 vec![
4954 outcome_model("x", Some(0.40)),
4955 outcome_model("y", Some(0.41)),
4956 ],
4957 InferenceTask::Code,
4958 &tracker,
4959 );
4960 assert!(!band.is_empty() && band.iter().any(|m| m.id == "y"));
4961 }
4962
4963 #[test]
4964 fn outcome_first_band_is_wired_into_selection_and_keeps_fallbacks() {
4965 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-route");
4972 unsafe {
4973 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
4974 }
4975 let mut reg = UnifiedRegistry::new_empty(tmp);
4976 let mut strong = outcome_model("strong-expensive", Some(0.90));
4977 strong.cost = crate::schema::CostModel {
4978 input_per_mtok: Some(20.0),
4979 output_per_mtok: Some(20.0),
4980 ..Default::default()
4981 };
4982 let mut weak = outcome_model("weak-cheap", Some(0.80));
4983 weak.cost = crate::schema::CostModel {
4984 input_per_mtok: Some(0.1),
4985 output_per_mtok: Some(0.1),
4986 ..Default::default()
4987 };
4988 reg.register_project_model(strong);
4989 reg.register_project_model(weak);
4990
4991 let router = AdaptiveRouter::new(
4992 test_hw(),
4993 RoutingConfig {
4994 prior_strength: 100.0, ..Default::default()
4996 },
4997 );
4998 let tracker = OutcomeTracker::new();
4999 let decision = router.route(
5001 "Analyze the architecture trade-offs in depth",
5002 ®,
5003 &tracker,
5004 );
5005
5006 assert_eq!(
5007 decision.model_id, "strong-expensive",
5008 "band must select the higher-reliability model even though it's pricier"
5009 );
5010 assert!(
5011 decision.fallbacks.contains(&"weak-cheap".to_string()),
5012 "the out-of-band weaker model must remain as a fallback, got {:?}",
5013 decision.fallbacks
5014 );
5015 }
5016
5017 #[test]
5018 fn outcome_first_fallbacks_rank_in_band_before_out_of_band() {
5019 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-fb-order");
5023 unsafe {
5024 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5025 }
5026 let mut reg = UnifiedRegistry::new_empty(tmp);
5027 reg.register_project_model(outcome_model("strong", Some(0.90)));
5028 let mut peer = outcome_model("peer-in-band", Some(0.89)); peer.cost = crate::schema::CostModel {
5030 input_per_mtok: Some(30.0),
5031 output_per_mtok: Some(30.0),
5032 ..Default::default()
5033 };
5034 reg.register_project_model(peer);
5035 let mut weak = outcome_model("weak-out-of-band", Some(0.70)); weak.cost = crate::schema::CostModel {
5037 input_per_mtok: Some(0.01),
5038 output_per_mtok: Some(0.01),
5039 ..Default::default()
5040 };
5041 reg.register_project_model(weak);
5042
5043 let router = AdaptiveRouter::new(
5044 test_hw(),
5045 RoutingConfig {
5046 prior_strength: 100.0,
5047 ..Default::default()
5048 },
5049 );
5050 let tracker = OutcomeTracker::new();
5051 let decision = router.route(
5052 "Analyze the architecture trade-offs in depth",
5053 ®,
5054 &tracker,
5055 );
5056
5057 assert!(
5059 decision.model_id == "strong" || decision.model_id == "peer-in-band",
5060 "primary must be from the band, got {}",
5061 decision.model_id
5062 );
5063 let pos = |id: &str| decision.fallbacks.iter().position(|x| x == id);
5064 let in_band_id = if decision.model_id == "strong" {
5065 "peer-in-band"
5066 } else {
5067 "strong"
5068 };
5069 let in_band_pos = pos(in_band_id).expect("in-band peer must be a fallback");
5070 let weak_pos = pos("weak-out-of-band").expect("weak must remain a fallback (availability)");
5071 assert!(
5072 in_band_pos < weak_pos,
5073 "in-band fallback ({in_band_id}@{in_band_pos}) must precede out-of-band weak (@{weak_pos})"
5074 );
5075 }
5076
5077 #[test]
5078 fn reliability_is_cost_free() {
5079 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5082 let tracker = OutcomeTracker::new();
5083 let mut cheap = outcome_model("cheap", Some(0.8));
5084 cheap.cost = crate::schema::CostModel {
5085 input_per_mtok: Some(0.1),
5086 output_per_mtok: Some(0.1),
5087 ..Default::default()
5088 };
5089 let mut pricey = outcome_model("pricey", Some(0.8));
5090 pricey.cost = crate::schema::CostModel {
5091 input_per_mtok: Some(50.0),
5092 output_per_mtok: Some(50.0),
5093 ..Default::default()
5094 };
5095 let rc = router.reliability(&cheap, InferenceTask::Code, &tracker);
5096 let rp = router.reliability(&pricey, InferenceTask::Code, &tracker);
5097 assert!(
5098 (rc - rp).abs() < 1e-9,
5099 "reliability must not depend on cost"
5100 );
5101 }
5102
5103 #[test]
5104 fn cold_start_with_no_sample_size_sits_at_schema_estimate() {
5105 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5110 let unmeasured = outcome_model("unmeasured", None); let schema = router.schema_quality_estimate(&unmeasured);
5112
5113 let mut tracker = OutcomeTracker::new();
5114 let mut p = crate::outcome::ModelProfile::new("unmeasured".into());
5115 p.ema_quality = 0.5; tracker.import_profiles(vec![p]);
5117
5118 let r = router.reliability(&unmeasured, InferenceTask::Code, &tracker);
5119 assert!(
5120 (r - schema).abs() < 1e-9,
5121 "no sample size ⇒ reliability == schema estimate ({schema}), got {r}"
5122 );
5123 }
5124
5125 #[test]
5126 fn sparse_seeded_prior_barely_perturbs_schema_anchor() {
5127 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5130 let benchmarked = outcome_model("frontier", Some(0.55)); let mut tracker = OutcomeTracker::new();
5133 let mut p = crate::outcome::ModelProfile::new("frontier".into());
5134 p.ema_quality = 1.0;
5135 p.prior_sample_size = 1;
5136 p.task_stats.insert(
5137 InferenceTask::Code.to_string(),
5138 crate::outcome::TaskStats {
5139 ema_quality: 1.0,
5140 prior_sample_size: 1,
5141 ..Default::default()
5142 },
5143 );
5144 tracker.import_profiles(vec![p]);
5145
5146 let r = router.reliability(&benchmarked, InferenceTask::Code, &tracker);
5147 assert!((r - 0.64).abs() < 0.01, "expected ~0.64, got {r}");
5149 assert!(
5150 r < 0.7,
5151 "a 1-case prior must not pull reliability near its raw 1.0, got {r}"
5152 );
5153 }
5154
5155 #[test]
5156 fn dense_seeded_prior_is_trusted_near_raw() {
5157 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5160 let m = outcome_model("dense", Some(0.55)); let mut tracker = OutcomeTracker::new();
5163 let mut p = crate::outcome::ModelProfile::new("dense".into());
5164 p.ema_quality = 0.95;
5165 p.prior_sample_size = 40;
5166 tracker.import_profiles(vec![p]);
5167
5168 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5169 assert!(
5171 r > 0.88,
5172 "dense prior should be trusted near raw 0.95, got {r}"
5173 );
5174 }
5175
5176 #[test]
5177 fn warm_ungraded_successes_sit_at_schema_not_neutral_ema() {
5178 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5183 let frontier = outcome_model("frontier", Some(0.55)); let schema = router.schema_quality_estimate(&frontier);
5185
5186 let mut tracker = OutcomeTracker::new();
5187 let mut p = crate::outcome::ModelProfile::new("frontier".into());
5188 p.total_calls = 9;
5189 p.success_count = 9; p.ema_quality = 0.5; p.quality_observations = 0;
5192 tracker.import_profiles(vec![p]);
5193
5194 let r = router.reliability(&frontier, InferenceTask::Code, &tracker);
5195 assert!(
5196 (r - schema).abs() < 1e-9,
5197 "ungraded successes ⇒ reliability == schema ({schema}), not neutral EMA; got {r}"
5198 );
5199 }
5200
5201 #[test]
5202 fn ungraded_task_stats_with_zero_ema_still_resolve_to_schema() {
5203 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5210 let m = outcome_model("mech-only", Some(0.55)); let schema = router.schema_quality_estimate(&m);
5212
5213 let mut tracker = OutcomeTracker::new();
5214 let mut p = crate::outcome::ModelProfile::new("mech-only".into());
5215 p.total_calls = 7;
5216 p.success_count = 7;
5217 p.task_stats.insert(
5220 InferenceTask::Code.to_string(),
5221 crate::outcome::TaskStats {
5222 calls: 7,
5223 successes: 7,
5224 ..Default::default()
5225 },
5226 );
5227 tracker.import_profiles(vec![p]);
5228
5229 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5230 assert!(
5231 (r - schema).abs() < 1e-9,
5232 "ungraded task stats (ema 0.0, e=0) must resolve to schema {schema}, got {r}"
5233 );
5234 }
5235
5236 #[test]
5237 fn graded_outcomes_earn_trust_in_the_live_ema() {
5238 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5241 let m = outcome_model("graded", Some(0.55)); let mut tracker = OutcomeTracker::new();
5244 let mut p = crate::outcome::ModelProfile::new("graded".into());
5245 p.total_calls = 30;
5246 p.ema_quality = 0.9;
5247 p.quality_observations = 30; tracker.import_profiles(vec![p]);
5249
5250 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5251 assert!(
5253 r > 0.83,
5254 "dense graded signal should trust the live EMA, got {r}"
5255 );
5256 }
5257
5258 #[test]
5259 fn live_grades_outweigh_benchmark_cases_of_equal_count() {
5260 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5266 let benched = outcome_model("benched", Some(0.7));
5267 let graded = outcome_model("graded", Some(0.7));
5268
5269 let mut tracker = OutcomeTracker::new();
5270 let mut pb = crate::outcome::ModelProfile::new("benched".into());
5271 pb.ema_quality = 0.2;
5272 pb.prior_sample_size = 4; let mut pg = crate::outcome::ModelProfile::new("graded".into());
5274 pg.ema_quality = 0.2;
5275 pg.total_calls = 4;
5276 pg.quality_observations = 4; tracker.import_profiles(vec![pb, pg]);
5278
5279 let r_bench = router.reliability(&benched, InferenceTask::Code, &tracker);
5280 let r_graded = router.reliability(&graded, InferenceTask::Code, &tracker);
5281 assert!(
5282 r_graded < r_bench,
5283 "4 live grades must distrust more than 4 benchmark cases: \
5284 graded={r_graded} bench={r_bench}"
5285 );
5286 }
5287
5288 #[test]
5289 fn graded_failures_count_as_evidence_and_pull_reliability_down() {
5290 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5294 let m = outcome_model("flaky", Some(0.8)); let schema = router.schema_quality_estimate(&m);
5296
5297 let mut tracker = OutcomeTracker::new();
5298 let mut p = crate::outcome::ModelProfile::new("flaky".into());
5299 p.total_calls = 12;
5300 p.fail_count = 12;
5301 p.ema_quality = 0.1; p.quality_observations = 12;
5303 tracker.import_profiles(vec![p]);
5304
5305 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5306 assert!(
5307 r < schema - 0.2,
5308 "12 graded failures must pull reliability well below schema {schema}, got {r}"
5309 );
5310 }
5311
5312 #[test]
5313 fn route_exposes_ranked_advisory_candidates() {
5314 let _environment = crate::openrouter::test_environment_scope();
5319 let tmp = std::path::PathBuf::from("/tmp/car-test-route-candidates");
5325 unsafe {
5326 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5327 }
5328 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
5329 reg.register_project_model(outcome_model("strong", Some(0.90)));
5330 let mut weak = outcome_model("weak-cheap", Some(0.70)); weak.cost = crate::schema::CostModel {
5332 input_per_mtok: Some(0.01),
5333 output_per_mtok: Some(0.01),
5334 ..Default::default()
5335 };
5336 reg.register_project_model(weak);
5337
5338 let router = AdaptiveRouter::new(
5339 test_hw(),
5340 RoutingConfig {
5342 prior_strength: 100.0,
5343 ..Default::default()
5344 },
5345 );
5346 let tracker = OutcomeTracker::new();
5347 let decision = router.route(
5348 "Analyze the architecture trade-offs in depth",
5349 ®,
5350 &tracker,
5351 );
5352
5353 assert!(
5355 decision.candidates.iter().any(|c| c.model_id == "strong"),
5356 "strong must appear as a candidate, got {:?}",
5357 decision.candidates
5358 );
5359 assert!(
5360 decision
5361 .candidates
5362 .iter()
5363 .any(|c| c.model_id == "weak-cheap"),
5364 "weak-cheap must appear as a candidate even though out of band"
5365 );
5366
5367 let selected: Vec<&RouteCandidate> =
5369 decision.candidates.iter().filter(|c| c.selected).collect();
5370 assert_eq!(selected.len(), 1, "exactly one candidate may be selected");
5371 assert_eq!(
5372 selected[0].model_id, decision.model_id,
5373 "selected flag must match model_id"
5374 );
5375
5376 let strong = decision
5378 .candidates
5379 .iter()
5380 .find(|c| c.model_id == "strong")
5381 .unwrap();
5382 let weak_c = decision
5383 .candidates
5384 .iter()
5385 .find(|c| c.model_id == "weak-cheap")
5386 .unwrap();
5387 assert!(strong.in_band, "strong must be in the reliability band");
5388 assert!(
5389 !weak_c.in_band,
5390 "weak-cheap (0.20 below) must be out of band"
5391 );
5392 assert!(
5394 strong.reliability > weak_c.reliability,
5395 "reliability must reflect quality, not the weak model's lower cost"
5396 );
5397 }
5398
5399 #[test]
5400 fn prefer_quality_selection_is_deterministic_no_exploration() {
5401 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5406 let reg = test_registry();
5407 let tracker = OutcomeTracker::new();
5408 let intent = crate::intent::IntentHint {
5409 task: Some(crate::intent::TaskHint::Code),
5410 require: vec![ModelCapability::Code],
5411 prefer_quality: true,
5412 ..Default::default()
5413 };
5414 let first = router.route_with_intent("write a function", ®, &tracker, &intent);
5415 for _ in 0..8 {
5416 let again = router.route_with_intent("write a function", ®, &tracker, &intent);
5417 assert_eq!(
5418 again.model_id, first.model_id,
5419 "prefer_quality must be deterministic (no Thompson exploration)"
5420 );
5421 }
5422 let chosen = reg
5424 .find_by_name(&first.model_name)
5425 .or_else(|| reg.list().into_iter().find(|m| m.id == first.model_id));
5426 if let Some(m) = chosen {
5427 assert!(m.has_capability(ModelCapability::Code));
5428 }
5429 }
5430
5431 fn remote_model(id: &str, name: &str, caps: Vec<ModelCapability>) -> ModelSchema {
5433 ModelSchema {
5434 id: id.into(),
5435 name: name.into(),
5436 provider: "openai".into(),
5437 family: name.into(),
5438 version: "latest".into(),
5439 capabilities: caps,
5440 context_length: 128_000,
5441 max_output_tokens: None,
5442 param_count: "api".into(),
5443 quantization: None,
5444 performance: Default::default(),
5445 cost: Default::default(),
5446 source: crate::schema::ModelSource::RemoteApi {
5447 endpoint: "https://api.openai.com/v1".into(),
5448 api_key_env: "OPENAI_API_KEY".into(),
5449 api_key_envs: vec![],
5450 api_version: None,
5451 protocol: crate::schema::ApiProtocol::OpenAiCompat,
5452 },
5453 tags: vec!["trusted-remote".into()],
5454 supported_params: vec![],
5455 public_benchmarks: vec![],
5456 trust_tier: crate::schema::TrustTier::Curated,
5457 deprecated: false,
5458 available: true,
5459 weights_ready: true,
5460 }
5461 }
5462
5463 #[test]
5464 fn code_intent_filters_generate_only_models_on_simple_prompt() {
5465 let _environment = crate::openrouter::test_environment_scope();
5470 unsafe {
5477 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5478 }
5479 let intent_filter_dir = std::path::PathBuf::from("/tmp/car-test-code-intent-filter");
5480 let mut reg =
5481 UnifiedRegistry::new_with_state_root(intent_filter_dir.clone(), intent_filter_dir);
5482 reg.register_project_model(remote_model(
5483 "openai/gen-only:latest",
5484 "gen-only",
5485 vec![ModelCapability::Generate],
5486 ));
5487 reg.register_project_model(remote_model(
5488 "openai/coder:latest",
5489 "coder",
5490 vec![
5491 ModelCapability::Generate,
5492 ModelCapability::Code,
5493 ModelCapability::Reasoning,
5494 ],
5495 ));
5496
5497 let router = AdaptiveRouter::new(
5498 test_hw(),
5499 RoutingConfig {
5500 prior_strength: 100.0,
5501 ..Default::default()
5502 },
5503 );
5504 let tracker = OutcomeTracker::new();
5505 let intent = crate::intent::IntentHint {
5506 task: Some(crate::intent::TaskHint::Code),
5507 prefer_quality: true,
5508 ..Default::default() };
5510
5511 let decision = router.route_with_intent(
5513 "Give me a short two-sentence overview.",
5514 ®,
5515 &tracker,
5516 &intent,
5517 );
5518
5519 assert_eq!(
5520 decision.task,
5521 InferenceTask::Code,
5522 "explicit task=code must be honored"
5523 );
5524 assert!(
5527 !decision
5528 .candidates
5529 .iter()
5530 .any(|c| c.model_id == "openai/gen-only:latest"),
5531 "generate-only model must be filtered out for task=code; candidates: {:?}",
5532 decision
5533 .candidates
5534 .iter()
5535 .map(|c| &c.model_id)
5536 .collect::<Vec<_>>()
5537 );
5538 let schema = reg
5539 .find_by_name(&decision.model_name)
5540 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
5541 .expect("selected model exists");
5542 assert!(
5543 schema.has_capability(ModelCapability::Code),
5544 "task=code on a simple prompt must route to a code-capable model, got {}",
5545 decision.model_name,
5546 );
5547 }
5548
5549 #[test]
5550 fn intent_default_does_not_override_task_or_caps() {
5551 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5558 let reg = test_registry();
5559 let tracker = OutcomeTracker::new();
5560
5561 let baseline = router.route("write a haiku", ®, &tracker);
5562 let with_default = router.route_with_intent(
5563 "write a haiku",
5564 ®,
5565 &tracker,
5566 &crate::intent::IntentHint::default(),
5567 );
5568
5569 assert_eq!(
5570 baseline.task, with_default.task,
5571 "default IntentHint must not change the prompt-derived task"
5572 );
5573 assert_eq!(
5574 baseline.complexity, with_default.complexity,
5575 "default IntentHint must not change the prompt-derived complexity"
5576 );
5577 }
5578
5579 #[test]
5590 fn require_ready_skips_models_that_are_not_on_disk() {
5591 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5592 let reg = test_registry();
5593 let tracker = OutcomeTracker::new();
5594
5595 let ready: std::collections::HashSet<String> = reg
5597 .list()
5598 .into_iter()
5599 .filter(|m| m.weights_ready)
5600 .map(|m| m.id.clone())
5601 .collect();
5602 assert!(!ready.is_empty(), "fixture should have ready models");
5603
5604 let hint = crate::intent::IntentHint {
5605 require_ready: true,
5606 ..Default::default()
5607 };
5608 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
5609 assert!(
5610 ready.contains(&decision.model_id),
5611 "require_ready picked {} which is not on disk; ready set: {:?}",
5612 decision.model_id,
5613 ready
5614 );
5615 }
5616
5617 #[test]
5620 fn require_ready_falls_back_when_nothing_is_ready() {
5621 let _environment = crate::openrouter::test_environment_scope();
5626 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5627 let tracker = OutcomeTracker::new();
5628 let empty_dir = std::env::temp_dir().join("car-test-require-ready-empty");
5630 let _ = std::fs::create_dir_all(&empty_dir);
5631 let reg = UnifiedRegistry::new_with_state_root(empty_dir.clone(), empty_dir);
5632
5633 let hint = crate::intent::IntentHint {
5634 require_ready: true,
5635 ..Default::default()
5636 };
5637 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
5638 assert!(
5639 !decision.model_id.is_empty(),
5640 "soft constraint must still yield a model when nothing is ready"
5641 );
5642 }
5643
5644 #[test]
5645 fn exclude_models_picks_a_different_model() {
5646 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5651 let reg = test_registry();
5652 let tracker = OutcomeTracker::new();
5653 let base = crate::intent::IntentHint {
5654 prefer_quality: true,
5655 ..Default::default()
5656 };
5657 let author = router.route_with_intent("review this", ®, &tracker, &base);
5658
5659 let with_exclude = crate::intent::IntentHint {
5660 prefer_quality: true,
5661 exclude_models: vec![author.model_id.clone()],
5662 ..Default::default()
5663 };
5664 let reviewer = router.route_with_intent("review this", ®, &tracker, &with_exclude);
5665 assert_ne!(
5666 reviewer.model_id, author.model_id,
5667 "an excluded model must never be the routed pick"
5668 );
5669 assert!(
5672 !reviewer.fallbacks.contains(&author.model_id),
5673 "excluded model leaked into fallbacks"
5674 );
5675 assert!(
5676 !reviewer
5677 .candidates
5678 .iter()
5679 .any(|c| c.model_id == author.model_id),
5680 "excluded model leaked into candidates[]"
5681 );
5682 }
5683
5684 #[test]
5685 fn strict_exclusions_refuse_when_every_candidate_is_a_panel_seat() {
5686 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5687 let reg = test_registry();
5688 let tracker = OutcomeTracker::new();
5689 let panel: Vec<String> = reg
5690 .list()
5691 .into_iter()
5692 .map(|model| model.name.clone())
5693 .collect();
5694 assert!(!panel.is_empty(), "fixture must contain candidate models");
5695
5696 let decision = router.route_with_intent(
5697 "repair this code",
5698 ®,
5699 &tracker,
5700 &crate::intent::IntentHint {
5701 task: Some(crate::intent::TaskHint::Code),
5702 exclude_models: panel,
5703 strict_exclusions: true,
5704 ..Default::default()
5705 },
5706 );
5707
5708 assert!(decision.model_id.is_empty(), "{decision:?}");
5709 assert!(decision.model_name.is_empty(), "{decision:?}");
5710 assert!(decision.fallbacks.is_empty(), "{decision:?}");
5711 assert!(
5712 decision.reason.contains("no eligible model"),
5713 "the refusal must tell the operator what is missing: {decision:?}"
5714 );
5715 }
5716
5717 #[test]
5718 fn exclude_models_accepts_the_name_a_result_reports() {
5719 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5727 let reg = test_registry();
5728 let tracker = OutcomeTracker::new();
5729 let base = crate::intent::IntentHint {
5730 prefer_quality: true,
5731 ..Default::default()
5732 };
5733 let author = router.route_with_intent("write a python function", ®, &tracker, &base);
5734 let author_name = reg
5735 .get(&author.model_id)
5736 .expect("the routed pick is in the registry")
5737 .name
5738 .clone();
5739 assert_ne!(
5740 author_name, author.model_id,
5741 "this test is only meaningful on a model whose name and id differ"
5742 );
5743
5744 let by_name = crate::intent::IntentHint {
5745 prefer_quality: true,
5746 exclude_models: vec![author_name.clone()],
5747 ..Default::default()
5748 };
5749 let rerouted =
5750 router.route_with_intent("write a python function", ®, &tracker, &by_name);
5751 assert_ne!(
5752 rerouted.model_id, author.model_id,
5753 "excluding by the name a result reports ({author_name}) must route elsewhere"
5754 );
5755 assert!(
5756 !rerouted.fallbacks.contains(&author.model_id),
5757 "name-excluded model leaked into fallbacks"
5758 );
5759 }
5760
5761 #[test]
5762 fn cold_start_honors_exclusion() {
5763 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5768 let reg = test_registry();
5769 let caps = [ModelCapability::Generate];
5770 let base = router.cold_start_decision(
5771 TaskComplexity::Simple,
5772 InferenceTask::Generate,
5773 &caps,
5774 ®,
5775 false,
5776 &std::collections::HashSet::new(),
5777 false,
5778 AvailabilitySnapshot::capture(®),
5779 );
5780 let mut exclude = std::collections::HashSet::new();
5781 exclude.insert(base.model_id.clone());
5782 let excluded = router.cold_start_decision(
5783 TaskComplexity::Simple,
5784 InferenceTask::Generate,
5785 &caps,
5786 ®,
5787 false,
5788 &exclude,
5789 false,
5790 AvailabilitySnapshot::capture(®),
5791 );
5792 let other_generate_exists = reg
5796 .list()
5797 .into_iter()
5798 .filter(|m| m.available && m.has_capability(ModelCapability::Generate))
5799 .any(|m| m.id != base.model_id);
5800 if other_generate_exists {
5801 assert_ne!(
5802 excluded.model_id, base.model_id,
5803 "cold start re-picked the excluded model despite an alternative"
5804 );
5805 }
5806 }
5807
5808 #[test]
5809 fn exclude_all_models_falls_back_to_full_set() {
5810 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5813 let reg = test_registry();
5814 let tracker = OutcomeTracker::new();
5815 let exclude_everything: Vec<String> =
5816 reg.list().into_iter().map(|m| m.id.clone()).collect();
5817 let hint = crate::intent::IntentHint {
5818 exclude_models: exclude_everything,
5819 ..Default::default()
5820 };
5821 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
5822 assert!(
5823 !decision.model_id.is_empty(),
5824 "excluding everything must still yield a routed model (fallback)"
5825 );
5826 }
5827
5828 #[test]
5829 fn intent_task_hint_overrides_prompt_complexity() {
5830 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5833 let reg = test_registry();
5834 let tracker = OutcomeTracker::new();
5835
5836 let hint = crate::intent::IntentHint {
5837 task: Some(crate::intent::TaskHint::Reasoning),
5838 ..Default::default()
5839 };
5840 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
5841
5842 assert_eq!(
5843 decision.task,
5844 InferenceTask::Reasoning,
5845 "TaskHint::Reasoning should override the prompt-derived task"
5846 );
5847 }
5848
5849 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
5858 #[test]
5859 fn router_proposes_mlx_twin_not_pre_redirect_gguf() {
5860 let gguf_id = "qwen/qwen3-8b:q4_k_m";
5861 let mlx_id = "mlx/qwen3-8b:4bit";
5862
5863 let reg = test_registry();
5864 let gguf = reg.get(gguf_id).expect("gguf in builtin catalog").clone();
5867 assert_eq!(
5868 reg.resolve_mlx_equivalent(&gguf).map(|m| m.id.as_str()),
5869 Some(mlx_id),
5870 "GGUF must resolve to its SAME-SIZE MLX twin"
5871 );
5872
5873 let router = AdaptiveRouter::new(
5874 test_hw(),
5875 RoutingConfig {
5876 prior_strength: 0.5,
5877 min_observations: 3,
5878 ..Default::default()
5879 },
5880 );
5881 let mut tracker = OutcomeTracker::new();
5882 for _ in 0..20 {
5884 let t = tracker.record_start(mlx_id, InferenceTask::Code, "test");
5885 tracker.record_complete(&t, 500, 100, 50);
5886 tracker.record_inferred_outcome(&t, InferredOutcome::Accepted { confidence: 0.95 });
5887 }
5888
5889 let mut mlx_wins = 0;
5890 for _ in 0..20 {
5891 let d = router.route("Fix this bug in the parser", ®, &tracker);
5892 assert_ne!(
5895 d.model_id, gguf_id,
5896 "router proposed the pre-redirect GGUF id"
5897 );
5898 assert!(
5899 !d.fallbacks.iter().any(|f| f == gguf_id),
5900 "pre-redirect GGUF id leaked into the fallback chain"
5901 );
5902 if d.model_id == mlx_id {
5903 mlx_wins += 1;
5904 }
5905 }
5906 assert!(
5907 mlx_wins >= 12,
5908 "proven MLX twin should win the majority (the router reads the id it \
5909 learns/runs); won only {mlx_wins}/20"
5910 );
5911 }
5912}