1use rand::Rng;
13use serde::{Deserialize, Serialize};
14
15use std::sync::{Arc, Mutex};
16
17use crate::hardware::HardwareInfo;
18use crate::outcome::{InferenceTask, OutcomeTracker};
19use crate::registry::UnifiedRegistry;
20use crate::routing_ext::CircuitBreakerRegistry;
21use crate::schema::{ModelCapability, ModelSchema};
22use crate::tasks::RoutingWorkload;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
26#[serde(rename_all = "snake_case")]
27pub enum TaskComplexity {
28 Simple,
29 Medium,
30 Code,
31 Complex,
32}
33
34impl TaskComplexity {
35 pub fn assess(prompt: &str) -> Self {
42 let lower = prompt.to_lowercase();
43 let word_count = prompt.split_whitespace().count();
44 let estimated_tokens = (word_count as f64 * 1.3) as usize;
45
46 let has_code = Self::detect_code(prompt);
47
48 let repair_markers = [
49 "fix", "repair", "debug", "refactor", "broken", "failing", "error", "bug",
50 ];
51 let has_repair = repair_markers.iter().any(|m| lower.contains(m));
52
53 let reasoning_markers = [
54 "analyze",
55 "compare",
56 "explain why",
57 "step by step",
58 "think through",
59 "evaluate",
60 "trade-off",
61 "tradeoff",
62 "pros and cons",
63 "architecture",
64 "design",
65 "strategy",
66 "optimize",
67 "comprehensive",
68 ];
69 let has_reasoning = reasoning_markers.iter().any(|m| lower.contains(m));
70
71 let simple_patterns = [
72 "what is",
73 "who is",
74 "when did",
75 "where is",
76 "how many",
77 "yes or no",
78 "true or false",
79 "name the",
80 "list the",
81 "define ",
82 ];
83 let is_simple = simple_patterns.iter().any(|p| lower.contains(p));
84
85 if has_code || has_repair {
86 TaskComplexity::Code
87 } else if has_reasoning || estimated_tokens > 500 {
88 TaskComplexity::Complex
89 } else if is_simple || estimated_tokens < 30 {
90 TaskComplexity::Simple
91 } else {
92 TaskComplexity::Medium
93 }
94 }
95
96 fn detect_code(prompt: &str) -> bool {
102 #[cfg(feature = "ast")]
104 {
105 if let Some(is_code) = Self::detect_code_ast(prompt) {
106 return is_code;
107 }
108 }
109
110 let code_markers = [
112 "```",
113 "fn ",
114 "def ",
115 "class ",
116 "import ",
117 "require(",
118 "async fn",
119 "pub fn",
120 "function ",
121 "const ",
122 "let ",
123 "var ",
124 "#include",
125 "package ",
126 "impl ",
127 ];
128 code_markers.iter().any(|m| prompt.contains(m))
129 }
130
131 #[cfg(feature = "ast")]
135 fn detect_code_ast(prompt: &str) -> Option<bool> {
136 let mut blocks = Vec::new();
138 let mut rest = prompt;
139 while let Some(start) = rest.find("```") {
140 let after_fence = &rest[start + 3..];
141 let code_start = after_fence.find('\n').map(|i| i + 1).unwrap_or(0);
143 if let Some(end) = after_fence[code_start..].find("```") {
144 blocks.push(&after_fence[code_start..code_start + end]);
145 rest = &after_fence[code_start + end + 3..];
146 } else {
147 break;
148 }
149 }
150
151 if blocks.is_empty() {
152 return None; }
154
155 let languages = [
157 car_ast::Language::Rust,
158 car_ast::Language::Python,
159 car_ast::Language::TypeScript,
160 car_ast::Language::JavaScript,
161 car_ast::Language::Go,
162 ];
163
164 for block in &blocks {
165 let trimmed = block.trim();
166 if trimmed.is_empty() {
167 continue;
168 }
169
170 for lang in &languages {
171 if let Some(parsed) = car_ast::parse(trimmed, *lang) {
172 if !parsed.symbols.is_empty() {
174 return Some(true);
175 }
176 }
177 }
178 }
179
180 Some(false)
183 }
184
185 pub fn required_capabilities(&self) -> Vec<ModelCapability> {
187 match self {
188 TaskComplexity::Simple => vec![ModelCapability::Generate],
189 TaskComplexity::Medium => vec![ModelCapability::Generate],
190 TaskComplexity::Code => vec![ModelCapability::Code],
191 TaskComplexity::Complex => vec![ModelCapability::Reasoning],
192 }
193 }
194
195 pub fn inference_task(&self) -> InferenceTask {
197 match self {
198 TaskComplexity::Simple | TaskComplexity::Medium => InferenceTask::Generate,
199 TaskComplexity::Code => InferenceTask::Code,
200 TaskComplexity::Complex => InferenceTask::Reasoning,
201 }
202 }
203}
204
205#[derive(Debug, Clone, Serialize, Deserialize)]
207pub struct RoutingConfig {
208 pub min_observations: u64,
210 pub quality_weight: f64,
212 pub latency_weight: f64,
213 pub cost_weight: f64,
214 pub max_latency_ms: Option<u64>,
216 pub max_cost_usd: Option<f64>,
218 pub prefer_local: bool,
220 pub prior_strength: f64,
225 pub quality_first_cold_start: bool,
228 pub bootstrap_min_task_observations: u64,
231 pub bootstrap_quality_floor: f64,
234}
235
236impl Default for RoutingConfig {
237 fn default() -> Self {
238 Self {
239 min_observations: 2,
240 quality_weight: 0.45,
241 latency_weight: 0.4,
242 cost_weight: 0.15,
243 max_latency_ms: None,
244 max_cost_usd: None,
245 prefer_local: true,
246 prior_strength: 20.0,
247 quality_first_cold_start: true,
248 bootstrap_min_task_observations: 8,
249 bootstrap_quality_floor: 0.8,
250 }
251 }
252}
253
254#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
256#[serde(rename_all = "snake_case")]
257pub enum RoutingStrategy {
258 SchemaBased,
260 ProfileBased,
262 Exploration,
264 Explicit,
266}
267
268#[derive(Debug, Clone, Serialize, Deserialize)]
277pub struct RouteCandidate {
278 pub model_id: String,
280 pub reliability: f64,
282 pub score: f64,
284 pub selected: bool,
286 pub in_band: bool,
288 #[serde(default, skip_serializing_if = "Vec::is_empty")]
292 pub external_grades: Vec<RouteExternalGrade>,
293}
294
295#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
297pub struct RouteExternalGrade {
298 pub source: String,
302 pub quality: f64,
304 pub observations: u64,
306 pub evidence: u64,
310}
311
312#[derive(Debug, Clone, Copy, PartialEq, Eq)]
314pub enum RouterExclusionReason {
315 RateLimited,
317 CircuitOpen,
319}
320
321impl RouterExclusionReason {
322 pub fn as_str(self) -> &'static str {
324 match self {
325 Self::RateLimited => "rate_limited",
326 Self::CircuitOpen => "circuit_open",
327 }
328 }
329}
330
331#[derive(Debug, Clone, PartialEq, Eq)]
333pub struct RouterExclusion {
334 pub lane: String,
336 pub reason: RouterExclusionReason,
338 pub until: Option<u64>,
342}
343
344#[derive(Debug, Clone, Serialize, Deserialize)]
346pub struct AdaptiveRoutingDecision {
347 pub model_id: String,
349 pub model_name: String,
351 pub task: InferenceTask,
353 pub complexity: TaskComplexity,
355 pub reason: String,
357 pub strategy: RoutingStrategy,
359 pub predicted_quality: f64,
361 pub fallbacks: Vec<String>,
363 pub context_length: usize,
365 pub needs_compaction: bool,
367 #[serde(default)]
370 pub candidates: Vec<RouteCandidate>,
371}
372
373pub struct AdaptiveRouter {
375 hw: HardwareInfo,
376 config: RoutingConfig,
377 pub circuit_breakers: Arc<Mutex<CircuitBreakerRegistry>>,
379}
380
381pub struct RouteRequest<'a> {
397 pub prompt: &'a str,
398 pub registry: &'a UnifiedRegistry,
399 pub tracker: &'a OutcomeTracker,
400 pub estimated_total_tokens: usize,
403 pub estimated_input_tokens: usize,
406 pub estimated_output_tokens: usize,
408 pub estimated_cache_read_tokens: usize,
410 pub estimated_cache_write_tokens: usize,
412 pub has_tools: bool,
413 pub has_vision: bool,
414 pub workload: RoutingWorkload,
415 pub intent: Option<&'a crate::intent::IntentHint>,
418}
419
420impl<'a> RouteRequest<'a> {
421 pub fn new(
425 prompt: &'a str,
426 registry: &'a UnifiedRegistry,
427 tracker: &'a OutcomeTracker,
428 ) -> Self {
429 Self {
430 prompt,
431 registry,
432 tracker,
433 estimated_total_tokens: 0,
434 estimated_input_tokens: 0,
435 estimated_output_tokens: 0,
436 estimated_cache_read_tokens: 0,
437 estimated_cache_write_tokens: 0,
438 has_tools: false,
439 has_vision: false,
440 workload: RoutingWorkload::Interactive,
441 intent: None,
442 }
443 }
444}
445
446#[derive(Clone, Copy)]
450struct AvailabilitySnapshot {
451 openrouter: Option<crate::openrouter::CredentialSource>,
452}
453
454impl AvailabilitySnapshot {
455 fn capture(registry: &UnifiedRegistry) -> Self {
456 let has_openrouter = registry.list().into_iter().any(|schema| {
457 matches!(
458 schema.source,
459 crate::schema::ModelSource::RemoteApi {
460 protocol: crate::schema::ApiProtocol::OpenRouter,
461 ..
462 }
463 )
464 });
465 Self {
466 openrouter: has_openrouter
467 .then(crate::openrouter::credential_source)
468 .flatten(),
469 }
470 }
471
472 fn is_available(self, schema: &ModelSchema) -> bool {
473 match schema.source {
474 crate::schema::ModelSource::RemoteApi {
475 protocol: crate::schema::ApiProtocol::OpenRouter,
476 ..
477 } => self.openrouter.is_some(),
478 _ => schema.available,
479 }
480 }
481}
482
483static BENCHMARK_FRONTIER_REFS: std::sync::LazyLock<std::collections::HashMap<String, f64>> =
510 std::sync::LazyLock::new(|| compute_frontier_refs(crate::registry::builtin_catalog().iter()));
511
512fn compute_frontier_refs<'a>(
517 models: impl Iterator<Item = &'a ModelSchema>,
518) -> std::collections::HashMap<String, f64> {
519 let mut max_score: std::collections::HashMap<String, f64> = std::collections::HashMap::new();
523 let mut has_frontier_carrier: std::collections::HashMap<String, bool> =
524 std::collections::HashMap::new();
525 for model in models {
526 let frontier = model_is_frontier_class(model);
527 for b in &model.public_benchmarks {
528 let entry = max_score.entry(b.name.clone()).or_insert(0.0);
529 *entry = entry.max(b.score);
530 let seen = has_frontier_carrier.entry(b.name.clone()).or_insert(false);
531 *seen = *seen || frontier;
532 }
533 }
534 max_score
538 .into_iter()
539 .filter(|(name, _)| {
540 has_frontier_carrier.get(name).copied().unwrap_or(false) || {
541 tracing::warn!(
542 benchmark = %name,
543 "benchmark has no frontier-class carrier in the catalog — \
544 refusing to self-anchor (uncalibrated clamp). Add a \
545 frontier-tagged model that carries it, or an explicit \
546 reference, before it can calibrate the quality tier (#371)."
547 );
548 false
549 }
550 })
551 .collect()
552}
553
554fn model_is_frontier_class(model: &ModelSchema) -> bool {
565 model.tags.iter().any(|t| t == "frontier")
566}
567
568impl AdaptiveRouter {
569 pub fn new(hw: HardwareInfo, config: RoutingConfig) -> Self {
570 let circuit_breakers = Arc::new(Mutex::new(
571 CircuitBreakerRegistry::new(3, 300), ));
573 Self {
574 hw,
575 config,
576 circuit_breakers,
577 }
578 }
579
580 pub fn with_default_config(hw: HardwareInfo) -> Self {
581 Self::new(hw, RoutingConfig::default())
582 }
583
584 pub fn config(&self) -> &RoutingConfig {
585 &self.config
586 }
587
588 pub fn set_config(&mut self, config: RoutingConfig) {
589 self.config = config;
590 }
591
592 pub fn route_with(&self, req: RouteRequest<'_>) -> AdaptiveRoutingDecision {
598 let mut ignore_exclusion = |_: RouterExclusion| {};
599 self.route_with_exclusion_observer(req, &mut ignore_exclusion)
600 }
601
602 pub(crate) fn route_with_exclusion_observer(
606 &self,
607 req: RouteRequest<'_>,
608 exclusion_observer: &mut dyn FnMut(RouterExclusion),
609 ) -> AdaptiveRoutingDecision {
610 let workload = Self::resolve_workload(req.intent, req.workload);
616 self.route_inner_with_intent(
617 req.prompt,
618 req.registry,
619 req.tracker,
620 req.has_tools,
621 req.has_vision,
622 req.estimated_total_tokens,
623 req.estimated_input_tokens,
624 req.estimated_output_tokens,
625 req.estimated_cache_read_tokens,
626 req.estimated_cache_write_tokens,
627 workload,
628 req.intent,
629 exclusion_observer,
630 )
631 }
632
633 pub fn route(
636 &self,
637 prompt: &str,
638 registry: &UnifiedRegistry,
639 tracker: &OutcomeTracker,
640 ) -> AdaptiveRoutingDecision {
641 self.route_with(RouteRequest::new(prompt, registry, tracker))
642 }
643
644 pub fn route_editor(
650 &self,
651 prompt: &str,
652 registry: &UnifiedRegistry,
653 tracker: &OutcomeTracker,
654 ) -> AdaptiveRoutingDecision {
655 self.route_with(RouteRequest {
656 workload: RoutingWorkload::Background,
657 ..RouteRequest::new(prompt, registry, tracker)
658 })
659 }
660
661 pub fn route_with_tools(
663 &self,
664 prompt: &str,
665 registry: &UnifiedRegistry,
666 tracker: &OutcomeTracker,
667 ) -> AdaptiveRoutingDecision {
668 self.route_with(RouteRequest {
669 has_tools: true,
670 ..RouteRequest::new(prompt, registry, tracker)
671 })
672 }
673
674 pub fn route_with_vision(
676 &self,
677 prompt: &str,
678 registry: &UnifiedRegistry,
679 tracker: &OutcomeTracker,
680 has_tools: bool,
681 ) -> AdaptiveRoutingDecision {
682 self.route_with(RouteRequest {
683 has_tools,
684 has_vision: true,
685 ..RouteRequest::new(prompt, registry, tracker)
686 })
687 }
688
689 pub fn route_with_intent<'a>(
695 &self,
696 prompt: &'a str,
697 registry: &'a UnifiedRegistry,
698 tracker: &'a OutcomeTracker,
699 intent: &'a crate::intent::IntentHint,
700 ) -> AdaptiveRoutingDecision {
701 self.route_with(RouteRequest {
702 intent: Some(intent),
703 ..RouteRequest::new(prompt, registry, tracker)
704 })
705 }
706
707 pub fn route_context_aware(
710 &self,
711 prompt: &str,
712 estimated_total_tokens: usize,
713 registry: &UnifiedRegistry,
714 tracker: &OutcomeTracker,
715 has_tools: bool,
716 has_vision: bool,
717 workload: RoutingWorkload,
718 ) -> AdaptiveRoutingDecision {
719 self.route_with(RouteRequest {
720 estimated_total_tokens,
721 has_tools,
722 has_vision,
723 workload,
724 ..RouteRequest::new(prompt, registry, tracker)
725 })
726 }
727
728 pub fn route_context_aware_with_intent<'a>(
733 &self,
734 prompt: &'a str,
735 estimated_total_tokens: usize,
736 registry: &'a UnifiedRegistry,
737 tracker: &'a OutcomeTracker,
738 has_tools: bool,
739 has_vision: bool,
740 workload: RoutingWorkload,
741 intent: &'a crate::intent::IntentHint,
742 ) -> AdaptiveRoutingDecision {
743 self.route_with(RouteRequest {
744 estimated_total_tokens,
745 has_tools,
746 has_vision,
747 workload,
748 intent: Some(intent),
749 ..RouteRequest::new(prompt, registry, tracker)
750 })
751 }
752
753 fn route_inner_with_intent(
754 &self,
755 prompt: &str,
756 registry: &UnifiedRegistry,
757 tracker: &OutcomeTracker,
758 has_tools: bool,
759 has_vision: bool,
760 estimated_total_tokens: usize,
761 estimated_input_tokens: usize,
762 estimated_output_tokens: usize,
763 estimated_cache_read_tokens: usize,
764 estimated_cache_write_tokens: usize,
765 workload: RoutingWorkload,
766 intent: Option<&crate::intent::IntentHint>,
767 exclusion_observer: &mut dyn FnMut(RouterExclusion),
768 ) -> AdaptiveRoutingDecision {
769 let complexity = TaskComplexity::assess(prompt);
770 let task = intent
772 .and_then(|h| h.task)
773 .map(task_hint_to_inference_task)
774 .unwrap_or_else(|| complexity.inference_task());
775 let mut required_caps = base_required_capabilities(complexity, intent, task);
783 if let Some(hint) = intent {
784 for cap in &hint.require {
785 if !required_caps.contains(cap) {
786 required_caps.push(*cap);
787 }
788 }
789 }
790 if has_vision {
791 required_caps.push(ModelCapability::Vision);
792 }
793 if has_tools {
794 required_caps.push(ModelCapability::ToolUse);
795 if Self::needs_multi_tool_call(prompt) {
798 required_caps.push(ModelCapability::MultiToolCall);
799 }
800 }
801
802 let exclude_set = self.build_exclude_set(intent, registry);
810 let strict_exclusions = intent.is_some_and(|hint| hint.strict_exclusions);
811 let availability = AvailabilitySnapshot::capture(registry);
812
813 let want_ready = intent.is_some_and(|h| h.require_ready);
818 let mut candidates = self.filter_candidates(
819 &required_caps,
820 registry,
821 tracker,
822 has_vision,
823 availability,
824 want_ready,
825 exclusion_observer,
826 );
827 if candidates.is_empty() && want_ready {
828 candidates = self.filter_candidates(
829 &required_caps,
830 registry,
831 tracker,
832 has_vision,
833 availability,
834 false,
835 exclusion_observer,
836 );
837 }
838
839 if candidates.is_empty() && required_caps.contains(&ModelCapability::MultiToolCall) {
842 required_caps.retain(|c| *c != ModelCapability::MultiToolCall);
843 candidates = self.filter_candidates(
844 &required_caps,
845 registry,
846 tracker,
847 has_vision,
848 availability,
849 want_ready,
850 exclusion_observer,
851 );
852 if candidates.is_empty() && want_ready {
853 candidates = self.filter_candidates(
854 &required_caps,
855 registry,
856 tracker,
857 has_vision,
858 availability,
859 false,
860 exclusion_observer,
861 );
862 }
863 }
864
865 if intent.is_some_and(|hint| hint.on_device_only) {
871 candidates.retain(|model| model.is_local());
872 if candidates.is_empty() {
873 return Self::no_on_device_model_decision(complexity, task);
874 }
875 }
876
877 if candidates.is_empty() {
878 return self.cold_start_decision(
882 complexity,
883 task,
884 &required_caps,
885 registry,
886 has_vision,
887 &exclude_set,
888 strict_exclusions,
889 availability,
890 );
891 }
892
893 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
906 {
907 let mut seen = std::collections::HashSet::new();
908 candidates = candidates
909 .into_iter()
910 .map(|m| registry.resolve_mlx_equivalent(&m).cloned().unwrap_or(m))
911 .filter(|m| seen.insert(m.id.clone()))
912 .collect();
913 }
914
915 if !exclude_set.is_empty() {
924 let filtered: Vec<ModelSchema> = candidates
925 .iter()
926 .filter(|m| !exclude_set.contains(&m.id))
927 .cloned()
928 .collect();
929 if filtered.is_empty() {
930 if strict_exclusions {
931 return Self::no_eligible_model_decision(complexity, task, exclude_set.len());
932 }
933 tracing::debug!(
934 ?exclude_set,
935 "model exclusion left no candidates; keeping full set"
936 );
937 } else {
938 candidates = filtered;
939 }
940 }
941
942 candidates = self.apply_quality_first_bootstrap_policy(
943 candidates, task, tracker, has_vision, has_tools, workload,
944 );
945
946 let (fits, needs_compaction_candidates) = if estimated_total_tokens > 0 {
949 let mut fits = Vec::new();
950 let mut tight = Vec::new();
951 for m in &candidates {
952 if m.context_length == 0 || m.context_length >= estimated_total_tokens {
953 fits.push(m.clone());
954 } else {
955 tight.push(m.clone());
956 }
957 }
958 (fits, tight)
959 } else {
960 (candidates.clone(), Vec::new())
961 };
962
963 let (scoring_candidates, compaction_needed) = if !fits.is_empty() {
965 (fits, false)
966 } else if !needs_compaction_candidates.is_empty() {
967 tracing::info!(
968 prompt_tokens = estimated_total_tokens,
969 candidates = needs_compaction_candidates.len(),
970 "no model fits full prompt — compaction will be needed"
971 );
972 (needs_compaction_candidates.clone(), true)
973 } else {
974 (candidates.clone(), false)
975 };
976
977 let pricing_input_tokens = estimated_input_tokens
983 .max(estimated_total_tokens.saturating_sub(estimated_output_tokens));
984 let scored = self.score_candidates_context_aware(
985 &scoring_candidates,
986 task,
987 tracker,
988 estimated_total_tokens,
989 pricing_input_tokens,
990 estimated_output_tokens,
991 estimated_cache_read_tokens,
992 estimated_cache_write_tokens,
993 workload,
994 );
995 tracing::debug!(
1001 ?workload,
1002 ?task,
1003 candidates = ?scoring_candidates.iter().map(|m| m.id.as_str()).collect::<Vec<_>>(),
1004 ?scored,
1005 "route scoring"
1006 );
1007
1008 let band_ids: Option<std::collections::HashSet<String>> =
1020 if Self::applies_reliability_band(task, workload) {
1021 Some(
1022 self.outcome_first_band(scoring_candidates.clone(), task, tracker)
1023 .into_iter()
1024 .map(|m| m.id)
1025 .collect(),
1026 )
1027 } else {
1028 self.competent_candidates(&scoring_candidates, task, tracker)
1030 };
1031 if let Some(ref ids) = band_ids {
1032 tracing::debug!(
1037 ?task,
1038 band = Self::OUTCOME_FIRST_BAND,
1039 band_size = ids.len(),
1040 "outcome-first: selection restricted to reliability band"
1041 );
1042 }
1043 let selection_pool: Vec<(String, f64)> = if let Some(ref ids) = band_ids {
1044 let pool: Vec<(String, f64)> = scored
1045 .iter()
1046 .filter(|(id, _)| ids.contains(id))
1047 .cloned()
1048 .collect();
1049 if pool.is_empty() {
1050 debug_assert!(
1056 false,
1057 "reliability band produced no in-`scored` ids (workload {workload:?})"
1058 );
1059 scored.clone()
1060 } else {
1061 pool
1062 }
1063 } else {
1064 scored.clone()
1065 };
1066
1067 let (selected_id, strategy) = if workload == RoutingWorkload::Quality {
1080 selection_pool
1081 .iter()
1082 .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
1083 .map(|(id, _)| (id.clone(), RoutingStrategy::SchemaBased))
1084 .unwrap_or_else(|| self.select_with_thompson_sampling(&selection_pool, tracker))
1085 } else {
1086 self.select_with_thompson_sampling(&selection_pool, tracker)
1087 };
1088
1089 let mut fallbacks: Vec<String> = if let Some(ref ids) = band_ids {
1096 let (in_band, out_band): (Vec<String>, Vec<String>) = scored
1097 .iter()
1098 .filter(|(id, _)| *id != selected_id)
1099 .map(|(id, _)| id.clone())
1100 .partition(|id| ids.contains(id));
1101 in_band.into_iter().chain(out_band).collect()
1102 } else {
1103 scored
1104 .iter()
1105 .filter(|(id, _)| *id != selected_id)
1106 .map(|(id, _)| id.clone())
1107 .collect()
1108 };
1109 if !compaction_needed {
1111 for m in &needs_compaction_candidates {
1112 if m.id != selected_id && !fallbacks.contains(&m.id) {
1113 fallbacks.push(m.id.clone());
1114 }
1115 }
1116 }
1117
1118 let predicted_quality = scored
1119 .iter()
1120 .find(|(id, _)| *id == selected_id)
1121 .map(|(_, score)| *score)
1122 .unwrap_or(0.5);
1123
1124 let selected_schema = registry
1125 .get(&selected_id)
1126 .or_else(|| registry.find_by_name(&selected_id));
1127 let model_name = selected_schema
1128 .map(|m| m.name.clone())
1129 .unwrap_or_else(|| selected_id.clone());
1130 let context_length = selected_schema.map(|m| m.context_length).unwrap_or(0);
1131
1132 let needs_compact = compaction_needed
1133 || (estimated_total_tokens > 0
1134 && context_length > 0
1135 && estimated_total_tokens > context_length);
1136
1137 let compaction_note = if needs_compact {
1138 format!(
1139 " [compaction needed: {}→{}tok]",
1140 estimated_total_tokens, context_length
1141 )
1142 } else {
1143 String::new()
1144 };
1145
1146 let cost_quality_note = selected_schema
1147 .map(|schema| {
1148 let is_openrouter = schema.tags.iter().any(|tag| tag == "openrouter");
1149 if is_openrouter && schema.tags.iter().any(|tag| tag == "cheap") {
1150 let prices = schema.cost.prices_for(pricing_input_tokens);
1151 format!(
1152 ", low-cost tier (${:.3}/MTok input, ${:.3}/MTok output at {} prompt tokens)",
1153 prices.input_per_mtok.unwrap_or(0.0),
1154 prices.output_per_mtok.unwrap_or(0.0),
1155 pricing_input_tokens,
1156 )
1157 } else if is_openrouter && model_is_frontier_class(schema) {
1158 ", frontier quality tier".to_string()
1159 } else {
1160 String::new()
1161 }
1162 })
1163 .unwrap_or_default();
1164
1165 let reason = format!(
1166 "{:?} task → {} via {:?} (quality: {:.2}, {} candidates{}){}",
1167 complexity,
1168 model_name,
1169 strategy,
1170 predicted_quality,
1171 scoring_candidates.len(),
1172 cost_quality_note,
1173 compaction_note,
1174 );
1175
1176 let candidates: Vec<RouteCandidate> = scored
1183 .iter()
1184 .map(|(id, score)| {
1185 let reliability = match scoring_candidates.iter().find(|m| m.id == *id) {
1191 Some(m) => self.reliability(m, task, tracker),
1192 None => {
1193 debug_assert!(false, "scored id {id} absent from scoring_candidates");
1194 0.0
1195 }
1196 };
1197 let now = crate::outcome::now_unix();
1198 let external_grades = tracker
1199 .profile(id)
1200 .and_then(|p| p.task_stats(task))
1201 .map(|ts| {
1202 ts.external_grades
1203 .iter()
1204 .map(|(source, grade)| RouteExternalGrade {
1205 source: source.clone(),
1206 quality: grade.quality,
1207 observations: grade.observations,
1208 evidence: if grade.is_stale_at(now) {
1209 0
1210 } else {
1211 grade.evidence()
1212 },
1213 })
1214 .collect()
1215 })
1216 .unwrap_or_default();
1217 RouteCandidate {
1218 model_id: id.clone(),
1219 reliability,
1220 score: *score,
1221 selected: *id == selected_id,
1222 in_band: band_ids.as_ref().is_none_or(|b| b.contains(id)),
1223 external_grades,
1224 }
1225 })
1226 .collect();
1227
1228 AdaptiveRoutingDecision {
1229 model_id: selected_id,
1230 model_name,
1231 task,
1232 complexity,
1233 reason,
1234 strategy,
1235 predicted_quality,
1236 fallbacks,
1237 context_length,
1238 needs_compaction: needs_compact,
1239 candidates,
1240 }
1241 }
1242
1243 pub fn route_embedding(&self, registry: &UnifiedRegistry) -> String {
1245 let embed_models = registry.query_by_capability(ModelCapability::Embed);
1246 embed_models
1247 .first()
1248 .map(|m| m.name.clone())
1249 .unwrap_or_else(|| "Qwen3-Embedding-0.6B".to_string())
1250 }
1251
1252 pub fn route_small(&self, registry: &UnifiedRegistry) -> String {
1254 let gen_models = registry.query_by_capability(ModelCapability::Generate);
1255 gen_models
1257 .iter()
1258 .filter(|m| m.is_local())
1259 .min_by_key(|m| m.size_mb())
1260 .map(|m| m.name.clone())
1261 .unwrap_or_else(|| "Qwen3-0.6B".to_string())
1262 }
1263
1264 const LATENCY_CEILING_MS: f64 = 10000.0;
1270 const UNMEASURED_REMOTE_TPS: f64 = 93.0;
1274 const _TPS_CEILING: f64 = 150.0;
1276 const MOE_TPS_MULTIPLIER: f64 = 0.10;
1280 const MLX_MOE_TPS_MULTIPLIER: f64 = 0.50;
1283 const COST_CEILING_PER_1K: f64 = 0.1;
1285 const LOCAL_BONUS: f64 = 0.15;
1287 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1295 const HAS_GPU_BACKEND: bool = true;
1296 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
1297 const HAS_GPU_BACKEND: bool = false;
1298 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1301 const MLX_BONUS: f64 = 0.10;
1302 const SYSTEM_LLM_BONUS: f64 = 0.12;
1313
1314 pub(crate) fn build_exclude_set(
1333 &self,
1334 intent: Option<&crate::intent::IntentHint>,
1335 registry: &UnifiedRegistry,
1336 ) -> std::collections::HashSet<String> {
1337 let mut set = std::collections::HashSet::new();
1338 let Some(hint) = intent else {
1339 return set;
1340 };
1341 for id in &hint.exclude_models {
1342 set.insert(id.clone());
1343 let Some(schema) = registry.get(id).or_else(|| registry.find_by_name(id)) else {
1344 continue;
1345 };
1346 set.insert(schema.id.clone());
1347 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1348 {
1349 if let Some(mlx) = registry.resolve_mlx_equivalent(schema) {
1350 set.insert(mlx.id.clone());
1351 }
1352 }
1353 }
1354 set
1355 }
1356
1357 fn filter_candidates(
1358 &self,
1359 required_caps: &[ModelCapability],
1360 registry: &UnifiedRegistry,
1361 tracker: &OutcomeTracker,
1362 has_vision: bool,
1363 availability: AvailabilitySnapshot,
1364 require_ready: bool,
1365 exclusion_observer: &mut dyn FnMut(RouterExclusion),
1366 ) -> Vec<ModelSchema> {
1367 let mut candidates = registry
1368 .list()
1369 .into_iter()
1370 .filter(|m| {
1371 if !required_caps.iter().all(|c| m.has_capability(*c)) {
1373 return false;
1374 }
1375 if !has_vision && m.tags.iter().any(|t| t == "mlx-vlm-cli") {
1385 return false;
1386 }
1387 if !availability.is_available(m) {
1389 return false;
1390 }
1391 if require_ready && !m.weights_ready {
1398 return false;
1399 }
1400 if m.is_local() && m.size_mb() >= self.hw.max_model_mb {
1402 return false;
1403 }
1404 if let Some(max) = self.config.max_latency_ms {
1406 if let Some(p50) = m.performance.latency_p50_ms {
1407 if p50 > max {
1408 return false;
1409 }
1410 }
1411 }
1412 if let Some(max) = self.config.max_cost_usd {
1414 if m.cost_per_1k_output() > max {
1415 return false;
1416 }
1417 }
1418 if !self.config.prefer_local && m.is_local() {
1420 return false;
1421 }
1422 if tracker.is_excluded(&m.id) {
1424 exclusion_observer(RouterExclusion {
1425 lane: m.id.clone(),
1426 reason: RouterExclusionReason::RateLimited,
1427 until: None,
1428 });
1429 return false;
1430 }
1431 if let Ok(mut cb) = self.circuit_breakers.lock() {
1433 if !cb.allow_request(&m.id) {
1434 let until = cb.blocked_until(&m.id);
1435 tracing::debug!(model = %m.id, ?until, "skipped by circuit breaker");
1436 exclusion_observer(RouterExclusion {
1437 lane: m.id.clone(),
1438 reason: RouterExclusionReason::CircuitOpen,
1439 until,
1440 });
1441 return false;
1442 }
1443 }
1444 true
1445 })
1446 .cloned()
1447 .collect::<Vec<_>>();
1448 if candidates.iter().any(|m| !m.deprecated) {
1454 candidates.retain(|m| !m.deprecated);
1455 }
1456 candidates
1457 }
1458
1459 fn apply_quality_first_bootstrap_policy(
1460 &self,
1461 candidates: Vec<ModelSchema>,
1462 task: InferenceTask,
1463 tracker: &OutcomeTracker,
1464 has_vision: bool,
1465 has_tools: bool,
1466 workload: RoutingWorkload,
1467 ) -> Vec<ModelSchema> {
1468 if !self.config.quality_first_cold_start
1474 || workload != RoutingWorkload::Interactive
1475 || !self.is_quality_critical_bootstrap_task(task, has_vision, has_tools)
1476 {
1477 return candidates;
1478 }
1479
1480 let trusted_remote: Vec<ModelSchema> = candidates
1481 .iter()
1482 .filter(|model| self.is_trusted_quality_remote(model))
1483 .cloned()
1484 .collect();
1485
1486 if trusted_remote.is_empty() {
1487 return candidates;
1488 }
1489
1490 let best_remote_quality = trusted_remote
1494 .iter()
1495 .map(|model| self.schema_quality_estimate(model))
1496 .fold(0.0_f64, f64::max);
1497 let proven_local: Vec<ModelSchema> = candidates
1498 .iter()
1499 .filter(|model| {
1500 model.is_local()
1501 && (self.is_local_model_proven_for_task(model, task, tracker)
1502 || (!model.public_benchmarks.is_empty()
1503 && self.schema_quality_estimate(model)
1504 >= best_remote_quality - Self::OUTCOME_FIRST_BAND))
1505 })
1506 .cloned()
1507 .collect();
1508
1509 if !proven_local.is_empty() {
1510 return trusted_remote.into_iter().chain(proven_local).collect();
1514 }
1515
1516 trusted_remote
1517 }
1518
1519 fn score_candidates_context_aware(
1523 &self,
1524 candidates: &[ModelSchema],
1525 task: InferenceTask,
1526 tracker: &OutcomeTracker,
1527 estimated_total_tokens: usize,
1528 estimated_input_tokens: usize,
1529 estimated_output_tokens: usize,
1530 estimated_cache_read_tokens: usize,
1531 estimated_cache_write_tokens: usize,
1532 workload: RoutingWorkload,
1533 ) -> Vec<(String, f64)> {
1534 let mut scored: Vec<(String, f64)> = candidates
1535 .iter()
1536 .map(|m| {
1537 let base_score = self.score_model(
1538 m,
1539 task,
1540 tracker,
1541 workload,
1542 estimated_input_tokens,
1543 estimated_output_tokens,
1544 estimated_cache_read_tokens,
1545 estimated_cache_write_tokens,
1546 );
1547 let headroom_bonus = if estimated_total_tokens > 0 && m.context_length > 0 {
1550 let ratio = m.context_length as f64 / estimated_total_tokens as f64;
1551 if ratio >= 1.0 {
1552 (ratio.min(4.0) - 1.0) / 3.0 * 0.10 } else {
1554 -0.15 }
1556 } else {
1557 0.0
1558 };
1559 (m.id.clone(), base_score + headroom_bonus)
1560 })
1561 .collect();
1562
1563 scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
1564 scored
1565 }
1566
1567 fn task_aware_weights(task: InferenceTask, workload: RoutingWorkload) -> (f64, f64, f64) {
1583 match (task, workload) {
1584 (
1585 InferenceTask::Code,
1586 RoutingWorkload::Interactive | RoutingWorkload::LocalPreferred,
1587 ) => (0.62, 0.28, 0.10),
1588 _ => workload.weights(),
1589 }
1590 }
1591
1592 fn reliability(
1606 &self,
1607 model: &ModelSchema,
1608 task: InferenceTask,
1609 tracker: &OutcomeTracker,
1610 ) -> f64 {
1611 let completion = tracker.profile(&model.id).map_or(1.0, |p| {
1612 (p.success_count as f64 + 1.0) / ((p.success_count + p.fail_count) as f64 + 1.0)
1613 });
1614 self.answer_quality(model, task, tracker) * completion
1615 }
1616
1617 fn answer_quality(
1620 &self,
1621 model: &ModelSchema,
1622 task: InferenceTask,
1623 tracker: &OutcomeTracker,
1624 ) -> f64 {
1625 let schema_quality = self.schema_quality_estimate(model);
1626 let Some(p) = tracker.profile(&model.id) else {
1627 return schema_quality;
1629 };
1630
1631 let task_stats = p.task_stats(task);
1644 let (ema, n, g, unseeded) = task_stats
1645 .filter(|ts| ts.has_served_evidence())
1646 .map(|ts| {
1647 (
1648 ts.ema_quality,
1649 ts.prior_sample_size,
1650 ts.quality_observations,
1651 (ts.unseeded_ema_quality, ts.unseeded_quality_observations),
1652 )
1653 })
1654 .unwrap_or((
1655 p.ema_quality,
1656 p.prior_sample_size,
1657 p.quality_observations,
1658 (p.unseeded_ema_quality, p.unseeded_quality_observations),
1659 ));
1660 let (ema, n, g) = if model.public_benchmarks.is_empty() || n == 0 {
1669 (ema, n, g)
1670 } else {
1671 (unseeded.0, 0, unseeded.1)
1672 };
1673 let (external, external_mean) = task_stats
1679 .map(|ts| ts.external_evidence())
1680 .unwrap_or((0.0, 0.0));
1681
1682 let served = n as f64 + Self::LIVE_GRADE_WEIGHT * g as f64;
1713 let evidence = served + external;
1714 if evidence <= 0.0 {
1715 return schema_quality;
1716 }
1717 let observed = (served * ema + external * external_mean) / evidence;
1718 let confidence = evidence / (evidence + Self::PRIOR_SHRINK_PSEUDOCOUNT);
1719 (schema_quality + confidence * (observed - schema_quality)).clamp(0.0, 1.0)
1720 }
1721
1722 const BENCH_TIER_FLOOR: f64 = 0.40;
1730 const BENCH_TIER_KNEE: f64 = 0.5;
1737 const BENCH_TIER_CEIL: f64 = 0.92;
1738
1739 const LOCAL_BENCH_TIER_FLOOR: f64 = 0.30;
1753 const LOCAL_BENCH_TIER_CEIL: f64 = 0.60;
1754
1755 const UNMEASURED_REMOTE_CURATED: f64 = 0.60;
1760 const UNMEASURED_REMOTE_COMMUNITY: f64 = 0.48;
1761 const CURATED_FRONTIER_TAG_PRIOR: f64 = 0.85;
1768 const CURATED_BALANCED_TAG_PRIOR: f64 = 0.72;
1769
1770 const LIVE_GRADE_WEIGHT: f64 = 2.0;
1781
1782 const PRIOR_SHRINK_PSEUDOCOUNT: f64 = 4.0;
1792
1793 const OUTCOME_FIRST_BAND: f64 = 0.02;
1809
1810 const MIN_ANSWER_QUALITY: f64 = 0.25;
1822
1823 fn resolve_workload(
1829 intent: Option<&crate::intent::IntentHint>,
1830 base: RoutingWorkload,
1831 ) -> RoutingWorkload {
1832 match intent {
1833 Some(h) if h.high_stakes => RoutingWorkload::Quality,
1834 Some(h) if h.prefer_fast => RoutingWorkload::Fastest,
1835 Some(h) if h.prefer_quality => RoutingWorkload::Quality,
1836 Some(h) if h.prefer_local => RoutingWorkload::LocalPreferred,
1837 _ => base,
1838 }
1839 }
1840
1841 fn applies_reliability_band(task: InferenceTask, workload: RoutingWorkload) -> bool {
1857 match workload {
1858 RoutingWorkload::Quality => true,
1859 RoutingWorkload::Fastest
1860 | RoutingWorkload::Background
1861 | RoutingWorkload::LocalPreferred => false,
1862 _ => matches!(task, InferenceTask::Code | InferenceTask::Reasoning),
1863 }
1864 }
1865
1866 fn competent_candidates(
1869 &self,
1870 candidates: &[ModelSchema],
1871 task: InferenceTask,
1872 tracker: &OutcomeTracker,
1873 ) -> Option<std::collections::HashSet<String>> {
1874 let competent: std::collections::HashSet<String> = candidates
1875 .iter()
1876 .filter(|m| self.answer_quality(m, task, tracker) >= Self::MIN_ANSWER_QUALITY)
1877 .map(|m| m.id.clone())
1878 .collect();
1879 (!competent.is_empty() && competent.len() < candidates.len()).then_some(competent)
1880 }
1881
1882 fn outcome_first_band(
1887 &self,
1888 candidates: Vec<ModelSchema>,
1889 task: InferenceTask,
1890 tracker: &OutcomeTracker,
1891 ) -> Vec<ModelSchema> {
1892 if candidates.len() <= 1 {
1893 return candidates;
1894 }
1895 let best = candidates
1896 .iter()
1897 .map(|m| self.reliability(m, task, tracker))
1898 .fold(f64::MIN, f64::max);
1899 candidates
1900 .into_iter()
1901 .filter(|m| {
1904 best - self.reliability(m, task, tracker) <= Self::OUTCOME_FIRST_BAND + 1e-9
1905 })
1906 .collect()
1907 }
1908
1909 fn score_model(
1910 &self,
1911 model: &ModelSchema,
1912 task: InferenceTask,
1913 tracker: &OutcomeTracker,
1914 workload: RoutingWorkload,
1915 estimated_input_tokens: usize,
1916 estimated_output_tokens: usize,
1917 estimated_cache_read_tokens: usize,
1918 estimated_cache_write_tokens: usize,
1919 ) -> f64 {
1920 let profile = tracker.profile(&model.id);
1921 let schema_latency = self.schema_latency_estimate(model);
1922 let (quality_weight, latency_weight, cost_weight) =
1923 Self::task_aware_weights(task, workload);
1924
1925 let quality = self.reliability(model, task, tracker);
1929
1930 let latency = match profile {
1933 Some(p) if p.total_calls >= self.config.min_observations => {
1934 let avg = p
1935 .task_stats(task)
1936 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1937 .map(|ts| ts.avg_latency_ms)
1938 .unwrap_or_else(|| p.avg_latency_ms());
1939 self.latency_ms_to_score(avg)
1940 }
1941 Some(p) if p.total_calls == 0 => p
1942 .task_stats(task)
1943 .filter(|ts| ts.avg_latency_ms > 0.0)
1944 .map(|ts| self.latency_ms_to_score(ts.avg_latency_ms))
1945 .unwrap_or(schema_latency),
1946 Some(p) if p.total_calls > 0 => {
1947 let observed = self.latency_ms_to_score(
1948 p.task_stats(task)
1949 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1950 .map(|ts| ts.avg_latency_ms)
1951 .unwrap_or_else(|| p.avg_latency_ms()),
1952 );
1953 let w = p.total_calls as f64 / self.config.min_observations as f64;
1954 schema_latency * (1.0 - w) + observed * w
1955 }
1956 _ => schema_latency,
1957 };
1958
1959 let cost = if model.is_local() {
1961 1.0
1962 } else if model.cost.output_per_mtok.is_none() {
1963 0.5
1970 } else {
1971 let normalized_cost_per_1k = if estimated_input_tokens
1972 + estimated_output_tokens
1973 + estimated_cache_read_tokens
1974 + estimated_cache_write_tokens
1975 > 0
1976 {
1977 let estimated_cost = model.cost.estimated_usd(
1978 estimated_input_tokens,
1979 estimated_output_tokens,
1980 estimated_cache_read_tokens,
1981 estimated_cache_write_tokens,
1982 );
1983 estimated_cost * 1000.0
1984 / (estimated_input_tokens + estimated_output_tokens).max(1) as f64
1985 } else {
1986 model.cost_per_1k_output()
1987 };
1988 (1.0 - (normalized_cost_per_1k / Self::COST_CEILING_PER_1K)).clamp(0.0, 1.0)
1989 };
1990
1991 let gpu_backend_or_latency_tolerant =
2010 Self::HAS_GPU_BACKEND || !workload.is_latency_sensitive();
2011 let local_bonus = if self.config.prefer_local
2012 && model.is_local()
2013 && gpu_backend_or_latency_tolerant
2014 {
2015 Self::LOCAL_BONUS
2016 } else {
2017 if self.config.prefer_local && model.is_local() && !Self::HAS_GPU_BACKEND {
2018 tracing::debug!(
2019 model = %model.id,
2020 "LOCAL_BONUS suppressed: no GPU backend on this host (Intel Mac or car_skip_mlx) for a latency-sensitive workload; cloud models will rank higher"
2021 );
2022 }
2023 0.0
2024 };
2025 let workload_local_bonus = if model.is_local() {
2026 workload.local_bonus()
2027 } else {
2028 0.0
2029 };
2030
2031 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2033 let mlx_bonus = if model.is_mlx() { Self::MLX_BONUS } else { 0.0 };
2034 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2035 let mlx_bonus = 0.0;
2036
2037 let vllm_mlx_bonus = if model.is_vllm_mlx() {
2039 Self::LOCAL_BONUS + 0.05
2040 } else {
2041 0.0
2042 };
2043
2044 let system_llm_bonus = if model.tags.iter().any(|t| t == "low_latency")
2052 && model.tags.iter().any(|t| t == "private")
2053 {
2054 Self::SYSTEM_LLM_BONUS
2055 } else {
2056 0.0
2057 };
2058
2059 let (local_bonus, workload_local_bonus, mlx_bonus, vllm_mlx_bonus, system_llm_bonus) =
2070 if workload == RoutingWorkload::Quality {
2071 (0.0, 0.0, 0.0, 0.0, 0.0)
2072 } else {
2073 (
2074 local_bonus,
2075 workload_local_bonus,
2076 mlx_bonus,
2077 vllm_mlx_bonus,
2078 system_llm_bonus,
2079 )
2080 };
2081
2082 quality_weight * quality
2083 + latency_weight * latency
2084 + cost_weight * cost
2085 + local_bonus
2086 + workload_local_bonus
2087 + mlx_bonus
2088 + vllm_mlx_bonus
2089 + system_llm_bonus
2090 }
2091
2092 fn latency_ms_to_score(&self, ms: f64) -> f64 {
2095 (1.0 - (ms / Self::LATENCY_CEILING_MS)).clamp(0.0, 1.0)
2096 }
2097
2098 fn tps_to_latency_ms(tps: f64) -> f64 {
2100 if tps <= 0.0 {
2101 return Self::LATENCY_CEILING_MS;
2102 }
2103 (200.0 / tps) * 1000.0
2105 }
2106
2107 fn needs_multi_tool_call(prompt: &str) -> bool {
2110 let lower = prompt.to_lowercase();
2111
2112 let has_numbered_list = {
2114 let mut count = 0u32;
2115 for i in 1..=5u32 {
2116 if lower.contains(&format!("{}) ", i)) || lower.contains(&format!("{}. ", i)) {
2117 count += 1;
2118 }
2119 }
2120 count >= 2
2121 };
2122
2123 let multi_keywords = [
2125 "multiple edits",
2126 "several changes",
2127 "three changes",
2128 "two changes",
2129 "all of the following",
2130 "each of these",
2131 "do both",
2132 "do all",
2133 "and also",
2134 "additionally",
2135 "as well as",
2136 "then also",
2137 ];
2138 let has_multi_keywords = multi_keywords.iter().any(|kw| lower.contains(kw));
2139
2140 let bullet_actions = lower.matches("- add ").count()
2142 + lower.matches("- update ").count()
2143 + lower.matches("- change ").count()
2144 + lower.matches("- remove ").count()
2145 + lower.matches("- fix ").count()
2146 + lower.matches("- edit ").count()
2147 + lower.matches("- implement ").count()
2148 + lower.matches("- create ").count();
2149 let has_bullet_list = bullet_actions >= 2;
2150
2151 has_numbered_list || has_multi_keywords || has_bullet_list
2152 }
2153
2154 fn schema_quality_estimate(&self, model: &ModelSchema) -> f64 {
2177 if !model.public_benchmarks.is_empty() {
2178 return self.normalized_benchmark_quality(model).clamp(0.0, 1.0);
2179 }
2180 if model.is_remote() {
2181 if model.trust_tier == crate::schema::TrustTier::Curated {
2196 if model_is_frontier_class(model) {
2197 return Self::CURATED_FRONTIER_TAG_PRIOR;
2198 }
2199 if model.tags.iter().any(|tag| tag == "balanced") {
2200 return Self::CURATED_BALANCED_TAG_PRIOR;
2201 }
2202 }
2203 return match model.trust_tier {
2204 crate::schema::TrustTier::Community => Self::UNMEASURED_REMOTE_COMMUNITY,
2205 _ => Self::UNMEASURED_REMOTE_CURATED,
2206 };
2207 }
2208 match model.size_mb() {
2212 0 => 0.35, s if s < 1000 => 0.30, s if s < 2000 => 0.38, s if s < 3000 => 0.45, s if s < 6000 => 0.52, _ => 0.60, }
2219 }
2220
2221 fn normalized_benchmark_quality(&self, model: &ModelSchema) -> f64 {
2232 let is_local = model.is_local();
2233 let n = model.public_benchmarks.len() as f64;
2234 let sum: f64 = model
2235 .public_benchmarks
2236 .iter()
2237 .map(|b| match Self::benchmark_frontier_ref(&b.name) {
2238 Some(frontier_ref) => {
2239 if b.score > frontier_ref {
2248 tracing::warn!(
2249 model = %model.id,
2250 benchmark = %b.name,
2251 score = b.score,
2252 frontier_ref,
2253 "model scores ABOVE the catalog frontier_ref — re-anchor \
2254 candidate: update builtin_catalog.json so this benchmark's \
2255 champion calibrates the quality tier (#370)"
2256 );
2257 }
2258 let frac = (b.score / frontier_ref).clamp(0.0, 1.0);
2259 let tier = |f: f64| {
2260 Self::BENCH_TIER_FLOOR
2261 + f * (Self::BENCH_TIER_CEIL - Self::BENCH_TIER_FLOOR)
2262 };
2263 if frac >= Self::BENCH_TIER_KNEE {
2264 tier(frac)
2265 } else {
2266 tier(Self::BENCH_TIER_KNEE) * frac / Self::BENCH_TIER_KNEE
2267 }
2268 }
2269 None if is_local => {
2275 b.score
2282 .clamp(Self::LOCAL_BENCH_TIER_FLOOR, Self::LOCAL_BENCH_TIER_CEIL)
2283 }
2284 None => b.score.clamp(Self::BENCH_TIER_FLOOR, Self::BENCH_TIER_CEIL),
2289 })
2290 .sum();
2291 sum / n
2298 }
2299
2300 fn benchmark_frontier_ref(name: &str) -> Option<f64> {
2312 BENCHMARK_FRONTIER_REFS
2313 .get(name)
2314 .copied()
2315 .filter(|r| *r > 0.0)
2316 }
2317
2318 fn schema_latency_estimate(&self, model: &ModelSchema) -> f64 {
2323 let is_moe = model.tags.contains(&"moe".to_string());
2324
2325 if model.is_local() {
2326 if let Some(tps) = model.performance.tokens_per_second {
2327 let measured = model.performance.latency_p50_ms.is_some();
2340 let multiplier = match &model.source {
2346 _ if !is_moe || measured => 1.0,
2347 crate::schema::ModelSource::Mlx { .. } => Self::MLX_MOE_TPS_MULTIPLIER,
2348 crate::schema::ModelSource::Local { .. } => Self::MOE_TPS_MULTIPLIER,
2349 _ => 1.0,
2350 };
2351 let effective_tps = tps * multiplier;
2352 let first_token_ms = model.performance.latency_p50_ms.unwrap_or(0) as f64;
2355 let estimated_ms = first_token_ms + Self::tps_to_latency_ms(effective_tps);
2356 return self.latency_ms_to_score(estimated_ms);
2357 }
2358 return 0.5; }
2360
2361 if let Some(tps) = model.performance.tokens_per_second {
2369 return self.latency_ms_to_score(Self::tps_to_latency_ms(tps));
2370 }
2371 if let Some(p50) = model.performance.latency_p50_ms {
2372 return self.latency_ms_to_score(
2373 p50 as f64 + Self::tps_to_latency_ms(Self::UNMEASURED_REMOTE_TPS),
2374 );
2375 }
2376 0.3 }
2378
2379 fn is_quality_critical_bootstrap_task(
2380 &self,
2381 task: InferenceTask,
2382 has_vision: bool,
2383 has_tools: bool,
2384 ) -> bool {
2385 has_vision
2386 || has_tools
2387 || matches!(
2388 task,
2389 InferenceTask::Generate | InferenceTask::Code | InferenceTask::Reasoning
2390 )
2391 }
2392
2393 fn is_trusted_quality_remote(&self, model: &ModelSchema) -> bool {
2394 let trusted_provider = matches!(
2395 model.provider.as_str(),
2396 "openai" | "anthropic" | "google" | "openrouter"
2397 );
2398 model.is_remote()
2399 && model.trust_tier == crate::schema::TrustTier::Curated
2400 && (trusted_provider || crate::openrouter::is_managed_gateway_schema(model))
2401 && !model.has_capability(ModelCapability::SpeechToText)
2402 && !model.has_capability(ModelCapability::TextToSpeech)
2403 }
2404
2405 fn is_local_model_proven_for_task(
2406 &self,
2407 model: &ModelSchema,
2408 task: InferenceTask,
2409 tracker: &OutcomeTracker,
2410 ) -> bool {
2411 let Some(profile) = tracker.profile(&model.id) else {
2412 return false;
2413 };
2414 if let Some(task_stats) = profile.task_stats(task) {
2415 if task_stats.calls >= self.config.bootstrap_min_task_observations
2416 && task_stats.ema_quality >= self.config.bootstrap_quality_floor
2417 {
2418 return true;
2419 }
2420 }
2421
2422 profile.total_calls >= self.config.bootstrap_min_task_observations
2423 && profile.ema_quality >= self.config.bootstrap_quality_floor
2424 }
2425
2426 const THOMPSON_TOP_MEAN: f64 = 0.95;
2429
2430 fn select_with_thompson_sampling(
2438 &self,
2439 scored: &[(String, f64)],
2440 tracker: &OutcomeTracker,
2441 ) -> (String, RoutingStrategy) {
2442 if scored.is_empty() {
2443 return (String::new(), RoutingStrategy::SchemaBased);
2444 }
2445
2446 let mut rng = rand::rng();
2447 let mut best_sample = f64::NEG_INFINITY;
2448 let mut best_id = scored[0].0.clone();
2449 let mut best_strategy = RoutingStrategy::SchemaBased;
2450
2451 let top_score = scored
2455 .iter()
2456 .map(|(_, score)| *score)
2457 .fold(f64::MIN_POSITIVE, f64::max);
2458
2459 for (id, phase2_score) in scored {
2460 let profile = tracker.profile(id);
2461
2462 let calls = profile.map_or(0, |p| p.total_calls);
2467 let mean = (Self::THOMPSON_TOP_MEAN * (phase2_score / top_score))
2468 .clamp(0.01, Self::THOMPSON_TOP_MEAN);
2469
2470 let concentration = self.config.prior_strength + calls as f64;
2473 let alpha = (concentration * mean).max(0.01);
2474 let beta = (concentration * (1.0 - mean)).max(0.01);
2475
2476 let sample = sample_beta(&mut rng, alpha, beta);
2478
2479 if sample > best_sample {
2480 best_sample = sample;
2481 best_id = id.clone();
2482 best_strategy = match profile {
2483 Some(p) if p.total_calls >= self.config.min_observations => {
2484 RoutingStrategy::ProfileBased
2485 }
2486 Some(p) if p.total_calls > 0 => {
2487 RoutingStrategy::Exploration
2489 }
2490 _ => RoutingStrategy::SchemaBased,
2491 };
2492 }
2493 }
2494
2495 (best_id, best_strategy)
2496 }
2497
2498 fn cold_start_decision(
2515 &self,
2516 complexity: TaskComplexity,
2517 task: InferenceTask,
2518 required_caps: &[ModelCapability],
2519 registry: &UnifiedRegistry,
2520 has_vision: bool,
2521 exclude: &std::collections::HashSet<String>,
2522 strict_exclusions: bool,
2523 availability: AvailabilitySnapshot,
2524 ) -> AdaptiveRoutingDecision {
2525 if has_vision {
2530 if let Some(model) = registry
2531 .query_by_capability(ModelCapability::Vision)
2532 .into_iter()
2533 .filter(|model| !exclude.contains(&model.id))
2534 .find(|model| {
2535 availability.is_available(model) && self.is_trusted_quality_remote(model)
2536 })
2537 .or_else(|| {
2538 registry
2539 .query_by_capability(ModelCapability::Vision)
2540 .into_iter()
2541 .find(|model| !exclude.contains(&model.id))
2542 })
2543 {
2544 return AdaptiveRoutingDecision {
2545 model_id: model.id.clone(),
2546 model_name: model.name.clone(),
2547 task,
2548 complexity,
2549 reason: format!(
2550 "{:?} task → {} (cold start, vision fallback)",
2551 complexity, model.name
2552 ),
2553 strategy: RoutingStrategy::SchemaBased,
2554 predicted_quality: 0.5,
2555 fallbacks: vec![],
2556 context_length: model.context_length,
2557 needs_compaction: false,
2558 candidates: vec![],
2559 };
2560 }
2561 }
2562
2563 if self.config.quality_first_cold_start {
2564 if let Some(model) = registry
2565 .list()
2566 .into_iter()
2567 .filter(|model| {
2568 availability.is_available(model)
2569 && !exclude.contains(&model.id)
2570 && required_caps.iter().all(|cap| model.has_capability(*cap))
2571 && self.is_trusted_quality_remote(model)
2572 })
2573 .max_by(|a, b| {
2574 self.schema_quality_estimate(a)
2575 .partial_cmp(&self.schema_quality_estimate(b))
2576 .unwrap_or(std::cmp::Ordering::Equal)
2577 })
2578 {
2579 return AdaptiveRoutingDecision {
2580 model_id: model.id.clone(),
2581 model_name: model.name.clone(),
2582 task,
2583 complexity,
2584 reason: format!(
2585 "{:?} task → {} (quality-first cold start)",
2586 complexity, model.name
2587 ),
2588 strategy: RoutingStrategy::SchemaBased,
2589 predicted_quality: self.schema_quality_estimate(model),
2590 fallbacks: vec![],
2591 context_length: model.context_length,
2592 needs_compaction: false,
2593 candidates: vec![],
2594 };
2595 }
2596 }
2597
2598 if let Some(model) = registry
2606 .list()
2607 .into_iter()
2608 .filter(|model| {
2609 availability.is_available(model)
2610 && !exclude.contains(&model.id)
2611 && model.is_local()
2612 && model.size_mb() < self.hw.max_model_mb
2613 && required_caps.iter().all(|cap| model.has_capability(*cap))
2614 })
2615 .max_by(|a, b| {
2616 self.schema_quality_estimate(a)
2617 .partial_cmp(&self.schema_quality_estimate(b))
2618 .unwrap_or(std::cmp::Ordering::Equal)
2619 })
2620 {
2621 return AdaptiveRoutingDecision {
2622 model_id: model.id.clone(),
2623 model_name: model.name.clone(),
2624 task,
2625 complexity,
2626 reason: format!(
2627 "{:?} task → {} (capability-satisfying local cold start)",
2628 complexity, model.name
2629 ),
2630 strategy: RoutingStrategy::SchemaBased,
2631 predicted_quality: self.schema_quality_estimate(model),
2632 fallbacks: vec![],
2633 context_length: model.context_length,
2634 needs_compaction: false,
2635 candidates: vec![],
2636 };
2637 }
2638
2639 if let Some(model) = registry
2650 .list()
2651 .into_iter()
2652 .filter(|model| {
2653 availability.is_available(model)
2654 && !exclude.contains(&model.id)
2655 && required_caps.iter().all(|cap| model.has_capability(*cap))
2656 })
2657 .max_by(|a, b| {
2658 let qa = (
2659 self.is_trusted_quality_remote(a),
2660 self.schema_quality_estimate(a),
2661 );
2662 let qb = (
2663 self.is_trusted_quality_remote(b),
2664 self.schema_quality_estimate(b),
2665 );
2666 qa.partial_cmp(&qb).unwrap_or(std::cmp::Ordering::Equal)
2667 })
2668 {
2669 return AdaptiveRoutingDecision {
2670 model_id: model.id.clone(),
2671 model_name: model.name.clone(),
2672 task,
2673 complexity,
2674 reason: format!(
2675 "{:?} task → {} (capability-satisfying cold start)",
2676 complexity, model.name
2677 ),
2678 strategy: RoutingStrategy::SchemaBased,
2679 predicted_quality: self.schema_quality_estimate(model),
2680 fallbacks: vec![],
2681 context_length: model.context_length,
2682 needs_compaction: false,
2683 candidates: vec![],
2684 };
2685 }
2686
2687 if strict_exclusions && !exclude.is_empty() {
2691 return Self::no_eligible_model_decision(complexity, task, exclude.len());
2692 }
2693
2694 let default_name = match complexity {
2701 TaskComplexity::Simple => "Qwen3-0.6B",
2702 TaskComplexity::Medium => "Qwen3-1.7B",
2703 TaskComplexity::Code => "Qwen3-4B",
2704 TaskComplexity::Complex => &self.hw.recommended_model,
2705 };
2706 let model_name = if registry.find_by_name(default_name).is_some() {
2711 default_name
2712 } else {
2713 crate::router::tier_fallback_order(default_name)
2714 .find(|name| registry.find_by_name(name).is_some())
2715 .unwrap_or(default_name)
2716 };
2717
2718 let model_id = registry
2719 .find_by_name(model_name)
2720 .map(|m| m.id.clone())
2721 .unwrap_or_else(|| model_name.to_string());
2722
2723 let context_length = registry
2724 .find_by_name(model_name)
2725 .map(|m| m.context_length)
2726 .unwrap_or(0);
2727
2728 AdaptiveRoutingDecision {
2729 model_id,
2730 model_name: model_name.to_string(),
2731 task,
2732 complexity,
2733 reason: format!(
2734 "{:?} task → {} (cold start, no candidates)",
2735 complexity, model_name
2736 ),
2737 strategy: RoutingStrategy::SchemaBased,
2738 predicted_quality: 0.5,
2739 fallbacks: vec![],
2740 context_length,
2741 needs_compaction: false,
2742 candidates: vec![],
2743 }
2744 }
2745
2746 fn no_on_device_model_decision(
2748 complexity: TaskComplexity,
2749 task: InferenceTask,
2750 ) -> AdaptiveRoutingDecision {
2751 AdaptiveRoutingDecision {
2752 reason: "on_device_only left no model on this device".into(),
2753 ..Self::no_eligible_model_decision(complexity, task, 0)
2754 }
2755 }
2756
2757 fn no_eligible_model_decision(
2758 complexity: TaskComplexity,
2759 task: InferenceTask,
2760 excluded_count: usize,
2761 ) -> AdaptiveRoutingDecision {
2762 AdaptiveRoutingDecision {
2763 model_id: String::new(),
2764 model_name: String::new(),
2765 task,
2766 complexity,
2767 reason: format!(
2768 "strict model exclusions left no eligible model ({excluded_count} catalog ids excluded)"
2769 ),
2770 strategy: RoutingStrategy::SchemaBased,
2771 predicted_quality: 0.0,
2772 fallbacks: vec![],
2773 context_length: 0,
2774 needs_compaction: false,
2775 candidates: vec![],
2776 }
2777 }
2778}
2779
2780fn task_hint_to_inference_task(hint: crate::intent::TaskHint) -> InferenceTask {
2786 use crate::intent::TaskHint;
2787 match hint {
2788 TaskHint::Chat => InferenceTask::Generate,
2789 TaskHint::Classify => InferenceTask::Classify,
2790 TaskHint::Reasoning => InferenceTask::Reasoning,
2791 TaskHint::Code => InferenceTask::Code,
2792 }
2793}
2794
2795fn base_required_capabilities(
2804 complexity: TaskComplexity,
2805 intent: Option<&crate::intent::IntentHint>,
2806 task: InferenceTask,
2807) -> Vec<ModelCapability> {
2808 match intent.and_then(|h| h.task) {
2809 Some(_) => vec![inference_task_required_capability(task)],
2810 None => complexity.required_capabilities(),
2811 }
2812}
2813
2814fn inference_task_required_capability(task: InferenceTask) -> ModelCapability {
2815 match task {
2816 InferenceTask::Generate => ModelCapability::Generate,
2817 InferenceTask::Code => ModelCapability::Code,
2818 InferenceTask::Reasoning => ModelCapability::Reasoning,
2819 InferenceTask::Classify => ModelCapability::Classify,
2820 InferenceTask::Embed => ModelCapability::Embed,
2821 }
2822}
2823
2824fn sample_beta(rng: &mut impl Rng, alpha: f64, beta: f64) -> f64 {
2832 let x = sample_gamma(rng, alpha);
2833 let y = sample_gamma(rng, beta);
2834 if x + y == 0.0 {
2835 0.5 } else {
2837 x / (x + y)
2838 }
2839}
2840
2841fn sample_gamma(rng: &mut impl Rng, shape: f64) -> f64 {
2844 if shape < 1.0 {
2845 let u: f64 = rng.random();
2847 return sample_gamma(rng, shape + 1.0) * u.powf(1.0 / shape);
2848 }
2849
2850 let d = shape - 1.0 / 3.0;
2852 let c = 1.0 / (9.0 * d).sqrt();
2853
2854 loop {
2855 let x: f64 = loop {
2856 let n = sample_standard_normal(rng);
2857 if 1.0 + c * n > 0.0 {
2858 break n;
2859 }
2860 };
2861
2862 let v = (1.0 + c * x).powi(3);
2863 let u: f64 = rng.random();
2864
2865 if u < 1.0 - 0.0331 * x.powi(4) {
2866 return d * v;
2867 }
2868 if u.ln() < 0.5 * x * x + d * (1.0 - v + v.ln()) {
2869 return d * v;
2870 }
2871 }
2872}
2873
2874fn sample_standard_normal(rng: &mut impl Rng) -> f64 {
2876 let u1: f64 = rng.random();
2877 let u2: f64 = rng.random();
2878 (-2.0 * u1.max(1e-300).ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
2879}
2880
2881#[cfg(test)]
2882mod tests {
2883 use super::*;
2884 use crate::outcome::InferredOutcome;
2885 use std::ffi::OsString;
2886
2887 fn route_model_ids(decision: &AdaptiveRoutingDecision) -> impl Iterator<Item = &str> {
2888 std::iter::once(decision.model_id.as_str())
2889 .chain(
2890 decision
2891 .candidates
2892 .iter()
2893 .map(|candidate| candidate.model_id.as_str()),
2894 )
2895 .chain(decision.fallbacks.iter().map(String::as_str))
2896 }
2897
2898 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
2899
2900 impl RestoredEnvironment {
2901 fn capture(names: &[&'static str]) -> Self {
2902 Self(
2903 names
2904 .iter()
2905 .map(|name| (*name, std::env::var_os(name)))
2906 .collect(),
2907 )
2908 }
2909 }
2910
2911 impl Drop for RestoredEnvironment {
2912 fn drop(&mut self) {
2913 for (name, value) in &self.0 {
2914 unsafe {
2915 match value {
2916 Some(value) => std::env::set_var(name, value),
2917 None => std::env::remove_var(name),
2918 }
2919 }
2920 }
2921 }
2922 }
2923
2924 struct TestRegistry {
2925 registry: UnifiedRegistry,
2926 _restore: RestoredEnvironment,
2929 _environment: tokio::sync::MutexGuard<'static, ()>,
2930 }
2931
2932 impl std::ops::Deref for TestRegistry {
2933 type Target = UnifiedRegistry;
2934
2935 fn deref(&self) -> &Self::Target {
2936 &self.registry
2937 }
2938 }
2939
2940 impl std::ops::DerefMut for TestRegistry {
2941 fn deref_mut(&mut self) -> &mut Self::Target {
2942 &mut self.registry
2943 }
2944 }
2945
2946 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2952 const QWEN3_8B_ROUTED_ID: &str = "mlx/qwen3-8b:4bit";
2953 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2954 const QWEN3_8B_ROUTED_ID: &str = "qwen/qwen3-8b:q4_k_m";
2955 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2956 const QWEN3_4B_ROUTED_ID: &str = "mlx/qwen3-4b:4bit";
2957 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2958 const QWEN3_4B_ROUTED_ID: &str = "qwen/qwen3-4b:q4_k_m";
2959
2960 fn test_hw() -> HardwareInfo {
2961 HardwareInfo {
2962 os: "macos".into(),
2963 arch: "aarch64".into(),
2964 cpu_cores: 10,
2965 total_ram_mb: 32768,
2966 gpu_backend: crate::hardware::GpuBackend::Metal,
2967 gpu_memory_mb: Some(28672),
2968 gpu_devices: Vec::new(),
2969 recommended_model: "Qwen3-8B".into(),
2970 recommended_context: 8192,
2971 max_model_mb: 18000, }
2973 }
2974
2975 fn test_registry() -> TestRegistry {
2976 let environment = crate::openrouter::test_environment_scope();
2977 let restore = RestoredEnvironment::capture(&["OPENAI_API_KEY", "CAR_SECRETS_FILE_DIR"]);
2978 let tmp = std::path::PathBuf::from("/tmp/car-test-adaptive-router");
2979 unsafe {
2980 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
2981 std::env::set_var(
2987 "CAR_SECRETS_FILE_DIR",
2988 std::env::temp_dir().join("car-test-adaptive-router-empty-secrets"),
2989 );
2990 }
2991 for name in &[
2993 "Qwen3-0.6B",
2994 "Qwen3-1.7B",
2995 "Qwen3-4B",
2996 "Qwen3-8B",
2997 "Qwen3-Embedding-0.6B",
2998 ] {
2999 let dir = tmp.join(name);
3000 let _ = std::fs::create_dir_all(&dir);
3001 let _ = std::fs::write(dir.join("model.gguf"), b"fake");
3002 let _ = std::fs::write(dir.join("tokenizer.json"), b"{}");
3003 }
3004 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
3005 reg.register_project_model(ModelSchema {
3006 id: "openai/gpt-5.4-mini:latest".into(),
3007 name: "gpt-5.4-mini".into(),
3008 provider: "openai".into(),
3009 family: "gpt-5.4".into(),
3010 version: "latest".into(),
3011 capabilities: vec![
3012 ModelCapability::Generate,
3013 ModelCapability::Code,
3014 ModelCapability::Reasoning,
3015 ModelCapability::ToolUse,
3016 ModelCapability::MultiToolCall,
3017 ModelCapability::Vision,
3018 ],
3019 context_length: 128_000,
3020 max_output_tokens: None,
3021 param_count: "api".into(),
3022 quantization: None,
3023 performance: Default::default(),
3024 cost: Default::default(),
3025 source: crate::schema::ModelSource::RemoteApi {
3026 endpoint: "https://api.openai.com/v1".into(),
3027 api_key_env: "OPENAI_API_KEY".into(),
3028 api_key_envs: vec![],
3029 api_version: None,
3030 protocol: crate::schema::ApiProtocol::OpenAiCompat,
3031 },
3032 tags: vec!["trusted-remote".into()],
3033 supported_params: vec![],
3034 public_benchmarks: vec![],
3035 trust_tier: crate::schema::TrustTier::Curated,
3036 deprecated: false,
3037 available: true,
3038 weights_ready: true,
3039 });
3040 TestRegistry {
3041 registry: reg,
3042 _restore: restore,
3043 _environment: environment,
3044 }
3045 }
3046
3047 #[test]
3048 fn route_observer_reports_a_429_exclusion_before_dispatch() {
3049 let registry = test_registry();
3050 let router = AdaptiveRouter::with_default_config(test_hw());
3051 let mut tracker = OutcomeTracker::new();
3052 let lane = "openai/gpt-5.4-mini:latest";
3053 let trace = tracker.record_start(lane, InferenceTask::Generate, "test 429");
3054 tracker.record_failure(&trace, "API returned 429 Too Many Requests");
3055 assert!(
3056 tracker.is_excluded(lane),
3057 "429 must seed the router exclusion"
3058 );
3059
3060 let mut exclusions = Vec::new();
3061 let mut observe = |exclusion| exclusions.push(exclusion);
3062 let decision = router.route_with_exclusion_observer(
3063 RouteRequest::new("Say hello.", ®istry, &tracker),
3064 &mut observe,
3065 );
3066
3067 assert_ne!(decision.model_id, lane);
3068 assert!(exclusions.contains(&RouterExclusion {
3069 lane: lane.into(),
3070 reason: RouterExclusionReason::RateLimited,
3071 until: None,
3072 }));
3073 }
3074
3075 #[test]
3076 fn route_observer_reports_an_open_circuit_with_its_cooldown() {
3077 let registry = test_registry();
3078 let router = AdaptiveRouter::with_default_config(test_hw());
3079 let tracker = OutcomeTracker::new();
3080 let lane = "openai/gpt-5.4-mini:latest";
3081 {
3082 let mut breakers = router.circuit_breakers.lock().unwrap();
3083 for _ in 0..3 {
3084 breakers.record_failure(lane);
3085 }
3086 }
3087
3088 let mut exclusions = Vec::new();
3089 let mut observe = |exclusion| exclusions.push(exclusion);
3090 let decision = router.route_with_exclusion_observer(
3091 RouteRequest::new("Say hello.", ®istry, &tracker),
3092 &mut observe,
3093 );
3094
3095 assert_ne!(decision.model_id, lane);
3096 let exclusion = exclusions
3097 .iter()
3098 .find(|exclusion| exclusion.lane == lane)
3099 .expect("open lane must be reported");
3100 assert_eq!(exclusion.reason, RouterExclusionReason::CircuitOpen);
3101 assert!(exclusion.until.is_some(), "open circuit has a cooldown end");
3102 }
3103
3104 #[test]
3105 fn openrouter_tags_drive_cost_quality_routing_and_live_availability() {
3106 let tmp = tempfile::tempdir().unwrap();
3107 let _credential_scope = crate::openrouter::test_credential_scope();
3108 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
3109 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
3110 for schema in crate::openrouter::curated_schemas()
3111 .into_iter()
3112 .filter(|schema| schema.provider == "openrouter")
3113 {
3114 registry.register_project_model(schema);
3115 }
3116 let router = AdaptiveRouter::new(
3117 test_hw(),
3118 RoutingConfig {
3119 prefer_local: false,
3120 prior_strength: 1_000_000.0,
3124 ..RoutingConfig::default()
3125 },
3126 );
3127 let tracker = OutcomeTracker::new();
3128 let mut simple_tracker = OutcomeTracker::new();
3129 let evidenced_cheap = "openrouter/qwen/qwen3-coder-next";
3130 for _ in 0..8 {
3131 let trace = simple_tracker.record_start(
3132 evidenced_cheap,
3133 InferenceTask::Generate,
3134 "OpenRouter cheap-model calibration",
3135 );
3136 simple_tracker.record_complete(&trace, 100, 10, 10);
3137 simple_tracker
3138 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
3139 }
3140
3141 crate::openrouter::reset_credential_source_call_count();
3142 let simple = router.route_with(RouteRequest {
3143 workload: RoutingWorkload::Background,
3144 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3145 });
3146 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
3147 let simple_schema = registry.get(&simple.model_id).unwrap();
3148 assert!(simple_schema.tags.iter().any(|tag| tag == "cheap"));
3149 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
3150 assert!(simple
3151 .candidates
3152 .iter()
3153 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
3154
3155 let quality_intent = crate::intent::IntentHint {
3156 task: Some(crate::intent::TaskHint::Reasoning),
3157 prefer_quality: true,
3158 ..Default::default()
3159 };
3160 let complex = router.route_with(RouteRequest {
3161 intent: Some(&quality_intent),
3162 ..RouteRequest::new(
3163 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
3164 ®istry,
3165 &tracker,
3166 )
3167 });
3168 let complex_schema = registry.get(&complex.model_id).unwrap();
3169 assert!(complex_schema.tags.iter().any(|tag| tag == "frontier"));
3170 assert!(
3171 complex.reason.contains("frontier quality tier"),
3172 "{}",
3173 complex.reason
3174 );
3175
3176 let vision_intent = crate::intent::IntentHint {
3177 require: vec![ModelCapability::Vision],
3178 ..Default::default()
3179 };
3180 let vision = router.route_with(RouteRequest {
3181 intent: Some(&vision_intent),
3182 ..RouteRequest::new("Inspect this image", ®istry, &tracker)
3183 });
3184 assert!(vision.candidates.iter().all(|candidate| registry
3185 .get(&candidate.model_id)
3186 .is_some_and(|schema| schema.has_capability(ModelCapability::Vision))));
3187
3188 let failed_model = simple.model_id.clone();
3189 {
3190 let mut breakers = router.circuit_breakers.lock().unwrap();
3191 for _ in 0..3 {
3192 breakers.record_failure(&failed_model);
3193 }
3194 }
3195 let after_failures = router.route_with(RouteRequest {
3196 workload: RoutingWorkload::Background,
3197 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3198 });
3199 assert_ne!(after_failures.model_id, failed_model);
3200 assert!(!after_failures
3201 .candidates
3202 .iter()
3203 .any(|candidate| candidate.model_id == failed_model));
3204
3205 crate::openrouter::set_test_credential(None);
3206 crate::openrouter::reset_credential_source_call_count();
3207 let without_key = router.route("Say hello.", ®istry, &tracker);
3208 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
3209 assert!(!without_key.model_id.starts_with("openrouter/"));
3210 assert!(!without_key
3211 .fallbacks
3212 .iter()
3213 .any(|model| model.starts_with("openrouter/")));
3214 assert!(!without_key
3215 .candidates
3216 .iter()
3217 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
3218 }
3219
3220 #[tokio::test(flavor = "current_thread")]
3221 async fn managed_openrouter_aliases_survive_quality_bootstrap_without_a_personal_key() {
3222 let tmp = tempfile::tempdir().unwrap();
3223 let (_credential_scope, _environment) =
3224 crate::openrouter::test_credential_and_environment_scope_async().await;
3225 let _restore = RestoredEnvironment::capture(&[
3226 "CAR_SECRETS_FILE_DIR",
3227 "CAR_MANAGED_ROUTING_TEST_KEY",
3228 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
3229 ]);
3230 let _home = crate::openrouter::StateRootScope::new();
3239 crate::openrouter::clear_gateway_unconfigured();
3240 crate::parslee_credential::clear_credential_rejected();
3241 let secrets_dir = tmp.path().join("secrets");
3242 unsafe {
3243 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
3244 std::env::set_var("CAR_MANAGED_ROUTING_TEST_KEY", "test-peer-key");
3245 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
3246 }
3247 crate::openrouter::set_test_credential(None);
3248
3249 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("models"));
3250 for schema in crate::openrouter::curated_schemas() {
3251 registry.register_project_model(schema);
3252 }
3253 let mut bootstrap_peer = remote_model(
3254 "openai/bootstrap-peer",
3255 "bootstrap-peer",
3256 vec![
3257 ModelCapability::Generate,
3258 ModelCapability::Code,
3259 ModelCapability::Reasoning,
3260 ],
3261 );
3262 if let crate::schema::ModelSource::RemoteApi {
3263 ref mut api_key_env,
3264 ..
3265 } = bootstrap_peer.source
3266 {
3267 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3268 }
3269 registry.register_project_model(bootstrap_peer);
3270 let mut unrelated = remote_model(
3271 "other/unrelated-curated",
3272 "unrelated-curated",
3273 vec![
3274 ModelCapability::Generate,
3275 ModelCapability::Code,
3276 ModelCapability::Reasoning,
3277 ],
3278 );
3279 unrelated.provider = "other".into();
3280 if let crate::schema::ModelSource::RemoteApi {
3281 ref mut api_key_env,
3282 ..
3283 } = unrelated.source
3284 {
3285 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3286 }
3287 registry.register_project_model(unrelated);
3288 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
3293
3294 let managed: Vec<_> = registry
3295 .list()
3296 .into_iter()
3297 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
3298 .collect();
3299 assert_eq!(managed.len(), crate::openrouter::managed_alias_count());
3300 assert!(managed.iter().all(|schema| {
3301 schema.available_now()
3302 && schema.tags.iter().any(|tag| tag == "openrouter")
3303 && schema.tags.iter().any(|tag| tag == "managed")
3304 && schema.cost.input_per_mtok.is_some_and(|price| price > 0.0)
3305 && schema.cost.output_per_mtok.is_some_and(|price| price > 0.0)
3306 && schema.has_capability(ModelCapability::Generate)
3307 }));
3308 assert!(registry
3309 .list()
3310 .into_iter()
3311 .filter(|schema| schema.id.starts_with("openrouter/"))
3312 .all(|schema| !schema.available_now()));
3313
3314 let router = AdaptiveRouter::new(
3315 test_hw(),
3316 RoutingConfig {
3317 prefer_local: false,
3318 prior_strength: 1_000_000.0,
3319 ..RoutingConfig::default()
3320 },
3321 );
3322 let tracker = OutcomeTracker::new();
3323
3324 let interactive = router.route("Say hello.", ®istry, &tracker);
3325 assert!(
3326 interactive
3327 .candidates
3328 .iter()
3329 .any(|candidate| candidate.model_id.starts_with("parslee/openrouter/")),
3330 "signed-in managed aliases must survive quality-first bootstrap: {interactive:?}"
3331 );
3332 assert!(
3333 interactive
3334 .fallbacks
3335 .iter()
3336 .any(|id| id.starts_with("parslee/openrouter/")),
3337 "signed-in managed aliases must remain fallback-eligible: {interactive:?}"
3338 );
3339 assert!(
3340 interactive
3341 .candidates
3342 .iter()
3343 .any(|candidate| candidate.model_id == "openai/bootstrap-peer"),
3344 "adding managed aliases must not remove existing trusted providers: {interactive:?}"
3345 );
3346 assert!(
3347 !interactive
3348 .candidates
3349 .iter()
3350 .any(|candidate| candidate.model_id == "other/unrelated-curated"),
3351 "managed-alias trust must not broaden the bootstrap to unrelated providers"
3352 );
3353
3354 let mut simple_tracker = OutcomeTracker::new();
3355 let evidenced_cheap = "parslee/openrouter/open-fast";
3356 for _ in 0..8 {
3357 let trace = simple_tracker.record_start(
3358 evidenced_cheap,
3359 InferenceTask::Generate,
3360 "managed OpenRouter cheap-model calibration",
3361 );
3362 simple_tracker.record_complete(&trace, 100, 10, 10);
3363 simple_tracker
3364 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
3365 }
3366 let simple = router.route_with(RouteRequest {
3367 workload: RoutingWorkload::Background,
3368 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3369 });
3370 let simple_schema = registry.get(&simple.model_id).unwrap();
3371 assert!(
3372 simple.model_id.starts_with("parslee/openrouter/")
3373 && simple_schema.tags.iter().any(|tag| tag == "cheap"),
3374 "simple work should prefer an evidenced cheap managed alias: {simple:?}"
3375 );
3376 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
3377
3378 let quality_intent = crate::intent::IntentHint {
3379 task: Some(crate::intent::TaskHint::Reasoning),
3380 prefer_quality: true,
3381 ..Default::default()
3382 };
3383 let complex = router.route_with(RouteRequest {
3384 intent: Some(&quality_intent),
3385 ..RouteRequest::new(
3386 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
3387 ®istry,
3388 &tracker,
3389 )
3390 });
3391 let complex_schema = registry.get(&complex.model_id).unwrap();
3392 assert!(
3393 complex.model_id.starts_with("parslee/openrouter/")
3394 && complex_schema.tags.iter().any(|tag| tag == "frontier"),
3395 "complex quality work should escalate to a managed frontier alias: {complex:?}"
3396 );
3397 assert!(
3398 complex.reason.contains("frontier quality tier"),
3399 "{}",
3400 complex.reason
3401 );
3402
3403 for decision in [&interactive, &simple, &complex] {
3404 assert!(
3405 !route_model_ids(decision).any(|id| id.starts_with("openrouter/")),
3406 "personal OpenRouter rows must stay excluded without a personal key: {decision:?}"
3407 );
3408 }
3409
3410 let spoofed_id = "parslee/openrouter/frontier-general";
3411 let spoofed_endpoint = "https://attacker.invalid/v1/chat/completions";
3412 let spoofed_value = serde_json::json!({
3413 "id": spoofed_id,
3414 "name": spoofed_id,
3415 "provider": "parslee",
3416 "family": "spoof",
3417 "capabilities": ["generate", "code", "reasoning"],
3418 "context_length": 1_000_000,
3419 "performance": {"latency_p50_ms": 1},
3420 "cost": {"input_per_mtok": 0.0, "output_per_mtok": 0.0},
3421 "source": {
3422 "type": "remote_api",
3423 "endpoint": spoofed_endpoint,
3424 "api_key_env": "CAR_MANAGED_ROUTING_TEST_KEY",
3425 "protocol": "open_ai_compat"
3426 },
3427 "tags": ["openrouter", "managed", "frontier"],
3428 "public_benchmarks": [{"name": "attacker-unanchored", "score": 1.0}]
3429 });
3430 let omitted_tier: ModelSchema = serde_json::from_value(spoofed_value.clone()).unwrap();
3431 assert_eq!(
3432 omitted_tier.trust_tier,
3433 crate::schema::TrustTier::Curated,
3434 "the regression must exercise the legacy omitted-tier default"
3435 );
3436 let mut direct_spoof_registry =
3437 UnifiedRegistry::new_empty(tmp.path().join("direct-spoof-models"));
3438 direct_spoof_registry.register(omitted_tier);
3439 assert!(
3440 direct_spoof_registry.get(spoofed_id).is_none(),
3441 "public registration must reject the reserved managed-alias namespace"
3442 );
3443 direct_spoof_registry
3444 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3445 let direct_spoof_route = router.route(
3446 "Analyze this architecture and prove the failure modes.",
3447 &direct_spoof_registry,
3448 &OutcomeTracker::new(),
3449 );
3450 assert!(
3451 !route_model_ids(&direct_spoof_route).any(|id| id == spoofed_id),
3452 "a rejected in-memory lookalike must not be routable: {direct_spoof_route:?}"
3453 );
3454 std::fs::write(
3455 tmp.path().join("models.json"),
3456 serde_json::to_vec_pretty(&vec![spoofed_value]).unwrap(),
3457 )
3458 .unwrap();
3459
3460 let mut reloaded = UnifiedRegistry::new_empty(tmp.path().join("reloaded-models"));
3461 reloaded.load_user_config().unwrap();
3462 assert!(
3463 reloaded.get(spoofed_id).is_none(),
3464 "persisted user config must reject the reserved managed-alias namespace"
3465 );
3466 reloaded.register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3467 let spoofed_route = router.route(
3468 "Analyze this architecture and prove the failure modes.",
3469 &reloaded,
3470 &OutcomeTracker::new(),
3471 );
3472 assert!(
3473 !route_model_ids(&spoofed_route).any(|id| id == spoofed_id),
3474 "a rejected persisted lookalike must not be routable: {spoofed_route:?}"
3475 );
3476
3477 let discovery_state_root = tmp.path().join("discovery-spoof-root");
3478 let discovery_models_dir = discovery_state_root.join("models");
3479 std::fs::create_dir_all(&discovery_models_dir).unwrap();
3480 let discovery_spoof_id = "openrouter/forged-discovery-cache";
3481 let mut discovery_spoof = remote_model(
3482 discovery_spoof_id,
3483 "forged-discovery-cache",
3484 vec![
3485 ModelCapability::Generate,
3486 ModelCapability::Code,
3487 ModelCapability::Reasoning,
3488 ],
3489 );
3490 discovery_spoof.provider = "openrouter".into();
3491 discovery_spoof.tags = vec!["openrouter".into(), "frontier".into()];
3492 discovery_spoof.trust_tier = crate::schema::TrustTier::Curated;
3493 discovery_spoof.performance.latency_p50_ms = Some(1);
3494 discovery_spoof.cost = crate::schema::CostModel::default();
3495 if let crate::schema::ModelSource::RemoteApi {
3496 ref mut api_key_env,
3497 ..
3498 } = discovery_spoof.source
3499 {
3500 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3501 }
3502 crate::discovery::save_cache(
3503 &crate::discovery::cache_path(&discovery_models_dir),
3504 &[discovery_spoof],
3505 )
3506 .unwrap();
3507
3508 let mut discovery_registry =
3509 UnifiedRegistry::new_with_state_root(discovery_state_root, discovery_models_dir);
3510 discovery_registry
3511 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3512 assert_eq!(
3513 discovery_registry
3514 .get(discovery_spoof_id)
3515 .unwrap()
3516 .trust_tier,
3517 crate::schema::TrustTier::Community,
3518 "unsigned discovery cache rows must be demoted before registration"
3519 );
3520 let discovery_spoof_route = router.route(
3521 "Analyze this architecture and prove the failure modes.",
3522 &discovery_registry,
3523 &OutcomeTracker::new(),
3524 );
3525 assert!(
3526 !discovery_spoof_route
3527 .candidates
3528 .iter()
3529 .any(|candidate| candidate.model_id == discovery_spoof_id),
3530 "a forged direct-provider discovery row must not enter trusted candidates: \
3531 {discovery_spoof_route:?}"
3532 );
3533 assert!(
3534 !discovery_spoof_route
3535 .fallbacks
3536 .iter()
3537 .any(|id| id == discovery_spoof_id),
3538 "a forged direct-provider discovery row must not enter trusted fallbacks: \
3539 {discovery_spoof_route:?}"
3540 );
3541 }
3542
3543 #[test]
3544 fn openrouter_becomes_require_ready_eligible_after_live_key_activation() {
3545 let tmp = tempfile::tempdir().unwrap();
3546 let _credential_scope = crate::openrouter::test_credential_scope();
3547 let _provider_env = crate::openrouter::test_environment_scope();
3548 crate::openrouter::set_test_credential(None);
3549 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
3550 let personal = crate::openrouter::curated_schemas()
3551 .into_iter()
3552 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3553 .unwrap();
3554 registry.register_project_model(personal);
3555
3556 unsafe {
3557 std::env::set_var("OPENAI_API_KEY", "ready-peer-key");
3558 }
3559 registry.register_project_model(remote_model(
3560 "ready-peer",
3561 "ready-peer",
3562 vec![ModelCapability::Generate, ModelCapability::Reasoning],
3563 ));
3564
3565 crate::openrouter::set_test_credential(Some("activated-after-register"));
3566 let hint = crate::intent::IntentHint {
3567 require_ready: true,
3568 prefer_quality: true,
3569 ..Default::default()
3570 };
3571 let router = AdaptiveRouter::new(
3572 test_hw(),
3573 RoutingConfig {
3574 prefer_local: false,
3575 prior_strength: 1_000_000.0,
3576 ..RoutingConfig::default()
3577 },
3578 );
3579 let tracker = OutcomeTracker::new();
3580 let decision = router.route_with(RouteRequest {
3581 intent: Some(&hint),
3582 ..RouteRequest::new("Prove the architecture tradeoff.", ®istry, &tracker)
3583 });
3584 assert!(
3585 decision
3586 .candidates
3587 .iter()
3588 .any(|candidate| candidate.model_id == "openrouter/openai/gpt-5.4"),
3589 "post-login OpenRouter model must remain require_ready-eligible: {:?}",
3590 decision.candidates
3591 );
3592 unsafe {
3593 std::env::remove_var("OPENAI_API_KEY");
3594 }
3595 }
3596
3597 #[test]
3598 fn route_cost_score_uses_prompt_tiers_and_model_cache_prices() {
3599 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
3600 let tracker = OutcomeTracker::new();
3601 let mut tiered = crate::openrouter::curated_schemas()
3602 .into_iter()
3603 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3604 .unwrap();
3605 let mut flat = tiered.clone();
3606 tiered.id = "tiered".into();
3607 flat.id = "flat".into();
3608 flat.cost = crate::schema::CostModel {
3609 input_per_mtok: Some(4.0),
3610 output_per_mtok: Some(22.5),
3611 cache_read_input_per_mtok: Some(4.0),
3612 ..Default::default()
3613 };
3614
3615 let score = |model: &ModelSchema, prompt, cache_read| {
3616 router.score_model(
3617 model,
3618 InferenceTask::Generate,
3619 &tracker,
3620 RoutingWorkload::Background,
3621 prompt,
3622 0,
3623 cache_read,
3624 0,
3625 )
3626 };
3627 assert!(score(&tiered, 100_000, 0) > score(&flat, 100_000, 0));
3628 assert!(score(&tiered, 300_000, 0) < score(&flat, 300_000, 0));
3629 assert!(score(&tiered, 300_000, 300_000) > score(&flat, 300_000, 300_000));
3630 }
3631
3632 fn assert_cold_start_pick_is_frontier_grade(
3636 router: &AdaptiveRouter,
3637 registry: &UnifiedRegistry,
3638 decision: &AdaptiveRoutingDecision,
3639 ) {
3640 let pick = registry
3641 .get(&decision.model_id)
3642 .expect("selected model should exist");
3643 if !pick.is_local() {
3644 assert!(
3645 router.is_trusted_quality_remote(pick),
3646 "cold start picked an untrusted remote: {decision:?}"
3647 );
3648 return;
3649 }
3650 let best_remote = registry
3651 .list()
3652 .into_iter()
3653 .filter(|model| model.available && router.is_trusted_quality_remote(model))
3654 .map(|model| router.schema_quality_estimate(model))
3655 .fold(0.0_f64, f64::max);
3656 assert!(
3657 !pick.public_benchmarks.is_empty()
3658 && router.schema_quality_estimate(pick)
3659 >= best_remote - AdaptiveRouter::OUTCOME_FIRST_BAND,
3660 "cold start picked a local below frontier grade: {decision:?}"
3661 );
3662 }
3663
3664 #[test]
3665 fn routes_simple_to_trusted_remote_during_cold_start() {
3666 let router = AdaptiveRouter::new(
3667 test_hw(),
3668 RoutingConfig {
3669 prior_strength: 100.0, ..Default::default()
3671 },
3672 );
3673 let reg = test_registry();
3674 let tracker = OutcomeTracker::new();
3675
3676 let decision = router.route("What is 2+2?", ®, &tracker);
3677 assert_eq!(decision.complexity, TaskComplexity::Simple);
3678 assert_eq!(decision.strategy, RoutingStrategy::SchemaBased);
3679 assert_cold_start_pick_is_frontier_grade(&router, ®, &decision);
3682 }
3683
3684 #[test]
3685 fn routes_code_to_code_capable_remote_during_cold_start() {
3686 let router = AdaptiveRouter::new(
3687 test_hw(),
3688 RoutingConfig {
3689 prior_strength: 100.0, ..Default::default()
3691 },
3692 );
3693 let reg = test_registry();
3694 let tracker = OutcomeTracker::new();
3695
3696 let decision = router.route(
3697 "Fix this function:\n```rust\nfn main() {}\n```",
3698 ®,
3699 &tracker,
3700 );
3701 assert_eq!(decision.complexity, TaskComplexity::Code);
3702 assert_eq!(decision.task, InferenceTask::Code);
3703 let schema = reg
3705 .find_by_name(&decision.model_name)
3706 .expect("model should exist");
3707 assert!(
3708 schema.has_capability(ModelCapability::Code),
3709 "selected model must support Code"
3710 );
3711 assert_cold_start_pick_is_frontier_grade(&router, ®, &decision);
3712 }
3713
3714 #[test]
3715 fn routes_images_to_vision_capable_model() {
3716 let router = AdaptiveRouter::new(
3717 test_hw(),
3718 RoutingConfig {
3719 prior_strength: 1_000_000.0,
3722 ..Default::default()
3723 },
3724 );
3725 let mut reg = test_registry();
3726 let tracker = OutcomeTracker::new();
3727
3728 reg.register_project_model(ModelSchema {
3729 id: "mlx-vlm/qwen3-vl-2b:bf16".into(),
3730 name: "Qwen3-VL-2B-mlx-vlm".into(),
3731 provider: "qwen".into(),
3732 family: "qwen3-vl".into(),
3733 version: "bf16".into(),
3734 capabilities: vec![
3735 ModelCapability::Generate,
3736 ModelCapability::Vision,
3737 ModelCapability::Grounding,
3738 ],
3739 context_length: 262_144,
3740 max_output_tokens: None,
3741 param_count: "2B".into(),
3742 quantization: None,
3743 performance: Default::default(),
3744 cost: Default::default(),
3745 source: crate::schema::ModelSource::Mlx {
3746 hf_repo: "Qwen/Qwen3-VL-2B-Instruct".into(),
3747 hf_weight_file: None,
3748 },
3749 tags: vec!["vision".into(), "mlx-vlm-cli".into()],
3750 supported_params: vec![],
3751 public_benchmarks: vec![],
3752 trust_tier: crate::schema::TrustTier::Curated,
3753 deprecated: false,
3754 available: true,
3755 weights_ready: true,
3756 });
3757
3758 let decision = router.route_with_vision("What is in this image?", ®, &tracker, false);
3759 let schema = reg
3760 .find_by_name(&decision.model_name)
3761 .expect("model should exist");
3762 assert!(
3763 schema.has_capability(ModelCapability::Vision),
3764 "selected model must support Vision"
3765 );
3766 }
3767
3768 #[test]
3769 fn profile_based_routing_favors_proven_model() {
3770 let router = AdaptiveRouter::new(
3771 test_hw(),
3772 RoutingConfig {
3776 min_observations: 3,
3777 ..Default::default()
3778 },
3779 );
3780 let reg = test_registry();
3781 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3782 let wins = |tracker: &OutcomeTracker| {
3784 (0..200)
3785 .filter(|_| {
3786 let decision = router.route("Fix this bug in the parser", ®, tracker);
3787 assert_eq!(decision.complexity, TaskComplexity::Code);
3788 decision.model_id == qwen_8b_id
3789 })
3790 .count()
3791 };
3792 let unproven = wins(&OutcomeTracker::new());
3793
3794 let mut tracker = OutcomeTracker::new();
3796 for _ in 0..20 {
3797 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Code, "test");
3798 tracker.record_complete(&trace, 500, 100, 50);
3799 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3800 }
3801 let proven = wins(&tracker);
3803 assert!(
3804 proven >= 100 && proven >= unproven + 60,
3805 "proven model won {proven}/200 (without its outcomes: {unproven}/200)"
3806 );
3807 }
3808
3809 #[test]
3810 fn proven_local_model_can_displace_bootstrap_remote() {
3811 let router = AdaptiveRouter::new(
3812 test_hw(),
3813 RoutingConfig {
3814 prior_strength: 100.0,
3815 bootstrap_min_task_observations: 6,
3816 bootstrap_quality_floor: 0.8,
3817 ..Default::default()
3818 },
3819 );
3820 let reg = test_registry();
3821 let mut tracker = OutcomeTracker::new();
3822
3823 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3824 for _ in 0..12 {
3825 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Generate, "test");
3826 tracker.record_complete(&trace, 300, 50, 20);
3827 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3828 }
3829
3830 let local_wins = |tracker: &OutcomeTracker| {
3836 (0..20)
3837 .filter(|_| {
3838 let decision = router.route("Summarize this paragraph.", ®, tracker);
3839 reg.get(&decision.model_id)
3840 .expect("selected model should exist")
3841 .is_local()
3842 })
3843 .count()
3844 };
3845
3846 let proven = local_wins(&tracker);
3847 assert!(
3848 proven >= 12,
3849 "proven local model won only {proven}/20 times (expected >= 12)"
3850 );
3851 let unproven = local_wins(&OutcomeTracker::new());
3854 assert!(
3855 unproven < 12,
3856 "an unobserved local model won {unproven}/20 times; the gate is not what this measures"
3857 );
3858 }
3859
3860 #[test]
3885 fn background_routing_prefers_local_models() {
3886 let router = AdaptiveRouter::new(
3887 test_hw(),
3888 RoutingConfig {
3889 prior_strength: 1_000_000.0,
3890 ..Default::default()
3891 },
3892 );
3893 let reg = test_registry();
3894 let tracker = OutcomeTracker::new();
3895
3896 let draws = 200;
3911 let mut local_wins = 0;
3912 for _ in 0..draws {
3913 let decision = router.route_context_aware(
3914 "Write a Python fibonacci function.",
3915 128,
3916 ®,
3917 &tracker,
3918 false,
3919 false,
3920 RoutingWorkload::Background,
3921 );
3922 let schema = reg
3923 .get(&decision.model_id)
3924 .expect("selected model should exist");
3925 if schema.is_local() {
3926 local_wins += 1;
3927 }
3928 }
3929
3930 assert!(
3931 local_wins >= 170,
3932 "background routing should keep work local; local won only \
3933 {local_wins}/{draws} (measured baseline: 96.2% off Apple Silicon, \
3934 100% on it)"
3935 );
3936 }
3937
3938 #[test]
3965 fn benchmark_prior_with_evidence_steers_background_routing() {
3966 fn target_score(profile: Option<crate::outcome::ModelProfile>) -> f64 {
3967 let router = AdaptiveRouter::new(
3968 test_hw(),
3969 RoutingConfig {
3970 prior_strength: 100.0,
3971 ..Default::default()
3972 },
3973 );
3974 let reg = test_registry();
3975 let mut tracker = OutcomeTracker::new();
3976 if let Some(p) = profile {
3977 tracker.import_profiles(vec![p]);
3978 }
3979 let decision = router.route_context_aware(
3980 "Write a Python fibonacci function.",
3981 128,
3982 ®,
3983 &tracker,
3984 false,
3985 false,
3986 RoutingWorkload::Background,
3987 );
3988 decision
3989 .candidates
3990 .iter()
3991 .find(|c| c.model_id == QWEN3_8B_ROUTED_ID)
3992 .unwrap_or_else(|| panic!("{QWEN3_8B_ROUTED_ID} should be a candidate"))
3993 .score
3994 }
3995
3996 fn profile_with(ema: f64) -> crate::outcome::ModelProfile {
3997 let mut p = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3998 p.ema_quality = ema;
3999 p.prior_sample_size = 30;
4003 p.quality_observations = 30;
4004 p.unseeded_ema_quality = ema;
4007 p.unseeded_quality_observations = 30;
4008 p
4009 }
4010
4011 let control = target_score(None);
4012 let good = target_score(Some(profile_with(0.99)));
4013 let bad = target_score(Some(profile_with(0.05)));
4014
4015 assert!(
4016 good > control,
4017 "an evidence-backed good prior must raise the score it vouches for \
4018 ({good} vs control {control}); equal means this test would pass with \
4019 the prior wiring removed"
4020 );
4021 assert!(
4022 bad < control,
4023 "an evidence-backed poor prior must lower the score ({bad} vs \
4024 control {control})"
4025 );
4026
4027 let mut unsupported = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
4031 unsupported.ema_quality = 0.99;
4032 assert_eq!(
4033 target_score(Some(unsupported)),
4034 control,
4035 "an EMA with no observations behind it must not move the score"
4036 );
4037 }
4038
4039 #[test]
4040 fn task_specific_benchmark_prior_informs_cold_start_routing() {
4041 let router = AdaptiveRouter::new(
4042 test_hw(),
4043 RoutingConfig {
4044 prior_strength: 100.0,
4045 ..Default::default()
4046 },
4047 );
4048 let reg = test_registry();
4049 let mut tracker = OutcomeTracker::new();
4050 let mut profile = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
4055 profile.task_stats.insert(
4056 crate::outcome::InferenceTask::Code.to_string(),
4057 crate::outcome::TaskStats {
4058 ema_quality: 0.95,
4059 prior_sample_size: 30,
4060 quality_observations: 30,
4061 ..Default::default()
4062 },
4063 );
4064 tracker.import_profiles(vec![profile]);
4065
4066 let decision = router.route_context_aware(
4067 "Write a Python fibonacci function.",
4068 128,
4069 ®,
4070 &tracker,
4071 false,
4072 false,
4073 RoutingWorkload::Background,
4074 );
4075
4076 let top_scored = decision
4080 .candidates
4081 .first()
4082 .expect("routing should expose ranked candidates");
4083 let schema = reg
4084 .get(&top_scored.model_id)
4085 .expect("top-scored model should exist");
4086 assert!(
4087 schema.is_local(),
4088 "background routing should score the task-specific local code prior highest"
4089 );
4090 }
4091
4092 #[test]
4093 fn task_specific_latency_prior_affects_cold_start_score() {
4094 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4095 let reg = test_registry();
4096 let model = reg
4097 .get("qwen/qwen3-8b:q4_k_m")
4098 .expect("local test model should exist");
4099
4100 let mut fast_tracker = OutcomeTracker::new();
4101 let mut fast_profile = crate::outcome::ModelProfile::new(model.id.clone());
4102 fast_profile.task_stats.insert(
4103 crate::outcome::InferenceTask::Generate.to_string(),
4104 crate::outcome::TaskStats {
4105 ema_quality: 0.95,
4106 avg_latency_ms: 1200.0,
4107 ..Default::default()
4108 },
4109 );
4110 fast_tracker.import_profiles(vec![fast_profile]);
4111
4112 let mut slow_tracker = OutcomeTracker::new();
4113 let mut slow_profile = crate::outcome::ModelProfile::new(model.id.clone());
4114 slow_profile.task_stats.insert(
4115 crate::outcome::InferenceTask::Generate.to_string(),
4116 crate::outcome::TaskStats {
4117 ema_quality: 0.95,
4118 avg_latency_ms: 120_000.0,
4119 ..Default::default()
4120 },
4121 );
4122 slow_tracker.import_profiles(vec![slow_profile]);
4123
4124 let fast_score = router.score_model(
4125 model,
4126 InferenceTask::Generate,
4127 &fast_tracker,
4128 RoutingWorkload::Interactive,
4129 0,
4130 0,
4131 0,
4132 0,
4133 );
4134 let slow_score = router.score_model(
4135 model,
4136 InferenceTask::Generate,
4137 &slow_tracker,
4138 RoutingWorkload::Interactive,
4139 0,
4140 0,
4141 0,
4142 0,
4143 );
4144
4145 assert!(
4146 fast_score > slow_score,
4147 "faster task latency prior should improve cold-start score ({fast_score} <= {slow_score})"
4148 );
4149 }
4150
4151 #[test]
4152 fn interactive_workload_keeps_remote_bootstrap_bias() {
4153 let router = AdaptiveRouter::new(
4154 test_hw(),
4155 RoutingConfig {
4156 prior_strength: 100.0,
4157 ..Default::default()
4158 },
4159 );
4160 let reg = test_registry();
4161 let mut tracker = OutcomeTracker::new();
4162 let mut profile = crate::outcome::ModelProfile::new("qwen/qwen3-8b:q4_k_m".into());
4163 profile.ema_quality = 0.95;
4164 tracker.import_profiles(vec![profile]);
4165
4166 let decision = router.route_context_aware(
4167 "Write a Python fibonacci function.",
4168 128,
4169 ®,
4170 &tracker,
4171 false,
4172 false,
4173 RoutingWorkload::Interactive,
4174 );
4175
4176 assert_ne!(decision.model_id, "qwen/qwen3-8b:q4_k_m", "{decision:?}");
4179 assert_cold_start_pick_is_frontier_grade(&router, ®, &decision);
4180 }
4181
4182 #[test]
4183 fn code_interactive_weights_prioritise_quality_over_speed_over_cost() {
4184 let (q, lat, cost) =
4190 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Interactive);
4191 assert!(
4192 q > lat && lat > cost,
4193 "expected quality > speed > cost, got ({q}, {lat}, {cost})"
4194 );
4195 assert!(
4196 (q + lat + cost - 1.0).abs() < 1e-9,
4197 "weights must sum to 1.0"
4198 );
4199 let (default_q, default_lat, default_cost) = RoutingWorkload::Interactive.weights();
4200 assert!(
4201 q > default_q && lat < default_lat && cost < default_cost,
4202 "code weighting must lift quality and cut latency/cost vs the generic interactive profile"
4203 );
4204
4205 assert_eq!(
4207 AdaptiveRouter::task_aware_weights(
4208 InferenceTask::Code,
4209 RoutingWorkload::LocalPreferred,
4210 ),
4211 (q, lat, cost),
4212 );
4213
4214 assert_eq!(
4217 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Fastest),
4218 RoutingWorkload::Fastest.weights(),
4219 );
4220 assert_eq!(
4222 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Background),
4223 RoutingWorkload::Background.weights(),
4224 );
4225 assert_eq!(
4227 AdaptiveRouter::task_aware_weights(
4228 InferenceTask::Generate,
4229 RoutingWorkload::Interactive,
4230 ),
4231 RoutingWorkload::Interactive.weights(),
4232 );
4233 }
4234
4235 #[test]
4236 fn fallback_chain_has_alternatives() {
4237 let router = AdaptiveRouter::new(
4238 test_hw(),
4239 RoutingConfig {
4240 prior_strength: 100.0, ..Default::default()
4242 },
4243 );
4244 let reg = test_registry();
4245 let tracker = OutcomeTracker::new();
4246
4247 let decision = router.route("Analyze the architecture trade-offs", ®, &tracker);
4248 assert!(!decision.fallbacks.is_empty());
4249 assert!(!decision.fallbacks.contains(&decision.model_id));
4251 }
4252
4253 #[test]
4254 fn latency_scoring_is_consistent() {
4255 let router = AdaptiveRouter::with_default_config(test_hw());
4257
4258 let schema_score = router.latency_ms_to_score(AdaptiveRouter::tps_to_latency_ms(25.0));
4260 let observed_score = router.latency_ms_to_score(8000.0);
4262 assert!(
4263 (schema_score - observed_score).abs() < 0.01,
4264 "schema ({schema_score}) and observed ({observed_score}) should match"
4265 );
4266 }
4267
4268 #[test]
4269 fn complexity_assessment() {
4270 assert_eq!(
4271 TaskComplexity::assess("What is the capital of France?"),
4272 TaskComplexity::Simple
4273 );
4274 assert_eq!(
4275 TaskComplexity::assess("Fix this broken test"),
4276 TaskComplexity::Code
4277 );
4278 assert_eq!(
4279 TaskComplexity::assess("Analyze the trade-offs between A and B"),
4280 TaskComplexity::Complex
4281 );
4282 }
4283
4284 #[test]
4285 fn beta_sampling_produces_valid_values() {
4286 let mut rng = rand::rng();
4287 for _ in 0..100 {
4289 let s = sample_beta(&mut rng, 2.0, 5.0);
4290 assert!((0.0..=1.0).contains(&s), "sample {s} out of [0,1] range");
4291 }
4292 let samples: Vec<f64> = (0..1000).map(|_| sample_beta(&mut rng, 1.0, 1.0)).collect();
4294 let mean = samples.iter().sum::<f64>() / samples.len() as f64;
4295 assert!(
4296 (mean - 0.5).abs() < 0.05,
4297 "Beta(1,1) mean {mean} should be ~0.5"
4298 );
4299 }
4300
4301 fn thompson_wins(scored: &[(String, f64)], tracker: &OutcomeTracker, id: &str) -> usize {
4304 let router = AdaptiveRouter::with_default_config(test_hw());
4305 (0..200)
4306 .filter(|_| router.select_with_thompson_sampling(scored, tracker).0 == id)
4307 .count()
4308 }
4309
4310 fn record_calls(tracker: &mut OutcomeTracker, id: &str, completed: usize) {
4311 for _ in 0..completed {
4312 let trace = tracker.record_start(id, InferenceTask::Code, "test");
4313 tracker.record_complete(&trace, 500, 100, 50);
4314 }
4315 }
4316
4317 #[test]
4321 fn many_completed_calls_do_not_outvote_a_better_score() {
4322 let mut tracker = OutcomeTracker::new();
4323 record_calls(&mut tracker, "busy", 900);
4324 let scored = vec![("better".to_string(), 0.92), ("busy".to_string(), 0.80)];
4325 let wins = thompson_wins(&scored, &tracker, "better");
4326 assert!(wins >= 160, "the better-scored model won only {wins}/200");
4327 }
4328
4329 #[test]
4332 fn scores_above_one_still_rank() {
4333 let tracker = OutcomeTracker::new();
4334 let scored = vec![("plain".to_string(), 1.0), ("bonused".to_string(), 1.15)];
4336 let wins = thompson_wins(&scored, &tracker, "bonused");
4337 assert!(wins >= 130, "the higher-scored model won only {wins}/200");
4338 }
4339
4340 #[test]
4342 fn unobserved_near_ties_are_both_explored() {
4343 let tracker = OutcomeTracker::new();
4344 let scored = vec![("a".to_string(), 0.90), ("b".to_string(), 0.88)];
4345 let wins = thompson_wins(&scored, &tracker, "a");
4346 assert!((40..=160).contains(&wins), "a won {wins}/200 of a near-tie");
4347 }
4348
4349 #[test]
4350 fn thompson_sampling_converges_to_best() {
4351 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4354 let reg = test_registry();
4355 let qwen_4b_id = QWEN3_4B_ROUTED_ID;
4356 let wins = |tracker: &OutcomeTracker| {
4357 (0..200)
4358 .filter(|_| {
4359 router.route("Fix this parser bug", ®, tracker).model_id == qwen_4b_id
4360 })
4361 .count()
4362 };
4363 let unproven = wins(&OutcomeTracker::new());
4364
4365 let mut tracker = OutcomeTracker::new();
4367 for _ in 0..20 {
4368 let trace = tracker.record_start(qwen_4b_id, InferenceTask::Code, "test");
4369 tracker.record_complete(&trace, 500, 100, 50);
4370 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
4371 }
4372 let proven = wins(&tracker);
4376 assert!(
4377 proven >= 100 && proven >= unproven + 60,
4378 "strong model won {proven}/200 (without its outcomes: {unproven}/200)"
4379 );
4380 }
4381
4382 #[test]
4385 fn cold_start_never_returns_a_model_lacking_a_required_capability() {
4386 let router = AdaptiveRouter::new(
4394 test_hw(),
4395 RoutingConfig {
4396 quality_first_cold_start: false,
4397 ..Default::default()
4398 },
4399 );
4400 let reg = test_registry();
4401
4402 for caps in [
4403 vec![ModelCapability::Code],
4404 vec![ModelCapability::Code, ModelCapability::ToolUse],
4405 ] {
4406 let decision = router.cold_start_decision(
4409 TaskComplexity::Simple,
4410 InferenceTask::Code,
4411 &caps,
4412 ®,
4413 false,
4414 &std::collections::HashSet::new(),
4415 false,
4416 AvailabilitySnapshot::capture(®),
4417 );
4418 let chosen = reg
4419 .find_by_name(&decision.model_name)
4420 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
4421 .unwrap_or_else(|| panic!("cold start named an unknown model: {decision:?}"));
4422 for cap in &caps {
4423 assert!(
4424 chosen.has_capability(*cap),
4425 "cold start chose {} which lacks required {:?} (caps {:?})",
4426 chosen.name,
4427 cap,
4428 caps
4429 );
4430 }
4431 assert_eq!(
4438 decision.strategy,
4439 RoutingStrategy::SchemaBased,
4440 "expected schema-based cold start"
4441 );
4442 assert!(
4443 decision.reason.contains("capability-satisfying"),
4444 "expected a capability-satisfying cold-start branch, got: {}",
4445 decision.reason
4446 );
4447 }
4448 }
4449
4450 #[test]
4451 fn cold_start_skips_a_capable_local_that_exceeds_the_memory_budget() {
4452 let mut hw = test_hw();
4457 hw.max_model_mb = 300; let router = AdaptiveRouter::new(
4459 hw,
4460 RoutingConfig {
4461 quality_first_cold_start: false,
4462 ..Default::default()
4463 },
4464 );
4465 let reg = test_registry();
4466 let decision = router.cold_start_decision(
4467 TaskComplexity::Code,
4468 InferenceTask::Code,
4469 &[ModelCapability::Code],
4470 ®,
4471 false,
4472 &std::collections::HashSet::new(),
4473 false,
4474 AvailabilitySnapshot::capture(®),
4475 );
4476 assert!(
4477 !decision.reason.contains("local cold start"),
4478 "no local fits the 300MB budget, so the local branch must not fire: {}",
4479 decision.reason
4480 );
4481 }
4482
4483 #[test]
4484 fn intent_require_filters_out_models_lacking_capability() {
4485 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4490 let reg = test_registry();
4491 let tracker = OutcomeTracker::new();
4492
4493 let intent = crate::intent::IntentHint {
4494 require: vec![ModelCapability::Vision],
4495 ..Default::default()
4496 };
4497 let decision = router.route_with_intent("hello", ®, &tracker, &intent);
4498
4499 assert_eq!(
4504 decision.strategy,
4505 RoutingStrategy::SchemaBased,
4506 "require=[vision] with no vision-capable candidates must drop to schema cold-start"
4507 );
4508 }
4509
4510 #[test]
4511 fn unbenchmarked_remote_is_conservative_not_assumed_frontier() {
4512 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4523 let reg = test_registry();
4524 let remote = reg
4525 .list()
4526 .into_iter()
4527 .find(|m| {
4528 m.is_remote()
4529 && m.public_benchmarks.is_empty()
4530 && !m.tags.iter().any(|t| t == "frontier" || t == "balanced")
4531 })
4532 .expect("a benchmark-less, tier-untagged remote in the fixture")
4533 .clone();
4534 let q = router.schema_quality_estimate(&remote);
4535 assert!(
4536 (q - 0.60).abs() < 1e-9,
4537 "unmeasured curated remote should be a conservative 0.60, got {q}"
4538 );
4539
4540 let mut benched = remote.clone();
4544 benched.public_benchmarks = vec![
4545 crate::schema::BenchmarkScore {
4546 name: "A".into(),
4547 score: 0.9,
4548 harness: None,
4549 source_url: None,
4550 measured_at: None,
4551 runs: None,
4552 spread: None,
4553 },
4554 crate::schema::BenchmarkScore {
4555 name: "B".into(),
4556 score: 0.7,
4557 harness: None,
4558 source_url: None,
4559 measured_at: None,
4560 runs: None,
4561 spread: None,
4562 },
4563 ];
4564 let qb = router.schema_quality_estimate(&benched);
4565 assert!((qb - 0.8).abs() < 1e-9);
4566 assert!(
4567 qb > q,
4568 "real benchmarks must lift a model above the unmeasured default"
4569 );
4570 }
4571
4572 #[test]
4573 fn calibrated_benchmark_is_normalized_onto_the_tier_scale() {
4574 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4578 let bench = |raw: f64| {
4579 let mut m = outcome_model("m", None);
4580 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4581 name: "car-judged".into(),
4582 score: raw,
4583 harness: None,
4584 source_url: None,
4585 measured_at: None,
4586 runs: None,
4587 spread: None,
4588 }];
4589 router.schema_quality_estimate(&m)
4590 };
4591 let frontier = bench(1.0); let strong = bench(0.54); assert!(
4595 (frontier - 0.92).abs() < 0.01,
4596 "best-in-class → ~CEIL, got {frontier}"
4597 );
4598 assert!(
4599 (strong - 0.68).abs() < 0.02,
4600 "0.54 raw → ~0.68 tier, got {strong}"
4601 );
4602 assert!(
4603 strong > router.schema_quality_estimate(&local_model("local-30b", 18_000)),
4604 "a measured frontier model must out-rank an unmeasured 30B local"
4605 );
4606 }
4607
4608 #[test]
4612 fn a_measured_failing_score_ranks_below_its_unmeasured_size_proxy() {
4613 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4614 let catalog = crate::registry::builtin_catalog();
4615 for id in ["vllm-mlx/qwen3.5-0.8b:4bit", "vllm-mlx/qwen3.5-2b:4bit"] {
4616 let measured = catalog.iter().find(|m| m.id == id).unwrap();
4617 assert!(measured.public_benchmarks[0].score < 0.2, "{id}");
4618 let mut unmeasured = measured.clone();
4619 unmeasured.public_benchmarks.clear();
4620 assert!(
4621 router.schema_quality_estimate(measured)
4622 < router.schema_quality_estimate(&unmeasured),
4623 "{id}: measured {} vs size proxy {}",
4624 router.schema_quality_estimate(measured),
4625 router.schema_quality_estimate(&unmeasured)
4626 );
4627 }
4628 let knee = AdaptiveRouter::BENCH_TIER_KNEE;
4630 let mut m = outcome_model("m", None);
4631 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4632 name: "car-judged".into(),
4633 score: knee,
4634 harness: None,
4635 source_url: None,
4636 measured_at: None,
4637 runs: None,
4638 spread: None,
4639 }];
4640 let at_knee = router.schema_quality_estimate(&m);
4641 m.public_benchmarks[0].score = knee - 1e-6;
4642 assert!((router.schema_quality_estimate(&m) - at_knee).abs() < 1e-5);
4643 assert!(
4644 (at_knee
4645 - (AdaptiveRouter::BENCH_TIER_FLOOR
4646 + knee * (AdaptiveRouter::BENCH_TIER_CEIL - AdaptiveRouter::BENCH_TIER_FLOOR)))
4647 .abs()
4648 < 1e-9
4649 );
4650 }
4651
4652 #[test]
4653 fn uncalibrated_benchmark_is_clamped_into_the_tier_band() {
4654 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4658 let q = |raw: f64| {
4659 let mut m = outcome_model("m", None);
4660 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4661 name: "some-new-unknown-bench".into(),
4662 score: raw,
4663 harness: None,
4664 source_url: None,
4665 measured_at: None,
4666 runs: None,
4667 spread: None,
4668 }];
4669 router.schema_quality_estimate(&m)
4670 };
4671 assert!((q(0.73) - 0.73).abs() < 1e-9);
4674 assert!((q(0.10) - AdaptiveRouter::BENCH_TIER_FLOOR).abs() < 1e-9);
4675 assert!((q(0.99) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9);
4676 }
4677
4678 #[test]
4679 fn local_uncalibrated_benchmark_maps_into_the_local_tier_band() {
4680 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4686 let q = |raw: f64| {
4687 let mut m = local_model("local-bench", 2500); m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4689 name: "car-agentic-suite".into(), score: raw,
4691 harness: None,
4692 source_url: None,
4693 measured_at: None,
4694 runs: None,
4695 spread: None,
4696 }];
4697 router.schema_quality_estimate(&m)
4698 };
4699 assert!(
4703 (q(0.45) - 0.45).abs() < 1e-9,
4704 "in-band local score passes through"
4705 );
4706 assert!(
4707 (q(0.05) - AdaptiveRouter::LOCAL_BENCH_TIER_FLOOR).abs() < 1e-9,
4708 "sub-floor clamps to LOCAL floor 0.30, not frontier FLOOR 0.40"
4709 );
4710 assert!(
4711 (q(0.95) - AdaptiveRouter::LOCAL_BENCH_TIER_CEIL).abs() < 1e-9,
4712 "a perfect local-suite score tops at LOCAL ceil 0.60, not frontier CEIL 0.92"
4713 );
4714 assert!(q(0.55) > q(0.35));
4716 let mut frontier = outcome_model("frontier", None);
4719 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
4720 name: "car-judged".into(),
4721 score: 1.0,
4722 harness: None,
4723 source_url: None,
4724 measured_at: None,
4725 runs: None,
4726 spread: None,
4727 }];
4728 assert!(
4729 router.schema_quality_estimate(&frontier) > q(0.95),
4730 "a benchmarked frontier model out-ranks even a perfect-scoring local"
4731 );
4732 }
4733
4734 #[test]
4735 fn frontier_ref_is_derived_from_catalog_best_in_class() {
4736 use std::collections::HashMap;
4741 let mut max_by_bench: HashMap<String, f64> = HashMap::new();
4742 for model in crate::registry::builtin_catalog() {
4743 for b in &model.public_benchmarks {
4744 let e = max_by_bench.entry(b.name.clone()).or_insert(0.0);
4745 *e = e.max(b.score);
4746 }
4747 }
4748 assert!(!max_by_bench.is_empty(), "catalog should carry benchmarks");
4749 for (name, max) in max_by_bench {
4750 let derived = AdaptiveRouter::benchmark_frontier_ref(&name)
4751 .unwrap_or_else(|| panic!("benchmark '{name}' must derive a frontier_ref"));
4752 assert!(
4753 (derived - max).abs() < 1e-9,
4754 "frontier_ref for '{name}' should be the catalog max {max}, got {derived}"
4755 );
4756 }
4757 assert_eq!(
4759 AdaptiveRouter::benchmark_frontier_ref("car-judged"),
4760 Some(1.0)
4761 );
4762 assert_eq!(
4764 AdaptiveRouter::benchmark_frontier_ref("nonexistent-bench"),
4765 None
4766 );
4767 }
4768
4769 fn benchmarked_model(id: &str, bench: &str, score: f64, frontier: bool) -> ModelSchema {
4773 let mut m = outcome_model(id, None);
4774 if frontier {
4775 m.tags.push("frontier".into());
4776 }
4777 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4778 name: bench.into(),
4779 score,
4780 harness: None,
4781 source_url: None,
4782 measured_at: None,
4783 runs: None,
4784 spread: None,
4785 }];
4786 m
4787 }
4788
4789 #[test]
4790 fn frontier_ref_guard_drops_frontier_less_benchmark() {
4791 let models = [
4796 benchmarked_model("mid-a", "midtier-only-bench", 0.50, false),
4797 benchmarked_model("mid-b", "midtier-only-bench", 0.40, false),
4798 ];
4799 let refs = compute_frontier_refs(models.iter());
4800 assert!(
4801 !refs.contains_key("midtier-only-bench"),
4802 "a benchmark with no frontier-class carrier must not self-anchor, got {refs:?}"
4803 );
4804 }
4805
4806 #[test]
4807 fn frontier_ref_guard_anchors_when_a_frontier_model_carries_the_benchmark() {
4808 let models = [
4813 benchmarked_model("frontier-x", "shared-bench", 0.60, true),
4814 benchmarked_model("mid-y", "shared-bench", 0.70, false),
4815 ];
4816 let refs = compute_frontier_refs(models.iter());
4817 assert_eq!(
4818 refs.get("shared-bench").copied(),
4819 Some(0.70),
4820 "anchor is the catalog max across all carriers once a frontier carrier exists"
4821 );
4822 }
4823
4824 #[test]
4825 fn frontier_ref_guard_is_noop_on_todays_catalog() {
4826 use std::collections::HashMap;
4831 let catalog = crate::registry::builtin_catalog();
4832 let mut raw_max: HashMap<String, f64> = HashMap::new();
4833 for m in &catalog {
4834 for b in &m.public_benchmarks {
4835 let e = raw_max.entry(b.name.clone()).or_insert(0.0);
4836 *e = e.max(b.score);
4837 }
4838 }
4839 let guarded = compute_frontier_refs(catalog.iter());
4840 assert_eq!(
4841 raw_max, guarded,
4842 "guard dropped a benchmark — a catalog benchmark lost its frontier-class carrier"
4843 );
4844 assert!(
4845 model_is_frontier_class(
4846 catalog
4847 .iter()
4848 .find(|m| m.id == "openai/gpt-6.1-sol:latest")
4849 .expect("gpt-6.1-sol in catalog")
4850 ),
4851 "the car-judged champion must be frontier-tagged for the anchor to be valid"
4852 );
4853 }
4854
4855 #[test]
4856 fn score_above_the_frontier_clamps_to_ceil() {
4857 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4861 let mut m = outcome_model("future-sota", None);
4862 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4863 name: "car-judged".into(),
4864 score: 1.05,
4867 harness: None,
4868 source_url: None,
4869 measured_at: None,
4870 runs: None,
4871 spread: None,
4872 }];
4873 assert!(
4874 (router.schema_quality_estimate(&m) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9,
4875 "above-frontier score should clamp to CEIL"
4876 );
4877 }
4878
4879 #[test]
4887 fn curated_native_frontier_row_outranks_measured_small_models_on_quality_code() {
4888 let _environment = crate::openrouter::test_environment_scope();
4889 let _restore = RestoredEnvironment::capture(&["OPENAI_API_KEY", "CAR_SECRETS_FILE_DIR"]);
4890 unsafe {
4891 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
4892 std::env::set_var(
4893 "CAR_SECRETS_FILE_DIR",
4894 std::env::temp_dir().join("car-test-frontier-prior-empty-secrets"),
4895 );
4896 }
4897 let catalog: Vec<ModelSchema> =
4898 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
4899 let row = |id: &str| {
4900 let mut m = catalog
4901 .iter()
4902 .find(|m| m.id == id)
4903 .unwrap_or_else(|| panic!("{id} missing from builtin catalog"))
4904 .clone();
4905 m.available = true;
4906 m.weights_ready = true;
4907 m
4908 };
4909 let mut frontier = row("openai/gpt-5.6-sol:latest");
4912 frontier.public_benchmarks.clear();
4913 let mini = row("openai/gpt-5.4-mini:latest");
4914 let local = row(QWEN3_4B_ROUTED_ID);
4915 assert_eq!(frontier.trust_tier, crate::schema::TrustTier::Curated);
4918 assert!(model_is_frontier_class(&frontier));
4919 assert!(!frontier.tags.iter().any(|t| t == "openrouter"));
4920 assert!(frontier.public_benchmarks.is_empty(), "must be unmeasured");
4921 assert!(matches!(
4922 frontier.source,
4923 crate::schema::ModelSource::RemoteApi { .. }
4924 ));
4925 assert_eq!(mini.public_benchmarks[0].name, "car-judged");
4926 assert_eq!(local.public_benchmarks[0].name, "car-judged");
4927 assert!(local.is_local());
4928
4929 let tmp = tempfile::tempdir().unwrap();
4930 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("models"));
4931 for m in [frontier.clone(), mini.clone(), local.clone()] {
4932 assert!(registry.register_project_model(m));
4933 }
4934 let local_routable = !cfg!(all(
4939 target_os = "macos",
4940 target_arch = "aarch64",
4941 not(car_skip_mlx)
4942 )) || crate::backend::local::in_process_loader_available();
4943 assert_eq!(
4944 registry.get(&local.id).map(|m| m.available),
4945 Some(local_routable),
4946 "admission must agree with whether this build can run {}",
4947 local.id
4948 );
4949
4950 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4951 let tracker = OutcomeTracker::new();
4952 let intent = crate::intent::IntentHint {
4953 task: Some(crate::intent::TaskHint::Code),
4954 require: vec![ModelCapability::Code],
4955 prefer_quality: true,
4956 ..Default::default()
4957 };
4958 let decision = router.route_with_intent(
4959 "Refactor this module and explain the failure modes.",
4960 ®istry,
4961 &tracker,
4962 &intent,
4963 );
4964 let reliability = |id: &str| {
4965 decision
4966 .candidates
4967 .iter()
4968 .find(|c| c.model_id == id)
4969 .unwrap_or_else(|| panic!("{id} not a candidate: {:?}", decision.candidates))
4970 .reliability
4971 };
4972 let (f, m) = (reliability(&frontier.id), reliability(&mini.id));
4973 assert!(
4974 (f - AdaptiveRouter::CURATED_FRONTIER_TAG_PRIOR).abs() < 1e-9,
4975 "unmeasured curated frontier row must take the tier prior, got {f}"
4976 );
4977 if local_routable {
4978 let l = reliability(&local.id);
4979 assert!(
4980 f > l,
4981 "frontier {f} must outrank the car-judged 4B local {l}"
4982 );
4983 } else {
4984 assert!(
4985 !decision.candidates.iter().any(|c| c.model_id == local.id),
4986 "{} cannot run in this build, so it must not be a candidate: {:?}",
4987 local.id,
4988 decision.candidates
4989 );
4990 }
4991 assert!(f > m, "frontier {f} must outrank the measured mini {m}");
4992 assert_eq!(
4993 decision.model_id, frontier.id,
4994 "prefer_quality code must pick the frontier row: {:?}",
4995 decision.candidates
4996 );
4997 }
4998
4999 #[test]
5000 fn local_size_heuristic_stays_below_measured_frontier_floor() {
5001 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5005 for size_mb in [500u64, 1500, 2500, 4500, 18_000] {
5006 let q = router.schema_quality_estimate(&local_model("local", size_mb));
5007 assert!(
5008 q <= AdaptiveRouter::BENCH_TIER_FLOOR + 0.21, "local {size_mb}MB quality {q} must stay near/below the measured floor"
5010 );
5011 }
5012 }
5013
5014 #[test]
5015 fn community_remote_quality_is_damped_below_curated() {
5016 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5021 let reg = test_registry();
5022 let mut remote = reg
5023 .list()
5024 .into_iter()
5025 .find(|m| {
5026 m.is_remote()
5027 && m.public_benchmarks.is_empty()
5028 && !m.tags.iter().any(|t| t == "frontier" || t == "balanced")
5029 })
5030 .expect("a benchmark-less, tier-untagged remote in the fixture")
5031 .clone();
5032
5033 remote.trust_tier = crate::schema::TrustTier::Curated;
5035 let qc = router.schema_quality_estimate(&remote);
5036 assert!(
5037 (qc - 0.60).abs() < 1e-9,
5038 "curated unmeasured remote should be 0.60, got {qc}"
5039 );
5040
5041 remote.trust_tier = crate::schema::TrustTier::Community;
5043 let q = router.schema_quality_estimate(&remote);
5044 assert!(
5045 (q - 0.48).abs() < 1e-9,
5046 "community remote should be damped to 0.48, got {q}"
5047 );
5048 assert!(
5049 q < qc,
5050 "community remote must rank below an unmeasured curated remote"
5051 );
5052
5053 remote.tags.push("frontier".into());
5057 let q = router.schema_quality_estimate(&remote);
5058 assert!(
5059 (q - 0.48).abs() < 1e-9,
5060 "community row with an inherited frontier tag stays at 0.48, got {q}"
5061 );
5062 remote.trust_tier = crate::schema::TrustTier::Curated;
5063 let q = router.schema_quality_estimate(&remote);
5064 assert!(
5065 (q - AdaptiveRouter::CURATED_FRONTIER_TAG_PRIOR).abs() < 1e-9,
5066 "curated frontier-tagged row takes the tier prior, got {q}"
5067 );
5068 }
5069
5070 #[test]
5071 fn quality_workload_is_quality_dominant() {
5072 let (q, lat, cost) = RoutingWorkload::Quality.weights();
5075 assert!(
5076 (q + lat + cost - 1.0).abs() < 1e-9,
5077 "weights must sum to 1.0"
5078 );
5079 assert!(q >= 0.8, "quality must dominate, got {q}");
5080 assert!(
5081 lat <= 0.1 && cost <= 0.15,
5082 "latency/cost near floor: {lat}/{cost}"
5083 );
5084 for w in [
5086 RoutingWorkload::Interactive,
5087 RoutingWorkload::Batch,
5088 RoutingWorkload::Background,
5089 RoutingWorkload::LocalPreferred,
5090 RoutingWorkload::Fastest,
5091 ] {
5092 assert!(
5093 q > w.weights().0,
5094 "Quality must out-weight {w:?} on quality"
5095 );
5096 }
5097 }
5098
5099 #[test]
5100 fn prefer_quality_scores_the_more_capable_model_higher() {
5101 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5106 let reg = test_registry();
5107 let tracker = OutcomeTracker::new();
5108 let small = reg
5109 .find_by_name("Qwen3-1.7B")
5110 .expect("1.7B in fixture")
5111 .clone();
5112 let big = reg.find_by_name("Qwen3-4B").expect("4B in fixture").clone();
5113 assert!(small.has_capability(ModelCapability::Code));
5114 assert!(big.has_capability(ModelCapability::Code));
5115
5116 let score = |m: &ModelSchema, w: RoutingWorkload| {
5117 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
5118 };
5119 assert!(
5121 score(&big, RoutingWorkload::Quality) > score(&small, RoutingWorkload::Quality),
5122 "prefer_quality must rank the capable 4B above the cheap 1.7B"
5123 );
5124 }
5125
5126 fn local_mlx_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
5129 let mut m = outcome_model(id, benchmark);
5130 m.provider = "local".into();
5131 m.source = crate::schema::ModelSource::Mlx {
5132 hf_repo: "mlx-community/test".into(),
5133 hf_weight_file: None,
5134 };
5135 m
5136 }
5137
5138 #[test]
5139 fn quality_workload_suppresses_local_mlx_bonus_vs_capable_remote() {
5140 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5145 let tracker = OutcomeTracker::new();
5146 let score = |m: &ModelSchema, w: RoutingWorkload| {
5147 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
5148 };
5149
5150 let remote_capable = outcome_model("remote/frontier", Some(0.85));
5151 let mlx_tiny = local_mlx_model("mlx/tiny", Some(0.55));
5152 let remote_peer = outcome_model("remote/peer", Some(0.55));
5154 assert!(mlx_tiny.is_mlx() && mlx_tiny.is_local());
5155
5156 assert!(
5160 score(&mlx_tiny, RoutingWorkload::Background)
5161 > score(&remote_peer, RoutingWorkload::Background),
5162 "the local/MLX bonus should still apply on non-Quality workloads"
5163 );
5164
5165 assert!(
5168 score(&remote_capable, RoutingWorkload::Quality)
5169 > score(&mlx_tiny, RoutingWorkload::Quality),
5170 "prefer_quality must suppress the locality bonus so the capable remote wins"
5171 );
5172 }
5173
5174 fn local_model(id: &str, size_mb: u64) -> ModelSchema {
5179 let mut m = outcome_model(id, None);
5180 m.provider = "local".into();
5181 m.source = crate::schema::ModelSource::Local {
5182 hf_repo: "test/repo".into(),
5183 hf_filename: "model.gguf".into(),
5184 tokenizer_repo: "test/repo".into(),
5185 };
5186 m.cost.size_mb = Some(size_mb);
5187 m
5188 }
5189
5190 #[test]
5201 fn a_measured_moe_rate_is_not_discounted_again() {
5202 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5203
5204 let mut measured = local_model("local/moe-measured", 17_000);
5205 measured.tags.push("moe".into());
5206 measured.performance.tokens_per_second = Some(35.0);
5207 measured.performance.latency_p50_ms = Some(1_200);
5208
5209 let mut declared = local_model("local/moe-declared", 17_000);
5211 declared.tags.push("moe".into());
5212 declared.performance.tokens_per_second = Some(35.0);
5213
5214 let mut dense = local_model("local/dense", 17_000);
5216 dense.performance.tokens_per_second = Some(35.0);
5217 dense.performance.latency_p50_ms = Some(1_200);
5218
5219 let measured_score = router.schema_latency_estimate(&measured);
5220 let declared_score = router.schema_latency_estimate(&declared);
5221 let dense_score = router.schema_latency_estimate(&dense);
5222
5223 assert_eq!(
5224 measured_score, dense_score,
5225 "a measured MoE rate must score as the rate it measured"
5226 );
5227 assert!(
5228 declared_score < measured_score,
5229 "a declared MoE rate still gets the discount: {declared_score} vs {measured_score}"
5230 );
5231 }
5232
5233 #[test]
5234 fn a_named_task_decides_the_capability_filter_over_the_keyword_guess() {
5235 use crate::intent::{IntentHint, TaskHint};
5236 let prompt = "Let me know if there's an error in the prefix list.";
5237 let guess = TaskComplexity::assess(prompt);
5238 assert_eq!(
5239 guess,
5240 TaskComplexity::Code,
5241 "the keyword guess reads this as code"
5242 );
5243 let chat = IntentHint {
5244 task: Some(TaskHint::Chat),
5245 ..Default::default()
5246 };
5247 let chat_task = task_hint_to_inference_task(TaskHint::Chat);
5248 assert_eq!(
5249 base_required_capabilities(guess, Some(&chat), chat_task),
5250 vec![inference_task_required_capability(chat_task)]
5251 );
5252 let code = IntentHint {
5254 task: Some(TaskHint::Code),
5255 ..Default::default()
5256 };
5257 assert_eq!(
5258 base_required_capabilities(TaskComplexity::Simple, Some(&code), InferenceTask::Code),
5259 vec![ModelCapability::Code]
5260 );
5261 assert_eq!(
5263 base_required_capabilities(guess, None, InferenceTask::Code),
5264 vec![ModelCapability::Code]
5265 );
5266 }
5267
5268 #[test]
5273 fn a_vllm_mlx_moe_row_scores_at_its_benchmarked_rate() {
5274 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5275 let catalog = crate::registry::builtin_catalog();
5276 let moe = catalog
5277 .iter()
5278 .find(|m| m.id == "vllm-mlx/qwen3.6-35b-a3b:4bit")
5279 .unwrap();
5280 let rate = moe.performance.tokens_per_second.unwrap();
5281 assert!(moe.tags.iter().any(|t| t == "moe"));
5282 assert!(moe.performance.latency_p50_ms.is_none());
5283
5284 let mut dense = local_model("local/dense", 17_000);
5285 dense.performance.tokens_per_second = Some(rate);
5286 assert_eq!(
5287 router.schema_latency_estimate(moe),
5288 router.schema_latency_estimate(&dense),
5289 "scored as if it decoded at {} tok/s",
5290 rate * AdaptiveRouter::MOE_TPS_MULTIPLIER
5291 );
5292 }
5293
5294 #[test]
5298 fn an_ignored_seed_stays_out_after_live_grades() {
5299 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5300 let seeded_model = outcome_model("remote/seeded", Some(0.70));
5301 let plain_model = outcome_model("remote/plain", Some(0.70));
5302 let mut tracker = OutcomeTracker::new();
5303 let mut seeded = crate::outcome::ModelProfile::new("remote/seeded".into());
5304 seeded.ema_quality = 1.0;
5305 seeded.prior_sample_size = 4;
5306 tracker.import_profiles(vec![seeded]);
5307 let task = InferenceTask::Generate;
5308 for id in ["remote/seeded", "remote/plain"] {
5309 for confidence in [0.4, 0.5] {
5310 let trace = tracker.record_start(id, task, "test");
5311 tracker.record_complete(&trace, 100, 10, 10);
5312 tracker.record_inferred_outcome(&trace, InferredOutcome::Rejected { confidence });
5313 }
5314 }
5315 let seeded_reliability = router.reliability(&seeded_model, task, &tracker);
5316 let plain_reliability = router.reliability(&plain_model, task, &tracker);
5317 assert!(
5318 (seeded_reliability - plain_reliability).abs() < 1e-12,
5319 "seeded {seeded_reliability} against never seeded {plain_reliability}"
5320 );
5321 assert!(seeded_reliability < router.schema_quality_estimate(&seeded_model));
5323 }
5324
5325 #[test]
5329 fn a_raw_benchmark_seed_never_overrides_a_catalog_benchmark() {
5330 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5331 let measured = outcome_model("remote/measured", Some(0.70));
5332 let unmeasured = outcome_model("remote/unmeasured", None);
5333 let mut tracker = OutcomeTracker::new();
5334 let seeded = |id: &str| {
5335 let mut p = crate::outcome::ModelProfile::new(id.into());
5336 p.ema_quality = 1.0;
5337 p.prior_sample_size = 4;
5338 p
5339 };
5340 tracker.import_profiles(vec![seeded("remote/measured"), seeded("remote/unmeasured")]);
5341
5342 let task = InferenceTask::Generate;
5343 assert_eq!(
5344 router.reliability(&measured, task, &tracker),
5345 router.schema_quality_estimate(&measured)
5346 );
5347 assert!(
5348 router.reliability(&unmeasured, task, &tracker)
5349 > router.schema_quality_estimate(&unmeasured)
5350 );
5351 }
5352
5353 #[test]
5358 fn remote_and_local_latency_are_the_time_to_a_whole_answer() {
5359 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5360 let mut local = local_model("local/dense", 17_000);
5361 local.performance.tokens_per_second = Some(100.0);
5362 let mut remote = outcome_model("remote/measured", Some(0.8));
5363 remote.performance.tokens_per_second = Some(100.0);
5364 assert_eq!(
5365 router.schema_latency_estimate(&remote),
5366 router.schema_latency_estimate(&local)
5367 );
5368
5369 let mut declared = outcome_model("remote/declared", Some(0.8));
5371 declared.performance.latency_p50_ms = Some(600);
5372 let answer_ms = 600.0 + 200.0 / AdaptiveRouter::UNMEASURED_REMOTE_TPS * 1000.0;
5373 assert!(
5374 (router.schema_latency_estimate(&declared) - router.latency_ms_to_score(answer_ms))
5375 .abs()
5376 < 1e-9
5377 );
5378
5379 let mut slow_start = local.clone();
5381 slow_start.performance.latency_p50_ms = Some(1_500);
5382 assert!(
5383 router.schema_latency_estimate(&slow_start) < router.schema_latency_estimate(&local)
5384 );
5385 }
5386
5387 fn outcome_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
5391 ModelSchema {
5392 id: id.into(),
5393 name: id.into(),
5394 provider: "openai".into(),
5395 family: "test".into(),
5396 version: "latest".into(),
5397 capabilities: vec![
5398 ModelCapability::Generate,
5399 ModelCapability::Code,
5400 ModelCapability::Reasoning,
5401 ],
5402 context_length: 128_000,
5403 max_output_tokens: None,
5404 param_count: "api".into(),
5405 quantization: None,
5406 performance: Default::default(),
5407 cost: Default::default(),
5408 source: crate::schema::ModelSource::RemoteApi {
5409 endpoint: "https://api.openai.com/v1".into(),
5410 api_key_env: "OPENAI_API_KEY".into(),
5411 api_key_envs: vec![],
5412 api_version: None,
5413 protocol: crate::schema::ApiProtocol::OpenAiCompat,
5414 },
5415 tags: vec![],
5416 supported_params: vec![],
5417 public_benchmarks: benchmark
5418 .map(|s| {
5419 vec![crate::schema::BenchmarkScore {
5420 name: "t".into(),
5421 score: s,
5422 harness: None,
5423 source_url: None,
5424 measured_at: None,
5425 runs: None,
5426 spread: None,
5427 }]
5428 })
5429 .unwrap_or_default(),
5430 trust_tier: crate::schema::TrustTier::Curated,
5431 deprecated: false,
5432 available: true,
5433 weights_ready: true,
5434 }
5435 }
5436
5437 #[test]
5438 fn reliability_band_applies_to_quality_and_substantive_normal_lanes() {
5439 use RoutingWorkload as W;
5440 assert!(AdaptiveRouter::applies_reliability_band(
5442 InferenceTask::Code,
5443 W::Interactive
5444 ));
5445 assert!(AdaptiveRouter::applies_reliability_band(
5446 InferenceTask::Reasoning,
5447 W::Batch
5448 ));
5449 assert!(AdaptiveRouter::applies_reliability_band(
5452 InferenceTask::Code,
5453 W::Quality
5454 ));
5455 assert!(AdaptiveRouter::applies_reliability_band(
5456 InferenceTask::Generate,
5457 W::Quality
5458 ));
5459 assert!(!AdaptiveRouter::applies_reliability_band(
5461 InferenceTask::Generate,
5462 W::Interactive
5463 ));
5464 assert!(!AdaptiveRouter::applies_reliability_band(
5466 InferenceTask::Code,
5467 W::Background
5468 ));
5469 assert!(!AdaptiveRouter::applies_reliability_band(
5470 InferenceTask::Code,
5471 W::Fastest
5472 ));
5473 assert!(!AdaptiveRouter::applies_reliability_band(
5475 InferenceTask::Code,
5476 W::LocalPreferred
5477 ));
5478 }
5479
5480 #[test]
5481 fn high_stakes_intent_forces_quality_over_everything() {
5482 use crate::intent::IntentHint;
5483 let hs = IntentHint {
5486 high_stakes: true,
5487 prefer_fast: true,
5488 prefer_local: true,
5489 ..Default::default()
5490 };
5491 assert_eq!(
5492 AdaptiveRouter::resolve_workload(Some(&hs), RoutingWorkload::Interactive),
5493 RoutingWorkload::Quality
5494 );
5495 let f = IntentHint {
5497 prefer_fast: true,
5498 ..Default::default()
5499 };
5500 assert_eq!(
5501 AdaptiveRouter::resolve_workload(Some(&f), RoutingWorkload::Interactive),
5502 RoutingWorkload::Fastest
5503 );
5504 assert_eq!(
5506 AdaptiveRouter::resolve_workload(None, RoutingWorkload::Batch),
5507 RoutingWorkload::Batch
5508 );
5509 }
5510
5511 #[test]
5512 fn band_excludes_a_real_quality_gap_even_if_cheaper() {
5513 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5515 let tracker = OutcomeTracker::new();
5516 let strong = outcome_model("strong", Some(0.90));
5517 let weak_cheap = outcome_model("weak-cheap", Some(0.83));
5518 let band =
5519 router.outcome_first_band(vec![strong, weak_cheap], InferenceTask::Code, &tracker);
5520 assert_eq!(
5521 band.len(),
5522 1,
5523 "0.07 gap (> band 0.02) must drop the weaker model"
5524 );
5525 assert_eq!(band[0].id, "strong");
5526 }
5527
5528 #[test]
5529 fn band_keeps_genuinely_near_equal_models() {
5530 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5532 let tracker = OutcomeTracker::new();
5533 let band = router.outcome_first_band(
5534 vec![
5535 outcome_model("a", Some(0.90)),
5536 outcome_model("a2", Some(0.89)),
5537 ],
5538 InferenceTask::Code,
5539 &tracker,
5540 );
5541 assert_eq!(
5542 band.len(),
5543 2,
5544 "0.01 gap (<= band 0.02) keeps both for tie-break"
5545 );
5546 }
5547
5548 #[test]
5549 fn band_cold_start_frontier_beats_cheap_unknown() {
5550 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5553 let tracker = OutcomeTracker::new();
5554 let band = router.outcome_first_band(
5555 vec![
5556 outcome_model("frontier", Some(0.85)),
5557 outcome_model("unknown", None),
5558 ],
5559 InferenceTask::Code,
5560 &tracker,
5561 );
5562 assert_eq!(band.len(), 1);
5563 assert_eq!(
5564 band[0].id, "frontier",
5565 "0.85 vs 0.60 unknown → unknown excluded"
5566 );
5567 }
5568
5569 #[test]
5570 fn quality_lane_bands_out_a_weak_local_but_local_preferred_keeps_it() {
5571 let mut reg = test_registry();
5578 let mut frontier = outcome_model("frontier-remote", None);
5579 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
5580 name: "car-judged".into(),
5581 score: 1.0, harness: None,
5583 source_url: None,
5584 measured_at: None,
5585 runs: None,
5586 spread: None,
5587 }];
5588 reg.register_project_model(frontier);
5589
5590 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5591 let tracker = OutcomeTracker::new();
5592 let prompt = "Implement a balanced binary search tree with deletion.";
5593
5594 let q = router.route_context_aware(
5597 prompt,
5598 256,
5599 ®,
5600 &tracker,
5601 false,
5602 false,
5603 RoutingWorkload::Quality,
5604 );
5605 let picked = reg.get(&q.model_id).expect("picked model is registered");
5608 assert!(
5609 !picked.is_local() && !picked.public_benchmarks.is_empty(),
5610 "prefer_quality must pick a benchmarked frontier over locals-on-bonuses, got {}",
5611 q.model_id
5612 );
5613 assert_eq!(
5614 q.strategy,
5615 RoutingStrategy::SchemaBased,
5616 "Quality lane must use deterministic argmax, not Thompson exploration"
5617 );
5618 assert!(
5619 q.candidates.iter().any(|c| !c.in_band),
5620 "Quality lane must apply a reliability band (some candidates excluded)"
5621 );
5622
5623 let lp = router.route_context_aware(
5626 prompt,
5627 256,
5628 ®,
5629 &tracker,
5630 false,
5631 false,
5632 RoutingWorkload::LocalPreferred,
5633 );
5634 assert!(
5637 lp.candidates
5638 .iter()
5639 .all(|c| c.in_band == (c.reliability >= AdaptiveRouter::MIN_ANSWER_QUALITY)),
5640 "LocalPreferred must NOT band — every answering candidate eligible"
5641 );
5642 }
5643
5644 #[test]
5648 fn prefer_fast_never_picks_a_model_that_fails_most_tasks() {
5649 let reg = test_registry();
5652 let failing = ["vllm-mlx/qwen3.5-0.8b:4bit", "vllm-mlx/qwen3.5-2b:4bit"];
5653 let judged = |score: f64| {
5654 vec![crate::schema::BenchmarkScore {
5655 name: "car-judged".into(),
5656 score,
5657 harness: None,
5658 source_url: None,
5659 measured_at: None,
5660 runs: None,
5661 spread: None,
5662 }]
5663 };
5664 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5665 let tracker = OutcomeTracker::new();
5666 for _ in 0..50 {
5667 let d = router.route_context_aware(
5668 "Say hello.",
5669 64,
5670 ®,
5671 &tracker,
5672 false,
5673 false,
5674 RoutingWorkload::Fastest,
5675 );
5676 assert!(!failing.contains(&d.model_id.as_str()), "{}", d.model_id);
5677 }
5678
5679 let mut failed = OutcomeTracker::new();
5682 let strong = "vllm-mlx/gpt-oss-20b:mxfp4";
5683 for _ in 0..3 {
5684 let trace = failed.record_start(strong, InferenceTask::Generate, "test");
5685 failed.record_failure(&trace, "connection reset");
5686 }
5687 let d = router.route_context_aware(
5688 "Say hello.",
5689 64,
5690 ®,
5691 &failed,
5692 false,
5693 false,
5694 RoutingWorkload::Fastest,
5695 );
5696 let found = d.candidates.iter().find(|c| c.model_id == strong);
5697 if cfg!(all(
5699 target_os = "macos",
5700 target_arch = "aarch64",
5701 not(car_skip_mlx)
5702 )) {
5703 assert!(found.is_some(), "{strong} is not a candidate here");
5704 }
5705 if let Some(c) = found {
5706 assert!(c.reliability < AdaptiveRouter::MIN_ANSWER_QUALITY, "{c:?}");
5707 assert!(c.in_band, "a model with failed calls was gated out: {c:?}");
5708 }
5709
5710 let mut alone = UnifiedRegistry::new_empty(tempfile::tempdir().unwrap().path().join("m"));
5712 let mut only = outcome_model("fast-but-failing", None);
5713 only.public_benchmarks = judged(0.054);
5714 alone.register_project_model(only);
5715 let d = router.route_context_aware(
5716 "Say hello.",
5717 64,
5718 &alone,
5719 &tracker,
5720 false,
5721 false,
5722 RoutingWorkload::Fastest,
5723 );
5724 assert_eq!(d.model_id, "fast-but-failing");
5725 }
5726
5727 #[test]
5730 fn a_deprecated_row_is_chosen_only_when_nothing_else_is() {
5731 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5732 let tracker = OutcomeTracker::new();
5733 let route = |reg: &UnifiedRegistry| {
5734 router.route_context_aware(
5735 "Say hello.",
5736 64,
5737 reg,
5738 &tracker,
5739 false,
5740 false,
5741 RoutingWorkload::Fastest,
5742 )
5743 };
5744 let mut reg = test_registry();
5745 let mut retired = outcome_model("retired-but-fast", None);
5746 retired.deprecated = true;
5747 retired.performance.tokens_per_second = Some(2_000.0);
5748 retired.performance.latency_p50_ms = Some(20);
5749 retired.public_benchmarks = vec![crate::schema::BenchmarkScore {
5750 name: "car-judged".into(),
5751 score: 1.0,
5752 harness: None,
5753 source_url: None,
5754 measured_at: None,
5755 runs: None,
5756 spread: None,
5757 }];
5758 reg.register_project_model(retired.clone());
5759 for _ in 0..50 {
5760 let d = route(®);
5761 assert_ne!(d.model_id, "retired-but-fast");
5762 assert!(!d
5763 .candidates
5764 .iter()
5765 .any(|c| c.model_id == "retired-but-fast"));
5766 }
5767
5768 let mut alone = UnifiedRegistry::new_empty(tempfile::tempdir().unwrap().path().join("m"));
5769 alone.register_project_model(retired);
5770 assert_eq!(route(&alone).model_id, "retired-but-fast");
5771 }
5772
5773 #[test]
5774 fn band_never_empties() {
5775 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5776 let tracker = OutcomeTracker::new();
5777 assert_eq!(
5779 router
5780 .outcome_first_band(
5781 vec![outcome_model("only", Some(0.5))],
5782 InferenceTask::Code,
5783 &tracker
5784 )
5785 .len(),
5786 1
5787 );
5788 let band = router.outcome_first_band(
5790 vec![
5791 outcome_model("x", Some(0.40)),
5792 outcome_model("y", Some(0.41)),
5793 ],
5794 InferenceTask::Code,
5795 &tracker,
5796 );
5797 assert!(!band.is_empty() && band.iter().any(|m| m.id == "y"));
5798 }
5799
5800 #[test]
5801 fn outcome_first_band_is_wired_into_selection_and_keeps_fallbacks() {
5802 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-route");
5809 unsafe {
5810 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5811 }
5812 let mut reg = UnifiedRegistry::new_empty(tmp);
5813 let mut strong = outcome_model("strong-expensive", Some(0.90));
5814 strong.cost = crate::schema::CostModel {
5815 input_per_mtok: Some(20.0),
5816 output_per_mtok: Some(20.0),
5817 ..Default::default()
5818 };
5819 let mut weak = outcome_model("weak-cheap", Some(0.80));
5820 weak.cost = crate::schema::CostModel {
5821 input_per_mtok: Some(0.1),
5822 output_per_mtok: Some(0.1),
5823 ..Default::default()
5824 };
5825 reg.register_project_model(strong);
5826 reg.register_project_model(weak);
5827
5828 let router = AdaptiveRouter::new(
5829 test_hw(),
5830 RoutingConfig {
5831 prior_strength: 100.0, ..Default::default()
5833 },
5834 );
5835 let tracker = OutcomeTracker::new();
5836 let decision = router.route(
5838 "Analyze the architecture trade-offs in depth",
5839 ®,
5840 &tracker,
5841 );
5842
5843 assert_eq!(
5844 decision.model_id, "strong-expensive",
5845 "band must select the higher-reliability model even though it's pricier"
5846 );
5847 assert!(
5848 decision.fallbacks.contains(&"weak-cheap".to_string()),
5849 "the out-of-band weaker model must remain as a fallback, got {:?}",
5850 decision.fallbacks
5851 );
5852 }
5853
5854 #[test]
5855 fn outcome_first_fallbacks_rank_in_band_before_out_of_band() {
5856 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-fb-order");
5860 unsafe {
5861 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5862 }
5863 let mut reg = UnifiedRegistry::new_empty(tmp);
5864 reg.register_project_model(outcome_model("strong", Some(0.90)));
5865 let mut peer = outcome_model("peer-in-band", Some(0.89)); peer.cost = crate::schema::CostModel {
5867 input_per_mtok: Some(30.0),
5868 output_per_mtok: Some(30.0),
5869 ..Default::default()
5870 };
5871 reg.register_project_model(peer);
5872 let mut weak = outcome_model("weak-out-of-band", Some(0.70)); weak.cost = crate::schema::CostModel {
5874 input_per_mtok: Some(0.01),
5875 output_per_mtok: Some(0.01),
5876 ..Default::default()
5877 };
5878 reg.register_project_model(weak);
5879
5880 let router = AdaptiveRouter::new(
5881 test_hw(),
5882 RoutingConfig {
5883 prior_strength: 100.0,
5884 ..Default::default()
5885 },
5886 );
5887 let tracker = OutcomeTracker::new();
5888 let decision = router.route(
5889 "Analyze the architecture trade-offs in depth",
5890 ®,
5891 &tracker,
5892 );
5893
5894 assert!(
5896 decision.model_id == "strong" || decision.model_id == "peer-in-band",
5897 "primary must be from the band, got {}",
5898 decision.model_id
5899 );
5900 let pos = |id: &str| decision.fallbacks.iter().position(|x| x == id);
5901 let in_band_id = if decision.model_id == "strong" {
5902 "peer-in-band"
5903 } else {
5904 "strong"
5905 };
5906 let in_band_pos = pos(in_band_id).expect("in-band peer must be a fallback");
5907 let weak_pos = pos("weak-out-of-band").expect("weak must remain a fallback (availability)");
5908 assert!(
5909 in_band_pos < weak_pos,
5910 "in-band fallback ({in_band_id}@{in_band_pos}) must precede out-of-band weak (@{weak_pos})"
5911 );
5912 }
5913
5914 #[test]
5915 fn reliability_is_cost_free() {
5916 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5919 let tracker = OutcomeTracker::new();
5920 let mut cheap = outcome_model("cheap", Some(0.8));
5921 cheap.cost = crate::schema::CostModel {
5922 input_per_mtok: Some(0.1),
5923 output_per_mtok: Some(0.1),
5924 ..Default::default()
5925 };
5926 let mut pricey = outcome_model("pricey", Some(0.8));
5927 pricey.cost = crate::schema::CostModel {
5928 input_per_mtok: Some(50.0),
5929 output_per_mtok: Some(50.0),
5930 ..Default::default()
5931 };
5932 let rc = router.reliability(&cheap, InferenceTask::Code, &tracker);
5933 let rp = router.reliability(&pricey, InferenceTask::Code, &tracker);
5934 assert!(
5935 (rc - rp).abs() < 1e-9,
5936 "reliability must not depend on cost"
5937 );
5938 }
5939
5940 #[test]
5941 fn cold_start_with_no_sample_size_sits_at_schema_estimate() {
5942 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5947 let unmeasured = outcome_model("unmeasured", None); let schema = router.schema_quality_estimate(&unmeasured);
5949
5950 let mut tracker = OutcomeTracker::new();
5951 let mut p = crate::outcome::ModelProfile::new("unmeasured".into());
5952 p.ema_quality = 0.5; tracker.import_profiles(vec![p]);
5954
5955 let r = router.reliability(&unmeasured, InferenceTask::Code, &tracker);
5956 assert!(
5957 (r - schema).abs() < 1e-9,
5958 "no sample size ⇒ reliability == schema estimate ({schema}), got {r}"
5959 );
5960 }
5961
5962 #[test]
5963 fn sparse_seeded_prior_barely_perturbs_schema_anchor() {
5964 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5967 let benchmarked = outcome_model("frontier", None); let mut tracker = OutcomeTracker::new();
5971 let mut p = crate::outcome::ModelProfile::new("frontier".into());
5972 p.ema_quality = 1.0;
5973 p.prior_sample_size = 1;
5974 p.task_stats.insert(
5975 InferenceTask::Code.to_string(),
5976 crate::outcome::TaskStats {
5977 ema_quality: 1.0,
5978 prior_sample_size: 1,
5979 ..Default::default()
5980 },
5981 );
5982 tracker.import_profiles(vec![p]);
5983
5984 let r = router.reliability(&benchmarked, InferenceTask::Code, &tracker);
5985 assert!((r - 0.68).abs() < 0.01, "expected ~0.68, got {r}");
5987 assert!(
5988 r < 0.7,
5989 "a 1-case prior must not pull reliability near its raw 1.0, got {r}"
5990 );
5991 }
5992
5993 #[test]
5994 fn dense_seeded_prior_is_trusted_near_raw() {
5995 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5998 let m = outcome_model("dense", None); let mut tracker = OutcomeTracker::new();
6002 let mut p = crate::outcome::ModelProfile::new("dense".into());
6003 p.ema_quality = 0.95;
6004 p.prior_sample_size = 40;
6005 tracker.import_profiles(vec![p]);
6006
6007 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6008 assert!(
6010 r > 0.88,
6011 "dense prior should be trusted near raw 0.95, got {r}"
6012 );
6013 }
6014
6015 #[test]
6016 fn warm_ungraded_successes_sit_at_schema_not_neutral_ema() {
6017 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6022 let frontier = outcome_model("frontier", Some(0.55)); let schema = router.schema_quality_estimate(&frontier);
6024
6025 let mut tracker = OutcomeTracker::new();
6026 let mut p = crate::outcome::ModelProfile::new("frontier".into());
6027 p.total_calls = 9;
6028 p.success_count = 9; p.ema_quality = 0.5; p.quality_observations = 0;
6031 tracker.import_profiles(vec![p]);
6032
6033 let r = router.reliability(&frontier, InferenceTask::Code, &tracker);
6034 assert!(
6035 (r - schema).abs() < 1e-9,
6036 "ungraded successes ⇒ reliability == schema ({schema}), not neutral EMA; got {r}"
6037 );
6038 }
6039
6040 #[test]
6041 fn ungraded_task_stats_with_zero_ema_still_resolve_to_schema() {
6042 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6049 let m = outcome_model("mech-only", Some(0.55)); let schema = router.schema_quality_estimate(&m);
6051
6052 let mut tracker = OutcomeTracker::new();
6053 let mut p = crate::outcome::ModelProfile::new("mech-only".into());
6054 p.total_calls = 7;
6055 p.success_count = 7;
6056 p.task_stats.insert(
6059 InferenceTask::Code.to_string(),
6060 crate::outcome::TaskStats {
6061 calls: 7,
6062 successes: 7,
6063 ..Default::default()
6064 },
6065 );
6066 tracker.import_profiles(vec![p]);
6067
6068 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6069 assert!(
6070 (r - schema).abs() < 1e-9,
6071 "ungraded task stats (ema 0.0, e=0) must resolve to schema {schema}, got {r}"
6072 );
6073 }
6074
6075 #[test]
6076 fn graded_outcomes_earn_trust_in_the_live_ema() {
6077 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6080 let m = outcome_model("graded", Some(0.55)); let mut tracker = OutcomeTracker::new();
6083 let mut p = crate::outcome::ModelProfile::new("graded".into());
6084 p.total_calls = 30;
6085 p.ema_quality = 0.9;
6086 p.quality_observations = 30; tracker.import_profiles(vec![p]);
6088
6089 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6090 assert!(
6092 r > 0.83,
6093 "dense graded signal should trust the live EMA, got {r}"
6094 );
6095 }
6096
6097 #[test]
6098 fn live_grades_outweigh_benchmark_cases_of_equal_count() {
6099 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6105 let benched = outcome_model("benched", Some(0.7));
6106 let graded = outcome_model("graded", Some(0.7));
6107
6108 let mut tracker = OutcomeTracker::new();
6109 let mut pb = crate::outcome::ModelProfile::new("benched".into());
6110 pb.ema_quality = 0.2;
6111 pb.prior_sample_size = 4; let mut pg = crate::outcome::ModelProfile::new("graded".into());
6113 pg.ema_quality = 0.2;
6114 pg.total_calls = 4;
6115 pg.quality_observations = 4; tracker.import_profiles(vec![pb, pg]);
6117
6118 let r_bench = router.reliability(&benched, InferenceTask::Code, &tracker);
6119 let r_graded = router.reliability(&graded, InferenceTask::Code, &tracker);
6120 assert!(
6121 r_graded < r_bench,
6122 "4 live grades must distrust more than 4 benchmark cases: \
6123 graded={r_graded} bench={r_bench}"
6124 );
6125 }
6126
6127 #[test]
6128 fn graded_failures_count_as_evidence_and_pull_reliability_down() {
6129 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6133 let m = outcome_model("flaky", Some(0.8)); let schema = router.schema_quality_estimate(&m);
6135
6136 let mut tracker = OutcomeTracker::new();
6137 let mut p = crate::outcome::ModelProfile::new("flaky".into());
6138 p.total_calls = 12;
6139 p.fail_count = 12;
6140 p.ema_quality = 0.1; p.quality_observations = 12;
6142 tracker.import_profiles(vec![p]);
6143
6144 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6145 assert!(
6146 r < schema - 0.2,
6147 "12 graded failures must pull reliability well below schema {schema}, got {r}"
6148 );
6149 }
6150
6151 #[test]
6156 fn calls_that_fail_lower_reliability_without_touching_quality() {
6157 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6158 let failing = outcome_model("failing", Some(0.92));
6159 let answering = outcome_model("answering", Some(0.91));
6160 let mut tracker = OutcomeTracker::new();
6161 for _ in 0..3 {
6162 let trace = tracker.record_start("failing", InferenceTask::Code, "test");
6163 tracker.record_failure(&trace, "upstream returned an error");
6164 }
6165
6166 let quality = router.answer_quality(&failing, InferenceTask::Code, &tracker);
6167 assert!(
6168 (quality - router.schema_quality_estimate(&failing)).abs() < 1e-9,
6169 "failures moved answer quality to {quality}"
6170 );
6171 let failing_reliability = router.reliability(&failing, InferenceTask::Code, &tracker);
6172 assert!((failing_reliability - quality / 4.0).abs() < 1e-9);
6173 assert!(
6174 failing_reliability < router.reliability(&answering, InferenceTask::Code, &tracker),
6175 "a model that answers must outrank one that has failed every call"
6176 );
6177
6178 for _ in 0..30 {
6180 let trace = tracker.record_start("failing", InferenceTask::Code, "test");
6181 tracker.record_complete(&trace, 500, 100, 50);
6182 }
6183 let recovered = router.reliability(&failing, InferenceTask::Code, &tracker);
6184 assert!(recovered > 0.9 * quality, "{recovered} after 30 answers");
6185 }
6186
6187 #[test]
6188 fn external_grade_moves_reliability_of_a_model_with_no_history() {
6189 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6193 let m = outcome_model("graded-outside", Some(0.55));
6194 let schema = router.schema_quality_estimate(&m);
6195 let mut tracker = OutcomeTracker::new();
6196 tracker
6197 .record_external_grade("graded-outside", InferenceTask::Code, 0.9, 8, "neo")
6198 .unwrap();
6199
6200 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6201 let expected = schema + (8.0 / 12.0) * (0.9 - schema);
6202 assert!(
6203 (r - expected).abs() < 1e-9,
6204 "expected {expected} (schema {schema} shrunk toward 0.9 at 8/12), got {r}"
6205 );
6206 let other = router.reliability(&m, InferenceTask::Generate, &tracker);
6208 assert!((other - schema).abs() < 1e-9);
6209 }
6210
6211 #[test]
6212 fn external_grade_survives_a_restart_that_applies_benchmark_priors() {
6213 let dir = tempfile::tempdir().unwrap();
6217 let path = dir.path().join("outcome_profiles.json");
6218 let mut tracker = OutcomeTracker::new();
6219 tracker
6220 .record_external_grade("graded-outside", InferenceTask::Code, 0.9, 8, "neo")
6221 .unwrap();
6222 tracker
6223 .record_external_grade("graded-outside", InferenceTask::Reasoning, 0.1, 8, "neo")
6224 .unwrap();
6225 tracker.save_to_file(&path).unwrap();
6226
6227 let mut restarted = OutcomeTracker::new();
6228 restarted.load_from_file(&path).unwrap();
6229 let mut prior = crate::routing_ext::BenchmarkPrior {
6230 overall_score: 0.6,
6231 sample_size: 10,
6232 ..Default::default()
6233 };
6234 prior.task_scores.insert("code".into(), 0.6);
6235 prior.task_sample_size.insert("code".into(), 10);
6236 let priors = std::collections::HashMap::from([("graded-outside".to_string(), prior)]);
6237 crate::routing_ext::apply_benchmark_priors(&mut restarted, &priors);
6238
6239 let p = restarted.profile("graded-outside").unwrap();
6240 assert_eq!(p.prior_sample_size, 10, "the prior was applied");
6241 let code = p.task_stats(InferenceTask::Code).unwrap();
6242 assert_eq!(code.prior_sample_size, 10);
6243 assert_eq!(code.external_grades["neo"].quality, 0.9);
6244 let reasoning = p.task_stats(InferenceTask::Reasoning).unwrap();
6245 assert_eq!(reasoning.external_grades["neo"].quality, 0.1);
6246
6247 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6250 let m = outcome_model("graded-outside", None);
6252 let schema = router.schema_quality_estimate(&m);
6253 let r = router.reliability(&m, InferenceTask::Code, &restarted);
6254 let observed = (10.0 * 0.6 + 8.0 * 0.9) / 18.0;
6255 let expected = schema + (18.0 / 22.0) * (observed - schema);
6256 assert!((r - expected).abs() < 1e-9, "expected {expected}, got {r}");
6257 }
6258
6259 #[test]
6260 fn external_grade_pools_with_served_evidence_instead_of_replacing_it() {
6261 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6262 let m = outcome_model("served", Some(0.55));
6263 let schema = router.schema_quality_estimate(&m);
6264 let mut tracker = OutcomeTracker::new();
6265 let mut p = crate::outcome::ModelProfile::new("served".into());
6266 p.total_calls = 10;
6267 p.ema_quality = 0.2;
6268 p.quality_observations = 10; tracker.import_profiles(vec![p]);
6270 tracker
6271 .record_external_grade("served", InferenceTask::Code, 1.0, 4, "neo")
6272 .unwrap();
6273
6274 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6275 let observed = (20.0 * 0.2 + 4.0 * 1.0) / 24.0;
6277 let expected = schema + (24.0 / 28.0) * (observed - schema);
6278 assert!(
6279 (r - expected).abs() < 1e-9,
6280 "expected pooled {expected}, got {r}"
6281 );
6282 }
6283
6284 #[test]
6285 fn external_grade_reaches_route_candidates_and_their_ranking() {
6286 let _environment = crate::openrouter::test_environment_scope();
6287 let tmp = std::path::PathBuf::from("/tmp/car-test-route-external-grade");
6288 unsafe {
6289 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
6290 }
6291 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
6292 reg.register_project_model(outcome_model("strong", Some(0.90)));
6293 reg.register_project_model(outcome_model("runner-up", Some(0.70)));
6294 let router = AdaptiveRouter::new(
6295 test_hw(),
6296 RoutingConfig {
6297 prior_strength: 100.0,
6298 ..Default::default()
6299 },
6300 );
6301 let prompt = "Analyze the architecture trade-offs in depth";
6302 let mut tracker = OutcomeTracker::new();
6303 let before = router.route(prompt, ®, &tracker);
6304 assert!(before
6305 .candidates
6306 .iter()
6307 .all(|c| c.external_grades.is_empty()));
6308 let reliability = |d: &AdaptiveRoutingDecision, id: &str| {
6309 d.candidates
6310 .iter()
6311 .find(|c| c.model_id == id)
6312 .unwrap_or_else(|| panic!("{id} is a candidate: {d:?}"))
6313 .reliability
6314 };
6315 assert!(reliability(&before, "strong") > reliability(&before, "runner-up"));
6316
6317 tracker
6319 .record_external_grade("strong", before.task, 0.0, 8, "neo")
6320 .unwrap();
6321 let after = router.route(prompt, ®, &tracker);
6322 let strong = after
6323 .candidates
6324 .iter()
6325 .find(|c| c.model_id == "strong")
6326 .expect("strong is still a candidate");
6327 assert_eq!(
6328 strong.external_grades,
6329 vec![RouteExternalGrade {
6330 source: "neo".into(),
6331 quality: 0.0,
6332 observations: 8,
6333 evidence: 8,
6334 }]
6335 );
6336 assert!(strong.reliability < reliability(&before, "strong") - 0.3);
6337 assert!(
6338 reliability(&after, "strong") < reliability(&after, "runner-up"),
6339 "the outside grade reorders the two models"
6340 );
6341 assert_ne!(
6342 after.model_id, "strong",
6343 "a model that lost every case is not chosen"
6344 );
6345 let wire = serde_json::to_value(&after).unwrap();
6346 assert!(wire["candidates"]
6347 .as_array()
6348 .unwrap()
6349 .iter()
6350 .any(|c| c["external_grades"][0]["source"] == "neo"));
6351 }
6352
6353 #[test]
6354 fn route_exposes_ranked_advisory_candidates() {
6355 let _environment = crate::openrouter::test_environment_scope();
6360 let tmp = std::path::PathBuf::from("/tmp/car-test-route-candidates");
6366 unsafe {
6367 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
6368 }
6369 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
6370 reg.register_project_model(outcome_model("strong", Some(0.90)));
6371 let mut weak = outcome_model("weak-cheap", Some(0.70)); weak.cost = crate::schema::CostModel {
6373 input_per_mtok: Some(0.01),
6374 output_per_mtok: Some(0.01),
6375 ..Default::default()
6376 };
6377 reg.register_project_model(weak);
6378
6379 let router = AdaptiveRouter::new(
6380 test_hw(),
6381 RoutingConfig {
6383 prior_strength: 100.0,
6384 ..Default::default()
6385 },
6386 );
6387 let tracker = OutcomeTracker::new();
6388 let decision = router.route(
6389 "Analyze the architecture trade-offs in depth",
6390 ®,
6391 &tracker,
6392 );
6393
6394 assert!(
6396 decision.candidates.iter().any(|c| c.model_id == "strong"),
6397 "strong must appear as a candidate, got {:?}",
6398 decision.candidates
6399 );
6400 assert!(
6401 decision
6402 .candidates
6403 .iter()
6404 .any(|c| c.model_id == "weak-cheap"),
6405 "weak-cheap must appear as a candidate even though out of band"
6406 );
6407
6408 let selected: Vec<&RouteCandidate> =
6410 decision.candidates.iter().filter(|c| c.selected).collect();
6411 assert_eq!(selected.len(), 1, "exactly one candidate may be selected");
6412 assert_eq!(
6413 selected[0].model_id, decision.model_id,
6414 "selected flag must match model_id"
6415 );
6416
6417 let strong = decision
6419 .candidates
6420 .iter()
6421 .find(|c| c.model_id == "strong")
6422 .unwrap();
6423 let weak_c = decision
6424 .candidates
6425 .iter()
6426 .find(|c| c.model_id == "weak-cheap")
6427 .unwrap();
6428 assert!(strong.in_band, "strong must be in the reliability band");
6429 assert!(
6430 !weak_c.in_band,
6431 "weak-cheap (0.20 below) must be out of band"
6432 );
6433 assert!(
6435 strong.reliability > weak_c.reliability,
6436 "reliability must reflect quality, not the weak model's lower cost"
6437 );
6438 }
6439
6440 #[test]
6441 fn prefer_quality_selection_is_deterministic_no_exploration() {
6442 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6447 let reg = test_registry();
6448 let tracker = OutcomeTracker::new();
6449 let intent = crate::intent::IntentHint {
6450 task: Some(crate::intent::TaskHint::Code),
6451 require: vec![ModelCapability::Code],
6452 prefer_quality: true,
6453 ..Default::default()
6454 };
6455 let first = router.route_with_intent("write a function", ®, &tracker, &intent);
6456 for _ in 0..8 {
6457 let again = router.route_with_intent("write a function", ®, &tracker, &intent);
6458 assert_eq!(
6459 again.model_id, first.model_id,
6460 "prefer_quality must be deterministic (no Thompson exploration)"
6461 );
6462 }
6463 let chosen = reg
6465 .find_by_name(&first.model_name)
6466 .or_else(|| reg.list().into_iter().find(|m| m.id == first.model_id));
6467 if let Some(m) = chosen {
6468 assert!(m.has_capability(ModelCapability::Code));
6469 }
6470 }
6471
6472 fn remote_model(id: &str, name: &str, caps: Vec<ModelCapability>) -> ModelSchema {
6474 ModelSchema {
6475 id: id.into(),
6476 name: name.into(),
6477 provider: "openai".into(),
6478 family: name.into(),
6479 version: "latest".into(),
6480 capabilities: caps,
6481 context_length: 128_000,
6482 max_output_tokens: None,
6483 param_count: "api".into(),
6484 quantization: None,
6485 performance: Default::default(),
6486 cost: Default::default(),
6487 source: crate::schema::ModelSource::RemoteApi {
6488 endpoint: "https://api.openai.com/v1".into(),
6489 api_key_env: "OPENAI_API_KEY".into(),
6490 api_key_envs: vec![],
6491 api_version: None,
6492 protocol: crate::schema::ApiProtocol::OpenAiCompat,
6493 },
6494 tags: vec!["trusted-remote".into()],
6495 supported_params: vec![],
6496 public_benchmarks: vec![],
6497 trust_tier: crate::schema::TrustTier::Curated,
6498 deprecated: false,
6499 available: true,
6500 weights_ready: true,
6501 }
6502 }
6503
6504 #[test]
6505 fn code_intent_filters_generate_only_models_on_simple_prompt() {
6506 let _environment = crate::openrouter::test_environment_scope();
6511 unsafe {
6518 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
6519 }
6520 let intent_filter_dir = std::path::PathBuf::from("/tmp/car-test-code-intent-filter");
6521 let mut reg =
6522 UnifiedRegistry::new_with_state_root(intent_filter_dir.clone(), intent_filter_dir);
6523 reg.register_project_model(remote_model(
6524 "openai/gen-only:latest",
6525 "gen-only",
6526 vec![ModelCapability::Generate],
6527 ));
6528 reg.register_project_model(remote_model(
6529 "openai/coder:latest",
6530 "coder",
6531 vec![
6532 ModelCapability::Generate,
6533 ModelCapability::Code,
6534 ModelCapability::Reasoning,
6535 ],
6536 ));
6537
6538 let router = AdaptiveRouter::new(
6539 test_hw(),
6540 RoutingConfig {
6541 prior_strength: 100.0,
6542 ..Default::default()
6543 },
6544 );
6545 let tracker = OutcomeTracker::new();
6546 let intent = crate::intent::IntentHint {
6547 task: Some(crate::intent::TaskHint::Code),
6548 prefer_quality: true,
6549 ..Default::default() };
6551
6552 let decision = router.route_with_intent(
6554 "Give me a short two-sentence overview.",
6555 ®,
6556 &tracker,
6557 &intent,
6558 );
6559
6560 assert_eq!(
6561 decision.task,
6562 InferenceTask::Code,
6563 "explicit task=code must be honored"
6564 );
6565 assert!(
6568 !decision
6569 .candidates
6570 .iter()
6571 .any(|c| c.model_id == "openai/gen-only:latest"),
6572 "generate-only model must be filtered out for task=code; candidates: {:?}",
6573 decision
6574 .candidates
6575 .iter()
6576 .map(|c| &c.model_id)
6577 .collect::<Vec<_>>()
6578 );
6579 let schema = reg
6580 .find_by_name(&decision.model_name)
6581 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
6582 .expect("selected model exists");
6583 assert!(
6584 schema.has_capability(ModelCapability::Code),
6585 "task=code on a simple prompt must route to a code-capable model, got {}",
6586 decision.model_name,
6587 );
6588 }
6589
6590 #[test]
6591 fn intent_default_does_not_override_task_or_caps() {
6592 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6599 let reg = test_registry();
6600 let tracker = OutcomeTracker::new();
6601
6602 let baseline = router.route("write a haiku", ®, &tracker);
6603 let with_default = router.route_with_intent(
6604 "write a haiku",
6605 ®,
6606 &tracker,
6607 &crate::intent::IntentHint::default(),
6608 );
6609
6610 assert_eq!(
6611 baseline.task, with_default.task,
6612 "default IntentHint must not change the prompt-derived task"
6613 );
6614 assert_eq!(
6615 baseline.complexity, with_default.complexity,
6616 "default IntentHint must not change the prompt-derived complexity"
6617 );
6618 }
6619
6620 #[test]
6631 fn require_ready_skips_models_that_are_not_on_disk() {
6632 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6633 let reg = test_registry();
6634 let tracker = OutcomeTracker::new();
6635
6636 let ready: std::collections::HashSet<String> = reg
6638 .list()
6639 .into_iter()
6640 .filter(|m| m.weights_ready)
6641 .map(|m| m.id.clone())
6642 .collect();
6643 assert!(!ready.is_empty(), "fixture should have ready models");
6644
6645 let hint = crate::intent::IntentHint {
6646 require_ready: true,
6647 ..Default::default()
6648 };
6649 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
6650 assert!(
6651 ready.contains(&decision.model_id),
6652 "require_ready picked {} which is not on disk; ready set: {:?}",
6653 decision.model_id,
6654 ready
6655 );
6656 }
6657
6658 #[test]
6661 fn require_ready_falls_back_when_nothing_is_ready() {
6662 let _environment = crate::openrouter::test_environment_scope();
6667 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6668 let tracker = OutcomeTracker::new();
6669 let empty_dir = std::env::temp_dir().join("car-test-require-ready-empty");
6671 let _ = std::fs::create_dir_all(&empty_dir);
6672 let reg = UnifiedRegistry::new_with_state_root(empty_dir.clone(), empty_dir);
6673
6674 let hint = crate::intent::IntentHint {
6675 require_ready: true,
6676 ..Default::default()
6677 };
6678 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
6679 assert!(
6680 !decision.model_id.is_empty(),
6681 "soft constraint must still yield a model when nothing is ready"
6682 );
6683 }
6684
6685 #[test]
6686 fn exclude_models_picks_a_different_model() {
6687 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6692 let reg = test_registry();
6693 let tracker = OutcomeTracker::new();
6694 let base = crate::intent::IntentHint {
6695 prefer_quality: true,
6696 ..Default::default()
6697 };
6698 let author = router.route_with_intent("review this", ®, &tracker, &base);
6699
6700 let with_exclude = crate::intent::IntentHint {
6701 prefer_quality: true,
6702 exclude_models: vec![author.model_id.clone()],
6703 ..Default::default()
6704 };
6705 let reviewer = router.route_with_intent("review this", ®, &tracker, &with_exclude);
6706 assert_ne!(
6707 reviewer.model_id, author.model_id,
6708 "an excluded model must never be the routed pick"
6709 );
6710 assert!(
6713 !reviewer.fallbacks.contains(&author.model_id),
6714 "excluded model leaked into fallbacks"
6715 );
6716 assert!(
6717 !reviewer
6718 .candidates
6719 .iter()
6720 .any(|c| c.model_id == author.model_id),
6721 "excluded model leaked into candidates[]"
6722 );
6723 }
6724
6725 #[test]
6728 fn on_device_only_picks_only_a_model_on_this_machine() {
6729 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6730 let reg = test_registry();
6731 let tracker = OutcomeTracker::new();
6732 assert!(
6733 reg.list().iter().any(|model| !model.is_local()),
6734 "fixture lists a remote model"
6735 );
6736 let decision = router.route_with_intent(
6737 "write a function",
6738 ®,
6739 &tracker,
6740 &crate::intent::IntentHint {
6741 on_device_only: true,
6742 ..Default::default()
6743 },
6744 );
6745 for id in std::iter::once(&decision.model_id).chain(decision.fallbacks.iter()) {
6746 if id.is_empty() {
6747 continue;
6748 }
6749 let schema = reg.get(id).or_else(|| reg.find_by_name(id));
6750 assert!(
6751 schema.is_some_and(|schema| schema.is_local()),
6752 "{id}: {decision:?}"
6753 );
6754 }
6755 }
6756
6757 #[test]
6761 fn on_device_only_with_nothing_on_this_machine_is_no_route() {
6762 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6763 let mut reg = test_registry();
6764 let local: Vec<String> = reg
6765 .registry
6766 .all()
6767 .filter(|model| model.is_local())
6768 .map(|model| model.id.clone())
6769 .collect();
6770 for id in local {
6771 reg.registry.unregister(&id);
6772 }
6773 let tracker = OutcomeTracker::new();
6774 let decision = router.route_with_intent(
6775 "write a function",
6776 ®,
6777 &tracker,
6778 &crate::intent::IntentHint {
6779 on_device_only: true,
6780 ..Default::default()
6781 },
6782 );
6783 assert!(decision.model_id.is_empty(), "{decision:?}");
6784 assert!(decision.fallbacks.is_empty(), "{decision:?}");
6785 assert!(decision.reason.contains("on_device_only"), "{decision:?}");
6786 let ordinary = router.route_with_intent(
6788 "write a function",
6789 ®,
6790 &tracker,
6791 &crate::intent::IntentHint::default(),
6792 );
6793 assert!(!ordinary.model_id.is_empty(), "{ordinary:?}");
6794 }
6795
6796 #[test]
6797 fn strict_exclusions_refuse_when_every_candidate_is_a_panel_seat() {
6798 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6799 let reg = test_registry();
6800 let tracker = OutcomeTracker::new();
6801 let panel: Vec<String> = reg
6802 .list()
6803 .into_iter()
6804 .map(|model| model.name.clone())
6805 .collect();
6806 assert!(!panel.is_empty(), "fixture must contain candidate models");
6807
6808 let decision = router.route_with_intent(
6809 "repair this code",
6810 ®,
6811 &tracker,
6812 &crate::intent::IntentHint {
6813 task: Some(crate::intent::TaskHint::Code),
6814 exclude_models: panel,
6815 strict_exclusions: true,
6816 ..Default::default()
6817 },
6818 );
6819
6820 assert!(decision.model_id.is_empty(), "{decision:?}");
6821 assert!(decision.model_name.is_empty(), "{decision:?}");
6822 assert!(decision.fallbacks.is_empty(), "{decision:?}");
6823 assert!(
6824 decision.reason.contains("no eligible model"),
6825 "the refusal must tell the operator what is missing: {decision:?}"
6826 );
6827 }
6828
6829 #[test]
6830 fn exclude_models_accepts_the_name_a_result_reports() {
6831 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6839 let reg = test_registry();
6840 let tracker = OutcomeTracker::new();
6841 let base = crate::intent::IntentHint {
6842 prefer_quality: true,
6843 ..Default::default()
6844 };
6845 let author = router.route_with_intent("write a python function", ®, &tracker, &base);
6846 let author_name = reg
6847 .get(&author.model_id)
6848 .expect("the routed pick is in the registry")
6849 .name
6850 .clone();
6851 assert_ne!(
6852 author_name, author.model_id,
6853 "this test is only meaningful on a model whose name and id differ"
6854 );
6855
6856 let by_name = crate::intent::IntentHint {
6857 prefer_quality: true,
6858 exclude_models: vec![author_name.clone()],
6859 ..Default::default()
6860 };
6861 let rerouted =
6862 router.route_with_intent("write a python function", ®, &tracker, &by_name);
6863 assert_ne!(
6864 rerouted.model_id, author.model_id,
6865 "excluding by the name a result reports ({author_name}) must route elsewhere"
6866 );
6867 assert!(
6868 !rerouted.fallbacks.contains(&author.model_id),
6869 "name-excluded model leaked into fallbacks"
6870 );
6871 }
6872
6873 #[test]
6874 fn cold_start_honors_exclusion() {
6875 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6880 let reg = test_registry();
6881 let caps = [ModelCapability::Generate];
6882 let base = router.cold_start_decision(
6883 TaskComplexity::Simple,
6884 InferenceTask::Generate,
6885 &caps,
6886 ®,
6887 false,
6888 &std::collections::HashSet::new(),
6889 false,
6890 AvailabilitySnapshot::capture(®),
6891 );
6892 let mut exclude = std::collections::HashSet::new();
6893 exclude.insert(base.model_id.clone());
6894 let excluded = router.cold_start_decision(
6895 TaskComplexity::Simple,
6896 InferenceTask::Generate,
6897 &caps,
6898 ®,
6899 false,
6900 &exclude,
6901 false,
6902 AvailabilitySnapshot::capture(®),
6903 );
6904 let other_generate_exists = reg
6908 .list()
6909 .into_iter()
6910 .filter(|m| m.available && m.has_capability(ModelCapability::Generate))
6911 .any(|m| m.id != base.model_id);
6912 if other_generate_exists {
6913 assert_ne!(
6914 excluded.model_id, base.model_id,
6915 "cold start re-picked the excluded model despite an alternative"
6916 );
6917 }
6918 }
6919
6920 #[test]
6921 fn exclude_all_models_falls_back_to_full_set() {
6922 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6925 let reg = test_registry();
6926 let tracker = OutcomeTracker::new();
6927 let exclude_everything: Vec<String> =
6928 reg.list().into_iter().map(|m| m.id.clone()).collect();
6929 let hint = crate::intent::IntentHint {
6930 exclude_models: exclude_everything,
6931 ..Default::default()
6932 };
6933 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
6934 assert!(
6935 !decision.model_id.is_empty(),
6936 "excluding everything must still yield a routed model (fallback)"
6937 );
6938 }
6939
6940 #[test]
6941 fn intent_task_hint_overrides_prompt_complexity() {
6942 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6945 let reg = test_registry();
6946 let tracker = OutcomeTracker::new();
6947
6948 let hint = crate::intent::IntentHint {
6949 task: Some(crate::intent::TaskHint::Reasoning),
6950 ..Default::default()
6951 };
6952 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
6953
6954 assert_eq!(
6955 decision.task,
6956 InferenceTask::Reasoning,
6957 "TaskHint::Reasoning should override the prompt-derived task"
6958 );
6959 }
6960
6961 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6965 #[test]
6966 fn a_current_model_is_never_redirected_to_a_deprecated_twin() {
6967 let mut reg = test_registry();
6968 let gguf = reg.get("qwen/qwen3-8b:q4_k_m").expect("gguf").clone();
6969 let mut retired = reg.get("mlx/qwen3-8b:4bit").expect("mlx twin").clone();
6970 retired.deprecated = true;
6971 assert!(reg.register_project_model(retired.clone()));
6972 assert_eq!(
6973 reg.resolve_mlx_equivalent(&gguf).map(|m| m.id.as_str()),
6974 None
6975 );
6976
6977 let mut current = retired.clone();
6978 current.id = "mlx/qwen3-8b:4bit-current".into();
6979 current.deprecated = false;
6980 assert!(reg.register_project_model(current));
6981 assert_eq!(
6982 reg.resolve_mlx_equivalent(&gguf).map(|m| m.id.as_str()),
6983 Some("mlx/qwen3-8b:4bit-current")
6984 );
6985
6986 let mut old_gguf = gguf.clone();
6988 old_gguf.deprecated = true;
6989 assert_eq!(
6990 reg.resolve_mlx_equivalent(&old_gguf).map(|m| m.id.as_str()),
6991 Some("mlx/qwen3-8b:4bit-current")
6992 );
6993 }
6994
6995 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
7004 #[test]
7005 fn router_proposes_mlx_twin_not_pre_redirect_gguf() {
7006 let gguf_id = "qwen/qwen3-8b:q4_k_m";
7007 let mlx_id = "mlx/qwen3-8b:4bit";
7008
7009 let reg = test_registry();
7010 let gguf = reg.get(gguf_id).expect("gguf in builtin catalog").clone();
7013 assert_eq!(
7014 reg.resolve_mlx_equivalent(&gguf).map(|m| m.id.as_str()),
7015 Some(mlx_id),
7016 "GGUF must resolve to its SAME-SIZE MLX twin"
7017 );
7018
7019 let router = AdaptiveRouter::new(
7020 test_hw(),
7021 RoutingConfig {
7022 min_observations: 3,
7023 ..Default::default()
7024 },
7025 );
7026 let mut tracker = OutcomeTracker::new();
7027 for _ in 0..20 {
7029 let t = tracker.record_start(mlx_id, InferenceTask::Code, "test");
7030 tracker.record_complete(&t, 500, 100, 50);
7031 tracker.record_inferred_outcome(&t, InferredOutcome::Accepted { confidence: 0.95 });
7032 }
7033
7034 let mut mlx_wins = 0;
7035 for _ in 0..20 {
7036 let d = router.route("Fix this bug in the parser", ®, &tracker);
7037 assert_ne!(
7040 d.model_id, gguf_id,
7041 "router proposed the pre-redirect GGUF id"
7042 );
7043 assert!(
7044 !d.fallbacks.iter().any(|f| f == gguf_id),
7045 "pre-redirect GGUF id leaked into the fallback chain"
7046 );
7047 if d.model_id == mlx_id {
7048 mlx_wins += 1;
7049 }
7050 }
7051 assert!(
7052 mlx_wins >= 12,
7053 "proven MLX twin should win the majority (the router reads the id it \
7054 learns/runs); won only {mlx_wins}/20"
7055 );
7056 }
7057
7058 fn m5_pro_64gb() -> HardwareInfo {
7065 HardwareInfo {
7066 os: "macos".into(),
7067 arch: "aarch64".into(),
7068 cpu_cores: 14,
7069 total_ram_mb: 64 * 1024,
7070 gpu_backend: crate::hardware::GpuBackend::Metal,
7071 gpu_memory_mb: None,
7072 gpu_devices: Vec::new(),
7073 recommended_model: "fixture".into(),
7074 recommended_context: 8192,
7075 max_model_mb: 64 * 1024,
7076 }
7077 }
7078
7079 const CLOUD_KEYS: [&str; 3] = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY"];
7080
7081 fn scenario_registry(keys: &[&str]) -> TestRegistry {
7084 let environment = crate::openrouter::test_environment_scope();
7085 let mut names = CLOUD_KEYS.to_vec();
7086 names.extend([
7087 "CAR_SECRETS_FILE_DIR",
7088 "OPENROUTER_API_KEY",
7089 "GOOGLE_API_KEY",
7090 "CAR_CODEX_BIN",
7091 ]);
7092 let restore = RestoredEnvironment::capture(&names);
7093 let root = std::env::temp_dir().join(format!(
7094 "car-routing-scenarios-{}",
7095 keys.join("-").to_ascii_lowercase()
7096 ));
7097 unsafe {
7098 for name in &names {
7099 std::env::remove_var(name);
7100 }
7101 std::env::set_var("CAR_SECRETS_FILE_DIR", root.join("empty-secrets"));
7102 std::env::set_var("CAR_CODEX_BIN", root.join("no-codex"));
7105 for key in keys {
7106 std::env::set_var(key, "scenario-key");
7107 }
7108 }
7109 let _ = std::fs::create_dir_all(&root);
7110 let registry =
7111 UnifiedRegistry::new_with_state_root(root.join("models"), root.join("state"));
7112 TestRegistry {
7113 registry,
7114 _restore: restore,
7115 _environment: environment,
7116 }
7117 }
7118
7119 struct Scenario {
7120 name: &'static str,
7121 prompt: &'static str,
7122 workload: RoutingWorkload,
7123 intent: crate::intent::IntentHint,
7124 has_tools: bool,
7125 }
7126
7127 fn scenarios() -> Vec<Scenario> {
7128 use crate::intent::{IntentHint, TaskHint};
7129 vec![
7130 Scenario {
7131 name: "chat, interactive",
7132 prompt: "What's a good name for a golden retriever?",
7133 workload: RoutingWorkload::Interactive,
7134 intent: IntentHint::default(),
7135 has_tools: false,
7136 },
7137 Scenario {
7138 name: "code, interactive",
7139 prompt: "Write a Python function that merges two sorted lists.",
7140 workload: RoutingWorkload::Interactive,
7141 intent: IntentHint::default(),
7142 has_tools: false,
7143 },
7144 Scenario {
7145 name: "agent turn with tools, interactive",
7146 prompt: "Read notes.txt and summarize it.",
7147 workload: RoutingWorkload::Interactive,
7148 intent: IntentHint::default(),
7149 has_tools: true,
7150 },
7151 Scenario {
7152 name: "code, prefer_local",
7153 prompt: "Write a Python function that merges two sorted lists.",
7154 workload: RoutingWorkload::Interactive,
7155 intent: IntentHint {
7156 prefer_local: true,
7157 ..Default::default()
7158 },
7159 has_tools: false,
7160 },
7161 Scenario {
7162 name: "agent turn with tools, prefer_local",
7163 prompt: "Read notes.txt and summarize it.",
7164 workload: RoutingWorkload::Interactive,
7165 intent: IntentHint {
7166 prefer_local: true,
7167 ..Default::default()
7168 },
7169 has_tools: true,
7170 },
7171 Scenario {
7172 name: "voice reply, prefer_fast",
7173 prompt: "What time is it in Tokyo?",
7174 workload: RoutingWorkload::Interactive,
7175 intent: IntentHint {
7176 prefer_fast: true,
7177 task: Some(TaskHint::Chat),
7178 ..Default::default()
7179 },
7180 has_tools: false,
7181 },
7182 Scenario {
7183 name: "hard reasoning, prefer_quality",
7184 prompt: "Prove that the halting problem is undecidable, step by step.",
7185 workload: RoutingWorkload::Interactive,
7186 intent: IntentHint {
7187 prefer_quality: true,
7188 task: Some(TaskHint::Reasoning),
7189 ..Default::default()
7190 },
7191 has_tools: false,
7192 },
7193 Scenario {
7194 name: "summary, background",
7195 prompt: "Summarize this meeting transcript in three bullets.",
7196 workload: RoutingWorkload::Background,
7197 intent: IntentHint::default(),
7198 has_tools: false,
7199 },
7200 ]
7201 }
7202
7203 fn route_scenario(
7204 router: &AdaptiveRouter,
7205 registry: &UnifiedRegistry,
7206 tracker: &OutcomeTracker,
7207 scenario: &Scenario,
7208 ) -> AdaptiveRoutingDecision {
7209 router.route_with(RouteRequest {
7210 intent: Some(&scenario.intent),
7211 has_tools: scenario.has_tools,
7212 workload: scenario.workload,
7213 ..RouteRequest::new(scenario.prompt, registry, tracker)
7214 })
7215 }
7216
7217 const SCENARIO_KEY_SETS: [&[&str]; 4] = [
7218 &[],
7219 &["OPENAI_API_KEY"],
7220 &["ANTHROPIC_API_KEY"],
7221 &["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY"],
7222 ];
7223
7224 #[test]
7231 fn a_stated_local_or_fast_preference_keeps_local_candidates_under_every_key_set() {
7232 let frontier_grade_locals = cfg!(all(
7233 target_os = "macos",
7234 target_arch = "aarch64",
7235 not(car_skip_mlx)
7236 ));
7237 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7238 for keys in SCENARIO_KEY_SETS {
7239 let registry = scenario_registry(keys);
7240 let tracker = OutcomeTracker::new();
7241 for scenario in scenarios()
7242 .iter()
7243 .filter(|s| s.intent.prefer_local || s.intent.prefer_fast)
7244 {
7245 for _ in 0..10 {
7247 let decision = route_scenario(&router, ®istry, &tracker, scenario);
7248 let is_local = |id: &String| registry.get(id).is_some_and(|m| m.is_local());
7249 let picked_local = is_local(&decision.model_id);
7250 assert!(
7251 picked_local || decision.fallbacks.iter().any(is_local),
7252 "{} with keys {keys:?} had no local candidate: {} ({})",
7253 scenario.name,
7254 decision.model_id,
7255 decision.reason
7256 );
7257 assert!(
7258 picked_local || !frontier_grade_locals,
7259 "{} with keys {keys:?} routed to {} ({})",
7260 scenario.name,
7261 decision.model_id,
7262 decision.reason
7263 );
7264 }
7265 }
7266 }
7267 }
7268
7269 #[test]
7275 fn a_tool_turn_always_goes_to_a_model_that_can_call_tools() {
7276 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7277 for keys in SCENARIO_KEY_SETS {
7278 let registry = scenario_registry(keys);
7279 let tracker = OutcomeTracker::new();
7280 for scenario in scenarios().iter().filter(|s| s.has_tools) {
7281 for _ in 0..10 {
7282 let decision = route_scenario(&router, ®istry, &tracker, scenario);
7283 let model = registry.get(&decision.model_id).expect("a routed model");
7284 assert!(
7285 model.has_capability(ModelCapability::ToolUse),
7286 "{} with keys {keys:?} routed to {}, which cannot call tools",
7287 scenario.name,
7288 decision.model_id
7289 );
7290 }
7291 }
7292 }
7293 }
7294
7295 #[test]
7299 fn the_quality_lane_follows_the_measured_scores_with_no_keys() {
7300 let car_judged = |m: &ModelSchema| {
7301 m.public_benchmarks
7302 .iter()
7303 .find(|b| b.name == "car-judged")
7304 .map(|b| b.score)
7305 };
7306 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7307 let registry = scenario_registry(&[]);
7308 let tracker = OutcomeTracker::new();
7309 let mut compared = 0;
7310 for scenario in scenarios().iter().filter(|s| s.intent.prefer_quality) {
7311 let decision = route_scenario(&router, ®istry, &tracker, scenario);
7312 let picked = registry.get(&decision.model_id).expect("a routed model");
7313 let picked_score = car_judged(picked).unwrap_or(0.0);
7314 for id in &decision.fallbacks {
7315 let Some(alt) = registry.get(id).filter(|m| m.is_local()) else {
7316 continue;
7317 };
7318 if let Some(score) = car_judged(alt) {
7319 compared += 1;
7320 assert!(
7321 picked_score + 1e-9 >= score,
7322 "{}: picked {} (car-judged {picked_score:.3}) over {} ({score:.3})",
7323 scenario.name,
7324 decision.model_id,
7325 id
7326 );
7327 }
7328 }
7329 }
7330 assert!(compared > 0, "no measured local alternative was compared");
7331 }
7332
7333 #[test]
7336 #[ignore = "report: prints routing picks, asserts nothing"]
7337 fn routing_scenario_report() {
7338 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7339 for keys in SCENARIO_KEY_SETS {
7340 let registry = scenario_registry(keys);
7341 let tracker = OutcomeTracker::new();
7342 println!("== keys: {keys:?}");
7343 for scenario in scenarios() {
7344 let decision = route_scenario(&router, ®istry, &tracker, &scenario);
7345 let local = registry
7346 .get(&decision.model_id)
7347 .is_some_and(|m| m.is_local());
7348 println!(
7349 " {:<38} -> {:<40} {} | {}",
7350 scenario.name,
7351 decision.model_id,
7352 if local { "local " } else { "remote" },
7353 decision.reason.chars().take(110).collect::<String>()
7354 );
7355 }
7356 }
7357 }
7358}