1use rand::Rng;
13use serde::{Deserialize, Serialize};
14
15use std::sync::{Arc, Mutex};
16
17use crate::hardware::HardwareInfo;
18use crate::outcome::{InferenceTask, OutcomeTracker};
19use crate::registry::UnifiedRegistry;
20use crate::routing_ext::CircuitBreakerRegistry;
21use crate::schema::{ModelCapability, ModelSchema};
22use crate::tasks::RoutingWorkload;
23
24#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
26#[serde(rename_all = "snake_case")]
27pub enum TaskComplexity {
28 Simple,
29 Medium,
30 Code,
31 Complex,
32}
33
34impl TaskComplexity {
35 pub fn assess(prompt: &str) -> Self {
42 let lower = prompt.to_lowercase();
43 let word_count = prompt.split_whitespace().count();
44 let estimated_tokens = (word_count as f64 * 1.3) as usize;
45
46 let has_code = Self::detect_code(prompt);
47
48 let repair_markers = [
49 "fix", "repair", "debug", "refactor", "broken", "failing", "error", "bug",
50 ];
51 let has_repair = repair_markers.iter().any(|m| lower.contains(m));
52
53 let reasoning_markers = [
54 "analyze",
55 "compare",
56 "explain why",
57 "step by step",
58 "think through",
59 "evaluate",
60 "trade-off",
61 "tradeoff",
62 "pros and cons",
63 "architecture",
64 "design",
65 "strategy",
66 "optimize",
67 "comprehensive",
68 ];
69 let has_reasoning = reasoning_markers.iter().any(|m| lower.contains(m));
70
71 let simple_patterns = [
72 "what is",
73 "who is",
74 "when did",
75 "where is",
76 "how many",
77 "yes or no",
78 "true or false",
79 "name the",
80 "list the",
81 "define ",
82 ];
83 let is_simple = simple_patterns.iter().any(|p| lower.contains(p));
84
85 if has_code || has_repair {
86 TaskComplexity::Code
87 } else if has_reasoning || estimated_tokens > 500 {
88 TaskComplexity::Complex
89 } else if is_simple || estimated_tokens < 30 {
90 TaskComplexity::Simple
91 } else {
92 TaskComplexity::Medium
93 }
94 }
95
96 fn detect_code(prompt: &str) -> bool {
102 #[cfg(feature = "ast")]
104 {
105 if let Some(is_code) = Self::detect_code_ast(prompt) {
106 return is_code;
107 }
108 }
109
110 let code_markers = [
112 "```",
113 "fn ",
114 "def ",
115 "class ",
116 "import ",
117 "require(",
118 "async fn",
119 "pub fn",
120 "function ",
121 "const ",
122 "let ",
123 "var ",
124 "#include",
125 "package ",
126 "impl ",
127 ];
128 code_markers.iter().any(|m| prompt.contains(m))
129 }
130
131 #[cfg(feature = "ast")]
135 fn detect_code_ast(prompt: &str) -> Option<bool> {
136 let mut blocks = Vec::new();
138 let mut rest = prompt;
139 while let Some(start) = rest.find("```") {
140 let after_fence = &rest[start + 3..];
141 let code_start = after_fence.find('\n').map(|i| i + 1).unwrap_or(0);
143 if let Some(end) = after_fence[code_start..].find("```") {
144 blocks.push(&after_fence[code_start..code_start + end]);
145 rest = &after_fence[code_start + end + 3..];
146 } else {
147 break;
148 }
149 }
150
151 if blocks.is_empty() {
152 return None; }
154
155 let languages = [
157 car_ast::Language::Rust,
158 car_ast::Language::Python,
159 car_ast::Language::TypeScript,
160 car_ast::Language::JavaScript,
161 car_ast::Language::Go,
162 ];
163
164 for block in &blocks {
165 let trimmed = block.trim();
166 if trimmed.is_empty() {
167 continue;
168 }
169
170 for lang in &languages {
171 if let Some(parsed) = car_ast::parse(trimmed, *lang) {
172 if !parsed.symbols.is_empty() {
174 return Some(true);
175 }
176 }
177 }
178 }
179
180 Some(false)
183 }
184
185 pub fn required_capabilities(&self) -> Vec<ModelCapability> {
187 match self {
188 TaskComplexity::Simple => vec![ModelCapability::Generate],
189 TaskComplexity::Medium => vec![ModelCapability::Generate],
190 TaskComplexity::Code => vec![ModelCapability::Code],
191 TaskComplexity::Complex => vec![ModelCapability::Reasoning],
192 }
193 }
194
195 pub fn inference_task(&self) -> InferenceTask {
197 match self {
198 TaskComplexity::Simple | TaskComplexity::Medium => InferenceTask::Generate,
199 TaskComplexity::Code => InferenceTask::Code,
200 TaskComplexity::Complex => InferenceTask::Reasoning,
201 }
202 }
203}
204
205#[derive(Debug, Clone, Serialize, Deserialize)]
207pub struct RoutingConfig {
208 pub min_observations: u64,
210 pub quality_weight: f64,
212 pub latency_weight: f64,
213 pub cost_weight: f64,
214 pub max_latency_ms: Option<u64>,
216 pub max_cost_usd: Option<f64>,
218 pub prefer_local: bool,
220 pub prior_strength: f64,
225 pub quality_first_cold_start: bool,
228 pub bootstrap_min_task_observations: u64,
231 pub bootstrap_quality_floor: f64,
234}
235
236impl Default for RoutingConfig {
237 fn default() -> Self {
238 Self {
239 min_observations: 2,
240 quality_weight: 0.45,
241 latency_weight: 0.4,
242 cost_weight: 0.15,
243 max_latency_ms: None,
244 max_cost_usd: None,
245 prefer_local: true,
246 prior_strength: 20.0,
247 quality_first_cold_start: true,
248 bootstrap_min_task_observations: 8,
249 bootstrap_quality_floor: 0.8,
250 }
251 }
252}
253
254#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
256#[serde(rename_all = "snake_case")]
257pub enum RoutingStrategy {
258 SchemaBased,
260 ProfileBased,
262 Exploration,
264 Explicit,
266}
267
268#[derive(Debug, Clone, Serialize, Deserialize)]
277pub struct RouteCandidate {
278 pub model_id: String,
280 pub reliability: f64,
282 pub score: f64,
284 pub selected: bool,
286 pub in_band: bool,
288 #[serde(default, skip_serializing_if = "Vec::is_empty")]
292 pub external_grades: Vec<RouteExternalGrade>,
293}
294
295#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
297pub struct RouteExternalGrade {
298 pub source: String,
302 pub quality: f64,
304 pub observations: u64,
306 pub evidence: u64,
310}
311
312#[derive(Debug, Clone, Copy, PartialEq, Eq)]
314pub enum RouterExclusionReason {
315 RateLimited,
317 CircuitOpen,
319}
320
321impl RouterExclusionReason {
322 pub fn as_str(self) -> &'static str {
324 match self {
325 Self::RateLimited => "rate_limited",
326 Self::CircuitOpen => "circuit_open",
327 }
328 }
329}
330
331#[derive(Debug, Clone, PartialEq, Eq)]
333pub struct RouterExclusion {
334 pub lane: String,
336 pub reason: RouterExclusionReason,
338 pub until: Option<u64>,
342}
343
344#[derive(Debug, Clone, Serialize, Deserialize)]
346pub struct AdaptiveRoutingDecision {
347 pub model_id: String,
349 pub model_name: String,
351 pub task: InferenceTask,
353 pub complexity: TaskComplexity,
355 pub reason: String,
357 pub strategy: RoutingStrategy,
359 pub predicted_quality: f64,
361 pub fallbacks: Vec<String>,
363 pub context_length: usize,
365 pub needs_compaction: bool,
367 #[serde(default)]
370 pub candidates: Vec<RouteCandidate>,
371}
372
373pub struct AdaptiveRouter {
375 hw: HardwareInfo,
376 config: RoutingConfig,
377 pub circuit_breakers: Arc<Mutex<CircuitBreakerRegistry>>,
379}
380
381pub struct RouteRequest<'a> {
397 pub prompt: &'a str,
398 pub registry: &'a UnifiedRegistry,
399 pub tracker: &'a OutcomeTracker,
400 pub estimated_total_tokens: usize,
403 pub estimated_input_tokens: usize,
406 pub estimated_output_tokens: usize,
408 pub estimated_cache_read_tokens: usize,
410 pub estimated_cache_write_tokens: usize,
412 pub has_tools: bool,
413 pub has_vision: bool,
414 pub workload: RoutingWorkload,
415 pub intent: Option<&'a crate::intent::IntentHint>,
418}
419
420impl<'a> RouteRequest<'a> {
421 pub fn new(
425 prompt: &'a str,
426 registry: &'a UnifiedRegistry,
427 tracker: &'a OutcomeTracker,
428 ) -> Self {
429 Self {
430 prompt,
431 registry,
432 tracker,
433 estimated_total_tokens: 0,
434 estimated_input_tokens: 0,
435 estimated_output_tokens: 0,
436 estimated_cache_read_tokens: 0,
437 estimated_cache_write_tokens: 0,
438 has_tools: false,
439 has_vision: false,
440 workload: RoutingWorkload::Interactive,
441 intent: None,
442 }
443 }
444}
445
446#[derive(Clone, Copy)]
450struct AvailabilitySnapshot {
451 openrouter: Option<crate::openrouter::CredentialSource>,
452}
453
454impl AvailabilitySnapshot {
455 fn capture(registry: &UnifiedRegistry) -> Self {
456 let has_openrouter = registry.list().into_iter().any(|schema| {
457 matches!(
458 schema.source,
459 crate::schema::ModelSource::RemoteApi {
460 protocol: crate::schema::ApiProtocol::OpenRouter,
461 ..
462 }
463 )
464 });
465 Self {
466 openrouter: has_openrouter
467 .then(crate::openrouter::credential_source)
468 .flatten(),
469 }
470 }
471
472 fn is_available(self, schema: &ModelSchema) -> bool {
473 match schema.source {
474 crate::schema::ModelSource::RemoteApi {
475 protocol: crate::schema::ApiProtocol::OpenRouter,
476 ..
477 } => self.openrouter.is_some(),
478 _ => schema.available,
479 }
480 }
481}
482
483static BENCHMARK_FRONTIER_REFS: std::sync::LazyLock<std::collections::HashMap<String, f64>> =
510 std::sync::LazyLock::new(|| compute_frontier_refs(crate::registry::builtin_catalog().iter()));
511
512fn compute_frontier_refs<'a>(
517 models: impl Iterator<Item = &'a ModelSchema>,
518) -> std::collections::HashMap<String, f64> {
519 let mut max_score: std::collections::HashMap<String, f64> = std::collections::HashMap::new();
523 let mut has_frontier_carrier: std::collections::HashMap<String, bool> =
524 std::collections::HashMap::new();
525 for model in models {
526 let frontier = model_is_frontier_class(model);
527 for b in &model.public_benchmarks {
528 let entry = max_score.entry(b.name.clone()).or_insert(0.0);
529 *entry = entry.max(b.score);
530 let seen = has_frontier_carrier.entry(b.name.clone()).or_insert(false);
531 *seen = *seen || frontier;
532 }
533 }
534 max_score
538 .into_iter()
539 .filter(|(name, _)| {
540 has_frontier_carrier.get(name).copied().unwrap_or(false) || {
541 tracing::warn!(
542 benchmark = %name,
543 "benchmark has no frontier-class carrier in the catalog — \
544 refusing to self-anchor (uncalibrated clamp). Add a \
545 frontier-tagged model that carries it, or an explicit \
546 reference, before it can calibrate the quality tier (#371)."
547 );
548 false
549 }
550 })
551 .collect()
552}
553
554fn model_is_frontier_class(model: &ModelSchema) -> bool {
565 model.tags.iter().any(|t| t == "frontier")
566}
567
568impl AdaptiveRouter {
569 pub fn new(hw: HardwareInfo, config: RoutingConfig) -> Self {
570 let circuit_breakers = Arc::new(Mutex::new(
571 CircuitBreakerRegistry::new(3, 300), ));
573 Self {
574 hw,
575 config,
576 circuit_breakers,
577 }
578 }
579
580 pub fn with_default_config(hw: HardwareInfo) -> Self {
581 Self::new(hw, RoutingConfig::default())
582 }
583
584 pub fn config(&self) -> &RoutingConfig {
585 &self.config
586 }
587
588 pub fn set_config(&mut self, config: RoutingConfig) {
589 self.config = config;
590 }
591
592 pub fn route_with(&self, req: RouteRequest<'_>) -> AdaptiveRoutingDecision {
598 let mut ignore_exclusion = |_: RouterExclusion| {};
599 self.route_with_exclusion_observer(req, &mut ignore_exclusion)
600 }
601
602 pub(crate) fn route_with_exclusion_observer(
606 &self,
607 req: RouteRequest<'_>,
608 exclusion_observer: &mut dyn FnMut(RouterExclusion),
609 ) -> AdaptiveRoutingDecision {
610 let workload = Self::resolve_workload(req.intent, req.workload);
616 self.route_inner_with_intent(
617 req.prompt,
618 req.registry,
619 req.tracker,
620 req.has_tools,
621 req.has_vision,
622 req.estimated_total_tokens,
623 req.estimated_input_tokens,
624 req.estimated_output_tokens,
625 req.estimated_cache_read_tokens,
626 req.estimated_cache_write_tokens,
627 workload,
628 req.intent,
629 exclusion_observer,
630 )
631 }
632
633 pub fn route(
636 &self,
637 prompt: &str,
638 registry: &UnifiedRegistry,
639 tracker: &OutcomeTracker,
640 ) -> AdaptiveRoutingDecision {
641 self.route_with(RouteRequest::new(prompt, registry, tracker))
642 }
643
644 pub fn route_editor(
650 &self,
651 prompt: &str,
652 registry: &UnifiedRegistry,
653 tracker: &OutcomeTracker,
654 ) -> AdaptiveRoutingDecision {
655 self.route_with(RouteRequest {
656 workload: RoutingWorkload::Background,
657 ..RouteRequest::new(prompt, registry, tracker)
658 })
659 }
660
661 pub fn route_with_tools(
663 &self,
664 prompt: &str,
665 registry: &UnifiedRegistry,
666 tracker: &OutcomeTracker,
667 ) -> AdaptiveRoutingDecision {
668 self.route_with(RouteRequest {
669 has_tools: true,
670 ..RouteRequest::new(prompt, registry, tracker)
671 })
672 }
673
674 pub fn route_with_vision(
676 &self,
677 prompt: &str,
678 registry: &UnifiedRegistry,
679 tracker: &OutcomeTracker,
680 has_tools: bool,
681 ) -> AdaptiveRoutingDecision {
682 self.route_with(RouteRequest {
683 has_tools,
684 has_vision: true,
685 ..RouteRequest::new(prompt, registry, tracker)
686 })
687 }
688
689 pub fn route_with_intent<'a>(
695 &self,
696 prompt: &'a str,
697 registry: &'a UnifiedRegistry,
698 tracker: &'a OutcomeTracker,
699 intent: &'a crate::intent::IntentHint,
700 ) -> AdaptiveRoutingDecision {
701 self.route_with(RouteRequest {
702 intent: Some(intent),
703 ..RouteRequest::new(prompt, registry, tracker)
704 })
705 }
706
707 pub fn route_context_aware(
710 &self,
711 prompt: &str,
712 estimated_total_tokens: usize,
713 registry: &UnifiedRegistry,
714 tracker: &OutcomeTracker,
715 has_tools: bool,
716 has_vision: bool,
717 workload: RoutingWorkload,
718 ) -> AdaptiveRoutingDecision {
719 self.route_with(RouteRequest {
720 estimated_total_tokens,
721 has_tools,
722 has_vision,
723 workload,
724 ..RouteRequest::new(prompt, registry, tracker)
725 })
726 }
727
728 pub fn route_context_aware_with_intent<'a>(
733 &self,
734 prompt: &'a str,
735 estimated_total_tokens: usize,
736 registry: &'a UnifiedRegistry,
737 tracker: &'a OutcomeTracker,
738 has_tools: bool,
739 has_vision: bool,
740 workload: RoutingWorkload,
741 intent: &'a crate::intent::IntentHint,
742 ) -> AdaptiveRoutingDecision {
743 self.route_with(RouteRequest {
744 estimated_total_tokens,
745 has_tools,
746 has_vision,
747 workload,
748 intent: Some(intent),
749 ..RouteRequest::new(prompt, registry, tracker)
750 })
751 }
752
753 fn route_inner_with_intent(
754 &self,
755 prompt: &str,
756 registry: &UnifiedRegistry,
757 tracker: &OutcomeTracker,
758 has_tools: bool,
759 has_vision: bool,
760 estimated_total_tokens: usize,
761 estimated_input_tokens: usize,
762 estimated_output_tokens: usize,
763 estimated_cache_read_tokens: usize,
764 estimated_cache_write_tokens: usize,
765 workload: RoutingWorkload,
766 intent: Option<&crate::intent::IntentHint>,
767 exclusion_observer: &mut dyn FnMut(RouterExclusion),
768 ) -> AdaptiveRoutingDecision {
769 let complexity = TaskComplexity::assess(prompt);
770 let task = intent
772 .and_then(|h| h.task)
773 .map(task_hint_to_inference_task)
774 .unwrap_or_else(|| complexity.inference_task());
775 let mut required_caps = base_required_capabilities(complexity, intent, task);
783 if let Some(hint) = intent {
784 for cap in &hint.require {
785 if !required_caps.contains(cap) {
786 required_caps.push(*cap);
787 }
788 }
789 }
790 if has_vision {
791 required_caps.push(ModelCapability::Vision);
792 }
793 if has_tools {
794 required_caps.push(ModelCapability::ToolUse);
795 if Self::needs_multi_tool_call(prompt) {
798 required_caps.push(ModelCapability::MultiToolCall);
799 }
800 }
801
802 let exclude_set = self.build_exclude_set(intent, registry);
810 let strict_exclusions = intent.is_some_and(|hint| hint.strict_exclusions);
811 let availability = AvailabilitySnapshot::capture(registry);
812
813 let want_ready = intent.is_some_and(|h| h.require_ready);
818 let mut candidates = self.filter_candidates(
819 &required_caps,
820 registry,
821 tracker,
822 has_vision,
823 availability,
824 want_ready,
825 exclusion_observer,
826 );
827 if candidates.is_empty() && want_ready {
828 candidates = self.filter_candidates(
829 &required_caps,
830 registry,
831 tracker,
832 has_vision,
833 availability,
834 false,
835 exclusion_observer,
836 );
837 }
838
839 if candidates.is_empty() && required_caps.contains(&ModelCapability::MultiToolCall) {
842 required_caps.retain(|c| *c != ModelCapability::MultiToolCall);
843 candidates = self.filter_candidates(
844 &required_caps,
845 registry,
846 tracker,
847 has_vision,
848 availability,
849 want_ready,
850 exclusion_observer,
851 );
852 if candidates.is_empty() && want_ready {
853 candidates = self.filter_candidates(
854 &required_caps,
855 registry,
856 tracker,
857 has_vision,
858 availability,
859 false,
860 exclusion_observer,
861 );
862 }
863 }
864
865 if intent.is_some_and(|hint| hint.on_device_only) {
871 candidates.retain(|model| model.is_local());
872 if candidates.is_empty() {
873 return Self::no_on_device_model_decision(complexity, task);
874 }
875 }
876
877 if candidates.is_empty() {
878 return self.cold_start_decision(
882 complexity,
883 task,
884 &required_caps,
885 registry,
886 has_vision,
887 &exclude_set,
888 strict_exclusions,
889 availability,
890 );
891 }
892
893 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
906 {
907 let mut seen = std::collections::HashSet::new();
908 candidates = candidates
909 .into_iter()
910 .map(|m| registry.resolve_mlx_equivalent(&m).cloned().unwrap_or(m))
911 .filter(|m| seen.insert(m.id.clone()))
912 .collect();
913 }
914
915 if !exclude_set.is_empty() {
924 let filtered: Vec<ModelSchema> = candidates
925 .iter()
926 .filter(|m| !exclude_set.contains(&m.id))
927 .cloned()
928 .collect();
929 if filtered.is_empty() {
930 if strict_exclusions {
931 return Self::no_eligible_model_decision(complexity, task, exclude_set.len());
932 }
933 tracing::debug!(
934 ?exclude_set,
935 "model exclusion left no candidates; keeping full set"
936 );
937 } else {
938 candidates = filtered;
939 }
940 }
941
942 candidates = self.apply_quality_first_bootstrap_policy(
943 candidates, task, tracker, has_vision, has_tools, workload,
944 );
945
946 let (fits, needs_compaction_candidates) = if estimated_total_tokens > 0 {
949 let mut fits = Vec::new();
950 let mut tight = Vec::new();
951 for m in &candidates {
952 if m.context_length == 0 || m.context_length >= estimated_total_tokens {
953 fits.push(m.clone());
954 } else {
955 tight.push(m.clone());
956 }
957 }
958 (fits, tight)
959 } else {
960 (candidates.clone(), Vec::new())
961 };
962
963 let (scoring_candidates, compaction_needed) = if !fits.is_empty() {
965 (fits, false)
966 } else if !needs_compaction_candidates.is_empty() {
967 tracing::info!(
968 prompt_tokens = estimated_total_tokens,
969 candidates = needs_compaction_candidates.len(),
970 "no model fits full prompt — compaction will be needed"
971 );
972 (needs_compaction_candidates.clone(), true)
973 } else {
974 (candidates.clone(), false)
975 };
976
977 let pricing_input_tokens = estimated_input_tokens
983 .max(estimated_total_tokens.saturating_sub(estimated_output_tokens));
984 let scored = self.score_candidates_context_aware(
985 &scoring_candidates,
986 task,
987 tracker,
988 estimated_total_tokens,
989 pricing_input_tokens,
990 estimated_output_tokens,
991 estimated_cache_read_tokens,
992 estimated_cache_write_tokens,
993 workload,
994 );
995 tracing::debug!(
1001 ?workload,
1002 ?task,
1003 candidates = ?scoring_candidates.iter().map(|m| m.id.as_str()).collect::<Vec<_>>(),
1004 ?scored,
1005 "route scoring"
1006 );
1007
1008 let band_ids: Option<std::collections::HashSet<String>> =
1020 if Self::applies_reliability_band(task, workload) {
1021 Some(
1022 self.outcome_first_band(scoring_candidates.clone(), task, tracker)
1023 .into_iter()
1024 .map(|m| m.id)
1025 .collect(),
1026 )
1027 } else {
1028 self.competent_candidates(&scoring_candidates, task, tracker)
1030 };
1031 if let Some(ref ids) = band_ids {
1032 tracing::debug!(
1037 ?task,
1038 band = Self::OUTCOME_FIRST_BAND,
1039 band_size = ids.len(),
1040 "outcome-first: selection restricted to reliability band"
1041 );
1042 }
1043 let selection_pool: Vec<(String, f64)> = if let Some(ref ids) = band_ids {
1044 let pool: Vec<(String, f64)> = scored
1045 .iter()
1046 .filter(|(id, _)| ids.contains(id))
1047 .cloned()
1048 .collect();
1049 if pool.is_empty() {
1050 debug_assert!(
1056 false,
1057 "reliability band produced no in-`scored` ids (workload {workload:?})"
1058 );
1059 scored.clone()
1060 } else {
1061 pool
1062 }
1063 } else {
1064 scored.clone()
1065 };
1066
1067 let (selected_id, strategy) = if workload == RoutingWorkload::Quality {
1080 selection_pool
1081 .iter()
1082 .max_by(|a, b| a.1.partial_cmp(&b.1).unwrap_or(std::cmp::Ordering::Equal))
1083 .map(|(id, _)| (id.clone(), RoutingStrategy::SchemaBased))
1084 .unwrap_or_else(|| self.select_with_thompson_sampling(&selection_pool, tracker))
1085 } else {
1086 self.select_with_thompson_sampling(&selection_pool, tracker)
1087 };
1088
1089 let mut fallbacks: Vec<String> = if let Some(ref ids) = band_ids {
1096 let (in_band, out_band): (Vec<String>, Vec<String>) = scored
1097 .iter()
1098 .filter(|(id, _)| *id != selected_id)
1099 .map(|(id, _)| id.clone())
1100 .partition(|id| ids.contains(id));
1101 in_band.into_iter().chain(out_band).collect()
1102 } else {
1103 scored
1104 .iter()
1105 .filter(|(id, _)| *id != selected_id)
1106 .map(|(id, _)| id.clone())
1107 .collect()
1108 };
1109 if !compaction_needed {
1111 for m in &needs_compaction_candidates {
1112 if m.id != selected_id && !fallbacks.contains(&m.id) {
1113 fallbacks.push(m.id.clone());
1114 }
1115 }
1116 }
1117
1118 let predicted_quality = scored
1119 .iter()
1120 .find(|(id, _)| *id == selected_id)
1121 .map(|(_, score)| *score)
1122 .unwrap_or(0.5);
1123
1124 let selected_schema = registry
1125 .get(&selected_id)
1126 .or_else(|| registry.find_by_name(&selected_id));
1127 let model_name = selected_schema
1128 .map(|m| m.name.clone())
1129 .unwrap_or_else(|| selected_id.clone());
1130 let context_length = selected_schema.map(|m| m.context_length).unwrap_or(0);
1131
1132 let needs_compact = compaction_needed
1133 || (estimated_total_tokens > 0
1134 && context_length > 0
1135 && estimated_total_tokens > context_length);
1136
1137 let compaction_note = if needs_compact {
1138 format!(
1139 " [compaction needed: {}→{}tok]",
1140 estimated_total_tokens, context_length
1141 )
1142 } else {
1143 String::new()
1144 };
1145
1146 let cost_quality_note = selected_schema
1147 .map(|schema| {
1148 let is_openrouter = schema.tags.iter().any(|tag| tag == "openrouter");
1149 if is_openrouter && schema.tags.iter().any(|tag| tag == "cheap") {
1150 let prices = schema.cost.prices_for(pricing_input_tokens);
1151 format!(
1152 ", low-cost tier (${:.3}/MTok input, ${:.3}/MTok output at {} prompt tokens)",
1153 prices.input_per_mtok.unwrap_or(0.0),
1154 prices.output_per_mtok.unwrap_or(0.0),
1155 pricing_input_tokens,
1156 )
1157 } else if is_openrouter && model_is_frontier_class(schema) {
1158 ", frontier quality tier".to_string()
1159 } else {
1160 String::new()
1161 }
1162 })
1163 .unwrap_or_default();
1164
1165 let reason = format!(
1166 "{:?} task → {} via {:?} (quality: {:.2}, {} candidates{}){}",
1167 complexity,
1168 model_name,
1169 strategy,
1170 predicted_quality,
1171 scoring_candidates.len(),
1172 cost_quality_note,
1173 compaction_note,
1174 );
1175
1176 let candidates: Vec<RouteCandidate> = scored
1183 .iter()
1184 .map(|(id, score)| {
1185 let reliability = match scoring_candidates.iter().find(|m| m.id == *id) {
1191 Some(m) => self.reliability(m, task, tracker),
1192 None => {
1193 debug_assert!(false, "scored id {id} absent from scoring_candidates");
1194 0.0
1195 }
1196 };
1197 let now = crate::outcome::now_unix();
1198 let external_grades = tracker
1199 .profile(id)
1200 .and_then(|p| p.task_stats(task))
1201 .map(|ts| {
1202 ts.external_grades
1203 .iter()
1204 .map(|(source, grade)| RouteExternalGrade {
1205 source: source.clone(),
1206 quality: grade.quality,
1207 observations: grade.observations,
1208 evidence: if grade.is_stale_at(now) {
1209 0
1210 } else {
1211 grade.evidence()
1212 },
1213 })
1214 .collect()
1215 })
1216 .unwrap_or_default();
1217 RouteCandidate {
1218 model_id: id.clone(),
1219 reliability,
1220 score: *score,
1221 selected: *id == selected_id,
1222 in_band: band_ids.as_ref().is_none_or(|b| b.contains(id)),
1223 external_grades,
1224 }
1225 })
1226 .collect();
1227
1228 AdaptiveRoutingDecision {
1229 model_id: selected_id,
1230 model_name,
1231 task,
1232 complexity,
1233 reason,
1234 strategy,
1235 predicted_quality,
1236 fallbacks,
1237 context_length,
1238 needs_compaction: needs_compact,
1239 candidates,
1240 }
1241 }
1242
1243 pub fn route_embedding(&self, registry: &UnifiedRegistry) -> String {
1245 let embed_models = registry.query_by_capability(ModelCapability::Embed);
1246 embed_models
1247 .first()
1248 .map(|m| m.name.clone())
1249 .unwrap_or_else(|| "Qwen3-Embedding-0.6B".to_string())
1250 }
1251
1252 pub fn route_small(&self, registry: &UnifiedRegistry) -> String {
1254 let gen_models = registry.query_by_capability(ModelCapability::Generate);
1255 gen_models
1257 .iter()
1258 .filter(|m| m.is_local())
1259 .min_by_key(|m| m.size_mb())
1260 .map(|m| m.name.clone())
1261 .unwrap_or_else(|| "Qwen3-0.6B".to_string())
1262 }
1263
1264 const LATENCY_CEILING_MS: f64 = 10000.0;
1270 const UNMEASURED_REMOTE_TPS: f64 = 93.0;
1274 const _TPS_CEILING: f64 = 150.0;
1276 const MOE_TPS_MULTIPLIER: f64 = 0.10;
1280 const MLX_MOE_TPS_MULTIPLIER: f64 = 0.50;
1283 const COST_CEILING_PER_1K: f64 = 0.1;
1285 const LOCAL_BONUS: f64 = 0.15;
1287 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1295 const HAS_GPU_BACKEND: bool = true;
1296 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
1297 const HAS_GPU_BACKEND: bool = false;
1298 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1301 const MLX_BONUS: f64 = 0.10;
1302 const SYSTEM_LLM_BONUS: f64 = 0.12;
1313
1314 pub(crate) fn build_exclude_set(
1333 &self,
1334 intent: Option<&crate::intent::IntentHint>,
1335 registry: &UnifiedRegistry,
1336 ) -> std::collections::HashSet<String> {
1337 let mut set = std::collections::HashSet::new();
1338 let Some(hint) = intent else {
1339 return set;
1340 };
1341 for id in &hint.exclude_models {
1342 set.insert(id.clone());
1343 let Some(schema) = registry.get(id).or_else(|| registry.find_by_name(id)) else {
1344 continue;
1345 };
1346 set.insert(schema.id.clone());
1347 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
1348 {
1349 if let Some(mlx) = registry.resolve_mlx_equivalent(schema) {
1350 set.insert(mlx.id.clone());
1351 }
1352 }
1353 }
1354 set
1355 }
1356
1357 fn filter_candidates(
1358 &self,
1359 required_caps: &[ModelCapability],
1360 registry: &UnifiedRegistry,
1361 tracker: &OutcomeTracker,
1362 has_vision: bool,
1363 availability: AvailabilitySnapshot,
1364 require_ready: bool,
1365 exclusion_observer: &mut dyn FnMut(RouterExclusion),
1366 ) -> Vec<ModelSchema> {
1367 let mut candidates = registry
1368 .list()
1369 .into_iter()
1370 .filter(|m| {
1371 if !required_caps.iter().all(|c| m.has_capability(*c)) {
1373 return false;
1374 }
1375 if !has_vision && m.tags.iter().any(|t| t == "mlx-vlm-cli") {
1385 return false;
1386 }
1387 if !availability.is_available(m) {
1389 return false;
1390 }
1391 if require_ready && !m.weights_ready {
1398 return false;
1399 }
1400 if m.is_local() && m.size_mb() >= self.hw.max_model_mb {
1402 return false;
1403 }
1404 if let Some(max) = self.config.max_latency_ms {
1406 if let Some(p50) = m.performance.latency_p50_ms {
1407 if p50 > max {
1408 return false;
1409 }
1410 }
1411 }
1412 if let Some(max) = self.config.max_cost_usd {
1414 if m.cost_per_1k_output() > max {
1415 return false;
1416 }
1417 }
1418 if !self.config.prefer_local && m.is_local() {
1420 return false;
1421 }
1422 if tracker.is_excluded(&m.id) {
1424 exclusion_observer(RouterExclusion {
1425 lane: m.id.clone(),
1426 reason: RouterExclusionReason::RateLimited,
1427 until: None,
1428 });
1429 return false;
1430 }
1431 if let Ok(mut cb) = self.circuit_breakers.lock() {
1433 if !cb.allow_request(&m.id) {
1434 let until = cb.blocked_until(&m.id);
1435 tracing::debug!(model = %m.id, ?until, "skipped by circuit breaker");
1436 exclusion_observer(RouterExclusion {
1437 lane: m.id.clone(),
1438 reason: RouterExclusionReason::CircuitOpen,
1439 until,
1440 });
1441 return false;
1442 }
1443 }
1444 true
1445 })
1446 .cloned()
1447 .collect::<Vec<_>>();
1448 if candidates.iter().any(|m| !m.deprecated) {
1454 candidates.retain(|m| !m.deprecated);
1455 }
1456 candidates
1457 }
1458
1459 fn apply_quality_first_bootstrap_policy(
1460 &self,
1461 candidates: Vec<ModelSchema>,
1462 task: InferenceTask,
1463 tracker: &OutcomeTracker,
1464 has_vision: bool,
1465 has_tools: bool,
1466 workload: RoutingWorkload,
1467 ) -> Vec<ModelSchema> {
1468 if !self.config.quality_first_cold_start
1474 || workload != RoutingWorkload::Interactive
1475 || !self.is_quality_critical_bootstrap_task(task, has_vision, has_tools)
1476 {
1477 return candidates;
1478 }
1479
1480 let trusted_remote: Vec<ModelSchema> = candidates
1481 .iter()
1482 .filter(|model| self.is_trusted_quality_remote(model))
1483 .cloned()
1484 .collect();
1485
1486 if trusted_remote.is_empty() {
1487 return candidates;
1488 }
1489
1490 let best_remote_quality = trusted_remote
1494 .iter()
1495 .map(|model| self.schema_quality_estimate(model))
1496 .fold(0.0_f64, f64::max);
1497 let proven_local: Vec<ModelSchema> = candidates
1498 .iter()
1499 .filter(|model| {
1500 model.is_local()
1501 && (self.is_local_model_proven_for_task(model, task, tracker)
1502 || (!model.public_benchmarks.is_empty()
1503 && self.schema_quality_estimate(model)
1504 >= best_remote_quality - Self::OUTCOME_FIRST_BAND))
1505 })
1506 .cloned()
1507 .collect();
1508
1509 if !proven_local.is_empty() {
1510 return trusted_remote.into_iter().chain(proven_local).collect();
1514 }
1515
1516 trusted_remote
1517 }
1518
1519 fn score_candidates_context_aware(
1523 &self,
1524 candidates: &[ModelSchema],
1525 task: InferenceTask,
1526 tracker: &OutcomeTracker,
1527 estimated_total_tokens: usize,
1528 estimated_input_tokens: usize,
1529 estimated_output_tokens: usize,
1530 estimated_cache_read_tokens: usize,
1531 estimated_cache_write_tokens: usize,
1532 workload: RoutingWorkload,
1533 ) -> Vec<(String, f64)> {
1534 let mut scored: Vec<(String, f64)> = candidates
1535 .iter()
1536 .map(|m| {
1537 let base_score = self.score_model(
1538 m,
1539 task,
1540 tracker,
1541 workload,
1542 estimated_input_tokens,
1543 estimated_output_tokens,
1544 estimated_cache_read_tokens,
1545 estimated_cache_write_tokens,
1546 );
1547 let headroom_bonus = if estimated_total_tokens > 0 && m.context_length > 0 {
1550 let ratio = m.context_length as f64 / estimated_total_tokens as f64;
1551 if ratio >= 1.0 {
1552 (ratio.min(4.0) - 1.0) / 3.0 * 0.10 } else {
1554 -0.15 }
1556 } else {
1557 0.0
1558 };
1559 (m.id.clone(), base_score + headroom_bonus)
1560 })
1561 .collect();
1562
1563 scored.sort_by(|a, b| b.1.partial_cmp(&a.1).unwrap_or(std::cmp::Ordering::Equal));
1564 scored
1565 }
1566
1567 fn task_aware_weights(task: InferenceTask, workload: RoutingWorkload) -> (f64, f64, f64) {
1583 match (task, workload) {
1584 (
1585 InferenceTask::Code,
1586 RoutingWorkload::Interactive | RoutingWorkload::LocalPreferred,
1587 ) => (0.62, 0.28, 0.10),
1588 _ => workload.weights(),
1589 }
1590 }
1591
1592 fn reliability(
1606 &self,
1607 model: &ModelSchema,
1608 task: InferenceTask,
1609 tracker: &OutcomeTracker,
1610 ) -> f64 {
1611 let completion = tracker.profile(&model.id).map_or(1.0, |p| {
1612 (p.success_count as f64 + 1.0) / ((p.success_count + p.fail_count) as f64 + 1.0)
1613 });
1614 self.answer_quality(model, task, tracker) * completion
1615 }
1616
1617 fn answer_quality(
1620 &self,
1621 model: &ModelSchema,
1622 task: InferenceTask,
1623 tracker: &OutcomeTracker,
1624 ) -> f64 {
1625 let schema_quality = self.schema_quality_estimate(model);
1626 let Some(p) = tracker.profile(&model.id) else {
1627 return schema_quality;
1629 };
1630
1631 let task_stats = p.task_stats(task);
1644 let (ema, n, g) = task_stats
1645 .filter(|ts| ts.has_served_evidence())
1646 .map(|ts| {
1647 (
1648 ts.ema_quality,
1649 ts.prior_sample_size,
1650 ts.quality_observations,
1651 )
1652 })
1653 .unwrap_or((p.ema_quality, p.prior_sample_size, p.quality_observations));
1654 let n = if model.public_benchmarks.is_empty() {
1660 n
1661 } else {
1662 0
1663 };
1664 let (external, external_mean) = task_stats
1670 .map(|ts| ts.external_evidence())
1671 .unwrap_or((0.0, 0.0));
1672
1673 let served = n as f64 + Self::LIVE_GRADE_WEIGHT * g as f64;
1704 let evidence = served + external;
1705 if evidence <= 0.0 {
1706 return schema_quality;
1707 }
1708 let observed = (served * ema + external * external_mean) / evidence;
1709 let confidence = evidence / (evidence + Self::PRIOR_SHRINK_PSEUDOCOUNT);
1710 (schema_quality + confidence * (observed - schema_quality)).clamp(0.0, 1.0)
1711 }
1712
1713 const BENCH_TIER_FLOOR: f64 = 0.40;
1721 const BENCH_TIER_KNEE: f64 = 0.5;
1728 const BENCH_TIER_CEIL: f64 = 0.92;
1729
1730 const LOCAL_BENCH_TIER_FLOOR: f64 = 0.30;
1744 const LOCAL_BENCH_TIER_CEIL: f64 = 0.60;
1745
1746 const UNMEASURED_REMOTE_CURATED: f64 = 0.60;
1751 const UNMEASURED_REMOTE_COMMUNITY: f64 = 0.48;
1752 const CURATED_FRONTIER_TAG_PRIOR: f64 = 0.85;
1759 const CURATED_BALANCED_TAG_PRIOR: f64 = 0.72;
1760
1761 const LIVE_GRADE_WEIGHT: f64 = 2.0;
1772
1773 const PRIOR_SHRINK_PSEUDOCOUNT: f64 = 4.0;
1783
1784 const OUTCOME_FIRST_BAND: f64 = 0.02;
1800
1801 const MIN_ANSWER_QUALITY: f64 = 0.25;
1813
1814 fn resolve_workload(
1820 intent: Option<&crate::intent::IntentHint>,
1821 base: RoutingWorkload,
1822 ) -> RoutingWorkload {
1823 match intent {
1824 Some(h) if h.high_stakes => RoutingWorkload::Quality,
1825 Some(h) if h.prefer_fast => RoutingWorkload::Fastest,
1826 Some(h) if h.prefer_quality => RoutingWorkload::Quality,
1827 Some(h) if h.prefer_local => RoutingWorkload::LocalPreferred,
1828 _ => base,
1829 }
1830 }
1831
1832 fn applies_reliability_band(task: InferenceTask, workload: RoutingWorkload) -> bool {
1848 match workload {
1849 RoutingWorkload::Quality => true,
1850 RoutingWorkload::Fastest
1851 | RoutingWorkload::Background
1852 | RoutingWorkload::LocalPreferred => false,
1853 _ => matches!(task, InferenceTask::Code | InferenceTask::Reasoning),
1854 }
1855 }
1856
1857 fn competent_candidates(
1860 &self,
1861 candidates: &[ModelSchema],
1862 task: InferenceTask,
1863 tracker: &OutcomeTracker,
1864 ) -> Option<std::collections::HashSet<String>> {
1865 let competent: std::collections::HashSet<String> = candidates
1866 .iter()
1867 .filter(|m| self.answer_quality(m, task, tracker) >= Self::MIN_ANSWER_QUALITY)
1868 .map(|m| m.id.clone())
1869 .collect();
1870 (!competent.is_empty() && competent.len() < candidates.len()).then_some(competent)
1871 }
1872
1873 fn outcome_first_band(
1878 &self,
1879 candidates: Vec<ModelSchema>,
1880 task: InferenceTask,
1881 tracker: &OutcomeTracker,
1882 ) -> Vec<ModelSchema> {
1883 if candidates.len() <= 1 {
1884 return candidates;
1885 }
1886 let best = candidates
1887 .iter()
1888 .map(|m| self.reliability(m, task, tracker))
1889 .fold(f64::MIN, f64::max);
1890 candidates
1891 .into_iter()
1892 .filter(|m| {
1895 best - self.reliability(m, task, tracker) <= Self::OUTCOME_FIRST_BAND + 1e-9
1896 })
1897 .collect()
1898 }
1899
1900 fn score_model(
1901 &self,
1902 model: &ModelSchema,
1903 task: InferenceTask,
1904 tracker: &OutcomeTracker,
1905 workload: RoutingWorkload,
1906 estimated_input_tokens: usize,
1907 estimated_output_tokens: usize,
1908 estimated_cache_read_tokens: usize,
1909 estimated_cache_write_tokens: usize,
1910 ) -> f64 {
1911 let profile = tracker.profile(&model.id);
1912 let schema_latency = self.schema_latency_estimate(model);
1913 let (quality_weight, latency_weight, cost_weight) =
1914 Self::task_aware_weights(task, workload);
1915
1916 let quality = self.reliability(model, task, tracker);
1920
1921 let latency = match profile {
1924 Some(p) if p.total_calls >= self.config.min_observations => {
1925 let avg = p
1926 .task_stats(task)
1927 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1928 .map(|ts| ts.avg_latency_ms)
1929 .unwrap_or_else(|| p.avg_latency_ms());
1930 self.latency_ms_to_score(avg)
1931 }
1932 Some(p) if p.total_calls == 0 => p
1933 .task_stats(task)
1934 .filter(|ts| ts.avg_latency_ms > 0.0)
1935 .map(|ts| self.latency_ms_to_score(ts.avg_latency_ms))
1936 .unwrap_or(schema_latency),
1937 Some(p) if p.total_calls > 0 => {
1938 let observed = self.latency_ms_to_score(
1939 p.task_stats(task)
1940 .filter(|ts| ts.calls > 0 || ts.avg_latency_ms > 0.0)
1941 .map(|ts| ts.avg_latency_ms)
1942 .unwrap_or_else(|| p.avg_latency_ms()),
1943 );
1944 let w = p.total_calls as f64 / self.config.min_observations as f64;
1945 schema_latency * (1.0 - w) + observed * w
1946 }
1947 _ => schema_latency,
1948 };
1949
1950 let cost = if model.is_local() {
1952 1.0
1953 } else if model.cost.output_per_mtok.is_none() {
1954 0.5
1961 } else {
1962 let normalized_cost_per_1k = if estimated_input_tokens
1963 + estimated_output_tokens
1964 + estimated_cache_read_tokens
1965 + estimated_cache_write_tokens
1966 > 0
1967 {
1968 let estimated_cost = model.cost.estimated_usd(
1969 estimated_input_tokens,
1970 estimated_output_tokens,
1971 estimated_cache_read_tokens,
1972 estimated_cache_write_tokens,
1973 );
1974 estimated_cost * 1000.0
1975 / (estimated_input_tokens + estimated_output_tokens).max(1) as f64
1976 } else {
1977 model.cost_per_1k_output()
1978 };
1979 (1.0 - (normalized_cost_per_1k / Self::COST_CEILING_PER_1K)).clamp(0.0, 1.0)
1980 };
1981
1982 let gpu_backend_or_latency_tolerant =
2001 Self::HAS_GPU_BACKEND || !workload.is_latency_sensitive();
2002 let local_bonus = if self.config.prefer_local
2003 && model.is_local()
2004 && gpu_backend_or_latency_tolerant
2005 {
2006 Self::LOCAL_BONUS
2007 } else {
2008 if self.config.prefer_local && model.is_local() && !Self::HAS_GPU_BACKEND {
2009 tracing::debug!(
2010 model = %model.id,
2011 "LOCAL_BONUS suppressed: no GPU backend on this host (Intel Mac or car_skip_mlx) for a latency-sensitive workload; cloud models will rank higher"
2012 );
2013 }
2014 0.0
2015 };
2016 let workload_local_bonus = if model.is_local() {
2017 workload.local_bonus()
2018 } else {
2019 0.0
2020 };
2021
2022 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2024 let mlx_bonus = if model.is_mlx() { Self::MLX_BONUS } else { 0.0 };
2025 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2026 let mlx_bonus = 0.0;
2027
2028 let vllm_mlx_bonus = if model.is_vllm_mlx() {
2030 Self::LOCAL_BONUS + 0.05
2031 } else {
2032 0.0
2033 };
2034
2035 let system_llm_bonus = if model.tags.iter().any(|t| t == "low_latency")
2043 && model.tags.iter().any(|t| t == "private")
2044 {
2045 Self::SYSTEM_LLM_BONUS
2046 } else {
2047 0.0
2048 };
2049
2050 let (local_bonus, workload_local_bonus, mlx_bonus, vllm_mlx_bonus, system_llm_bonus) =
2061 if workload == RoutingWorkload::Quality {
2062 (0.0, 0.0, 0.0, 0.0, 0.0)
2063 } else {
2064 (
2065 local_bonus,
2066 workload_local_bonus,
2067 mlx_bonus,
2068 vllm_mlx_bonus,
2069 system_llm_bonus,
2070 )
2071 };
2072
2073 quality_weight * quality
2074 + latency_weight * latency
2075 + cost_weight * cost
2076 + local_bonus
2077 + workload_local_bonus
2078 + mlx_bonus
2079 + vllm_mlx_bonus
2080 + system_llm_bonus
2081 }
2082
2083 fn latency_ms_to_score(&self, ms: f64) -> f64 {
2086 (1.0 - (ms / Self::LATENCY_CEILING_MS)).clamp(0.0, 1.0)
2087 }
2088
2089 fn tps_to_latency_ms(tps: f64) -> f64 {
2091 if tps <= 0.0 {
2092 return Self::LATENCY_CEILING_MS;
2093 }
2094 (200.0 / tps) * 1000.0
2096 }
2097
2098 fn needs_multi_tool_call(prompt: &str) -> bool {
2101 let lower = prompt.to_lowercase();
2102
2103 let has_numbered_list = {
2105 let mut count = 0u32;
2106 for i in 1..=5u32 {
2107 if lower.contains(&format!("{}) ", i)) || lower.contains(&format!("{}. ", i)) {
2108 count += 1;
2109 }
2110 }
2111 count >= 2
2112 };
2113
2114 let multi_keywords = [
2116 "multiple edits",
2117 "several changes",
2118 "three changes",
2119 "two changes",
2120 "all of the following",
2121 "each of these",
2122 "do both",
2123 "do all",
2124 "and also",
2125 "additionally",
2126 "as well as",
2127 "then also",
2128 ];
2129 let has_multi_keywords = multi_keywords.iter().any(|kw| lower.contains(kw));
2130
2131 let bullet_actions = lower.matches("- add ").count()
2133 + lower.matches("- update ").count()
2134 + lower.matches("- change ").count()
2135 + lower.matches("- remove ").count()
2136 + lower.matches("- fix ").count()
2137 + lower.matches("- edit ").count()
2138 + lower.matches("- implement ").count()
2139 + lower.matches("- create ").count();
2140 let has_bullet_list = bullet_actions >= 2;
2141
2142 has_numbered_list || has_multi_keywords || has_bullet_list
2143 }
2144
2145 fn schema_quality_estimate(&self, model: &ModelSchema) -> f64 {
2168 if !model.public_benchmarks.is_empty() {
2169 return self.normalized_benchmark_quality(model).clamp(0.0, 1.0);
2170 }
2171 if model.is_remote() {
2172 if model.trust_tier == crate::schema::TrustTier::Curated {
2187 if model_is_frontier_class(model) {
2188 return Self::CURATED_FRONTIER_TAG_PRIOR;
2189 }
2190 if model.tags.iter().any(|tag| tag == "balanced") {
2191 return Self::CURATED_BALANCED_TAG_PRIOR;
2192 }
2193 }
2194 return match model.trust_tier {
2195 crate::schema::TrustTier::Community => Self::UNMEASURED_REMOTE_COMMUNITY,
2196 _ => Self::UNMEASURED_REMOTE_CURATED,
2197 };
2198 }
2199 match model.size_mb() {
2203 0 => 0.35, s if s < 1000 => 0.30, s if s < 2000 => 0.38, s if s < 3000 => 0.45, s if s < 6000 => 0.52, _ => 0.60, }
2210 }
2211
2212 fn normalized_benchmark_quality(&self, model: &ModelSchema) -> f64 {
2223 let is_local = model.is_local();
2224 let n = model.public_benchmarks.len() as f64;
2225 let sum: f64 = model
2226 .public_benchmarks
2227 .iter()
2228 .map(|b| match Self::benchmark_frontier_ref(&b.name) {
2229 Some(frontier_ref) => {
2230 if b.score > frontier_ref {
2239 tracing::warn!(
2240 model = %model.id,
2241 benchmark = %b.name,
2242 score = b.score,
2243 frontier_ref,
2244 "model scores ABOVE the catalog frontier_ref — re-anchor \
2245 candidate: update builtin_catalog.json so this benchmark's \
2246 champion calibrates the quality tier (#370)"
2247 );
2248 }
2249 let frac = (b.score / frontier_ref).clamp(0.0, 1.0);
2250 let tier = |f: f64| {
2251 Self::BENCH_TIER_FLOOR
2252 + f * (Self::BENCH_TIER_CEIL - Self::BENCH_TIER_FLOOR)
2253 };
2254 if frac >= Self::BENCH_TIER_KNEE {
2255 tier(frac)
2256 } else {
2257 tier(Self::BENCH_TIER_KNEE) * frac / Self::BENCH_TIER_KNEE
2258 }
2259 }
2260 None if is_local => {
2266 b.score
2273 .clamp(Self::LOCAL_BENCH_TIER_FLOOR, Self::LOCAL_BENCH_TIER_CEIL)
2274 }
2275 None => b.score.clamp(Self::BENCH_TIER_FLOOR, Self::BENCH_TIER_CEIL),
2280 })
2281 .sum();
2282 sum / n
2289 }
2290
2291 fn benchmark_frontier_ref(name: &str) -> Option<f64> {
2303 BENCHMARK_FRONTIER_REFS
2304 .get(name)
2305 .copied()
2306 .filter(|r| *r > 0.0)
2307 }
2308
2309 fn schema_latency_estimate(&self, model: &ModelSchema) -> f64 {
2314 let is_moe = model.tags.contains(&"moe".to_string());
2315
2316 if model.is_local() {
2317 if let Some(tps) = model.performance.tokens_per_second {
2318 let measured = model.performance.latency_p50_ms.is_some();
2331 let multiplier = match &model.source {
2337 _ if !is_moe || measured => 1.0,
2338 crate::schema::ModelSource::Mlx { .. } => Self::MLX_MOE_TPS_MULTIPLIER,
2339 crate::schema::ModelSource::Local { .. } => Self::MOE_TPS_MULTIPLIER,
2340 _ => 1.0,
2341 };
2342 let effective_tps = tps * multiplier;
2343 let first_token_ms = model.performance.latency_p50_ms.unwrap_or(0) as f64;
2346 let estimated_ms = first_token_ms + Self::tps_to_latency_ms(effective_tps);
2347 return self.latency_ms_to_score(estimated_ms);
2348 }
2349 return 0.5; }
2351
2352 if let Some(tps) = model.performance.tokens_per_second {
2360 return self.latency_ms_to_score(Self::tps_to_latency_ms(tps));
2361 }
2362 if let Some(p50) = model.performance.latency_p50_ms {
2363 return self.latency_ms_to_score(
2364 p50 as f64 + Self::tps_to_latency_ms(Self::UNMEASURED_REMOTE_TPS),
2365 );
2366 }
2367 0.3 }
2369
2370 fn is_quality_critical_bootstrap_task(
2371 &self,
2372 task: InferenceTask,
2373 has_vision: bool,
2374 has_tools: bool,
2375 ) -> bool {
2376 has_vision
2377 || has_tools
2378 || matches!(
2379 task,
2380 InferenceTask::Generate | InferenceTask::Code | InferenceTask::Reasoning
2381 )
2382 }
2383
2384 fn is_trusted_quality_remote(&self, model: &ModelSchema) -> bool {
2385 let trusted_provider = matches!(
2386 model.provider.as_str(),
2387 "openai" | "anthropic" | "google" | "openrouter"
2388 );
2389 model.is_remote()
2390 && model.trust_tier == crate::schema::TrustTier::Curated
2391 && (trusted_provider || crate::openrouter::is_managed_gateway_schema(model))
2392 && !model.has_capability(ModelCapability::SpeechToText)
2393 && !model.has_capability(ModelCapability::TextToSpeech)
2394 }
2395
2396 fn is_local_model_proven_for_task(
2397 &self,
2398 model: &ModelSchema,
2399 task: InferenceTask,
2400 tracker: &OutcomeTracker,
2401 ) -> bool {
2402 let Some(profile) = tracker.profile(&model.id) else {
2403 return false;
2404 };
2405 if let Some(task_stats) = profile.task_stats(task) {
2406 if task_stats.calls >= self.config.bootstrap_min_task_observations
2407 && task_stats.ema_quality >= self.config.bootstrap_quality_floor
2408 {
2409 return true;
2410 }
2411 }
2412
2413 profile.total_calls >= self.config.bootstrap_min_task_observations
2414 && profile.ema_quality >= self.config.bootstrap_quality_floor
2415 }
2416
2417 const THOMPSON_TOP_MEAN: f64 = 0.95;
2420
2421 fn select_with_thompson_sampling(
2429 &self,
2430 scored: &[(String, f64)],
2431 tracker: &OutcomeTracker,
2432 ) -> (String, RoutingStrategy) {
2433 if scored.is_empty() {
2434 return (String::new(), RoutingStrategy::SchemaBased);
2435 }
2436
2437 let mut rng = rand::rng();
2438 let mut best_sample = f64::NEG_INFINITY;
2439 let mut best_id = scored[0].0.clone();
2440 let mut best_strategy = RoutingStrategy::SchemaBased;
2441
2442 let top_score = scored
2446 .iter()
2447 .map(|(_, score)| *score)
2448 .fold(f64::MIN_POSITIVE, f64::max);
2449
2450 for (id, phase2_score) in scored {
2451 let profile = tracker.profile(id);
2452
2453 let calls = profile.map_or(0, |p| p.total_calls);
2458 let mean = (Self::THOMPSON_TOP_MEAN * (phase2_score / top_score))
2459 .clamp(0.01, Self::THOMPSON_TOP_MEAN);
2460
2461 let concentration = self.config.prior_strength + calls as f64;
2464 let alpha = (concentration * mean).max(0.01);
2465 let beta = (concentration * (1.0 - mean)).max(0.01);
2466
2467 let sample = sample_beta(&mut rng, alpha, beta);
2469
2470 if sample > best_sample {
2471 best_sample = sample;
2472 best_id = id.clone();
2473 best_strategy = match profile {
2474 Some(p) if p.total_calls >= self.config.min_observations => {
2475 RoutingStrategy::ProfileBased
2476 }
2477 Some(p) if p.total_calls > 0 => {
2478 RoutingStrategy::Exploration
2480 }
2481 _ => RoutingStrategy::SchemaBased,
2482 };
2483 }
2484 }
2485
2486 (best_id, best_strategy)
2487 }
2488
2489 fn cold_start_decision(
2506 &self,
2507 complexity: TaskComplexity,
2508 task: InferenceTask,
2509 required_caps: &[ModelCapability],
2510 registry: &UnifiedRegistry,
2511 has_vision: bool,
2512 exclude: &std::collections::HashSet<String>,
2513 strict_exclusions: bool,
2514 availability: AvailabilitySnapshot,
2515 ) -> AdaptiveRoutingDecision {
2516 if has_vision {
2521 if let Some(model) = registry
2522 .query_by_capability(ModelCapability::Vision)
2523 .into_iter()
2524 .filter(|model| !exclude.contains(&model.id))
2525 .find(|model| {
2526 availability.is_available(model) && self.is_trusted_quality_remote(model)
2527 })
2528 .or_else(|| {
2529 registry
2530 .query_by_capability(ModelCapability::Vision)
2531 .into_iter()
2532 .find(|model| !exclude.contains(&model.id))
2533 })
2534 {
2535 return AdaptiveRoutingDecision {
2536 model_id: model.id.clone(),
2537 model_name: model.name.clone(),
2538 task,
2539 complexity,
2540 reason: format!(
2541 "{:?} task → {} (cold start, vision fallback)",
2542 complexity, model.name
2543 ),
2544 strategy: RoutingStrategy::SchemaBased,
2545 predicted_quality: 0.5,
2546 fallbacks: vec![],
2547 context_length: model.context_length,
2548 needs_compaction: false,
2549 candidates: vec![],
2550 };
2551 }
2552 }
2553
2554 if self.config.quality_first_cold_start {
2555 if let Some(model) = registry
2556 .list()
2557 .into_iter()
2558 .filter(|model| {
2559 availability.is_available(model)
2560 && !exclude.contains(&model.id)
2561 && required_caps.iter().all(|cap| model.has_capability(*cap))
2562 && self.is_trusted_quality_remote(model)
2563 })
2564 .max_by(|a, b| {
2565 self.schema_quality_estimate(a)
2566 .partial_cmp(&self.schema_quality_estimate(b))
2567 .unwrap_or(std::cmp::Ordering::Equal)
2568 })
2569 {
2570 return AdaptiveRoutingDecision {
2571 model_id: model.id.clone(),
2572 model_name: model.name.clone(),
2573 task,
2574 complexity,
2575 reason: format!(
2576 "{:?} task → {} (quality-first cold start)",
2577 complexity, model.name
2578 ),
2579 strategy: RoutingStrategy::SchemaBased,
2580 predicted_quality: self.schema_quality_estimate(model),
2581 fallbacks: vec![],
2582 context_length: model.context_length,
2583 needs_compaction: false,
2584 candidates: vec![],
2585 };
2586 }
2587 }
2588
2589 if let Some(model) = registry
2597 .list()
2598 .into_iter()
2599 .filter(|model| {
2600 availability.is_available(model)
2601 && !exclude.contains(&model.id)
2602 && model.is_local()
2603 && model.size_mb() < self.hw.max_model_mb
2604 && required_caps.iter().all(|cap| model.has_capability(*cap))
2605 })
2606 .max_by(|a, b| {
2607 self.schema_quality_estimate(a)
2608 .partial_cmp(&self.schema_quality_estimate(b))
2609 .unwrap_or(std::cmp::Ordering::Equal)
2610 })
2611 {
2612 return AdaptiveRoutingDecision {
2613 model_id: model.id.clone(),
2614 model_name: model.name.clone(),
2615 task,
2616 complexity,
2617 reason: format!(
2618 "{:?} task → {} (capability-satisfying local cold start)",
2619 complexity, model.name
2620 ),
2621 strategy: RoutingStrategy::SchemaBased,
2622 predicted_quality: self.schema_quality_estimate(model),
2623 fallbacks: vec![],
2624 context_length: model.context_length,
2625 needs_compaction: false,
2626 candidates: vec![],
2627 };
2628 }
2629
2630 if let Some(model) = registry
2641 .list()
2642 .into_iter()
2643 .filter(|model| {
2644 availability.is_available(model)
2645 && !exclude.contains(&model.id)
2646 && required_caps.iter().all(|cap| model.has_capability(*cap))
2647 })
2648 .max_by(|a, b| {
2649 let qa = (
2650 self.is_trusted_quality_remote(a),
2651 self.schema_quality_estimate(a),
2652 );
2653 let qb = (
2654 self.is_trusted_quality_remote(b),
2655 self.schema_quality_estimate(b),
2656 );
2657 qa.partial_cmp(&qb).unwrap_or(std::cmp::Ordering::Equal)
2658 })
2659 {
2660 return AdaptiveRoutingDecision {
2661 model_id: model.id.clone(),
2662 model_name: model.name.clone(),
2663 task,
2664 complexity,
2665 reason: format!(
2666 "{:?} task → {} (capability-satisfying cold start)",
2667 complexity, model.name
2668 ),
2669 strategy: RoutingStrategy::SchemaBased,
2670 predicted_quality: self.schema_quality_estimate(model),
2671 fallbacks: vec![],
2672 context_length: model.context_length,
2673 needs_compaction: false,
2674 candidates: vec![],
2675 };
2676 }
2677
2678 if strict_exclusions && !exclude.is_empty() {
2682 return Self::no_eligible_model_decision(complexity, task, exclude.len());
2683 }
2684
2685 let default_name = match complexity {
2692 TaskComplexity::Simple => "Qwen3-0.6B",
2693 TaskComplexity::Medium => "Qwen3-1.7B",
2694 TaskComplexity::Code => "Qwen3-4B",
2695 TaskComplexity::Complex => &self.hw.recommended_model,
2696 };
2697 let model_name = if registry.find_by_name(default_name).is_some() {
2702 default_name
2703 } else {
2704 crate::router::tier_fallback_order(default_name)
2705 .find(|name| registry.find_by_name(name).is_some())
2706 .unwrap_or(default_name)
2707 };
2708
2709 let model_id = registry
2710 .find_by_name(model_name)
2711 .map(|m| m.id.clone())
2712 .unwrap_or_else(|| model_name.to_string());
2713
2714 let context_length = registry
2715 .find_by_name(model_name)
2716 .map(|m| m.context_length)
2717 .unwrap_or(0);
2718
2719 AdaptiveRoutingDecision {
2720 model_id,
2721 model_name: model_name.to_string(),
2722 task,
2723 complexity,
2724 reason: format!(
2725 "{:?} task → {} (cold start, no candidates)",
2726 complexity, model_name
2727 ),
2728 strategy: RoutingStrategy::SchemaBased,
2729 predicted_quality: 0.5,
2730 fallbacks: vec![],
2731 context_length,
2732 needs_compaction: false,
2733 candidates: vec![],
2734 }
2735 }
2736
2737 fn no_on_device_model_decision(
2739 complexity: TaskComplexity,
2740 task: InferenceTask,
2741 ) -> AdaptiveRoutingDecision {
2742 AdaptiveRoutingDecision {
2743 reason: "on_device_only left no model on this device".into(),
2744 ..Self::no_eligible_model_decision(complexity, task, 0)
2745 }
2746 }
2747
2748 fn no_eligible_model_decision(
2749 complexity: TaskComplexity,
2750 task: InferenceTask,
2751 excluded_count: usize,
2752 ) -> AdaptiveRoutingDecision {
2753 AdaptiveRoutingDecision {
2754 model_id: String::new(),
2755 model_name: String::new(),
2756 task,
2757 complexity,
2758 reason: format!(
2759 "strict model exclusions left no eligible model ({excluded_count} catalog ids excluded)"
2760 ),
2761 strategy: RoutingStrategy::SchemaBased,
2762 predicted_quality: 0.0,
2763 fallbacks: vec![],
2764 context_length: 0,
2765 needs_compaction: false,
2766 candidates: vec![],
2767 }
2768 }
2769}
2770
2771fn task_hint_to_inference_task(hint: crate::intent::TaskHint) -> InferenceTask {
2777 use crate::intent::TaskHint;
2778 match hint {
2779 TaskHint::Chat => InferenceTask::Generate,
2780 TaskHint::Classify => InferenceTask::Classify,
2781 TaskHint::Reasoning => InferenceTask::Reasoning,
2782 TaskHint::Code => InferenceTask::Code,
2783 }
2784}
2785
2786fn base_required_capabilities(
2795 complexity: TaskComplexity,
2796 intent: Option<&crate::intent::IntentHint>,
2797 task: InferenceTask,
2798) -> Vec<ModelCapability> {
2799 match intent.and_then(|h| h.task) {
2800 Some(_) => vec![inference_task_required_capability(task)],
2801 None => complexity.required_capabilities(),
2802 }
2803}
2804
2805fn inference_task_required_capability(task: InferenceTask) -> ModelCapability {
2806 match task {
2807 InferenceTask::Generate => ModelCapability::Generate,
2808 InferenceTask::Code => ModelCapability::Code,
2809 InferenceTask::Reasoning => ModelCapability::Reasoning,
2810 InferenceTask::Classify => ModelCapability::Classify,
2811 InferenceTask::Embed => ModelCapability::Embed,
2812 }
2813}
2814
2815fn sample_beta(rng: &mut impl Rng, alpha: f64, beta: f64) -> f64 {
2823 let x = sample_gamma(rng, alpha);
2824 let y = sample_gamma(rng, beta);
2825 if x + y == 0.0 {
2826 0.5 } else {
2828 x / (x + y)
2829 }
2830}
2831
2832fn sample_gamma(rng: &mut impl Rng, shape: f64) -> f64 {
2835 if shape < 1.0 {
2836 let u: f64 = rng.random();
2838 return sample_gamma(rng, shape + 1.0) * u.powf(1.0 / shape);
2839 }
2840
2841 let d = shape - 1.0 / 3.0;
2843 let c = 1.0 / (9.0 * d).sqrt();
2844
2845 loop {
2846 let x: f64 = loop {
2847 let n = sample_standard_normal(rng);
2848 if 1.0 + c * n > 0.0 {
2849 break n;
2850 }
2851 };
2852
2853 let v = (1.0 + c * x).powi(3);
2854 let u: f64 = rng.random();
2855
2856 if u < 1.0 - 0.0331 * x.powi(4) {
2857 return d * v;
2858 }
2859 if u.ln() < 0.5 * x * x + d * (1.0 - v + v.ln()) {
2860 return d * v;
2861 }
2862 }
2863}
2864
2865fn sample_standard_normal(rng: &mut impl Rng) -> f64 {
2867 let u1: f64 = rng.random();
2868 let u2: f64 = rng.random();
2869 (-2.0 * u1.max(1e-300).ln()).sqrt() * (2.0 * std::f64::consts::PI * u2).cos()
2870}
2871
2872#[cfg(test)]
2873mod tests {
2874 use super::*;
2875 use crate::outcome::InferredOutcome;
2876 use std::ffi::OsString;
2877
2878 fn route_model_ids(decision: &AdaptiveRoutingDecision) -> impl Iterator<Item = &str> {
2879 std::iter::once(decision.model_id.as_str())
2880 .chain(
2881 decision
2882 .candidates
2883 .iter()
2884 .map(|candidate| candidate.model_id.as_str()),
2885 )
2886 .chain(decision.fallbacks.iter().map(String::as_str))
2887 }
2888
2889 struct RestoredEnvironment(Vec<(&'static str, Option<OsString>)>);
2890
2891 impl RestoredEnvironment {
2892 fn capture(names: &[&'static str]) -> Self {
2893 Self(
2894 names
2895 .iter()
2896 .map(|name| (*name, std::env::var_os(name)))
2897 .collect(),
2898 )
2899 }
2900 }
2901
2902 impl Drop for RestoredEnvironment {
2903 fn drop(&mut self) {
2904 for (name, value) in &self.0 {
2905 unsafe {
2906 match value {
2907 Some(value) => std::env::set_var(name, value),
2908 None => std::env::remove_var(name),
2909 }
2910 }
2911 }
2912 }
2913 }
2914
2915 struct TestRegistry {
2916 registry: UnifiedRegistry,
2917 _restore: RestoredEnvironment,
2920 _environment: tokio::sync::MutexGuard<'static, ()>,
2921 }
2922
2923 impl std::ops::Deref for TestRegistry {
2924 type Target = UnifiedRegistry;
2925
2926 fn deref(&self) -> &Self::Target {
2927 &self.registry
2928 }
2929 }
2930
2931 impl std::ops::DerefMut for TestRegistry {
2932 fn deref_mut(&mut self) -> &mut Self::Target {
2933 &mut self.registry
2934 }
2935 }
2936
2937 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2943 const QWEN3_8B_ROUTED_ID: &str = "mlx/qwen3-8b:4bit";
2944 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2945 const QWEN3_8B_ROUTED_ID: &str = "qwen/qwen3-8b:q4_k_m";
2946 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
2947 const QWEN3_4B_ROUTED_ID: &str = "mlx/qwen3-4b:4bit";
2948 #[cfg(not(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx))))]
2949 const QWEN3_4B_ROUTED_ID: &str = "qwen/qwen3-4b:q4_k_m";
2950
2951 fn test_hw() -> HardwareInfo {
2952 HardwareInfo {
2953 os: "macos".into(),
2954 arch: "aarch64".into(),
2955 cpu_cores: 10,
2956 total_ram_mb: 32768,
2957 gpu_backend: crate::hardware::GpuBackend::Metal,
2958 gpu_memory_mb: Some(28672),
2959 gpu_devices: Vec::new(),
2960 recommended_model: "Qwen3-8B".into(),
2961 recommended_context: 8192,
2962 max_model_mb: 18000, }
2964 }
2965
2966 fn test_registry() -> TestRegistry {
2967 let environment = crate::openrouter::test_environment_scope();
2968 let restore = RestoredEnvironment::capture(&["OPENAI_API_KEY", "CAR_SECRETS_FILE_DIR"]);
2969 let tmp = std::path::PathBuf::from("/tmp/car-test-adaptive-router");
2970 unsafe {
2971 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
2972 std::env::set_var(
2978 "CAR_SECRETS_FILE_DIR",
2979 std::env::temp_dir().join("car-test-adaptive-router-empty-secrets"),
2980 );
2981 }
2982 for name in &[
2984 "Qwen3-0.6B",
2985 "Qwen3-1.7B",
2986 "Qwen3-4B",
2987 "Qwen3-8B",
2988 "Qwen3-Embedding-0.6B",
2989 ] {
2990 let dir = tmp.join(name);
2991 let _ = std::fs::create_dir_all(&dir);
2992 let _ = std::fs::write(dir.join("model.gguf"), b"fake");
2993 let _ = std::fs::write(dir.join("tokenizer.json"), b"{}");
2994 }
2995 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
2996 reg.register_project_model(ModelSchema {
2997 id: "openai/gpt-5.4-mini:latest".into(),
2998 name: "gpt-5.4-mini".into(),
2999 provider: "openai".into(),
3000 family: "gpt-5.4".into(),
3001 version: "latest".into(),
3002 capabilities: vec![
3003 ModelCapability::Generate,
3004 ModelCapability::Code,
3005 ModelCapability::Reasoning,
3006 ModelCapability::ToolUse,
3007 ModelCapability::MultiToolCall,
3008 ModelCapability::Vision,
3009 ],
3010 context_length: 128_000,
3011 max_output_tokens: None,
3012 param_count: "api".into(),
3013 quantization: None,
3014 performance: Default::default(),
3015 cost: Default::default(),
3016 source: crate::schema::ModelSource::RemoteApi {
3017 endpoint: "https://api.openai.com/v1".into(),
3018 api_key_env: "OPENAI_API_KEY".into(),
3019 api_key_envs: vec![],
3020 api_version: None,
3021 protocol: crate::schema::ApiProtocol::OpenAiCompat,
3022 },
3023 tags: vec!["trusted-remote".into()],
3024 supported_params: vec![],
3025 public_benchmarks: vec![],
3026 trust_tier: crate::schema::TrustTier::Curated,
3027 deprecated: false,
3028 available: true,
3029 weights_ready: true,
3030 });
3031 TestRegistry {
3032 registry: reg,
3033 _restore: restore,
3034 _environment: environment,
3035 }
3036 }
3037
3038 #[test]
3039 fn route_observer_reports_a_429_exclusion_before_dispatch() {
3040 let registry = test_registry();
3041 let router = AdaptiveRouter::with_default_config(test_hw());
3042 let mut tracker = OutcomeTracker::new();
3043 let lane = "openai/gpt-5.4-mini:latest";
3044 let trace = tracker.record_start(lane, InferenceTask::Generate, "test 429");
3045 tracker.record_failure(&trace, "API returned 429 Too Many Requests");
3046 assert!(
3047 tracker.is_excluded(lane),
3048 "429 must seed the router exclusion"
3049 );
3050
3051 let mut exclusions = Vec::new();
3052 let mut observe = |exclusion| exclusions.push(exclusion);
3053 let decision = router.route_with_exclusion_observer(
3054 RouteRequest::new("Say hello.", ®istry, &tracker),
3055 &mut observe,
3056 );
3057
3058 assert_ne!(decision.model_id, lane);
3059 assert!(exclusions.contains(&RouterExclusion {
3060 lane: lane.into(),
3061 reason: RouterExclusionReason::RateLimited,
3062 until: None,
3063 }));
3064 }
3065
3066 #[test]
3067 fn route_observer_reports_an_open_circuit_with_its_cooldown() {
3068 let registry = test_registry();
3069 let router = AdaptiveRouter::with_default_config(test_hw());
3070 let tracker = OutcomeTracker::new();
3071 let lane = "openai/gpt-5.4-mini:latest";
3072 {
3073 let mut breakers = router.circuit_breakers.lock().unwrap();
3074 for _ in 0..3 {
3075 breakers.record_failure(lane);
3076 }
3077 }
3078
3079 let mut exclusions = Vec::new();
3080 let mut observe = |exclusion| exclusions.push(exclusion);
3081 let decision = router.route_with_exclusion_observer(
3082 RouteRequest::new("Say hello.", ®istry, &tracker),
3083 &mut observe,
3084 );
3085
3086 assert_ne!(decision.model_id, lane);
3087 let exclusion = exclusions
3088 .iter()
3089 .find(|exclusion| exclusion.lane == lane)
3090 .expect("open lane must be reported");
3091 assert_eq!(exclusion.reason, RouterExclusionReason::CircuitOpen);
3092 assert!(exclusion.until.is_some(), "open circuit has a cooldown end");
3093 }
3094
3095 #[test]
3096 fn openrouter_tags_drive_cost_quality_routing_and_live_availability() {
3097 let tmp = tempfile::tempdir().unwrap();
3098 let _credential_scope = crate::openrouter::test_credential_scope();
3099 crate::openrouter::set_test_credential(Some("test-openrouter-key"));
3100 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
3101 for schema in crate::openrouter::curated_schemas()
3102 .into_iter()
3103 .filter(|schema| schema.provider == "openrouter")
3104 {
3105 registry.register_project_model(schema);
3106 }
3107 let router = AdaptiveRouter::new(
3108 test_hw(),
3109 RoutingConfig {
3110 prefer_local: false,
3111 prior_strength: 1_000_000.0,
3115 ..RoutingConfig::default()
3116 },
3117 );
3118 let tracker = OutcomeTracker::new();
3119 let mut simple_tracker = OutcomeTracker::new();
3120 let evidenced_cheap = "openrouter/qwen/qwen3-coder-next";
3121 for _ in 0..8 {
3122 let trace = simple_tracker.record_start(
3123 evidenced_cheap,
3124 InferenceTask::Generate,
3125 "OpenRouter cheap-model calibration",
3126 );
3127 simple_tracker.record_complete(&trace, 100, 10, 10);
3128 simple_tracker
3129 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
3130 }
3131
3132 crate::openrouter::reset_credential_source_call_count();
3133 let simple = router.route_with(RouteRequest {
3134 workload: RoutingWorkload::Background,
3135 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3136 });
3137 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
3138 let simple_schema = registry.get(&simple.model_id).unwrap();
3139 assert!(simple_schema.tags.iter().any(|tag| tag == "cheap"));
3140 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
3141 assert!(simple
3142 .candidates
3143 .iter()
3144 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
3145
3146 let quality_intent = crate::intent::IntentHint {
3147 task: Some(crate::intent::TaskHint::Reasoning),
3148 prefer_quality: true,
3149 ..Default::default()
3150 };
3151 let complex = router.route_with(RouteRequest {
3152 intent: Some(&quality_intent),
3153 ..RouteRequest::new(
3154 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
3155 ®istry,
3156 &tracker,
3157 )
3158 });
3159 let complex_schema = registry.get(&complex.model_id).unwrap();
3160 assert!(complex_schema.tags.iter().any(|tag| tag == "frontier"));
3161 assert!(
3162 complex.reason.contains("frontier quality tier"),
3163 "{}",
3164 complex.reason
3165 );
3166
3167 let vision_intent = crate::intent::IntentHint {
3168 require: vec![ModelCapability::Vision],
3169 ..Default::default()
3170 };
3171 let vision = router.route_with(RouteRequest {
3172 intent: Some(&vision_intent),
3173 ..RouteRequest::new("Inspect this image", ®istry, &tracker)
3174 });
3175 assert!(vision.candidates.iter().all(|candidate| registry
3176 .get(&candidate.model_id)
3177 .is_some_and(|schema| schema.has_capability(ModelCapability::Vision))));
3178
3179 let failed_model = simple.model_id.clone();
3180 {
3181 let mut breakers = router.circuit_breakers.lock().unwrap();
3182 for _ in 0..3 {
3183 breakers.record_failure(&failed_model);
3184 }
3185 }
3186 let after_failures = router.route_with(RouteRequest {
3187 workload: RoutingWorkload::Background,
3188 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3189 });
3190 assert_ne!(after_failures.model_id, failed_model);
3191 assert!(!after_failures
3192 .candidates
3193 .iter()
3194 .any(|candidate| candidate.model_id == failed_model));
3195
3196 crate::openrouter::set_test_credential(None);
3197 crate::openrouter::reset_credential_source_call_count();
3198 let without_key = router.route("Say hello.", ®istry, &tracker);
3199 assert_eq!(crate::openrouter::credential_source_call_count(), 1);
3200 assert!(!without_key.model_id.starts_with("openrouter/"));
3201 assert!(!without_key
3202 .fallbacks
3203 .iter()
3204 .any(|model| model.starts_with("openrouter/")));
3205 assert!(!without_key
3206 .candidates
3207 .iter()
3208 .any(|candidate| candidate.model_id.starts_with("openrouter/")));
3209 }
3210
3211 #[tokio::test(flavor = "current_thread")]
3212 async fn managed_openrouter_aliases_survive_quality_bootstrap_without_a_personal_key() {
3213 let tmp = tempfile::tempdir().unwrap();
3214 let (_credential_scope, _environment) =
3215 crate::openrouter::test_credential_and_environment_scope_async().await;
3216 let _restore = RestoredEnvironment::capture(&[
3217 "CAR_SECRETS_FILE_DIR",
3218 "CAR_MANAGED_ROUTING_TEST_KEY",
3219 car_auth::PARSLEE_ACCESS_TOKEN_KEY,
3220 ]);
3221 let _home = crate::openrouter::StateRootScope::new();
3230 crate::openrouter::clear_gateway_unconfigured();
3231 crate::parslee_credential::clear_credential_rejected();
3232 let secrets_dir = tmp.path().join("secrets");
3233 unsafe {
3234 std::env::set_var("CAR_SECRETS_FILE_DIR", &secrets_dir);
3235 std::env::set_var("CAR_MANAGED_ROUTING_TEST_KEY", "test-peer-key");
3236 std::env::remove_var(car_auth::PARSLEE_ACCESS_TOKEN_KEY);
3237 }
3238 crate::openrouter::set_test_credential(None);
3239
3240 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("models"));
3241 for schema in crate::openrouter::curated_schemas() {
3242 registry.register_project_model(schema);
3243 }
3244 let mut bootstrap_peer = remote_model(
3245 "openai/bootstrap-peer",
3246 "bootstrap-peer",
3247 vec![
3248 ModelCapability::Generate,
3249 ModelCapability::Code,
3250 ModelCapability::Reasoning,
3251 ],
3252 );
3253 if let crate::schema::ModelSource::RemoteApi {
3254 ref mut api_key_env,
3255 ..
3256 } = bootstrap_peer.source
3257 {
3258 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3259 }
3260 registry.register_project_model(bootstrap_peer);
3261 let mut unrelated = remote_model(
3262 "other/unrelated-curated",
3263 "unrelated-curated",
3264 vec![
3265 ModelCapability::Generate,
3266 ModelCapability::Code,
3267 ModelCapability::Reasoning,
3268 ],
3269 );
3270 unrelated.provider = "other".into();
3271 if let crate::schema::ModelSource::RemoteApi {
3272 ref mut api_key_env,
3273 ..
3274 } = unrelated.source
3275 {
3276 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3277 }
3278 registry.register_project_model(unrelated);
3279 registry.refresh_routing_availability(Some("https://api.parslee.ai"), false);
3284
3285 let managed: Vec<_> = registry
3286 .list()
3287 .into_iter()
3288 .filter(|schema| schema.id.starts_with("parslee/openrouter/"))
3289 .collect();
3290 assert_eq!(managed.len(), crate::openrouter::managed_alias_count());
3291 assert!(managed.iter().all(|schema| {
3292 schema.available_now()
3293 && schema.tags.iter().any(|tag| tag == "openrouter")
3294 && schema.tags.iter().any(|tag| tag == "managed")
3295 && schema.cost.input_per_mtok.is_some_and(|price| price > 0.0)
3296 && schema.cost.output_per_mtok.is_some_and(|price| price > 0.0)
3297 && schema.has_capability(ModelCapability::Generate)
3298 }));
3299 assert!(registry
3300 .list()
3301 .into_iter()
3302 .filter(|schema| schema.id.starts_with("openrouter/"))
3303 .all(|schema| !schema.available_now()));
3304
3305 let router = AdaptiveRouter::new(
3306 test_hw(),
3307 RoutingConfig {
3308 prefer_local: false,
3309 prior_strength: 1_000_000.0,
3310 ..RoutingConfig::default()
3311 },
3312 );
3313 let tracker = OutcomeTracker::new();
3314
3315 let interactive = router.route("Say hello.", ®istry, &tracker);
3316 assert!(
3317 interactive
3318 .candidates
3319 .iter()
3320 .any(|candidate| candidate.model_id.starts_with("parslee/openrouter/")),
3321 "signed-in managed aliases must survive quality-first bootstrap: {interactive:?}"
3322 );
3323 assert!(
3324 interactive
3325 .fallbacks
3326 .iter()
3327 .any(|id| id.starts_with("parslee/openrouter/")),
3328 "signed-in managed aliases must remain fallback-eligible: {interactive:?}"
3329 );
3330 assert!(
3331 interactive
3332 .candidates
3333 .iter()
3334 .any(|candidate| candidate.model_id == "openai/bootstrap-peer"),
3335 "adding managed aliases must not remove existing trusted providers: {interactive:?}"
3336 );
3337 assert!(
3338 !interactive
3339 .candidates
3340 .iter()
3341 .any(|candidate| candidate.model_id == "other/unrelated-curated"),
3342 "managed-alias trust must not broaden the bootstrap to unrelated providers"
3343 );
3344
3345 let mut simple_tracker = OutcomeTracker::new();
3346 let evidenced_cheap = "parslee/openrouter/open-fast";
3347 for _ in 0..8 {
3348 let trace = simple_tracker.record_start(
3349 evidenced_cheap,
3350 InferenceTask::Generate,
3351 "managed OpenRouter cheap-model calibration",
3352 );
3353 simple_tracker.record_complete(&trace, 100, 10, 10);
3354 simple_tracker
3355 .record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 1.0 });
3356 }
3357 let simple = router.route_with(RouteRequest {
3358 workload: RoutingWorkload::Background,
3359 ..RouteRequest::new("Say hello.", ®istry, &simple_tracker)
3360 });
3361 let simple_schema = registry.get(&simple.model_id).unwrap();
3362 assert!(
3363 simple.model_id.starts_with("parslee/openrouter/")
3364 && simple_schema.tags.iter().any(|tag| tag == "cheap"),
3365 "simple work should prefer an evidenced cheap managed alias: {simple:?}"
3366 );
3367 assert!(simple.reason.contains("low-cost tier"), "{}", simple.reason);
3368
3369 let quality_intent = crate::intent::IntentHint {
3370 task: Some(crate::intent::TaskHint::Reasoning),
3371 prefer_quality: true,
3372 ..Default::default()
3373 };
3374 let complex = router.route_with(RouteRequest {
3375 intent: Some(&quality_intent),
3376 ..RouteRequest::new(
3377 "Analyze this multi-step architecture tradeoff and prove the failure modes.",
3378 ®istry,
3379 &tracker,
3380 )
3381 });
3382 let complex_schema = registry.get(&complex.model_id).unwrap();
3383 assert!(
3384 complex.model_id.starts_with("parslee/openrouter/")
3385 && complex_schema.tags.iter().any(|tag| tag == "frontier"),
3386 "complex quality work should escalate to a managed frontier alias: {complex:?}"
3387 );
3388 assert!(
3389 complex.reason.contains("frontier quality tier"),
3390 "{}",
3391 complex.reason
3392 );
3393
3394 for decision in [&interactive, &simple, &complex] {
3395 assert!(
3396 !route_model_ids(decision).any(|id| id.starts_with("openrouter/")),
3397 "personal OpenRouter rows must stay excluded without a personal key: {decision:?}"
3398 );
3399 }
3400
3401 let spoofed_id = "parslee/openrouter/frontier-general";
3402 let spoofed_endpoint = "https://attacker.invalid/v1/chat/completions";
3403 let spoofed_value = serde_json::json!({
3404 "id": spoofed_id,
3405 "name": spoofed_id,
3406 "provider": "parslee",
3407 "family": "spoof",
3408 "capabilities": ["generate", "code", "reasoning"],
3409 "context_length": 1_000_000,
3410 "performance": {"latency_p50_ms": 1},
3411 "cost": {"input_per_mtok": 0.0, "output_per_mtok": 0.0},
3412 "source": {
3413 "type": "remote_api",
3414 "endpoint": spoofed_endpoint,
3415 "api_key_env": "CAR_MANAGED_ROUTING_TEST_KEY",
3416 "protocol": "open_ai_compat"
3417 },
3418 "tags": ["openrouter", "managed", "frontier"],
3419 "public_benchmarks": [{"name": "attacker-unanchored", "score": 1.0}]
3420 });
3421 let omitted_tier: ModelSchema = serde_json::from_value(spoofed_value.clone()).unwrap();
3422 assert_eq!(
3423 omitted_tier.trust_tier,
3424 crate::schema::TrustTier::Curated,
3425 "the regression must exercise the legacy omitted-tier default"
3426 );
3427 let mut direct_spoof_registry =
3428 UnifiedRegistry::new_empty(tmp.path().join("direct-spoof-models"));
3429 direct_spoof_registry.register(omitted_tier);
3430 assert!(
3431 direct_spoof_registry.get(spoofed_id).is_none(),
3432 "public registration must reject the reserved managed-alias namespace"
3433 );
3434 direct_spoof_registry
3435 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3436 let direct_spoof_route = router.route(
3437 "Analyze this architecture and prove the failure modes.",
3438 &direct_spoof_registry,
3439 &OutcomeTracker::new(),
3440 );
3441 assert!(
3442 !route_model_ids(&direct_spoof_route).any(|id| id == spoofed_id),
3443 "a rejected in-memory lookalike must not be routable: {direct_spoof_route:?}"
3444 );
3445 std::fs::write(
3446 tmp.path().join("models.json"),
3447 serde_json::to_vec_pretty(&vec![spoofed_value]).unwrap(),
3448 )
3449 .unwrap();
3450
3451 let mut reloaded = UnifiedRegistry::new_empty(tmp.path().join("reloaded-models"));
3452 reloaded.load_user_config().unwrap();
3453 assert!(
3454 reloaded.get(spoofed_id).is_none(),
3455 "persisted user config must reject the reserved managed-alias namespace"
3456 );
3457 reloaded.register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3458 let spoofed_route = router.route(
3459 "Analyze this architecture and prove the failure modes.",
3460 &reloaded,
3461 &OutcomeTracker::new(),
3462 );
3463 assert!(
3464 !route_model_ids(&spoofed_route).any(|id| id == spoofed_id),
3465 "a rejected persisted lookalike must not be routable: {spoofed_route:?}"
3466 );
3467
3468 let discovery_state_root = tmp.path().join("discovery-spoof-root");
3469 let discovery_models_dir = discovery_state_root.join("models");
3470 std::fs::create_dir_all(&discovery_models_dir).unwrap();
3471 let discovery_spoof_id = "openrouter/forged-discovery-cache";
3472 let mut discovery_spoof = remote_model(
3473 discovery_spoof_id,
3474 "forged-discovery-cache",
3475 vec![
3476 ModelCapability::Generate,
3477 ModelCapability::Code,
3478 ModelCapability::Reasoning,
3479 ],
3480 );
3481 discovery_spoof.provider = "openrouter".into();
3482 discovery_spoof.tags = vec!["openrouter".into(), "frontier".into()];
3483 discovery_spoof.trust_tier = crate::schema::TrustTier::Curated;
3484 discovery_spoof.performance.latency_p50_ms = Some(1);
3485 discovery_spoof.cost = crate::schema::CostModel::default();
3486 if let crate::schema::ModelSource::RemoteApi {
3487 ref mut api_key_env,
3488 ..
3489 } = discovery_spoof.source
3490 {
3491 *api_key_env = "CAR_MANAGED_ROUTING_TEST_KEY".into();
3492 }
3493 crate::discovery::save_cache(
3494 &crate::discovery::cache_path(&discovery_models_dir),
3495 &[discovery_spoof],
3496 )
3497 .unwrap();
3498
3499 let mut discovery_registry =
3500 UnifiedRegistry::new_with_state_root(discovery_state_root, discovery_models_dir);
3501 discovery_registry
3502 .register_project_model(registry.get("openai/bootstrap-peer").unwrap().clone());
3503 assert_eq!(
3504 discovery_registry
3505 .get(discovery_spoof_id)
3506 .unwrap()
3507 .trust_tier,
3508 crate::schema::TrustTier::Community,
3509 "unsigned discovery cache rows must be demoted before registration"
3510 );
3511 let discovery_spoof_route = router.route(
3512 "Analyze this architecture and prove the failure modes.",
3513 &discovery_registry,
3514 &OutcomeTracker::new(),
3515 );
3516 assert!(
3517 !discovery_spoof_route
3518 .candidates
3519 .iter()
3520 .any(|candidate| candidate.model_id == discovery_spoof_id),
3521 "a forged direct-provider discovery row must not enter trusted candidates: \
3522 {discovery_spoof_route:?}"
3523 );
3524 assert!(
3525 !discovery_spoof_route
3526 .fallbacks
3527 .iter()
3528 .any(|id| id == discovery_spoof_id),
3529 "a forged direct-provider discovery row must not enter trusted fallbacks: \
3530 {discovery_spoof_route:?}"
3531 );
3532 }
3533
3534 #[test]
3535 fn openrouter_becomes_require_ready_eligible_after_live_key_activation() {
3536 let tmp = tempfile::tempdir().unwrap();
3537 let _credential_scope = crate::openrouter::test_credential_scope();
3538 let _provider_env = crate::openrouter::test_environment_scope();
3539 crate::openrouter::set_test_credential(None);
3540 let mut registry = UnifiedRegistry::new_empty(tmp.path().to_path_buf());
3541 let personal = crate::openrouter::curated_schemas()
3542 .into_iter()
3543 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3544 .unwrap();
3545 registry.register_project_model(personal);
3546
3547 unsafe {
3548 std::env::set_var("OPENAI_API_KEY", "ready-peer-key");
3549 }
3550 registry.register_project_model(remote_model(
3551 "ready-peer",
3552 "ready-peer",
3553 vec![ModelCapability::Generate, ModelCapability::Reasoning],
3554 ));
3555
3556 crate::openrouter::set_test_credential(Some("activated-after-register"));
3557 let hint = crate::intent::IntentHint {
3558 require_ready: true,
3559 prefer_quality: true,
3560 ..Default::default()
3561 };
3562 let router = AdaptiveRouter::new(
3563 test_hw(),
3564 RoutingConfig {
3565 prefer_local: false,
3566 prior_strength: 1_000_000.0,
3567 ..RoutingConfig::default()
3568 },
3569 );
3570 let tracker = OutcomeTracker::new();
3571 let decision = router.route_with(RouteRequest {
3572 intent: Some(&hint),
3573 ..RouteRequest::new("Prove the architecture tradeoff.", ®istry, &tracker)
3574 });
3575 assert!(
3576 decision
3577 .candidates
3578 .iter()
3579 .any(|candidate| candidate.model_id == "openrouter/openai/gpt-5.4"),
3580 "post-login OpenRouter model must remain require_ready-eligible: {:?}",
3581 decision.candidates
3582 );
3583 unsafe {
3584 std::env::remove_var("OPENAI_API_KEY");
3585 }
3586 }
3587
3588 #[test]
3589 fn route_cost_score_uses_prompt_tiers_and_model_cache_prices() {
3590 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
3591 let tracker = OutcomeTracker::new();
3592 let mut tiered = crate::openrouter::curated_schemas()
3593 .into_iter()
3594 .find(|schema| schema.id == "openrouter/openai/gpt-5.4")
3595 .unwrap();
3596 let mut flat = tiered.clone();
3597 tiered.id = "tiered".into();
3598 flat.id = "flat".into();
3599 flat.cost = crate::schema::CostModel {
3600 input_per_mtok: Some(4.0),
3601 output_per_mtok: Some(22.5),
3602 cache_read_input_per_mtok: Some(4.0),
3603 ..Default::default()
3604 };
3605
3606 let score = |model: &ModelSchema, prompt, cache_read| {
3607 router.score_model(
3608 model,
3609 InferenceTask::Generate,
3610 &tracker,
3611 RoutingWorkload::Background,
3612 prompt,
3613 0,
3614 cache_read,
3615 0,
3616 )
3617 };
3618 assert!(score(&tiered, 100_000, 0) > score(&flat, 100_000, 0));
3619 assert!(score(&tiered, 300_000, 0) < score(&flat, 300_000, 0));
3620 assert!(score(&tiered, 300_000, 300_000) > score(&flat, 300_000, 300_000));
3621 }
3622
3623 fn assert_cold_start_pick_is_frontier_grade(
3627 router: &AdaptiveRouter,
3628 registry: &UnifiedRegistry,
3629 decision: &AdaptiveRoutingDecision,
3630 ) {
3631 let pick = registry
3632 .get(&decision.model_id)
3633 .expect("selected model should exist");
3634 if !pick.is_local() {
3635 assert!(
3636 router.is_trusted_quality_remote(pick),
3637 "cold start picked an untrusted remote: {decision:?}"
3638 );
3639 return;
3640 }
3641 let best_remote = registry
3642 .list()
3643 .into_iter()
3644 .filter(|model| model.available && router.is_trusted_quality_remote(model))
3645 .map(|model| router.schema_quality_estimate(model))
3646 .fold(0.0_f64, f64::max);
3647 assert!(
3648 !pick.public_benchmarks.is_empty()
3649 && router.schema_quality_estimate(pick)
3650 >= best_remote - AdaptiveRouter::OUTCOME_FIRST_BAND,
3651 "cold start picked a local below frontier grade: {decision:?}"
3652 );
3653 }
3654
3655 #[test]
3656 fn routes_simple_to_trusted_remote_during_cold_start() {
3657 let router = AdaptiveRouter::new(
3658 test_hw(),
3659 RoutingConfig {
3660 prior_strength: 100.0, ..Default::default()
3662 },
3663 );
3664 let reg = test_registry();
3665 let tracker = OutcomeTracker::new();
3666
3667 let decision = router.route("What is 2+2?", ®, &tracker);
3668 assert_eq!(decision.complexity, TaskComplexity::Simple);
3669 assert_eq!(decision.strategy, RoutingStrategy::SchemaBased);
3670 assert_cold_start_pick_is_frontier_grade(&router, ®, &decision);
3673 }
3674
3675 #[test]
3676 fn routes_code_to_code_capable_remote_during_cold_start() {
3677 let router = AdaptiveRouter::new(
3678 test_hw(),
3679 RoutingConfig {
3680 prior_strength: 100.0, ..Default::default()
3682 },
3683 );
3684 let reg = test_registry();
3685 let tracker = OutcomeTracker::new();
3686
3687 let decision = router.route(
3688 "Fix this function:\n```rust\nfn main() {}\n```",
3689 ®,
3690 &tracker,
3691 );
3692 assert_eq!(decision.complexity, TaskComplexity::Code);
3693 assert_eq!(decision.task, InferenceTask::Code);
3694 let schema = reg
3696 .find_by_name(&decision.model_name)
3697 .expect("model should exist");
3698 assert!(
3699 schema.has_capability(ModelCapability::Code),
3700 "selected model must support Code"
3701 );
3702 assert_cold_start_pick_is_frontier_grade(&router, ®, &decision);
3703 }
3704
3705 #[test]
3706 fn routes_images_to_vision_capable_model() {
3707 let router = AdaptiveRouter::new(
3708 test_hw(),
3709 RoutingConfig {
3710 prior_strength: 1_000_000.0,
3713 ..Default::default()
3714 },
3715 );
3716 let mut reg = test_registry();
3717 let tracker = OutcomeTracker::new();
3718
3719 reg.register_project_model(ModelSchema {
3720 id: "mlx-vlm/qwen3-vl-2b:bf16".into(),
3721 name: "Qwen3-VL-2B-mlx-vlm".into(),
3722 provider: "qwen".into(),
3723 family: "qwen3-vl".into(),
3724 version: "bf16".into(),
3725 capabilities: vec![
3726 ModelCapability::Generate,
3727 ModelCapability::Vision,
3728 ModelCapability::Grounding,
3729 ],
3730 context_length: 262_144,
3731 max_output_tokens: None,
3732 param_count: "2B".into(),
3733 quantization: None,
3734 performance: Default::default(),
3735 cost: Default::default(),
3736 source: crate::schema::ModelSource::Mlx {
3737 hf_repo: "Qwen/Qwen3-VL-2B-Instruct".into(),
3738 hf_weight_file: None,
3739 },
3740 tags: vec!["vision".into(), "mlx-vlm-cli".into()],
3741 supported_params: vec![],
3742 public_benchmarks: vec![],
3743 trust_tier: crate::schema::TrustTier::Curated,
3744 deprecated: false,
3745 available: true,
3746 weights_ready: true,
3747 });
3748
3749 let decision = router.route_with_vision("What is in this image?", ®, &tracker, false);
3750 let schema = reg
3751 .find_by_name(&decision.model_name)
3752 .expect("model should exist");
3753 assert!(
3754 schema.has_capability(ModelCapability::Vision),
3755 "selected model must support Vision"
3756 );
3757 }
3758
3759 #[test]
3760 fn profile_based_routing_favors_proven_model() {
3761 let router = AdaptiveRouter::new(
3762 test_hw(),
3763 RoutingConfig {
3767 min_observations: 3,
3768 ..Default::default()
3769 },
3770 );
3771 let reg = test_registry();
3772 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3773 let wins = |tracker: &OutcomeTracker| {
3775 (0..200)
3776 .filter(|_| {
3777 let decision = router.route("Fix this bug in the parser", ®, tracker);
3778 assert_eq!(decision.complexity, TaskComplexity::Code);
3779 decision.model_id == qwen_8b_id
3780 })
3781 .count()
3782 };
3783 let unproven = wins(&OutcomeTracker::new());
3784
3785 let mut tracker = OutcomeTracker::new();
3787 for _ in 0..20 {
3788 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Code, "test");
3789 tracker.record_complete(&trace, 500, 100, 50);
3790 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3791 }
3792 let proven = wins(&tracker);
3794 assert!(
3795 proven >= 100 && proven >= unproven + 60,
3796 "proven model won {proven}/200 (without its outcomes: {unproven}/200)"
3797 );
3798 }
3799
3800 #[test]
3801 fn proven_local_model_can_displace_bootstrap_remote() {
3802 let router = AdaptiveRouter::new(
3803 test_hw(),
3804 RoutingConfig {
3805 prior_strength: 100.0,
3806 bootstrap_min_task_observations: 6,
3807 bootstrap_quality_floor: 0.8,
3808 ..Default::default()
3809 },
3810 );
3811 let reg = test_registry();
3812 let mut tracker = OutcomeTracker::new();
3813
3814 let qwen_8b_id = QWEN3_8B_ROUTED_ID;
3815 for _ in 0..12 {
3816 let trace = tracker.record_start(qwen_8b_id, InferenceTask::Generate, "test");
3817 tracker.record_complete(&trace, 300, 50, 20);
3818 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
3819 }
3820
3821 let local_wins = |tracker: &OutcomeTracker| {
3827 (0..20)
3828 .filter(|_| {
3829 let decision = router.route("Summarize this paragraph.", ®, tracker);
3830 reg.get(&decision.model_id)
3831 .expect("selected model should exist")
3832 .is_local()
3833 })
3834 .count()
3835 };
3836
3837 let proven = local_wins(&tracker);
3838 assert!(
3839 proven >= 12,
3840 "proven local model won only {proven}/20 times (expected >= 12)"
3841 );
3842 let unproven = local_wins(&OutcomeTracker::new());
3845 assert!(
3846 unproven < 12,
3847 "an unobserved local model won {unproven}/20 times; the gate is not what this measures"
3848 );
3849 }
3850
3851 #[test]
3876 fn background_routing_prefers_local_models() {
3877 let router = AdaptiveRouter::new(
3878 test_hw(),
3879 RoutingConfig {
3880 prior_strength: 1_000_000.0,
3881 ..Default::default()
3882 },
3883 );
3884 let reg = test_registry();
3885 let tracker = OutcomeTracker::new();
3886
3887 let draws = 200;
3902 let mut local_wins = 0;
3903 for _ in 0..draws {
3904 let decision = router.route_context_aware(
3905 "Write a Python fibonacci function.",
3906 128,
3907 ®,
3908 &tracker,
3909 false,
3910 false,
3911 RoutingWorkload::Background,
3912 );
3913 let schema = reg
3914 .get(&decision.model_id)
3915 .expect("selected model should exist");
3916 if schema.is_local() {
3917 local_wins += 1;
3918 }
3919 }
3920
3921 assert!(
3922 local_wins >= 170,
3923 "background routing should keep work local; local won only \
3924 {local_wins}/{draws} (measured baseline: 96.2% off Apple Silicon, \
3925 100% on it)"
3926 );
3927 }
3928
3929 #[test]
3956 fn benchmark_prior_with_evidence_steers_background_routing() {
3957 fn target_score(profile: Option<crate::outcome::ModelProfile>) -> f64 {
3958 let router = AdaptiveRouter::new(
3959 test_hw(),
3960 RoutingConfig {
3961 prior_strength: 100.0,
3962 ..Default::default()
3963 },
3964 );
3965 let reg = test_registry();
3966 let mut tracker = OutcomeTracker::new();
3967 if let Some(p) = profile {
3968 tracker.import_profiles(vec![p]);
3969 }
3970 let decision = router.route_context_aware(
3971 "Write a Python fibonacci function.",
3972 128,
3973 ®,
3974 &tracker,
3975 false,
3976 false,
3977 RoutingWorkload::Background,
3978 );
3979 decision
3980 .candidates
3981 .iter()
3982 .find(|c| c.model_id == QWEN3_8B_ROUTED_ID)
3983 .unwrap_or_else(|| panic!("{QWEN3_8B_ROUTED_ID} should be a candidate"))
3984 .score
3985 }
3986
3987 fn profile_with(ema: f64) -> crate::outcome::ModelProfile {
3988 let mut p = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
3989 p.ema_quality = ema;
3990 p.prior_sample_size = 30;
3994 p.quality_observations = 30;
3995 p
3996 }
3997
3998 let control = target_score(None);
3999 let good = target_score(Some(profile_with(0.99)));
4000 let bad = target_score(Some(profile_with(0.05)));
4001
4002 assert!(
4003 good > control,
4004 "an evidence-backed good prior must raise the score it vouches for \
4005 ({good} vs control {control}); equal means this test would pass with \
4006 the prior wiring removed"
4007 );
4008 assert!(
4009 bad < control,
4010 "an evidence-backed poor prior must lower the score ({bad} vs \
4011 control {control})"
4012 );
4013
4014 let mut unsupported = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
4018 unsupported.ema_quality = 0.99;
4019 assert_eq!(
4020 target_score(Some(unsupported)),
4021 control,
4022 "an EMA with no observations behind it must not move the score"
4023 );
4024 }
4025
4026 #[test]
4027 fn task_specific_benchmark_prior_informs_cold_start_routing() {
4028 let router = AdaptiveRouter::new(
4029 test_hw(),
4030 RoutingConfig {
4031 prior_strength: 100.0,
4032 ..Default::default()
4033 },
4034 );
4035 let reg = test_registry();
4036 let mut tracker = OutcomeTracker::new();
4037 let mut profile = crate::outcome::ModelProfile::new(QWEN3_8B_ROUTED_ID.into());
4042 profile.task_stats.insert(
4043 crate::outcome::InferenceTask::Code.to_string(),
4044 crate::outcome::TaskStats {
4045 ema_quality: 0.95,
4046 prior_sample_size: 30,
4047 quality_observations: 30,
4048 ..Default::default()
4049 },
4050 );
4051 tracker.import_profiles(vec![profile]);
4052
4053 let decision = router.route_context_aware(
4054 "Write a Python fibonacci function.",
4055 128,
4056 ®,
4057 &tracker,
4058 false,
4059 false,
4060 RoutingWorkload::Background,
4061 );
4062
4063 let top_scored = decision
4067 .candidates
4068 .first()
4069 .expect("routing should expose ranked candidates");
4070 let schema = reg
4071 .get(&top_scored.model_id)
4072 .expect("top-scored model should exist");
4073 assert!(
4074 schema.is_local(),
4075 "background routing should score the task-specific local code prior highest"
4076 );
4077 }
4078
4079 #[test]
4080 fn task_specific_latency_prior_affects_cold_start_score() {
4081 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4082 let reg = test_registry();
4083 let model = reg
4084 .get("qwen/qwen3-8b:q4_k_m")
4085 .expect("local test model should exist");
4086
4087 let mut fast_tracker = OutcomeTracker::new();
4088 let mut fast_profile = crate::outcome::ModelProfile::new(model.id.clone());
4089 fast_profile.task_stats.insert(
4090 crate::outcome::InferenceTask::Generate.to_string(),
4091 crate::outcome::TaskStats {
4092 ema_quality: 0.95,
4093 avg_latency_ms: 1200.0,
4094 ..Default::default()
4095 },
4096 );
4097 fast_tracker.import_profiles(vec![fast_profile]);
4098
4099 let mut slow_tracker = OutcomeTracker::new();
4100 let mut slow_profile = crate::outcome::ModelProfile::new(model.id.clone());
4101 slow_profile.task_stats.insert(
4102 crate::outcome::InferenceTask::Generate.to_string(),
4103 crate::outcome::TaskStats {
4104 ema_quality: 0.95,
4105 avg_latency_ms: 120_000.0,
4106 ..Default::default()
4107 },
4108 );
4109 slow_tracker.import_profiles(vec![slow_profile]);
4110
4111 let fast_score = router.score_model(
4112 model,
4113 InferenceTask::Generate,
4114 &fast_tracker,
4115 RoutingWorkload::Interactive,
4116 0,
4117 0,
4118 0,
4119 0,
4120 );
4121 let slow_score = router.score_model(
4122 model,
4123 InferenceTask::Generate,
4124 &slow_tracker,
4125 RoutingWorkload::Interactive,
4126 0,
4127 0,
4128 0,
4129 0,
4130 );
4131
4132 assert!(
4133 fast_score > slow_score,
4134 "faster task latency prior should improve cold-start score ({fast_score} <= {slow_score})"
4135 );
4136 }
4137
4138 #[test]
4139 fn interactive_workload_keeps_remote_bootstrap_bias() {
4140 let router = AdaptiveRouter::new(
4141 test_hw(),
4142 RoutingConfig {
4143 prior_strength: 100.0,
4144 ..Default::default()
4145 },
4146 );
4147 let reg = test_registry();
4148 let mut tracker = OutcomeTracker::new();
4149 let mut profile = crate::outcome::ModelProfile::new("qwen/qwen3-8b:q4_k_m".into());
4150 profile.ema_quality = 0.95;
4151 tracker.import_profiles(vec![profile]);
4152
4153 let decision = router.route_context_aware(
4154 "Write a Python fibonacci function.",
4155 128,
4156 ®,
4157 &tracker,
4158 false,
4159 false,
4160 RoutingWorkload::Interactive,
4161 );
4162
4163 assert_ne!(decision.model_id, "qwen/qwen3-8b:q4_k_m", "{decision:?}");
4166 assert_cold_start_pick_is_frontier_grade(&router, ®, &decision);
4167 }
4168
4169 #[test]
4170 fn code_interactive_weights_prioritise_quality_over_speed_over_cost() {
4171 let (q, lat, cost) =
4177 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Interactive);
4178 assert!(
4179 q > lat && lat > cost,
4180 "expected quality > speed > cost, got ({q}, {lat}, {cost})"
4181 );
4182 assert!(
4183 (q + lat + cost - 1.0).abs() < 1e-9,
4184 "weights must sum to 1.0"
4185 );
4186 let (default_q, default_lat, default_cost) = RoutingWorkload::Interactive.weights();
4187 assert!(
4188 q > default_q && lat < default_lat && cost < default_cost,
4189 "code weighting must lift quality and cut latency/cost vs the generic interactive profile"
4190 );
4191
4192 assert_eq!(
4194 AdaptiveRouter::task_aware_weights(
4195 InferenceTask::Code,
4196 RoutingWorkload::LocalPreferred,
4197 ),
4198 (q, lat, cost),
4199 );
4200
4201 assert_eq!(
4204 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Fastest),
4205 RoutingWorkload::Fastest.weights(),
4206 );
4207 assert_eq!(
4209 AdaptiveRouter::task_aware_weights(InferenceTask::Code, RoutingWorkload::Background),
4210 RoutingWorkload::Background.weights(),
4211 );
4212 assert_eq!(
4214 AdaptiveRouter::task_aware_weights(
4215 InferenceTask::Generate,
4216 RoutingWorkload::Interactive,
4217 ),
4218 RoutingWorkload::Interactive.weights(),
4219 );
4220 }
4221
4222 #[test]
4223 fn fallback_chain_has_alternatives() {
4224 let router = AdaptiveRouter::new(
4225 test_hw(),
4226 RoutingConfig {
4227 prior_strength: 100.0, ..Default::default()
4229 },
4230 );
4231 let reg = test_registry();
4232 let tracker = OutcomeTracker::new();
4233
4234 let decision = router.route("Analyze the architecture trade-offs", ®, &tracker);
4235 assert!(!decision.fallbacks.is_empty());
4236 assert!(!decision.fallbacks.contains(&decision.model_id));
4238 }
4239
4240 #[test]
4241 fn latency_scoring_is_consistent() {
4242 let router = AdaptiveRouter::with_default_config(test_hw());
4244
4245 let schema_score = router.latency_ms_to_score(AdaptiveRouter::tps_to_latency_ms(25.0));
4247 let observed_score = router.latency_ms_to_score(8000.0);
4249 assert!(
4250 (schema_score - observed_score).abs() < 0.01,
4251 "schema ({schema_score}) and observed ({observed_score}) should match"
4252 );
4253 }
4254
4255 #[test]
4256 fn complexity_assessment() {
4257 assert_eq!(
4258 TaskComplexity::assess("What is the capital of France?"),
4259 TaskComplexity::Simple
4260 );
4261 assert_eq!(
4262 TaskComplexity::assess("Fix this broken test"),
4263 TaskComplexity::Code
4264 );
4265 assert_eq!(
4266 TaskComplexity::assess("Analyze the trade-offs between A and B"),
4267 TaskComplexity::Complex
4268 );
4269 }
4270
4271 #[test]
4272 fn beta_sampling_produces_valid_values() {
4273 let mut rng = rand::rng();
4274 for _ in 0..100 {
4276 let s = sample_beta(&mut rng, 2.0, 5.0);
4277 assert!((0.0..=1.0).contains(&s), "sample {s} out of [0,1] range");
4278 }
4279 let samples: Vec<f64> = (0..1000).map(|_| sample_beta(&mut rng, 1.0, 1.0)).collect();
4281 let mean = samples.iter().sum::<f64>() / samples.len() as f64;
4282 assert!(
4283 (mean - 0.5).abs() < 0.05,
4284 "Beta(1,1) mean {mean} should be ~0.5"
4285 );
4286 }
4287
4288 fn thompson_wins(scored: &[(String, f64)], tracker: &OutcomeTracker, id: &str) -> usize {
4291 let router = AdaptiveRouter::with_default_config(test_hw());
4292 (0..200)
4293 .filter(|_| router.select_with_thompson_sampling(scored, tracker).0 == id)
4294 .count()
4295 }
4296
4297 fn record_calls(tracker: &mut OutcomeTracker, id: &str, completed: usize) {
4298 for _ in 0..completed {
4299 let trace = tracker.record_start(id, InferenceTask::Code, "test");
4300 tracker.record_complete(&trace, 500, 100, 50);
4301 }
4302 }
4303
4304 #[test]
4308 fn many_completed_calls_do_not_outvote_a_better_score() {
4309 let mut tracker = OutcomeTracker::new();
4310 record_calls(&mut tracker, "busy", 900);
4311 let scored = vec![("better".to_string(), 0.92), ("busy".to_string(), 0.80)];
4312 let wins = thompson_wins(&scored, &tracker, "better");
4313 assert!(wins >= 160, "the better-scored model won only {wins}/200");
4314 }
4315
4316 #[test]
4319 fn scores_above_one_still_rank() {
4320 let tracker = OutcomeTracker::new();
4321 let scored = vec![("plain".to_string(), 1.0), ("bonused".to_string(), 1.15)];
4323 let wins = thompson_wins(&scored, &tracker, "bonused");
4324 assert!(wins >= 130, "the higher-scored model won only {wins}/200");
4325 }
4326
4327 #[test]
4329 fn unobserved_near_ties_are_both_explored() {
4330 let tracker = OutcomeTracker::new();
4331 let scored = vec![("a".to_string(), 0.90), ("b".to_string(), 0.88)];
4332 let wins = thompson_wins(&scored, &tracker, "a");
4333 assert!((40..=160).contains(&wins), "a won {wins}/200 of a near-tie");
4334 }
4335
4336 #[test]
4337 fn thompson_sampling_converges_to_best() {
4338 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4341 let reg = test_registry();
4342 let qwen_4b_id = QWEN3_4B_ROUTED_ID;
4343 let wins = |tracker: &OutcomeTracker| {
4344 (0..200)
4345 .filter(|_| {
4346 router.route("Fix this parser bug", ®, tracker).model_id == qwen_4b_id
4347 })
4348 .count()
4349 };
4350 let unproven = wins(&OutcomeTracker::new());
4351
4352 let mut tracker = OutcomeTracker::new();
4354 for _ in 0..20 {
4355 let trace = tracker.record_start(qwen_4b_id, InferenceTask::Code, "test");
4356 tracker.record_complete(&trace, 500, 100, 50);
4357 tracker.record_inferred_outcome(&trace, InferredOutcome::Accepted { confidence: 0.95 });
4358 }
4359 let proven = wins(&tracker);
4363 assert!(
4364 proven >= 100 && proven >= unproven + 60,
4365 "strong model won {proven}/200 (without its outcomes: {unproven}/200)"
4366 );
4367 }
4368
4369 #[test]
4372 fn cold_start_never_returns_a_model_lacking_a_required_capability() {
4373 let router = AdaptiveRouter::new(
4381 test_hw(),
4382 RoutingConfig {
4383 quality_first_cold_start: false,
4384 ..Default::default()
4385 },
4386 );
4387 let reg = test_registry();
4388
4389 for caps in [
4390 vec![ModelCapability::Code],
4391 vec![ModelCapability::Code, ModelCapability::ToolUse],
4392 ] {
4393 let decision = router.cold_start_decision(
4396 TaskComplexity::Simple,
4397 InferenceTask::Code,
4398 &caps,
4399 ®,
4400 false,
4401 &std::collections::HashSet::new(),
4402 false,
4403 AvailabilitySnapshot::capture(®),
4404 );
4405 let chosen = reg
4406 .find_by_name(&decision.model_name)
4407 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
4408 .unwrap_or_else(|| panic!("cold start named an unknown model: {decision:?}"));
4409 for cap in &caps {
4410 assert!(
4411 chosen.has_capability(*cap),
4412 "cold start chose {} which lacks required {:?} (caps {:?})",
4413 chosen.name,
4414 cap,
4415 caps
4416 );
4417 }
4418 assert_eq!(
4425 decision.strategy,
4426 RoutingStrategy::SchemaBased,
4427 "expected schema-based cold start"
4428 );
4429 assert!(
4430 decision.reason.contains("capability-satisfying"),
4431 "expected a capability-satisfying cold-start branch, got: {}",
4432 decision.reason
4433 );
4434 }
4435 }
4436
4437 #[test]
4438 fn cold_start_skips_a_capable_local_that_exceeds_the_memory_budget() {
4439 let mut hw = test_hw();
4444 hw.max_model_mb = 300; let router = AdaptiveRouter::new(
4446 hw,
4447 RoutingConfig {
4448 quality_first_cold_start: false,
4449 ..Default::default()
4450 },
4451 );
4452 let reg = test_registry();
4453 let decision = router.cold_start_decision(
4454 TaskComplexity::Code,
4455 InferenceTask::Code,
4456 &[ModelCapability::Code],
4457 ®,
4458 false,
4459 &std::collections::HashSet::new(),
4460 false,
4461 AvailabilitySnapshot::capture(®),
4462 );
4463 assert!(
4464 !decision.reason.contains("local cold start"),
4465 "no local fits the 300MB budget, so the local branch must not fire: {}",
4466 decision.reason
4467 );
4468 }
4469
4470 #[test]
4471 fn intent_require_filters_out_models_lacking_capability() {
4472 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4477 let reg = test_registry();
4478 let tracker = OutcomeTracker::new();
4479
4480 let intent = crate::intent::IntentHint {
4481 require: vec![ModelCapability::Vision],
4482 ..Default::default()
4483 };
4484 let decision = router.route_with_intent("hello", ®, &tracker, &intent);
4485
4486 assert_eq!(
4491 decision.strategy,
4492 RoutingStrategy::SchemaBased,
4493 "require=[vision] with no vision-capable candidates must drop to schema cold-start"
4494 );
4495 }
4496
4497 #[test]
4498 fn unbenchmarked_remote_is_conservative_not_assumed_frontier() {
4499 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4510 let reg = test_registry();
4511 let remote = reg
4512 .list()
4513 .into_iter()
4514 .find(|m| {
4515 m.is_remote()
4516 && m.public_benchmarks.is_empty()
4517 && !m.tags.iter().any(|t| t == "frontier" || t == "balanced")
4518 })
4519 .expect("a benchmark-less, tier-untagged remote in the fixture")
4520 .clone();
4521 let q = router.schema_quality_estimate(&remote);
4522 assert!(
4523 (q - 0.60).abs() < 1e-9,
4524 "unmeasured curated remote should be a conservative 0.60, got {q}"
4525 );
4526
4527 let mut benched = remote.clone();
4531 benched.public_benchmarks = vec![
4532 crate::schema::BenchmarkScore {
4533 name: "A".into(),
4534 score: 0.9,
4535 harness: None,
4536 source_url: None,
4537 measured_at: None,
4538 runs: None,
4539 spread: None,
4540 },
4541 crate::schema::BenchmarkScore {
4542 name: "B".into(),
4543 score: 0.7,
4544 harness: None,
4545 source_url: None,
4546 measured_at: None,
4547 runs: None,
4548 spread: None,
4549 },
4550 ];
4551 let qb = router.schema_quality_estimate(&benched);
4552 assert!((qb - 0.8).abs() < 1e-9);
4553 assert!(
4554 qb > q,
4555 "real benchmarks must lift a model above the unmeasured default"
4556 );
4557 }
4558
4559 #[test]
4560 fn calibrated_benchmark_is_normalized_onto_the_tier_scale() {
4561 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4565 let bench = |raw: f64| {
4566 let mut m = outcome_model("m", None);
4567 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4568 name: "car-judged".into(),
4569 score: raw,
4570 harness: None,
4571 source_url: None,
4572 measured_at: None,
4573 runs: None,
4574 spread: None,
4575 }];
4576 router.schema_quality_estimate(&m)
4577 };
4578 let frontier = bench(1.0); let strong = bench(0.54); assert!(
4582 (frontier - 0.92).abs() < 0.01,
4583 "best-in-class → ~CEIL, got {frontier}"
4584 );
4585 assert!(
4586 (strong - 0.68).abs() < 0.02,
4587 "0.54 raw → ~0.68 tier, got {strong}"
4588 );
4589 assert!(
4590 strong > router.schema_quality_estimate(&local_model("local-30b", 18_000)),
4591 "a measured frontier model must out-rank an unmeasured 30B local"
4592 );
4593 }
4594
4595 #[test]
4599 fn a_measured_failing_score_ranks_below_its_unmeasured_size_proxy() {
4600 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4601 let catalog = crate::registry::builtin_catalog();
4602 for id in ["vllm-mlx/qwen3.5-0.8b:4bit", "vllm-mlx/qwen3.5-2b:4bit"] {
4603 let measured = catalog.iter().find(|m| m.id == id).unwrap();
4604 assert!(measured.public_benchmarks[0].score < 0.2, "{id}");
4605 let mut unmeasured = measured.clone();
4606 unmeasured.public_benchmarks.clear();
4607 assert!(
4608 router.schema_quality_estimate(measured)
4609 < router.schema_quality_estimate(&unmeasured),
4610 "{id}: measured {} vs size proxy {}",
4611 router.schema_quality_estimate(measured),
4612 router.schema_quality_estimate(&unmeasured)
4613 );
4614 }
4615 let knee = AdaptiveRouter::BENCH_TIER_KNEE;
4617 let mut m = outcome_model("m", None);
4618 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4619 name: "car-judged".into(),
4620 score: knee,
4621 harness: None,
4622 source_url: None,
4623 measured_at: None,
4624 runs: None,
4625 spread: None,
4626 }];
4627 let at_knee = router.schema_quality_estimate(&m);
4628 m.public_benchmarks[0].score = knee - 1e-6;
4629 assert!((router.schema_quality_estimate(&m) - at_knee).abs() < 1e-5);
4630 assert!(
4631 (at_knee
4632 - (AdaptiveRouter::BENCH_TIER_FLOOR
4633 + knee * (AdaptiveRouter::BENCH_TIER_CEIL - AdaptiveRouter::BENCH_TIER_FLOOR)))
4634 .abs()
4635 < 1e-9
4636 );
4637 }
4638
4639 #[test]
4640 fn uncalibrated_benchmark_is_clamped_into_the_tier_band() {
4641 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4645 let q = |raw: f64| {
4646 let mut m = outcome_model("m", None);
4647 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4648 name: "some-new-unknown-bench".into(),
4649 score: raw,
4650 harness: None,
4651 source_url: None,
4652 measured_at: None,
4653 runs: None,
4654 spread: None,
4655 }];
4656 router.schema_quality_estimate(&m)
4657 };
4658 assert!((q(0.73) - 0.73).abs() < 1e-9);
4661 assert!((q(0.10) - AdaptiveRouter::BENCH_TIER_FLOOR).abs() < 1e-9);
4662 assert!((q(0.99) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9);
4663 }
4664
4665 #[test]
4666 fn local_uncalibrated_benchmark_maps_into_the_local_tier_band() {
4667 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4673 let q = |raw: f64| {
4674 let mut m = local_model("local-bench", 2500); m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4676 name: "car-agentic-suite".into(), score: raw,
4678 harness: None,
4679 source_url: None,
4680 measured_at: None,
4681 runs: None,
4682 spread: None,
4683 }];
4684 router.schema_quality_estimate(&m)
4685 };
4686 assert!(
4690 (q(0.45) - 0.45).abs() < 1e-9,
4691 "in-band local score passes through"
4692 );
4693 assert!(
4694 (q(0.05) - AdaptiveRouter::LOCAL_BENCH_TIER_FLOOR).abs() < 1e-9,
4695 "sub-floor clamps to LOCAL floor 0.30, not frontier FLOOR 0.40"
4696 );
4697 assert!(
4698 (q(0.95) - AdaptiveRouter::LOCAL_BENCH_TIER_CEIL).abs() < 1e-9,
4699 "a perfect local-suite score tops at LOCAL ceil 0.60, not frontier CEIL 0.92"
4700 );
4701 assert!(q(0.55) > q(0.35));
4703 let mut frontier = outcome_model("frontier", None);
4706 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
4707 name: "car-judged".into(),
4708 score: 1.0,
4709 harness: None,
4710 source_url: None,
4711 measured_at: None,
4712 runs: None,
4713 spread: None,
4714 }];
4715 assert!(
4716 router.schema_quality_estimate(&frontier) > q(0.95),
4717 "a benchmarked frontier model out-ranks even a perfect-scoring local"
4718 );
4719 }
4720
4721 #[test]
4722 fn frontier_ref_is_derived_from_catalog_best_in_class() {
4723 use std::collections::HashMap;
4728 let mut max_by_bench: HashMap<String, f64> = HashMap::new();
4729 for model in crate::registry::builtin_catalog() {
4730 for b in &model.public_benchmarks {
4731 let e = max_by_bench.entry(b.name.clone()).or_insert(0.0);
4732 *e = e.max(b.score);
4733 }
4734 }
4735 assert!(!max_by_bench.is_empty(), "catalog should carry benchmarks");
4736 for (name, max) in max_by_bench {
4737 let derived = AdaptiveRouter::benchmark_frontier_ref(&name)
4738 .unwrap_or_else(|| panic!("benchmark '{name}' must derive a frontier_ref"));
4739 assert!(
4740 (derived - max).abs() < 1e-9,
4741 "frontier_ref for '{name}' should be the catalog max {max}, got {derived}"
4742 );
4743 }
4744 assert_eq!(
4746 AdaptiveRouter::benchmark_frontier_ref("car-judged"),
4747 Some(1.0)
4748 );
4749 assert_eq!(
4751 AdaptiveRouter::benchmark_frontier_ref("nonexistent-bench"),
4752 None
4753 );
4754 }
4755
4756 fn benchmarked_model(id: &str, bench: &str, score: f64, frontier: bool) -> ModelSchema {
4760 let mut m = outcome_model(id, None);
4761 if frontier {
4762 m.tags.push("frontier".into());
4763 }
4764 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4765 name: bench.into(),
4766 score,
4767 harness: None,
4768 source_url: None,
4769 measured_at: None,
4770 runs: None,
4771 spread: None,
4772 }];
4773 m
4774 }
4775
4776 #[test]
4777 fn frontier_ref_guard_drops_frontier_less_benchmark() {
4778 let models = [
4783 benchmarked_model("mid-a", "midtier-only-bench", 0.50, false),
4784 benchmarked_model("mid-b", "midtier-only-bench", 0.40, false),
4785 ];
4786 let refs = compute_frontier_refs(models.iter());
4787 assert!(
4788 !refs.contains_key("midtier-only-bench"),
4789 "a benchmark with no frontier-class carrier must not self-anchor, got {refs:?}"
4790 );
4791 }
4792
4793 #[test]
4794 fn frontier_ref_guard_anchors_when_a_frontier_model_carries_the_benchmark() {
4795 let models = [
4800 benchmarked_model("frontier-x", "shared-bench", 0.60, true),
4801 benchmarked_model("mid-y", "shared-bench", 0.70, false),
4802 ];
4803 let refs = compute_frontier_refs(models.iter());
4804 assert_eq!(
4805 refs.get("shared-bench").copied(),
4806 Some(0.70),
4807 "anchor is the catalog max across all carriers once a frontier carrier exists"
4808 );
4809 }
4810
4811 #[test]
4812 fn frontier_ref_guard_is_noop_on_todays_catalog() {
4813 use std::collections::HashMap;
4818 let catalog = crate::registry::builtin_catalog();
4819 let mut raw_max: HashMap<String, f64> = HashMap::new();
4820 for m in &catalog {
4821 for b in &m.public_benchmarks {
4822 let e = raw_max.entry(b.name.clone()).or_insert(0.0);
4823 *e = e.max(b.score);
4824 }
4825 }
4826 let guarded = compute_frontier_refs(catalog.iter());
4827 assert_eq!(
4828 raw_max, guarded,
4829 "guard dropped a benchmark — a catalog benchmark lost its frontier-class carrier"
4830 );
4831 assert!(
4832 model_is_frontier_class(
4833 catalog
4834 .iter()
4835 .find(|m| m.id == "openai/gpt-6.1-sol:latest")
4836 .expect("gpt-6.1-sol in catalog")
4837 ),
4838 "the car-judged champion must be frontier-tagged for the anchor to be valid"
4839 );
4840 }
4841
4842 #[test]
4843 fn score_above_the_frontier_clamps_to_ceil() {
4844 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4848 let mut m = outcome_model("future-sota", None);
4849 m.public_benchmarks = vec![crate::schema::BenchmarkScore {
4850 name: "car-judged".into(),
4851 score: 1.05,
4854 harness: None,
4855 source_url: None,
4856 measured_at: None,
4857 runs: None,
4858 spread: None,
4859 }];
4860 assert!(
4861 (router.schema_quality_estimate(&m) - AdaptiveRouter::BENCH_TIER_CEIL).abs() < 1e-9,
4862 "above-frontier score should clamp to CEIL"
4863 );
4864 }
4865
4866 #[test]
4874 fn curated_native_frontier_row_outranks_measured_small_models_on_quality_code() {
4875 let _environment = crate::openrouter::test_environment_scope();
4876 let _restore = RestoredEnvironment::capture(&["OPENAI_API_KEY", "CAR_SECRETS_FILE_DIR"]);
4877 unsafe {
4878 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
4879 std::env::set_var(
4880 "CAR_SECRETS_FILE_DIR",
4881 std::env::temp_dir().join("car-test-frontier-prior-empty-secrets"),
4882 );
4883 }
4884 let catalog: Vec<ModelSchema> =
4885 serde_json::from_str(include_str!("builtin_catalog.json")).unwrap();
4886 let row = |id: &str| {
4887 let mut m = catalog
4888 .iter()
4889 .find(|m| m.id == id)
4890 .unwrap_or_else(|| panic!("{id} missing from builtin catalog"))
4891 .clone();
4892 m.available = true;
4893 m.weights_ready = true;
4894 m
4895 };
4896 let mut frontier = row("openai/gpt-5.6-sol:latest");
4899 frontier.public_benchmarks.clear();
4900 let mini = row("openai/gpt-5.4-mini:latest");
4901 let local = row(QWEN3_4B_ROUTED_ID);
4902 assert_eq!(frontier.trust_tier, crate::schema::TrustTier::Curated);
4905 assert!(model_is_frontier_class(&frontier));
4906 assert!(!frontier.tags.iter().any(|t| t == "openrouter"));
4907 assert!(frontier.public_benchmarks.is_empty(), "must be unmeasured");
4908 assert!(matches!(
4909 frontier.source,
4910 crate::schema::ModelSource::RemoteApi { .. }
4911 ));
4912 assert_eq!(mini.public_benchmarks[0].name, "car-judged");
4913 assert_eq!(local.public_benchmarks[0].name, "car-judged");
4914 assert!(local.is_local());
4915
4916 let tmp = tempfile::tempdir().unwrap();
4917 let mut registry = UnifiedRegistry::new_empty(tmp.path().join("models"));
4918 for m in [frontier.clone(), mini.clone(), local.clone()] {
4919 assert!(registry.register_project_model(m));
4920 }
4921 let local_routable = !cfg!(all(
4926 target_os = "macos",
4927 target_arch = "aarch64",
4928 not(car_skip_mlx)
4929 )) || crate::backend::local::in_process_loader_available();
4930 assert_eq!(
4931 registry.get(&local.id).map(|m| m.available),
4932 Some(local_routable),
4933 "admission must agree with whether this build can run {}",
4934 local.id
4935 );
4936
4937 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4938 let tracker = OutcomeTracker::new();
4939 let intent = crate::intent::IntentHint {
4940 task: Some(crate::intent::TaskHint::Code),
4941 require: vec![ModelCapability::Code],
4942 prefer_quality: true,
4943 ..Default::default()
4944 };
4945 let decision = router.route_with_intent(
4946 "Refactor this module and explain the failure modes.",
4947 ®istry,
4948 &tracker,
4949 &intent,
4950 );
4951 let reliability = |id: &str| {
4952 decision
4953 .candidates
4954 .iter()
4955 .find(|c| c.model_id == id)
4956 .unwrap_or_else(|| panic!("{id} not a candidate: {:?}", decision.candidates))
4957 .reliability
4958 };
4959 let (f, m) = (reliability(&frontier.id), reliability(&mini.id));
4960 assert!(
4961 (f - AdaptiveRouter::CURATED_FRONTIER_TAG_PRIOR).abs() < 1e-9,
4962 "unmeasured curated frontier row must take the tier prior, got {f}"
4963 );
4964 if local_routable {
4965 let l = reliability(&local.id);
4966 assert!(
4967 f > l,
4968 "frontier {f} must outrank the car-judged 4B local {l}"
4969 );
4970 } else {
4971 assert!(
4972 !decision.candidates.iter().any(|c| c.model_id == local.id),
4973 "{} cannot run in this build, so it must not be a candidate: {:?}",
4974 local.id,
4975 decision.candidates
4976 );
4977 }
4978 assert!(f > m, "frontier {f} must outrank the measured mini {m}");
4979 assert_eq!(
4980 decision.model_id, frontier.id,
4981 "prefer_quality code must pick the frontier row: {:?}",
4982 decision.candidates
4983 );
4984 }
4985
4986 #[test]
4987 fn local_size_heuristic_stays_below_measured_frontier_floor() {
4988 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
4992 for size_mb in [500u64, 1500, 2500, 4500, 18_000] {
4993 let q = router.schema_quality_estimate(&local_model("local", size_mb));
4994 assert!(
4995 q <= AdaptiveRouter::BENCH_TIER_FLOOR + 0.21, "local {size_mb}MB quality {q} must stay near/below the measured floor"
4997 );
4998 }
4999 }
5000
5001 #[test]
5002 fn community_remote_quality_is_damped_below_curated() {
5003 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5008 let reg = test_registry();
5009 let mut remote = reg
5010 .list()
5011 .into_iter()
5012 .find(|m| {
5013 m.is_remote()
5014 && m.public_benchmarks.is_empty()
5015 && !m.tags.iter().any(|t| t == "frontier" || t == "balanced")
5016 })
5017 .expect("a benchmark-less, tier-untagged remote in the fixture")
5018 .clone();
5019
5020 remote.trust_tier = crate::schema::TrustTier::Curated;
5022 let qc = router.schema_quality_estimate(&remote);
5023 assert!(
5024 (qc - 0.60).abs() < 1e-9,
5025 "curated unmeasured remote should be 0.60, got {qc}"
5026 );
5027
5028 remote.trust_tier = crate::schema::TrustTier::Community;
5030 let q = router.schema_quality_estimate(&remote);
5031 assert!(
5032 (q - 0.48).abs() < 1e-9,
5033 "community remote should be damped to 0.48, got {q}"
5034 );
5035 assert!(
5036 q < qc,
5037 "community remote must rank below an unmeasured curated remote"
5038 );
5039
5040 remote.tags.push("frontier".into());
5044 let q = router.schema_quality_estimate(&remote);
5045 assert!(
5046 (q - 0.48).abs() < 1e-9,
5047 "community row with an inherited frontier tag stays at 0.48, got {q}"
5048 );
5049 remote.trust_tier = crate::schema::TrustTier::Curated;
5050 let q = router.schema_quality_estimate(&remote);
5051 assert!(
5052 (q - AdaptiveRouter::CURATED_FRONTIER_TAG_PRIOR).abs() < 1e-9,
5053 "curated frontier-tagged row takes the tier prior, got {q}"
5054 );
5055 }
5056
5057 #[test]
5058 fn quality_workload_is_quality_dominant() {
5059 let (q, lat, cost) = RoutingWorkload::Quality.weights();
5062 assert!(
5063 (q + lat + cost - 1.0).abs() < 1e-9,
5064 "weights must sum to 1.0"
5065 );
5066 assert!(q >= 0.8, "quality must dominate, got {q}");
5067 assert!(
5068 lat <= 0.1 && cost <= 0.15,
5069 "latency/cost near floor: {lat}/{cost}"
5070 );
5071 for w in [
5073 RoutingWorkload::Interactive,
5074 RoutingWorkload::Batch,
5075 RoutingWorkload::Background,
5076 RoutingWorkload::LocalPreferred,
5077 RoutingWorkload::Fastest,
5078 ] {
5079 assert!(
5080 q > w.weights().0,
5081 "Quality must out-weight {w:?} on quality"
5082 );
5083 }
5084 }
5085
5086 #[test]
5087 fn prefer_quality_scores_the_more_capable_model_higher() {
5088 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5093 let reg = test_registry();
5094 let tracker = OutcomeTracker::new();
5095 let small = reg
5096 .find_by_name("Qwen3-1.7B")
5097 .expect("1.7B in fixture")
5098 .clone();
5099 let big = reg.find_by_name("Qwen3-4B").expect("4B in fixture").clone();
5100 assert!(small.has_capability(ModelCapability::Code));
5101 assert!(big.has_capability(ModelCapability::Code));
5102
5103 let score = |m: &ModelSchema, w: RoutingWorkload| {
5104 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
5105 };
5106 assert!(
5108 score(&big, RoutingWorkload::Quality) > score(&small, RoutingWorkload::Quality),
5109 "prefer_quality must rank the capable 4B above the cheap 1.7B"
5110 );
5111 }
5112
5113 fn local_mlx_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
5116 let mut m = outcome_model(id, benchmark);
5117 m.provider = "local".into();
5118 m.source = crate::schema::ModelSource::Mlx {
5119 hf_repo: "mlx-community/test".into(),
5120 hf_weight_file: None,
5121 };
5122 m
5123 }
5124
5125 #[test]
5126 fn quality_workload_suppresses_local_mlx_bonus_vs_capable_remote() {
5127 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5132 let tracker = OutcomeTracker::new();
5133 let score = |m: &ModelSchema, w: RoutingWorkload| {
5134 router.score_model(m, InferenceTask::Code, &tracker, w, 0, 0, 0, 0)
5135 };
5136
5137 let remote_capable = outcome_model("remote/frontier", Some(0.85));
5138 let mlx_tiny = local_mlx_model("mlx/tiny", Some(0.55));
5139 let remote_peer = outcome_model("remote/peer", Some(0.55));
5141 assert!(mlx_tiny.is_mlx() && mlx_tiny.is_local());
5142
5143 assert!(
5147 score(&mlx_tiny, RoutingWorkload::Background)
5148 > score(&remote_peer, RoutingWorkload::Background),
5149 "the local/MLX bonus should still apply on non-Quality workloads"
5150 );
5151
5152 assert!(
5155 score(&remote_capable, RoutingWorkload::Quality)
5156 > score(&mlx_tiny, RoutingWorkload::Quality),
5157 "prefer_quality must suppress the locality bonus so the capable remote wins"
5158 );
5159 }
5160
5161 fn local_model(id: &str, size_mb: u64) -> ModelSchema {
5166 let mut m = outcome_model(id, None);
5167 m.provider = "local".into();
5168 m.source = crate::schema::ModelSource::Local {
5169 hf_repo: "test/repo".into(),
5170 hf_filename: "model.gguf".into(),
5171 tokenizer_repo: "test/repo".into(),
5172 };
5173 m.cost.size_mb = Some(size_mb);
5174 m
5175 }
5176
5177 #[test]
5188 fn a_measured_moe_rate_is_not_discounted_again() {
5189 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5190
5191 let mut measured = local_model("local/moe-measured", 17_000);
5192 measured.tags.push("moe".into());
5193 measured.performance.tokens_per_second = Some(35.0);
5194 measured.performance.latency_p50_ms = Some(1_200);
5195
5196 let mut declared = local_model("local/moe-declared", 17_000);
5198 declared.tags.push("moe".into());
5199 declared.performance.tokens_per_second = Some(35.0);
5200
5201 let mut dense = local_model("local/dense", 17_000);
5203 dense.performance.tokens_per_second = Some(35.0);
5204 dense.performance.latency_p50_ms = Some(1_200);
5205
5206 let measured_score = router.schema_latency_estimate(&measured);
5207 let declared_score = router.schema_latency_estimate(&declared);
5208 let dense_score = router.schema_latency_estimate(&dense);
5209
5210 assert_eq!(
5211 measured_score, dense_score,
5212 "a measured MoE rate must score as the rate it measured"
5213 );
5214 assert!(
5215 declared_score < measured_score,
5216 "a declared MoE rate still gets the discount: {declared_score} vs {measured_score}"
5217 );
5218 }
5219
5220 #[test]
5221 fn a_named_task_decides_the_capability_filter_over_the_keyword_guess() {
5222 use crate::intent::{IntentHint, TaskHint};
5223 let prompt = "Let me know if there's an error in the prefix list.";
5224 let guess = TaskComplexity::assess(prompt);
5225 assert_eq!(
5226 guess,
5227 TaskComplexity::Code,
5228 "the keyword guess reads this as code"
5229 );
5230 let chat = IntentHint {
5231 task: Some(TaskHint::Chat),
5232 ..Default::default()
5233 };
5234 let chat_task = task_hint_to_inference_task(TaskHint::Chat);
5235 assert_eq!(
5236 base_required_capabilities(guess, Some(&chat), chat_task),
5237 vec![inference_task_required_capability(chat_task)]
5238 );
5239 let code = IntentHint {
5241 task: Some(TaskHint::Code),
5242 ..Default::default()
5243 };
5244 assert_eq!(
5245 base_required_capabilities(TaskComplexity::Simple, Some(&code), InferenceTask::Code),
5246 vec![ModelCapability::Code]
5247 );
5248 assert_eq!(
5250 base_required_capabilities(guess, None, InferenceTask::Code),
5251 vec![ModelCapability::Code]
5252 );
5253 }
5254
5255 #[test]
5260 fn a_vllm_mlx_moe_row_scores_at_its_benchmarked_rate() {
5261 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5262 let catalog = crate::registry::builtin_catalog();
5263 let moe = catalog
5264 .iter()
5265 .find(|m| m.id == "vllm-mlx/qwen3.6-35b-a3b:4bit")
5266 .unwrap();
5267 let rate = moe.performance.tokens_per_second.unwrap();
5268 assert!(moe.tags.iter().any(|t| t == "moe"));
5269 assert!(moe.performance.latency_p50_ms.is_none());
5270
5271 let mut dense = local_model("local/dense", 17_000);
5272 dense.performance.tokens_per_second = Some(rate);
5273 assert_eq!(
5274 router.schema_latency_estimate(moe),
5275 router.schema_latency_estimate(&dense),
5276 "scored as if it decoded at {} tok/s",
5277 rate * AdaptiveRouter::MOE_TPS_MULTIPLIER
5278 );
5279 }
5280
5281 #[test]
5285 fn a_raw_benchmark_seed_never_overrides_a_catalog_benchmark() {
5286 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5287 let measured = outcome_model("remote/measured", Some(0.70));
5288 let unmeasured = outcome_model("remote/unmeasured", None);
5289 let mut tracker = OutcomeTracker::new();
5290 let seeded = |id: &str| {
5291 let mut p = crate::outcome::ModelProfile::new(id.into());
5292 p.ema_quality = 1.0;
5293 p.prior_sample_size = 4;
5294 p
5295 };
5296 tracker.import_profiles(vec![seeded("remote/measured"), seeded("remote/unmeasured")]);
5297
5298 let task = InferenceTask::Generate;
5299 assert_eq!(
5300 router.reliability(&measured, task, &tracker),
5301 router.schema_quality_estimate(&measured)
5302 );
5303 assert!(
5304 router.reliability(&unmeasured, task, &tracker)
5305 > router.schema_quality_estimate(&unmeasured)
5306 );
5307 }
5308
5309 #[test]
5314 fn remote_and_local_latency_are_the_time_to_a_whole_answer() {
5315 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5316 let mut local = local_model("local/dense", 17_000);
5317 local.performance.tokens_per_second = Some(100.0);
5318 let mut remote = outcome_model("remote/measured", Some(0.8));
5319 remote.performance.tokens_per_second = Some(100.0);
5320 assert_eq!(
5321 router.schema_latency_estimate(&remote),
5322 router.schema_latency_estimate(&local)
5323 );
5324
5325 let mut declared = outcome_model("remote/declared", Some(0.8));
5327 declared.performance.latency_p50_ms = Some(600);
5328 let answer_ms = 600.0 + 200.0 / AdaptiveRouter::UNMEASURED_REMOTE_TPS * 1000.0;
5329 assert!(
5330 (router.schema_latency_estimate(&declared) - router.latency_ms_to_score(answer_ms))
5331 .abs()
5332 < 1e-9
5333 );
5334
5335 let mut slow_start = local.clone();
5337 slow_start.performance.latency_p50_ms = Some(1_500);
5338 assert!(
5339 router.schema_latency_estimate(&slow_start) < router.schema_latency_estimate(&local)
5340 );
5341 }
5342
5343 fn outcome_model(id: &str, benchmark: Option<f64>) -> ModelSchema {
5347 ModelSchema {
5348 id: id.into(),
5349 name: id.into(),
5350 provider: "openai".into(),
5351 family: "test".into(),
5352 version: "latest".into(),
5353 capabilities: vec![
5354 ModelCapability::Generate,
5355 ModelCapability::Code,
5356 ModelCapability::Reasoning,
5357 ],
5358 context_length: 128_000,
5359 max_output_tokens: None,
5360 param_count: "api".into(),
5361 quantization: None,
5362 performance: Default::default(),
5363 cost: Default::default(),
5364 source: crate::schema::ModelSource::RemoteApi {
5365 endpoint: "https://api.openai.com/v1".into(),
5366 api_key_env: "OPENAI_API_KEY".into(),
5367 api_key_envs: vec![],
5368 api_version: None,
5369 protocol: crate::schema::ApiProtocol::OpenAiCompat,
5370 },
5371 tags: vec![],
5372 supported_params: vec![],
5373 public_benchmarks: benchmark
5374 .map(|s| {
5375 vec![crate::schema::BenchmarkScore {
5376 name: "t".into(),
5377 score: s,
5378 harness: None,
5379 source_url: None,
5380 measured_at: None,
5381 runs: None,
5382 spread: None,
5383 }]
5384 })
5385 .unwrap_or_default(),
5386 trust_tier: crate::schema::TrustTier::Curated,
5387 deprecated: false,
5388 available: true,
5389 weights_ready: true,
5390 }
5391 }
5392
5393 #[test]
5394 fn reliability_band_applies_to_quality_and_substantive_normal_lanes() {
5395 use RoutingWorkload as W;
5396 assert!(AdaptiveRouter::applies_reliability_band(
5398 InferenceTask::Code,
5399 W::Interactive
5400 ));
5401 assert!(AdaptiveRouter::applies_reliability_band(
5402 InferenceTask::Reasoning,
5403 W::Batch
5404 ));
5405 assert!(AdaptiveRouter::applies_reliability_band(
5408 InferenceTask::Code,
5409 W::Quality
5410 ));
5411 assert!(AdaptiveRouter::applies_reliability_band(
5412 InferenceTask::Generate,
5413 W::Quality
5414 ));
5415 assert!(!AdaptiveRouter::applies_reliability_band(
5417 InferenceTask::Generate,
5418 W::Interactive
5419 ));
5420 assert!(!AdaptiveRouter::applies_reliability_band(
5422 InferenceTask::Code,
5423 W::Background
5424 ));
5425 assert!(!AdaptiveRouter::applies_reliability_band(
5426 InferenceTask::Code,
5427 W::Fastest
5428 ));
5429 assert!(!AdaptiveRouter::applies_reliability_band(
5431 InferenceTask::Code,
5432 W::LocalPreferred
5433 ));
5434 }
5435
5436 #[test]
5437 fn high_stakes_intent_forces_quality_over_everything() {
5438 use crate::intent::IntentHint;
5439 let hs = IntentHint {
5442 high_stakes: true,
5443 prefer_fast: true,
5444 prefer_local: true,
5445 ..Default::default()
5446 };
5447 assert_eq!(
5448 AdaptiveRouter::resolve_workload(Some(&hs), RoutingWorkload::Interactive),
5449 RoutingWorkload::Quality
5450 );
5451 let f = IntentHint {
5453 prefer_fast: true,
5454 ..Default::default()
5455 };
5456 assert_eq!(
5457 AdaptiveRouter::resolve_workload(Some(&f), RoutingWorkload::Interactive),
5458 RoutingWorkload::Fastest
5459 );
5460 assert_eq!(
5462 AdaptiveRouter::resolve_workload(None, RoutingWorkload::Batch),
5463 RoutingWorkload::Batch
5464 );
5465 }
5466
5467 #[test]
5468 fn band_excludes_a_real_quality_gap_even_if_cheaper() {
5469 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5471 let tracker = OutcomeTracker::new();
5472 let strong = outcome_model("strong", Some(0.90));
5473 let weak_cheap = outcome_model("weak-cheap", Some(0.83));
5474 let band =
5475 router.outcome_first_band(vec![strong, weak_cheap], InferenceTask::Code, &tracker);
5476 assert_eq!(
5477 band.len(),
5478 1,
5479 "0.07 gap (> band 0.02) must drop the weaker model"
5480 );
5481 assert_eq!(band[0].id, "strong");
5482 }
5483
5484 #[test]
5485 fn band_keeps_genuinely_near_equal_models() {
5486 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5488 let tracker = OutcomeTracker::new();
5489 let band = router.outcome_first_band(
5490 vec![
5491 outcome_model("a", Some(0.90)),
5492 outcome_model("a2", Some(0.89)),
5493 ],
5494 InferenceTask::Code,
5495 &tracker,
5496 );
5497 assert_eq!(
5498 band.len(),
5499 2,
5500 "0.01 gap (<= band 0.02) keeps both for tie-break"
5501 );
5502 }
5503
5504 #[test]
5505 fn band_cold_start_frontier_beats_cheap_unknown() {
5506 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5509 let tracker = OutcomeTracker::new();
5510 let band = router.outcome_first_band(
5511 vec![
5512 outcome_model("frontier", Some(0.85)),
5513 outcome_model("unknown", None),
5514 ],
5515 InferenceTask::Code,
5516 &tracker,
5517 );
5518 assert_eq!(band.len(), 1);
5519 assert_eq!(
5520 band[0].id, "frontier",
5521 "0.85 vs 0.60 unknown → unknown excluded"
5522 );
5523 }
5524
5525 #[test]
5526 fn quality_lane_bands_out_a_weak_local_but_local_preferred_keeps_it() {
5527 let mut reg = test_registry();
5534 let mut frontier = outcome_model("frontier-remote", None);
5535 frontier.public_benchmarks = vec![crate::schema::BenchmarkScore {
5536 name: "car-judged".into(),
5537 score: 1.0, harness: None,
5539 source_url: None,
5540 measured_at: None,
5541 runs: None,
5542 spread: None,
5543 }];
5544 reg.register_project_model(frontier);
5545
5546 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5547 let tracker = OutcomeTracker::new();
5548 let prompt = "Implement a balanced binary search tree with deletion.";
5549
5550 let q = router.route_context_aware(
5553 prompt,
5554 256,
5555 ®,
5556 &tracker,
5557 false,
5558 false,
5559 RoutingWorkload::Quality,
5560 );
5561 let picked = reg.get(&q.model_id).expect("picked model is registered");
5564 assert!(
5565 !picked.is_local() && !picked.public_benchmarks.is_empty(),
5566 "prefer_quality must pick a benchmarked frontier over locals-on-bonuses, got {}",
5567 q.model_id
5568 );
5569 assert_eq!(
5570 q.strategy,
5571 RoutingStrategy::SchemaBased,
5572 "Quality lane must use deterministic argmax, not Thompson exploration"
5573 );
5574 assert!(
5575 q.candidates.iter().any(|c| !c.in_band),
5576 "Quality lane must apply a reliability band (some candidates excluded)"
5577 );
5578
5579 let lp = router.route_context_aware(
5582 prompt,
5583 256,
5584 ®,
5585 &tracker,
5586 false,
5587 false,
5588 RoutingWorkload::LocalPreferred,
5589 );
5590 assert!(
5593 lp.candidates
5594 .iter()
5595 .all(|c| c.in_band == (c.reliability >= AdaptiveRouter::MIN_ANSWER_QUALITY)),
5596 "LocalPreferred must NOT band — every answering candidate eligible"
5597 );
5598 }
5599
5600 #[test]
5604 fn prefer_fast_never_picks_a_model_that_fails_most_tasks() {
5605 let reg = test_registry();
5608 let failing = ["vllm-mlx/qwen3.5-0.8b:4bit", "vllm-mlx/qwen3.5-2b:4bit"];
5609 let judged = |score: f64| {
5610 vec![crate::schema::BenchmarkScore {
5611 name: "car-judged".into(),
5612 score,
5613 harness: None,
5614 source_url: None,
5615 measured_at: None,
5616 runs: None,
5617 spread: None,
5618 }]
5619 };
5620 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5621 let tracker = OutcomeTracker::new();
5622 for _ in 0..50 {
5623 let d = router.route_context_aware(
5624 "Say hello.",
5625 64,
5626 ®,
5627 &tracker,
5628 false,
5629 false,
5630 RoutingWorkload::Fastest,
5631 );
5632 assert!(!failing.contains(&d.model_id.as_str()), "{}", d.model_id);
5633 }
5634
5635 let mut failed = OutcomeTracker::new();
5638 let strong = "vllm-mlx/gpt-oss-20b:mxfp4";
5639 for _ in 0..3 {
5640 let trace = failed.record_start(strong, InferenceTask::Generate, "test");
5641 failed.record_failure(&trace, "connection reset");
5642 }
5643 let d = router.route_context_aware(
5644 "Say hello.",
5645 64,
5646 ®,
5647 &failed,
5648 false,
5649 false,
5650 RoutingWorkload::Fastest,
5651 );
5652 let found = d.candidates.iter().find(|c| c.model_id == strong);
5653 if cfg!(all(
5655 target_os = "macos",
5656 target_arch = "aarch64",
5657 not(car_skip_mlx)
5658 )) {
5659 assert!(found.is_some(), "{strong} is not a candidate here");
5660 }
5661 if let Some(c) = found {
5662 assert!(c.reliability < AdaptiveRouter::MIN_ANSWER_QUALITY, "{c:?}");
5663 assert!(c.in_band, "a model with failed calls was gated out: {c:?}");
5664 }
5665
5666 let mut alone = UnifiedRegistry::new_empty(tempfile::tempdir().unwrap().path().join("m"));
5668 let mut only = outcome_model("fast-but-failing", None);
5669 only.public_benchmarks = judged(0.054);
5670 alone.register_project_model(only);
5671 let d = router.route_context_aware(
5672 "Say hello.",
5673 64,
5674 &alone,
5675 &tracker,
5676 false,
5677 false,
5678 RoutingWorkload::Fastest,
5679 );
5680 assert_eq!(d.model_id, "fast-but-failing");
5681 }
5682
5683 #[test]
5686 fn a_deprecated_row_is_chosen_only_when_nothing_else_is() {
5687 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5688 let tracker = OutcomeTracker::new();
5689 let route = |reg: &UnifiedRegistry| {
5690 router.route_context_aware(
5691 "Say hello.",
5692 64,
5693 reg,
5694 &tracker,
5695 false,
5696 false,
5697 RoutingWorkload::Fastest,
5698 )
5699 };
5700 let mut reg = test_registry();
5701 let mut retired = outcome_model("retired-but-fast", None);
5702 retired.deprecated = true;
5703 retired.performance.tokens_per_second = Some(2_000.0);
5704 retired.performance.latency_p50_ms = Some(20);
5705 retired.public_benchmarks = vec![crate::schema::BenchmarkScore {
5706 name: "car-judged".into(),
5707 score: 1.0,
5708 harness: None,
5709 source_url: None,
5710 measured_at: None,
5711 runs: None,
5712 spread: None,
5713 }];
5714 reg.register_project_model(retired.clone());
5715 for _ in 0..50 {
5716 let d = route(®);
5717 assert_ne!(d.model_id, "retired-but-fast");
5718 assert!(!d
5719 .candidates
5720 .iter()
5721 .any(|c| c.model_id == "retired-but-fast"));
5722 }
5723
5724 let mut alone = UnifiedRegistry::new_empty(tempfile::tempdir().unwrap().path().join("m"));
5725 alone.register_project_model(retired);
5726 assert_eq!(route(&alone).model_id, "retired-but-fast");
5727 }
5728
5729 #[test]
5730 fn band_never_empties() {
5731 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5732 let tracker = OutcomeTracker::new();
5733 assert_eq!(
5735 router
5736 .outcome_first_band(
5737 vec![outcome_model("only", Some(0.5))],
5738 InferenceTask::Code,
5739 &tracker
5740 )
5741 .len(),
5742 1
5743 );
5744 let band = router.outcome_first_band(
5746 vec![
5747 outcome_model("x", Some(0.40)),
5748 outcome_model("y", Some(0.41)),
5749 ],
5750 InferenceTask::Code,
5751 &tracker,
5752 );
5753 assert!(!band.is_empty() && band.iter().any(|m| m.id == "y"));
5754 }
5755
5756 #[test]
5757 fn outcome_first_band_is_wired_into_selection_and_keeps_fallbacks() {
5758 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-route");
5765 unsafe {
5766 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5767 }
5768 let mut reg = UnifiedRegistry::new_empty(tmp);
5769 let mut strong = outcome_model("strong-expensive", Some(0.90));
5770 strong.cost = crate::schema::CostModel {
5771 input_per_mtok: Some(20.0),
5772 output_per_mtok: Some(20.0),
5773 ..Default::default()
5774 };
5775 let mut weak = outcome_model("weak-cheap", Some(0.80));
5776 weak.cost = crate::schema::CostModel {
5777 input_per_mtok: Some(0.1),
5778 output_per_mtok: Some(0.1),
5779 ..Default::default()
5780 };
5781 reg.register_project_model(strong);
5782 reg.register_project_model(weak);
5783
5784 let router = AdaptiveRouter::new(
5785 test_hw(),
5786 RoutingConfig {
5787 prior_strength: 100.0, ..Default::default()
5789 },
5790 );
5791 let tracker = OutcomeTracker::new();
5792 let decision = router.route(
5794 "Analyze the architecture trade-offs in depth",
5795 ®,
5796 &tracker,
5797 );
5798
5799 assert_eq!(
5800 decision.model_id, "strong-expensive",
5801 "band must select the higher-reliability model even though it's pricier"
5802 );
5803 assert!(
5804 decision.fallbacks.contains(&"weak-cheap".to_string()),
5805 "the out-of-band weaker model must remain as a fallback, got {:?}",
5806 decision.fallbacks
5807 );
5808 }
5809
5810 #[test]
5811 fn outcome_first_fallbacks_rank_in_band_before_out_of_band() {
5812 let tmp = std::path::PathBuf::from("/tmp/car-test-outcome-first-fb-order");
5816 unsafe {
5817 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
5818 }
5819 let mut reg = UnifiedRegistry::new_empty(tmp);
5820 reg.register_project_model(outcome_model("strong", Some(0.90)));
5821 let mut peer = outcome_model("peer-in-band", Some(0.89)); peer.cost = crate::schema::CostModel {
5823 input_per_mtok: Some(30.0),
5824 output_per_mtok: Some(30.0),
5825 ..Default::default()
5826 };
5827 reg.register_project_model(peer);
5828 let mut weak = outcome_model("weak-out-of-band", Some(0.70)); weak.cost = crate::schema::CostModel {
5830 input_per_mtok: Some(0.01),
5831 output_per_mtok: Some(0.01),
5832 ..Default::default()
5833 };
5834 reg.register_project_model(weak);
5835
5836 let router = AdaptiveRouter::new(
5837 test_hw(),
5838 RoutingConfig {
5839 prior_strength: 100.0,
5840 ..Default::default()
5841 },
5842 );
5843 let tracker = OutcomeTracker::new();
5844 let decision = router.route(
5845 "Analyze the architecture trade-offs in depth",
5846 ®,
5847 &tracker,
5848 );
5849
5850 assert!(
5852 decision.model_id == "strong" || decision.model_id == "peer-in-band",
5853 "primary must be from the band, got {}",
5854 decision.model_id
5855 );
5856 let pos = |id: &str| decision.fallbacks.iter().position(|x| x == id);
5857 let in_band_id = if decision.model_id == "strong" {
5858 "peer-in-band"
5859 } else {
5860 "strong"
5861 };
5862 let in_band_pos = pos(in_band_id).expect("in-band peer must be a fallback");
5863 let weak_pos = pos("weak-out-of-band").expect("weak must remain a fallback (availability)");
5864 assert!(
5865 in_band_pos < weak_pos,
5866 "in-band fallback ({in_band_id}@{in_band_pos}) must precede out-of-band weak (@{weak_pos})"
5867 );
5868 }
5869
5870 #[test]
5871 fn reliability_is_cost_free() {
5872 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5875 let tracker = OutcomeTracker::new();
5876 let mut cheap = outcome_model("cheap", Some(0.8));
5877 cheap.cost = crate::schema::CostModel {
5878 input_per_mtok: Some(0.1),
5879 output_per_mtok: Some(0.1),
5880 ..Default::default()
5881 };
5882 let mut pricey = outcome_model("pricey", Some(0.8));
5883 pricey.cost = crate::schema::CostModel {
5884 input_per_mtok: Some(50.0),
5885 output_per_mtok: Some(50.0),
5886 ..Default::default()
5887 };
5888 let rc = router.reliability(&cheap, InferenceTask::Code, &tracker);
5889 let rp = router.reliability(&pricey, InferenceTask::Code, &tracker);
5890 assert!(
5891 (rc - rp).abs() < 1e-9,
5892 "reliability must not depend on cost"
5893 );
5894 }
5895
5896 #[test]
5897 fn cold_start_with_no_sample_size_sits_at_schema_estimate() {
5898 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5903 let unmeasured = outcome_model("unmeasured", None); let schema = router.schema_quality_estimate(&unmeasured);
5905
5906 let mut tracker = OutcomeTracker::new();
5907 let mut p = crate::outcome::ModelProfile::new("unmeasured".into());
5908 p.ema_quality = 0.5; tracker.import_profiles(vec![p]);
5910
5911 let r = router.reliability(&unmeasured, InferenceTask::Code, &tracker);
5912 assert!(
5913 (r - schema).abs() < 1e-9,
5914 "no sample size ⇒ reliability == schema estimate ({schema}), got {r}"
5915 );
5916 }
5917
5918 #[test]
5919 fn sparse_seeded_prior_barely_perturbs_schema_anchor() {
5920 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5923 let benchmarked = outcome_model("frontier", None); let mut tracker = OutcomeTracker::new();
5927 let mut p = crate::outcome::ModelProfile::new("frontier".into());
5928 p.ema_quality = 1.0;
5929 p.prior_sample_size = 1;
5930 p.task_stats.insert(
5931 InferenceTask::Code.to_string(),
5932 crate::outcome::TaskStats {
5933 ema_quality: 1.0,
5934 prior_sample_size: 1,
5935 ..Default::default()
5936 },
5937 );
5938 tracker.import_profiles(vec![p]);
5939
5940 let r = router.reliability(&benchmarked, InferenceTask::Code, &tracker);
5941 assert!((r - 0.68).abs() < 0.01, "expected ~0.68, got {r}");
5943 assert!(
5944 r < 0.7,
5945 "a 1-case prior must not pull reliability near its raw 1.0, got {r}"
5946 );
5947 }
5948
5949 #[test]
5950 fn dense_seeded_prior_is_trusted_near_raw() {
5951 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5954 let m = outcome_model("dense", None); let mut tracker = OutcomeTracker::new();
5958 let mut p = crate::outcome::ModelProfile::new("dense".into());
5959 p.ema_quality = 0.95;
5960 p.prior_sample_size = 40;
5961 tracker.import_profiles(vec![p]);
5962
5963 let r = router.reliability(&m, InferenceTask::Code, &tracker);
5964 assert!(
5966 r > 0.88,
5967 "dense prior should be trusted near raw 0.95, got {r}"
5968 );
5969 }
5970
5971 #[test]
5972 fn warm_ungraded_successes_sit_at_schema_not_neutral_ema() {
5973 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
5978 let frontier = outcome_model("frontier", Some(0.55)); let schema = router.schema_quality_estimate(&frontier);
5980
5981 let mut tracker = OutcomeTracker::new();
5982 let mut p = crate::outcome::ModelProfile::new("frontier".into());
5983 p.total_calls = 9;
5984 p.success_count = 9; p.ema_quality = 0.5; p.quality_observations = 0;
5987 tracker.import_profiles(vec![p]);
5988
5989 let r = router.reliability(&frontier, InferenceTask::Code, &tracker);
5990 assert!(
5991 (r - schema).abs() < 1e-9,
5992 "ungraded successes ⇒ reliability == schema ({schema}), not neutral EMA; got {r}"
5993 );
5994 }
5995
5996 #[test]
5997 fn ungraded_task_stats_with_zero_ema_still_resolve_to_schema() {
5998 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6005 let m = outcome_model("mech-only", Some(0.55)); let schema = router.schema_quality_estimate(&m);
6007
6008 let mut tracker = OutcomeTracker::new();
6009 let mut p = crate::outcome::ModelProfile::new("mech-only".into());
6010 p.total_calls = 7;
6011 p.success_count = 7;
6012 p.task_stats.insert(
6015 InferenceTask::Code.to_string(),
6016 crate::outcome::TaskStats {
6017 calls: 7,
6018 successes: 7,
6019 ..Default::default()
6020 },
6021 );
6022 tracker.import_profiles(vec![p]);
6023
6024 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6025 assert!(
6026 (r - schema).abs() < 1e-9,
6027 "ungraded task stats (ema 0.0, e=0) must resolve to schema {schema}, got {r}"
6028 );
6029 }
6030
6031 #[test]
6032 fn graded_outcomes_earn_trust_in_the_live_ema() {
6033 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6036 let m = outcome_model("graded", Some(0.55)); let mut tracker = OutcomeTracker::new();
6039 let mut p = crate::outcome::ModelProfile::new("graded".into());
6040 p.total_calls = 30;
6041 p.ema_quality = 0.9;
6042 p.quality_observations = 30; tracker.import_profiles(vec![p]);
6044
6045 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6046 assert!(
6048 r > 0.83,
6049 "dense graded signal should trust the live EMA, got {r}"
6050 );
6051 }
6052
6053 #[test]
6054 fn live_grades_outweigh_benchmark_cases_of_equal_count() {
6055 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6061 let benched = outcome_model("benched", Some(0.7));
6062 let graded = outcome_model("graded", Some(0.7));
6063
6064 let mut tracker = OutcomeTracker::new();
6065 let mut pb = crate::outcome::ModelProfile::new("benched".into());
6066 pb.ema_quality = 0.2;
6067 pb.prior_sample_size = 4; let mut pg = crate::outcome::ModelProfile::new("graded".into());
6069 pg.ema_quality = 0.2;
6070 pg.total_calls = 4;
6071 pg.quality_observations = 4; tracker.import_profiles(vec![pb, pg]);
6073
6074 let r_bench = router.reliability(&benched, InferenceTask::Code, &tracker);
6075 let r_graded = router.reliability(&graded, InferenceTask::Code, &tracker);
6076 assert!(
6077 r_graded < r_bench,
6078 "4 live grades must distrust more than 4 benchmark cases: \
6079 graded={r_graded} bench={r_bench}"
6080 );
6081 }
6082
6083 #[test]
6084 fn graded_failures_count_as_evidence_and_pull_reliability_down() {
6085 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6089 let m = outcome_model("flaky", Some(0.8)); let schema = router.schema_quality_estimate(&m);
6091
6092 let mut tracker = OutcomeTracker::new();
6093 let mut p = crate::outcome::ModelProfile::new("flaky".into());
6094 p.total_calls = 12;
6095 p.fail_count = 12;
6096 p.ema_quality = 0.1; p.quality_observations = 12;
6098 tracker.import_profiles(vec![p]);
6099
6100 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6101 assert!(
6102 r < schema - 0.2,
6103 "12 graded failures must pull reliability well below schema {schema}, got {r}"
6104 );
6105 }
6106
6107 #[test]
6112 fn calls_that_fail_lower_reliability_without_touching_quality() {
6113 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6114 let failing = outcome_model("failing", Some(0.92));
6115 let answering = outcome_model("answering", Some(0.91));
6116 let mut tracker = OutcomeTracker::new();
6117 for _ in 0..3 {
6118 let trace = tracker.record_start("failing", InferenceTask::Code, "test");
6119 tracker.record_failure(&trace, "upstream returned an error");
6120 }
6121
6122 let quality = router.answer_quality(&failing, InferenceTask::Code, &tracker);
6123 assert!(
6124 (quality - router.schema_quality_estimate(&failing)).abs() < 1e-9,
6125 "failures moved answer quality to {quality}"
6126 );
6127 let failing_reliability = router.reliability(&failing, InferenceTask::Code, &tracker);
6128 assert!((failing_reliability - quality / 4.0).abs() < 1e-9);
6129 assert!(
6130 failing_reliability < router.reliability(&answering, InferenceTask::Code, &tracker),
6131 "a model that answers must outrank one that has failed every call"
6132 );
6133
6134 for _ in 0..30 {
6136 let trace = tracker.record_start("failing", InferenceTask::Code, "test");
6137 tracker.record_complete(&trace, 500, 100, 50);
6138 }
6139 let recovered = router.reliability(&failing, InferenceTask::Code, &tracker);
6140 assert!(recovered > 0.9 * quality, "{recovered} after 30 answers");
6141 }
6142
6143 #[test]
6144 fn external_grade_moves_reliability_of_a_model_with_no_history() {
6145 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6149 let m = outcome_model("graded-outside", Some(0.55));
6150 let schema = router.schema_quality_estimate(&m);
6151 let mut tracker = OutcomeTracker::new();
6152 tracker
6153 .record_external_grade("graded-outside", InferenceTask::Code, 0.9, 8, "neo")
6154 .unwrap();
6155
6156 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6157 let expected = schema + (8.0 / 12.0) * (0.9 - schema);
6158 assert!(
6159 (r - expected).abs() < 1e-9,
6160 "expected {expected} (schema {schema} shrunk toward 0.9 at 8/12), got {r}"
6161 );
6162 let other = router.reliability(&m, InferenceTask::Generate, &tracker);
6164 assert!((other - schema).abs() < 1e-9);
6165 }
6166
6167 #[test]
6168 fn external_grade_survives_a_restart_that_applies_benchmark_priors() {
6169 let dir = tempfile::tempdir().unwrap();
6173 let path = dir.path().join("outcome_profiles.json");
6174 let mut tracker = OutcomeTracker::new();
6175 tracker
6176 .record_external_grade("graded-outside", InferenceTask::Code, 0.9, 8, "neo")
6177 .unwrap();
6178 tracker
6179 .record_external_grade("graded-outside", InferenceTask::Reasoning, 0.1, 8, "neo")
6180 .unwrap();
6181 tracker.save_to_file(&path).unwrap();
6182
6183 let mut restarted = OutcomeTracker::new();
6184 restarted.load_from_file(&path).unwrap();
6185 let mut prior = crate::routing_ext::BenchmarkPrior {
6186 overall_score: 0.6,
6187 sample_size: 10,
6188 ..Default::default()
6189 };
6190 prior.task_scores.insert("code".into(), 0.6);
6191 prior.task_sample_size.insert("code".into(), 10);
6192 let priors = std::collections::HashMap::from([("graded-outside".to_string(), prior)]);
6193 crate::routing_ext::apply_benchmark_priors(&mut restarted, &priors);
6194
6195 let p = restarted.profile("graded-outside").unwrap();
6196 assert_eq!(p.prior_sample_size, 10, "the prior was applied");
6197 let code = p.task_stats(InferenceTask::Code).unwrap();
6198 assert_eq!(code.prior_sample_size, 10);
6199 assert_eq!(code.external_grades["neo"].quality, 0.9);
6200 let reasoning = p.task_stats(InferenceTask::Reasoning).unwrap();
6201 assert_eq!(reasoning.external_grades["neo"].quality, 0.1);
6202
6203 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6206 let m = outcome_model("graded-outside", None);
6208 let schema = router.schema_quality_estimate(&m);
6209 let r = router.reliability(&m, InferenceTask::Code, &restarted);
6210 let observed = (10.0 * 0.6 + 8.0 * 0.9) / 18.0;
6211 let expected = schema + (18.0 / 22.0) * (observed - schema);
6212 assert!((r - expected).abs() < 1e-9, "expected {expected}, got {r}");
6213 }
6214
6215 #[test]
6216 fn external_grade_pools_with_served_evidence_instead_of_replacing_it() {
6217 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6218 let m = outcome_model("served", Some(0.55));
6219 let schema = router.schema_quality_estimate(&m);
6220 let mut tracker = OutcomeTracker::new();
6221 let mut p = crate::outcome::ModelProfile::new("served".into());
6222 p.total_calls = 10;
6223 p.ema_quality = 0.2;
6224 p.quality_observations = 10; tracker.import_profiles(vec![p]);
6226 tracker
6227 .record_external_grade("served", InferenceTask::Code, 1.0, 4, "neo")
6228 .unwrap();
6229
6230 let r = router.reliability(&m, InferenceTask::Code, &tracker);
6231 let observed = (20.0 * 0.2 + 4.0 * 1.0) / 24.0;
6233 let expected = schema + (24.0 / 28.0) * (observed - schema);
6234 assert!(
6235 (r - expected).abs() < 1e-9,
6236 "expected pooled {expected}, got {r}"
6237 );
6238 }
6239
6240 #[test]
6241 fn external_grade_reaches_route_candidates_and_their_ranking() {
6242 let _environment = crate::openrouter::test_environment_scope();
6243 let tmp = std::path::PathBuf::from("/tmp/car-test-route-external-grade");
6244 unsafe {
6245 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
6246 }
6247 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
6248 reg.register_project_model(outcome_model("strong", Some(0.90)));
6249 reg.register_project_model(outcome_model("runner-up", Some(0.70)));
6250 let router = AdaptiveRouter::new(
6251 test_hw(),
6252 RoutingConfig {
6253 prior_strength: 100.0,
6254 ..Default::default()
6255 },
6256 );
6257 let prompt = "Analyze the architecture trade-offs in depth";
6258 let mut tracker = OutcomeTracker::new();
6259 let before = router.route(prompt, ®, &tracker);
6260 assert!(before
6261 .candidates
6262 .iter()
6263 .all(|c| c.external_grades.is_empty()));
6264 let reliability = |d: &AdaptiveRoutingDecision, id: &str| {
6265 d.candidates
6266 .iter()
6267 .find(|c| c.model_id == id)
6268 .unwrap_or_else(|| panic!("{id} is a candidate: {d:?}"))
6269 .reliability
6270 };
6271 assert!(reliability(&before, "strong") > reliability(&before, "runner-up"));
6272
6273 tracker
6275 .record_external_grade("strong", before.task, 0.0, 8, "neo")
6276 .unwrap();
6277 let after = router.route(prompt, ®, &tracker);
6278 let strong = after
6279 .candidates
6280 .iter()
6281 .find(|c| c.model_id == "strong")
6282 .expect("strong is still a candidate");
6283 assert_eq!(
6284 strong.external_grades,
6285 vec![RouteExternalGrade {
6286 source: "neo".into(),
6287 quality: 0.0,
6288 observations: 8,
6289 evidence: 8,
6290 }]
6291 );
6292 assert!(strong.reliability < reliability(&before, "strong") - 0.3);
6293 assert!(
6294 reliability(&after, "strong") < reliability(&after, "runner-up"),
6295 "the outside grade reorders the two models"
6296 );
6297 assert_ne!(
6298 after.model_id, "strong",
6299 "a model that lost every case is not chosen"
6300 );
6301 let wire = serde_json::to_value(&after).unwrap();
6302 assert!(wire["candidates"]
6303 .as_array()
6304 .unwrap()
6305 .iter()
6306 .any(|c| c["external_grades"][0]["source"] == "neo"));
6307 }
6308
6309 #[test]
6310 fn route_exposes_ranked_advisory_candidates() {
6311 let _environment = crate::openrouter::test_environment_scope();
6316 let tmp = std::path::PathBuf::from("/tmp/car-test-route-candidates");
6322 unsafe {
6323 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
6324 }
6325 let mut reg = UnifiedRegistry::new_with_state_root(tmp.clone(), tmp);
6326 reg.register_project_model(outcome_model("strong", Some(0.90)));
6327 let mut weak = outcome_model("weak-cheap", Some(0.70)); weak.cost = crate::schema::CostModel {
6329 input_per_mtok: Some(0.01),
6330 output_per_mtok: Some(0.01),
6331 ..Default::default()
6332 };
6333 reg.register_project_model(weak);
6334
6335 let router = AdaptiveRouter::new(
6336 test_hw(),
6337 RoutingConfig {
6339 prior_strength: 100.0,
6340 ..Default::default()
6341 },
6342 );
6343 let tracker = OutcomeTracker::new();
6344 let decision = router.route(
6345 "Analyze the architecture trade-offs in depth",
6346 ®,
6347 &tracker,
6348 );
6349
6350 assert!(
6352 decision.candidates.iter().any(|c| c.model_id == "strong"),
6353 "strong must appear as a candidate, got {:?}",
6354 decision.candidates
6355 );
6356 assert!(
6357 decision
6358 .candidates
6359 .iter()
6360 .any(|c| c.model_id == "weak-cheap"),
6361 "weak-cheap must appear as a candidate even though out of band"
6362 );
6363
6364 let selected: Vec<&RouteCandidate> =
6366 decision.candidates.iter().filter(|c| c.selected).collect();
6367 assert_eq!(selected.len(), 1, "exactly one candidate may be selected");
6368 assert_eq!(
6369 selected[0].model_id, decision.model_id,
6370 "selected flag must match model_id"
6371 );
6372
6373 let strong = decision
6375 .candidates
6376 .iter()
6377 .find(|c| c.model_id == "strong")
6378 .unwrap();
6379 let weak_c = decision
6380 .candidates
6381 .iter()
6382 .find(|c| c.model_id == "weak-cheap")
6383 .unwrap();
6384 assert!(strong.in_band, "strong must be in the reliability band");
6385 assert!(
6386 !weak_c.in_band,
6387 "weak-cheap (0.20 below) must be out of band"
6388 );
6389 assert!(
6391 strong.reliability > weak_c.reliability,
6392 "reliability must reflect quality, not the weak model's lower cost"
6393 );
6394 }
6395
6396 #[test]
6397 fn prefer_quality_selection_is_deterministic_no_exploration() {
6398 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6403 let reg = test_registry();
6404 let tracker = OutcomeTracker::new();
6405 let intent = crate::intent::IntentHint {
6406 task: Some(crate::intent::TaskHint::Code),
6407 require: vec![ModelCapability::Code],
6408 prefer_quality: true,
6409 ..Default::default()
6410 };
6411 let first = router.route_with_intent("write a function", ®, &tracker, &intent);
6412 for _ in 0..8 {
6413 let again = router.route_with_intent("write a function", ®, &tracker, &intent);
6414 assert_eq!(
6415 again.model_id, first.model_id,
6416 "prefer_quality must be deterministic (no Thompson exploration)"
6417 );
6418 }
6419 let chosen = reg
6421 .find_by_name(&first.model_name)
6422 .or_else(|| reg.list().into_iter().find(|m| m.id == first.model_id));
6423 if let Some(m) = chosen {
6424 assert!(m.has_capability(ModelCapability::Code));
6425 }
6426 }
6427
6428 fn remote_model(id: &str, name: &str, caps: Vec<ModelCapability>) -> ModelSchema {
6430 ModelSchema {
6431 id: id.into(),
6432 name: name.into(),
6433 provider: "openai".into(),
6434 family: name.into(),
6435 version: "latest".into(),
6436 capabilities: caps,
6437 context_length: 128_000,
6438 max_output_tokens: None,
6439 param_count: "api".into(),
6440 quantization: None,
6441 performance: Default::default(),
6442 cost: Default::default(),
6443 source: crate::schema::ModelSource::RemoteApi {
6444 endpoint: "https://api.openai.com/v1".into(),
6445 api_key_env: "OPENAI_API_KEY".into(),
6446 api_key_envs: vec![],
6447 api_version: None,
6448 protocol: crate::schema::ApiProtocol::OpenAiCompat,
6449 },
6450 tags: vec!["trusted-remote".into()],
6451 supported_params: vec![],
6452 public_benchmarks: vec![],
6453 trust_tier: crate::schema::TrustTier::Curated,
6454 deprecated: false,
6455 available: true,
6456 weights_ready: true,
6457 }
6458 }
6459
6460 #[test]
6461 fn code_intent_filters_generate_only_models_on_simple_prompt() {
6462 let _environment = crate::openrouter::test_environment_scope();
6467 unsafe {
6474 std::env::set_var("OPENAI_API_KEY", "test-openai-key");
6475 }
6476 let intent_filter_dir = std::path::PathBuf::from("/tmp/car-test-code-intent-filter");
6477 let mut reg =
6478 UnifiedRegistry::new_with_state_root(intent_filter_dir.clone(), intent_filter_dir);
6479 reg.register_project_model(remote_model(
6480 "openai/gen-only:latest",
6481 "gen-only",
6482 vec![ModelCapability::Generate],
6483 ));
6484 reg.register_project_model(remote_model(
6485 "openai/coder:latest",
6486 "coder",
6487 vec![
6488 ModelCapability::Generate,
6489 ModelCapability::Code,
6490 ModelCapability::Reasoning,
6491 ],
6492 ));
6493
6494 let router = AdaptiveRouter::new(
6495 test_hw(),
6496 RoutingConfig {
6497 prior_strength: 100.0,
6498 ..Default::default()
6499 },
6500 );
6501 let tracker = OutcomeTracker::new();
6502 let intent = crate::intent::IntentHint {
6503 task: Some(crate::intent::TaskHint::Code),
6504 prefer_quality: true,
6505 ..Default::default() };
6507
6508 let decision = router.route_with_intent(
6510 "Give me a short two-sentence overview.",
6511 ®,
6512 &tracker,
6513 &intent,
6514 );
6515
6516 assert_eq!(
6517 decision.task,
6518 InferenceTask::Code,
6519 "explicit task=code must be honored"
6520 );
6521 assert!(
6524 !decision
6525 .candidates
6526 .iter()
6527 .any(|c| c.model_id == "openai/gen-only:latest"),
6528 "generate-only model must be filtered out for task=code; candidates: {:?}",
6529 decision
6530 .candidates
6531 .iter()
6532 .map(|c| &c.model_id)
6533 .collect::<Vec<_>>()
6534 );
6535 let schema = reg
6536 .find_by_name(&decision.model_name)
6537 .or_else(|| reg.list().into_iter().find(|m| m.id == decision.model_id))
6538 .expect("selected model exists");
6539 assert!(
6540 schema.has_capability(ModelCapability::Code),
6541 "task=code on a simple prompt must route to a code-capable model, got {}",
6542 decision.model_name,
6543 );
6544 }
6545
6546 #[test]
6547 fn intent_default_does_not_override_task_or_caps() {
6548 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6555 let reg = test_registry();
6556 let tracker = OutcomeTracker::new();
6557
6558 let baseline = router.route("write a haiku", ®, &tracker);
6559 let with_default = router.route_with_intent(
6560 "write a haiku",
6561 ®,
6562 &tracker,
6563 &crate::intent::IntentHint::default(),
6564 );
6565
6566 assert_eq!(
6567 baseline.task, with_default.task,
6568 "default IntentHint must not change the prompt-derived task"
6569 );
6570 assert_eq!(
6571 baseline.complexity, with_default.complexity,
6572 "default IntentHint must not change the prompt-derived complexity"
6573 );
6574 }
6575
6576 #[test]
6587 fn require_ready_skips_models_that_are_not_on_disk() {
6588 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6589 let reg = test_registry();
6590 let tracker = OutcomeTracker::new();
6591
6592 let ready: std::collections::HashSet<String> = reg
6594 .list()
6595 .into_iter()
6596 .filter(|m| m.weights_ready)
6597 .map(|m| m.id.clone())
6598 .collect();
6599 assert!(!ready.is_empty(), "fixture should have ready models");
6600
6601 let hint = crate::intent::IntentHint {
6602 require_ready: true,
6603 ..Default::default()
6604 };
6605 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
6606 assert!(
6607 ready.contains(&decision.model_id),
6608 "require_ready picked {} which is not on disk; ready set: {:?}",
6609 decision.model_id,
6610 ready
6611 );
6612 }
6613
6614 #[test]
6617 fn require_ready_falls_back_when_nothing_is_ready() {
6618 let _environment = crate::openrouter::test_environment_scope();
6623 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6624 let tracker = OutcomeTracker::new();
6625 let empty_dir = std::env::temp_dir().join("car-test-require-ready-empty");
6627 let _ = std::fs::create_dir_all(&empty_dir);
6628 let reg = UnifiedRegistry::new_with_state_root(empty_dir.clone(), empty_dir);
6629
6630 let hint = crate::intent::IntentHint {
6631 require_ready: true,
6632 ..Default::default()
6633 };
6634 let decision = router.route_with_intent("write a python function", ®, &tracker, &hint);
6635 assert!(
6636 !decision.model_id.is_empty(),
6637 "soft constraint must still yield a model when nothing is ready"
6638 );
6639 }
6640
6641 #[test]
6642 fn exclude_models_picks_a_different_model() {
6643 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6648 let reg = test_registry();
6649 let tracker = OutcomeTracker::new();
6650 let base = crate::intent::IntentHint {
6651 prefer_quality: true,
6652 ..Default::default()
6653 };
6654 let author = router.route_with_intent("review this", ®, &tracker, &base);
6655
6656 let with_exclude = crate::intent::IntentHint {
6657 prefer_quality: true,
6658 exclude_models: vec![author.model_id.clone()],
6659 ..Default::default()
6660 };
6661 let reviewer = router.route_with_intent("review this", ®, &tracker, &with_exclude);
6662 assert_ne!(
6663 reviewer.model_id, author.model_id,
6664 "an excluded model must never be the routed pick"
6665 );
6666 assert!(
6669 !reviewer.fallbacks.contains(&author.model_id),
6670 "excluded model leaked into fallbacks"
6671 );
6672 assert!(
6673 !reviewer
6674 .candidates
6675 .iter()
6676 .any(|c| c.model_id == author.model_id),
6677 "excluded model leaked into candidates[]"
6678 );
6679 }
6680
6681 #[test]
6684 fn on_device_only_picks_only_a_model_on_this_machine() {
6685 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6686 let reg = test_registry();
6687 let tracker = OutcomeTracker::new();
6688 assert!(
6689 reg.list().iter().any(|model| !model.is_local()),
6690 "fixture lists a remote model"
6691 );
6692 let decision = router.route_with_intent(
6693 "write a function",
6694 ®,
6695 &tracker,
6696 &crate::intent::IntentHint {
6697 on_device_only: true,
6698 ..Default::default()
6699 },
6700 );
6701 for id in std::iter::once(&decision.model_id).chain(decision.fallbacks.iter()) {
6702 if id.is_empty() {
6703 continue;
6704 }
6705 let schema = reg.get(id).or_else(|| reg.find_by_name(id));
6706 assert!(
6707 schema.is_some_and(|schema| schema.is_local()),
6708 "{id}: {decision:?}"
6709 );
6710 }
6711 }
6712
6713 #[test]
6717 fn on_device_only_with_nothing_on_this_machine_is_no_route() {
6718 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6719 let mut reg = test_registry();
6720 let local: Vec<String> = reg
6721 .registry
6722 .all()
6723 .filter(|model| model.is_local())
6724 .map(|model| model.id.clone())
6725 .collect();
6726 for id in local {
6727 reg.registry.unregister(&id);
6728 }
6729 let tracker = OutcomeTracker::new();
6730 let decision = router.route_with_intent(
6731 "write a function",
6732 ®,
6733 &tracker,
6734 &crate::intent::IntentHint {
6735 on_device_only: true,
6736 ..Default::default()
6737 },
6738 );
6739 assert!(decision.model_id.is_empty(), "{decision:?}");
6740 assert!(decision.fallbacks.is_empty(), "{decision:?}");
6741 assert!(decision.reason.contains("on_device_only"), "{decision:?}");
6742 let ordinary = router.route_with_intent(
6744 "write a function",
6745 ®,
6746 &tracker,
6747 &crate::intent::IntentHint::default(),
6748 );
6749 assert!(!ordinary.model_id.is_empty(), "{ordinary:?}");
6750 }
6751
6752 #[test]
6753 fn strict_exclusions_refuse_when_every_candidate_is_a_panel_seat() {
6754 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6755 let reg = test_registry();
6756 let tracker = OutcomeTracker::new();
6757 let panel: Vec<String> = reg
6758 .list()
6759 .into_iter()
6760 .map(|model| model.name.clone())
6761 .collect();
6762 assert!(!panel.is_empty(), "fixture must contain candidate models");
6763
6764 let decision = router.route_with_intent(
6765 "repair this code",
6766 ®,
6767 &tracker,
6768 &crate::intent::IntentHint {
6769 task: Some(crate::intent::TaskHint::Code),
6770 exclude_models: panel,
6771 strict_exclusions: true,
6772 ..Default::default()
6773 },
6774 );
6775
6776 assert!(decision.model_id.is_empty(), "{decision:?}");
6777 assert!(decision.model_name.is_empty(), "{decision:?}");
6778 assert!(decision.fallbacks.is_empty(), "{decision:?}");
6779 assert!(
6780 decision.reason.contains("no eligible model"),
6781 "the refusal must tell the operator what is missing: {decision:?}"
6782 );
6783 }
6784
6785 #[test]
6786 fn exclude_models_accepts_the_name_a_result_reports() {
6787 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6795 let reg = test_registry();
6796 let tracker = OutcomeTracker::new();
6797 let base = crate::intent::IntentHint {
6798 prefer_quality: true,
6799 ..Default::default()
6800 };
6801 let author = router.route_with_intent("write a python function", ®, &tracker, &base);
6802 let author_name = reg
6803 .get(&author.model_id)
6804 .expect("the routed pick is in the registry")
6805 .name
6806 .clone();
6807 assert_ne!(
6808 author_name, author.model_id,
6809 "this test is only meaningful on a model whose name and id differ"
6810 );
6811
6812 let by_name = crate::intent::IntentHint {
6813 prefer_quality: true,
6814 exclude_models: vec![author_name.clone()],
6815 ..Default::default()
6816 };
6817 let rerouted =
6818 router.route_with_intent("write a python function", ®, &tracker, &by_name);
6819 assert_ne!(
6820 rerouted.model_id, author.model_id,
6821 "excluding by the name a result reports ({author_name}) must route elsewhere"
6822 );
6823 assert!(
6824 !rerouted.fallbacks.contains(&author.model_id),
6825 "name-excluded model leaked into fallbacks"
6826 );
6827 }
6828
6829 #[test]
6830 fn cold_start_honors_exclusion() {
6831 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6836 let reg = test_registry();
6837 let caps = [ModelCapability::Generate];
6838 let base = router.cold_start_decision(
6839 TaskComplexity::Simple,
6840 InferenceTask::Generate,
6841 &caps,
6842 ®,
6843 false,
6844 &std::collections::HashSet::new(),
6845 false,
6846 AvailabilitySnapshot::capture(®),
6847 );
6848 let mut exclude = std::collections::HashSet::new();
6849 exclude.insert(base.model_id.clone());
6850 let excluded = router.cold_start_decision(
6851 TaskComplexity::Simple,
6852 InferenceTask::Generate,
6853 &caps,
6854 ®,
6855 false,
6856 &exclude,
6857 false,
6858 AvailabilitySnapshot::capture(®),
6859 );
6860 let other_generate_exists = reg
6864 .list()
6865 .into_iter()
6866 .filter(|m| m.available && m.has_capability(ModelCapability::Generate))
6867 .any(|m| m.id != base.model_id);
6868 if other_generate_exists {
6869 assert_ne!(
6870 excluded.model_id, base.model_id,
6871 "cold start re-picked the excluded model despite an alternative"
6872 );
6873 }
6874 }
6875
6876 #[test]
6877 fn exclude_all_models_falls_back_to_full_set() {
6878 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6881 let reg = test_registry();
6882 let tracker = OutcomeTracker::new();
6883 let exclude_everything: Vec<String> =
6884 reg.list().into_iter().map(|m| m.id.clone()).collect();
6885 let hint = crate::intent::IntentHint {
6886 exclude_models: exclude_everything,
6887 ..Default::default()
6888 };
6889 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
6890 assert!(
6891 !decision.model_id.is_empty(),
6892 "excluding everything must still yield a routed model (fallback)"
6893 );
6894 }
6895
6896 #[test]
6897 fn intent_task_hint_overrides_prompt_complexity() {
6898 let router = AdaptiveRouter::new(test_hw(), RoutingConfig::default());
6901 let reg = test_registry();
6902 let tracker = OutcomeTracker::new();
6903
6904 let hint = crate::intent::IntentHint {
6905 task: Some(crate::intent::TaskHint::Reasoning),
6906 ..Default::default()
6907 };
6908 let decision = router.route_with_intent("hi", ®, &tracker, &hint);
6909
6910 assert_eq!(
6911 decision.task,
6912 InferenceTask::Reasoning,
6913 "TaskHint::Reasoning should override the prompt-derived task"
6914 );
6915 }
6916
6917 #[cfg(all(target_os = "macos", target_arch = "aarch64", not(car_skip_mlx)))]
6926 #[test]
6927 fn router_proposes_mlx_twin_not_pre_redirect_gguf() {
6928 let gguf_id = "qwen/qwen3-8b:q4_k_m";
6929 let mlx_id = "mlx/qwen3-8b:4bit";
6930
6931 let reg = test_registry();
6932 let gguf = reg.get(gguf_id).expect("gguf in builtin catalog").clone();
6935 assert_eq!(
6936 reg.resolve_mlx_equivalent(&gguf).map(|m| m.id.as_str()),
6937 Some(mlx_id),
6938 "GGUF must resolve to its SAME-SIZE MLX twin"
6939 );
6940
6941 let router = AdaptiveRouter::new(
6942 test_hw(),
6943 RoutingConfig {
6944 min_observations: 3,
6945 ..Default::default()
6946 },
6947 );
6948 let mut tracker = OutcomeTracker::new();
6949 for _ in 0..20 {
6951 let t = tracker.record_start(mlx_id, InferenceTask::Code, "test");
6952 tracker.record_complete(&t, 500, 100, 50);
6953 tracker.record_inferred_outcome(&t, InferredOutcome::Accepted { confidence: 0.95 });
6954 }
6955
6956 let mut mlx_wins = 0;
6957 for _ in 0..20 {
6958 let d = router.route("Fix this bug in the parser", ®, &tracker);
6959 assert_ne!(
6962 d.model_id, gguf_id,
6963 "router proposed the pre-redirect GGUF id"
6964 );
6965 assert!(
6966 !d.fallbacks.iter().any(|f| f == gguf_id),
6967 "pre-redirect GGUF id leaked into the fallback chain"
6968 );
6969 if d.model_id == mlx_id {
6970 mlx_wins += 1;
6971 }
6972 }
6973 assert!(
6974 mlx_wins >= 12,
6975 "proven MLX twin should win the majority (the router reads the id it \
6976 learns/runs); won only {mlx_wins}/20"
6977 );
6978 }
6979
6980 fn m5_pro_64gb() -> HardwareInfo {
6987 HardwareInfo {
6988 os: "macos".into(),
6989 arch: "aarch64".into(),
6990 cpu_cores: 14,
6991 total_ram_mb: 64 * 1024,
6992 gpu_backend: crate::hardware::GpuBackend::Metal,
6993 gpu_memory_mb: None,
6994 gpu_devices: Vec::new(),
6995 recommended_model: "fixture".into(),
6996 recommended_context: 8192,
6997 max_model_mb: 64 * 1024,
6998 }
6999 }
7000
7001 const CLOUD_KEYS: [&str; 3] = ["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY"];
7002
7003 fn scenario_registry(keys: &[&str]) -> TestRegistry {
7006 let environment = crate::openrouter::test_environment_scope();
7007 let mut names = CLOUD_KEYS.to_vec();
7008 names.extend([
7009 "CAR_SECRETS_FILE_DIR",
7010 "OPENROUTER_API_KEY",
7011 "GOOGLE_API_KEY",
7012 "CAR_CODEX_BIN",
7013 ]);
7014 let restore = RestoredEnvironment::capture(&names);
7015 let root = std::env::temp_dir().join(format!(
7016 "car-routing-scenarios-{}",
7017 keys.join("-").to_ascii_lowercase()
7018 ));
7019 unsafe {
7020 for name in &names {
7021 std::env::remove_var(name);
7022 }
7023 std::env::set_var("CAR_SECRETS_FILE_DIR", root.join("empty-secrets"));
7024 std::env::set_var("CAR_CODEX_BIN", root.join("no-codex"));
7027 for key in keys {
7028 std::env::set_var(key, "scenario-key");
7029 }
7030 }
7031 let _ = std::fs::create_dir_all(&root);
7032 let registry =
7033 UnifiedRegistry::new_with_state_root(root.join("models"), root.join("state"));
7034 TestRegistry {
7035 registry,
7036 _restore: restore,
7037 _environment: environment,
7038 }
7039 }
7040
7041 struct Scenario {
7042 name: &'static str,
7043 prompt: &'static str,
7044 workload: RoutingWorkload,
7045 intent: crate::intent::IntentHint,
7046 has_tools: bool,
7047 }
7048
7049 fn scenarios() -> Vec<Scenario> {
7050 use crate::intent::{IntentHint, TaskHint};
7051 vec![
7052 Scenario {
7053 name: "chat, interactive",
7054 prompt: "What's a good name for a golden retriever?",
7055 workload: RoutingWorkload::Interactive,
7056 intent: IntentHint::default(),
7057 has_tools: false,
7058 },
7059 Scenario {
7060 name: "code, interactive",
7061 prompt: "Write a Python function that merges two sorted lists.",
7062 workload: RoutingWorkload::Interactive,
7063 intent: IntentHint::default(),
7064 has_tools: false,
7065 },
7066 Scenario {
7067 name: "agent turn with tools, interactive",
7068 prompt: "Read notes.txt and summarize it.",
7069 workload: RoutingWorkload::Interactive,
7070 intent: IntentHint::default(),
7071 has_tools: true,
7072 },
7073 Scenario {
7074 name: "code, prefer_local",
7075 prompt: "Write a Python function that merges two sorted lists.",
7076 workload: RoutingWorkload::Interactive,
7077 intent: IntentHint {
7078 prefer_local: true,
7079 ..Default::default()
7080 },
7081 has_tools: false,
7082 },
7083 Scenario {
7084 name: "agent turn with tools, prefer_local",
7085 prompt: "Read notes.txt and summarize it.",
7086 workload: RoutingWorkload::Interactive,
7087 intent: IntentHint {
7088 prefer_local: true,
7089 ..Default::default()
7090 },
7091 has_tools: true,
7092 },
7093 Scenario {
7094 name: "voice reply, prefer_fast",
7095 prompt: "What time is it in Tokyo?",
7096 workload: RoutingWorkload::Interactive,
7097 intent: IntentHint {
7098 prefer_fast: true,
7099 task: Some(TaskHint::Chat),
7100 ..Default::default()
7101 },
7102 has_tools: false,
7103 },
7104 Scenario {
7105 name: "hard reasoning, prefer_quality",
7106 prompt: "Prove that the halting problem is undecidable, step by step.",
7107 workload: RoutingWorkload::Interactive,
7108 intent: IntentHint {
7109 prefer_quality: true,
7110 task: Some(TaskHint::Reasoning),
7111 ..Default::default()
7112 },
7113 has_tools: false,
7114 },
7115 Scenario {
7116 name: "summary, background",
7117 prompt: "Summarize this meeting transcript in three bullets.",
7118 workload: RoutingWorkload::Background,
7119 intent: IntentHint::default(),
7120 has_tools: false,
7121 },
7122 ]
7123 }
7124
7125 fn route_scenario(
7126 router: &AdaptiveRouter,
7127 registry: &UnifiedRegistry,
7128 tracker: &OutcomeTracker,
7129 scenario: &Scenario,
7130 ) -> AdaptiveRoutingDecision {
7131 router.route_with(RouteRequest {
7132 intent: Some(&scenario.intent),
7133 has_tools: scenario.has_tools,
7134 workload: scenario.workload,
7135 ..RouteRequest::new(scenario.prompt, registry, tracker)
7136 })
7137 }
7138
7139 const SCENARIO_KEY_SETS: [&[&str]; 4] = [
7140 &[],
7141 &["OPENAI_API_KEY"],
7142 &["ANTHROPIC_API_KEY"],
7143 &["OPENAI_API_KEY", "ANTHROPIC_API_KEY", "GEMINI_API_KEY"],
7144 ];
7145
7146 #[test]
7153 fn a_stated_local_or_fast_preference_keeps_local_candidates_under_every_key_set() {
7154 let frontier_grade_locals = cfg!(all(
7155 target_os = "macos",
7156 target_arch = "aarch64",
7157 not(car_skip_mlx)
7158 ));
7159 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7160 for keys in SCENARIO_KEY_SETS {
7161 let registry = scenario_registry(keys);
7162 let tracker = OutcomeTracker::new();
7163 for scenario in scenarios()
7164 .iter()
7165 .filter(|s| s.intent.prefer_local || s.intent.prefer_fast)
7166 {
7167 for _ in 0..10 {
7169 let decision = route_scenario(&router, ®istry, &tracker, scenario);
7170 let is_local = |id: &String| registry.get(id).is_some_and(|m| m.is_local());
7171 let picked_local = is_local(&decision.model_id);
7172 assert!(
7173 picked_local || decision.fallbacks.iter().any(is_local),
7174 "{} with keys {keys:?} had no local candidate: {} ({})",
7175 scenario.name,
7176 decision.model_id,
7177 decision.reason
7178 );
7179 assert!(
7180 picked_local || !frontier_grade_locals,
7181 "{} with keys {keys:?} routed to {} ({})",
7182 scenario.name,
7183 decision.model_id,
7184 decision.reason
7185 );
7186 }
7187 }
7188 }
7189 }
7190
7191 #[test]
7197 fn a_tool_turn_always_goes_to_a_model_that_can_call_tools() {
7198 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7199 for keys in SCENARIO_KEY_SETS {
7200 let registry = scenario_registry(keys);
7201 let tracker = OutcomeTracker::new();
7202 for scenario in scenarios().iter().filter(|s| s.has_tools) {
7203 for _ in 0..10 {
7204 let decision = route_scenario(&router, ®istry, &tracker, scenario);
7205 let model = registry.get(&decision.model_id).expect("a routed model");
7206 assert!(
7207 model.has_capability(ModelCapability::ToolUse),
7208 "{} with keys {keys:?} routed to {}, which cannot call tools",
7209 scenario.name,
7210 decision.model_id
7211 );
7212 }
7213 }
7214 }
7215 }
7216
7217 #[test]
7221 fn the_quality_lane_follows_the_measured_scores_with_no_keys() {
7222 let car_judged = |m: &ModelSchema| {
7223 m.public_benchmarks
7224 .iter()
7225 .find(|b| b.name == "car-judged")
7226 .map(|b| b.score)
7227 };
7228 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7229 let registry = scenario_registry(&[]);
7230 let tracker = OutcomeTracker::new();
7231 let mut compared = 0;
7232 for scenario in scenarios().iter().filter(|s| s.intent.prefer_quality) {
7233 let decision = route_scenario(&router, ®istry, &tracker, scenario);
7234 let picked = registry.get(&decision.model_id).expect("a routed model");
7235 let picked_score = car_judged(picked).unwrap_or(0.0);
7236 for id in &decision.fallbacks {
7237 let Some(alt) = registry.get(id).filter(|m| m.is_local()) else {
7238 continue;
7239 };
7240 if let Some(score) = car_judged(alt) {
7241 compared += 1;
7242 assert!(
7243 picked_score + 1e-9 >= score,
7244 "{}: picked {} (car-judged {picked_score:.3}) over {} ({score:.3})",
7245 scenario.name,
7246 decision.model_id,
7247 id
7248 );
7249 }
7250 }
7251 }
7252 assert!(compared > 0, "no measured local alternative was compared");
7253 }
7254
7255 #[test]
7258 #[ignore = "report: prints routing picks, asserts nothing"]
7259 fn routing_scenario_report() {
7260 let router = AdaptiveRouter::with_default_config(m5_pro_64gb());
7261 for keys in SCENARIO_KEY_SETS {
7262 let registry = scenario_registry(keys);
7263 let tracker = OutcomeTracker::new();
7264 println!("== keys: {keys:?}");
7265 for scenario in scenarios() {
7266 let decision = route_scenario(&router, ®istry, &tracker, &scenario);
7267 let local = registry
7268 .get(&decision.model_id)
7269 .is_some_and(|m| m.is_local());
7270 println!(
7271 " {:<38} -> {:<40} {} | {}",
7272 scenario.name,
7273 decision.model_id,
7274 if local { "local " } else { "remote" },
7275 decision.reason.chars().take(110).collect::<String>()
7276 );
7277 }
7278 }
7279 }
7280}