Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22use crate::backends::ExternalTask;
23use crate::cmd_cache;
24use crate::engine::cooldown;
25use crate::engine::cooldown::{
26    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
27    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
28};
29use crate::engine::runner::agents::opencode;
30use crate::engine::runner::response;
31use crate::store::WaitingOnDependency;
32use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
33pub use config::{RouterConfig, parse_pool_entry};
34use llm::{LlmRouter, TIMEOUT_PREFIX};
35use serde::{Deserialize, Serialize};
36use std::fmt;
37use std::fmt::Debug;
38use std::fmt::Formatter;
39use std::sync::Arc;
40use tokio::task;
41pub use weights::AgentWeights;
42
43fn fmt_degraded(degraded: &[String]) -> String {
44    if degraded.is_empty() {
45        String::new()
46    } else {
47        format!(" (degraded: {})", degraded.join(", "))
48    }
49}
50
51/// A turn-cap error comes from the router command line (`--max-turns 1`)
52fn is_max_turns_error(err_msg: &str) -> bool {
53    let lower = err_msg.to_ascii_lowercase();
54    lower.contains("max turns reached") || lower.contains("reached the maximum number of turns")
55}
56
57/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
58///
59/// This is a transient error that should not count against `max_route_attempts` since
60/// no agent was actually invoked. The caller should back off and retry later.
61#[derive(Debug, thiserror::Error)]
62pub enum AllAgentsCooledError {
63    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
64    AllCooled {
65        scope: String,
66        remaining_secs: Option<u64>,
67        degraded: Vec<String>,
68    },
69}
70
71impl AllAgentsCooledError {
72    /// Create a new error for the given scope, naming currently degraded agents
73    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
74        Self::AllCooled {
75            scope,
76            remaining_secs,
77            degraded: list_degraded_agents()
78                .into_iter()
79                .map(|(agent, _reason)| agent)
80                .collect(),
81        }
82    }
83
84    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
85
86    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
87    pub fn scope(&self) -> &str {
88        match self {
89            Self::AllCooled { scope, .. } => scope,
90        }
91    }
92
93    /// Remaining cooldown duration in seconds when available.
94    pub fn remaining_secs(&self) -> Option<u64> {
95        match self {
96            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
97        }
98    }
99
100    /// Agents currently marked degraded, shown by `orch cooldown list`
101    pub fn degraded(&self) -> &[String] {
102        match self {
103            Self::AllCooled { degraded, .. } => degraded,
104        }
105    }
106}
107
108/// Typed error returned by [`get_route_result`].
109#[derive(Debug, thiserror::Error)]
110pub enum RouteResultError {
111    /// The task has no agent field — it needs to be re-routed.
112    #[error("no agent configured for task {task_id}")]
113    NoAgent { task_id: String },
114    /// Any other failure (store lookup, SQL, etc.).
115    #[error(transparent)]
116    Other(#[from] anyhow::Error),
117}
118
119/// Result of routing a task to an agent.
120#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
121pub struct RouteResult {
122    /// The selected agent: "claude", "codex", or "opencode"
123    pub agent: String,
124    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
125    pub model: Option<String>,
126    /// Complexity level: "simple", "medium", or "complex"
127    pub complexity: String,
128    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
129    pub estimate: u8,
130    /// Why this agent was selected
131    pub reason: String,
132    /// Specialized agent profile (role, tools, constraints)
133    pub profile: AgentProfile,
134    /// Optional warning about routing decision
135    pub warning: Option<String>,
136}
137
138/// Specialized agent profile for a task.
139#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
140pub struct AgentProfile {
141    /// Role name, e.g., "backend specialist"
142    pub role: String,
143    /// Tools allowed for this task
144    pub tools: Vec<String>,
145    /// Constraints for this task
146    pub constraints: Vec<String>,
147}
148
149impl AgentProfile {
150    /// Build the default "general" profile from router config.
151    pub fn default_from_config(config: &RouterConfig) -> Self {
152        Self {
153            role: "general".to_string(),
154            tools: config.allowed_tools.clone(),
155            constraints: vec![],
156        }
157    }
158}
159
160impl RouteResult {
161    /// The model for the task's tier, or one from the nearest tier with a free model. The warning
162    /// names the tier used when it differs, so the switch shows in the route activity
163    pub(crate) fn resolve_model(
164        config: &RouterConfig,
165        agent: &str,
166        complexity: &str,
167        task_id: &str,
168    ) -> (Option<String>, Option<String>) {
169        match config.model_for_complexity_or_adjacent(agent, complexity, task_id) {
170            Some((model, tier)) if tier != complexity => {
171                let warning =
172                    format!("no available {complexity} model for {agent}; using {tier} tier");
173                (Some(model), Some(warning))
174            }
175            Some((model, _)) => (Some(model), None),
176            None => (None, None),
177        }
178    }
179
180    /// Build a standard route result from the three task-specific values.
181    ///
182    /// Fills `profile` and `warning` with router-config defaults so call sites
183    /// only need to supply what varies per task.
184    pub fn from_config(
185        agent: String,
186        complexity: String,
187        reason: String,
188        config: &RouterConfig,
189        task_id: &str,
190    ) -> Self {
191        let (model, warning) = Self::resolve_model(config, &agent, &complexity, task_id);
192        Self {
193            agent,
194            model,
195            complexity,
196            estimate: 0,
197            reason,
198            profile: AgentProfile::default_from_config(config),
199            warning,
200        }
201    }
202}
203
204/// The agent router.
205#[expect(
206    clippy::struct_field_names,
207    reason = "the router holds sub-routers whose names say which backend they use"
208)]
209pub struct Router {
210    /// Router configuration
211    pub config: RouterConfig,
212    /// Available agents discovered at runtime
213    pub available_agents: Vec<String>,
214    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
215    pub weights: AgentWeights,
216    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
217    /// without holding the router RwLock across async operations)
218    llm_router: Arc<LlmRouter>,
219    /// Ollama router for local routing (contains reusable reqwest::Client)
220    ollama_router: Option<ollama::OllamaRouter>,
221    /// Round-robin index for task routing
222    pub(crate) rr_index: usize,
223    /// Last agent routed to (for distribution tracking)
224    pub(crate) last_agent: Option<String>,
225    /// Round-robin index for review agent selection
226    pub(crate) review_rr_index: usize,
227    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
228    /// `opencode:free` entries are expanded at construction time.
229    pub(crate) router_pool: Vec<(String, String)>,
230    /// Current round-robin index into router_pool
231    pub(crate) pool_index: usize,
232}
233
234impl Debug for Router {
235    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
236        f.debug_struct("Router")
237            .field("config", &self.config)
238            .field("available_agents", &self.available_agents)
239            .field("weights", &self.weights)
240            .field("ollama_router", &self.ollama_router)
241            .field("rr_index", &self.rr_index)
242            .field("last_agent", &self.last_agent)
243            .field("review_rr_index", &self.review_rr_index)
244            .field("router_pool", &self.router_pool)
245            .field("pool_index", &self.pool_index)
246            .finish_non_exhaustive()
247    }
248}
249
250impl Router {
251    /// Create a new router with the given configuration.
252    pub fn new(config: RouterConfig) -> Self {
253        let available_agents = Self::discover_agents(&config.agents);
254        let mut weights = AgentWeights {
255            base_weights: config.weights.clone(),
256            ..Default::default()
257        };
258        weights.ensure_agents_with_weights(&available_agents, &config.weights);
259        if !config.weights.is_empty() {
260            tracing::info!(
261                weights = ?config.weights,
262                "configured agent routing weights"
263            );
264        }
265        let router_pool = Self::expand_pool(&config);
266        tracing::info!(
267            pool = ?router_pool,
268            fallback = %config.effective_fallback(),
269            "router LLM pool initialized"
270        );
271        // Prime only when this config names opencode, so a default or claude-only
272        // router never runs `opencode models`
273        if config.uses_opencode() {
274            opencode::prime_free_model_cache();
275        }
276        let ollama_router = if config.mode == "local" {
277            Some(ollama::OllamaRouter::new(&config))
278        } else {
279            None
280        };
281        Self {
282            config,
283            available_agents,
284            weights,
285            llm_router: Arc::new(LlmRouter::new()),
286            ollama_router,
287            rr_index: 0,
288            last_agent: None,
289            review_rr_index: 0,
290            router_pool,
291            pool_index: 0,
292        }
293    }
294
295    /// Create a router with default configuration loaded from files.
296    pub fn from_config() -> Self {
297        Self::new(RouterConfig::from_config())
298    }
299
300    /// Create a router with a forced agent list, bypassing CLI discovery.
301    /// Only for use in unit tests.
302    #[cfg(test)]
303    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
304        let mut weights = AgentWeights {
305            base_weights: config.weights.clone(),
306            ..Default::default()
307        };
308        weights.ensure_agents(&available_agents);
309        let router_pool = Self::expand_pool(&config);
310        Self {
311            config,
312            available_agents,
313            weights,
314            llm_router: Arc::new(LlmRouter::new()),
315            ollama_router: None,
316            rr_index: 0,
317            last_agent: None,
318            review_rr_index: 0,
319            router_pool,
320            pool_index: 0,
321        }
322    }
323
324    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
325        debug_assert!(pool_len > 0, "router pool must not be empty");
326        self.pool_index = (idx + 1) % pool_len;
327    }
328
329    /// Discover available agent CLIs in PATH.
330    /// Checks all agents from the configured list.
331    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
332        let mut agents = Vec::new();
333        for agent in configured_agents {
334            if cmd_cache::command_exists(agent) {
335                agents.push(agent.clone());
336            }
337        }
338        agents
339    }
340
341    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
342    ///
343    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
344    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
345    /// - If `opencode` is not installed or the command fails, the entry is skipped.
346    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
347    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
348        let raw_pool = config.effective_pool();
349        let mut expanded: Vec<(String, String)> = Vec::new();
350
351        for entry in &raw_pool {
352            if entry == "opencode:free" {
353                let free_models = Self::discover_free_opencode_models();
354                if free_models.is_empty() {
355                    tracing::debug!("opencode:free expanded to nothing — skipping");
356                } else {
357                    tracing::debug!(models = ?free_models, "opencode:free expanded");
358                    for model in free_models {
359                        expanded.push(("opencode".to_string(), model));
360                    }
361                }
362            } else {
363                let (agent, model) = parse_pool_entry(entry);
364                expanded.push((agent, model));
365            }
366        }
367
368        if expanded.is_empty() {
369            // All entries were opencode:free and opencode isn't installed — use fallback
370            let (agent, model) = parse_pool_entry(&config.effective_fallback());
371            tracing::debug!(
372                agent = %agent, model = %model,
373                "pool empty after expansion, using fallback as sole pool entry"
374            );
375            expanded.push((agent, model));
376        }
377
378        expanded
379    }
380
381    /// Async version of expand_pool which may query the opencode CLI without
382    /// blocking the Tokio runtime.
383    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
384        let raw_pool = config.effective_pool();
385        let mut expanded: Vec<(String, String)> = Vec::new();
386
387        for entry in &raw_pool {
388            if entry == "opencode:free" {
389                let free_models = Self::discover_free_opencode_models_async().await;
390                if free_models.is_empty() {
391                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
392                } else {
393                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
394                    for model in free_models {
395                        expanded.push(("opencode".to_string(), model));
396                    }
397                }
398            } else {
399                let (agent, model) = parse_pool_entry(entry);
400                expanded.push((agent, model));
401            }
402        }
403
404        if expanded.is_empty() {
405            // All entries were opencode:free and opencode isn't installed — use fallback
406            let (agent, model) = parse_pool_entry(&config.effective_fallback());
407            tracing::debug!(
408                agent = %agent, model = %model,
409                "pool empty after async expansion, using fallback as sole pool entry"
410            );
411            expanded.push((agent, model));
412        }
413
414        expanded
415    }
416
417    /// Async reload that uses async pool expansion to avoid blocking the runtime
418    /// when discovering opencode free models.
419    pub async fn reload_async(&mut self) {
420        let new_config = RouterConfig::from_config();
421        let new_agents = Self::discover_agents(&new_config.agents);
422        let new_pool = Self::expand_pool_async(&new_config).await;
423        tracing::info!(
424            mode = %new_config.mode,
425            agents = ?new_agents,
426            fallback = %new_config.fallback_executor,
427            weighted_rr = new_config.weighted_round_robin,
428            pool = ?new_pool,
429            "router reloaded (async)"
430        );
431        self.config = new_config;
432        self.available_agents.clone_from(&new_agents);
433        // Ensure new agents have weight entries (preserves existing weights)
434        self.weights.ensure_agents(&new_agents);
435        self.router_pool = new_pool;
436        self.pool_index = 0;
437        // Reinitialize Ollama router if mode changed to local
438        self.ollama_router = if self.config.mode == "local" {
439            Some(ollama::OllamaRouter::new(&self.config))
440        } else {
441            None
442        };
443    }
444
445    fn discover_free_opencode_models() -> Vec<String> {
446        opencode::discover_free_opencode_models()
447    }
448
449    async fn discover_free_opencode_models_async() -> Vec<String> {
450        task::spawn_blocking(Self::discover_free_opencode_models)
451            .await
452            .unwrap_or_default()
453    }
454
455    /// Run the pre-emptive health check, refreshing the degraded-agent set.
456    ///
457    /// Queries the `rate_limits` table for recent events and combines with
458    /// cooldown state to mark agents as degraded before routing attempts them.
459    pub async fn refresh_health(&self, store: &Arc<TaskStore>) {
460        // KV is the source of truth for cooldowns. Reconcile the in-memory map
461        // from KV every tick so external mutations (e.g. `orch cooldown clear`
462        // from a separate CLI process) actually take effect on the running
463        // service. Without this, the engine keeps stale entries until they
464        // expire on their own, producing the "all agents cooled" log with
465        // `orch cooldown list` simultaneously reporting "No active cooldowns".
466        sync_from_kv(store).await;
467
468        let agents = self.available_agents.clone();
469        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
470        refresh_degraded_agents(
471            store,
472            &agents,
473            &model_checker,
474            config::health_check_window_hours(),
475            config::degraded_rate_limit_threshold(),
476        )
477        .await;
478    }
479
480    /// Check if an agent is available.
481    pub fn is_agent_available(&self, agent: &str) -> bool {
482        self.available_agents.contains(&agent.to_string())
483    }
484
485    /// True if the agent has at least one uncooled model across any complexity
486    /// tier. Used where the eventual complexity isn't known yet (e.g. before
487    /// LLM classification), so we can't check a single tier.
488    fn agent_has_any_available_model(&self, agent: &str) -> bool {
489        ["simple", "medium", "complex", "review"]
490            .iter()
491            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
492    }
493
494    /// True if the agent has an available (uncooled) model for the `medium`
495    /// and `complex` tiers — the tiers real coding tasks are classified into
496    /// by the LLM classifier. Used to pre-filter the candidate list handed to
497    /// that classifier so an agent whose healthy tiers are only `simple` isn't
498    /// offered: the classifier can't be constrained to `simple`, so offering
499    /// such an agent produces a near-certain wasted classification call whenever
500    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
501    /// outcome of initial classification and is deliberately excluded.
502    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
503        ["medium", "complex"]
504            .iter()
505            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
506    }
507
508    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
509        if !self.is_agent_available(agent) {
510            return false;
511        }
512        if is_agent_in_cooldown(agent) {
513            tracing::debug!(agent, "agent skipped: in cooldown");
514            return false;
515        }
516        if is_agent_degraded(agent) {
517            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
518            return false;
519        }
520        // If weighted routing is used, consider the agent degraded when its
521        // weight has decayed below the threshold. This avoids
522        // proactively routing to agents that recently hit many rate limits.
523        if self.config.weighted_round_robin {
524            let weight = self.weights.get_weight(agent);
525            if weight < config::SKIP_LIMITED_THRESHOLD {
526                tracing::debug!(
527                    agent,
528                    weight,
529                    "agent considered degraded by weight threshold, skipping"
530                );
531                return false;
532            }
533        }
534        if !self
535            .config
536            .has_available_model_in_any_tier(agent, complexity)
537        {
538            tracing::debug!(agent, complexity, "agent skipped: no available model");
539            return false;
540        }
541        true
542    }
543
544    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
545        self.available_agents
546            .iter()
547            .filter(|agent| self.agent_is_routable(agent, complexity))
548            .cloned()
549            .collect()
550    }
551
552    /// Count of healthy (routable) agents for a given complexity.
553    /// Returns the number of agents that are available, not in cooldown, and have an available model.
554    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
555        self.available_agents_for_complexity(complexity).len()
556    }
557
558    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
559    /// Uses "simple" complexity as the baseline since it has the widest model support.
560    pub fn is_degraded(&self, threshold: usize) -> bool {
561        self.healthy_agent_count("simple") < threshold
562    }
563
564    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
565        let mut earliest: Option<i64> = None;
566
567        for agent in &self.available_agents {
568            if let Some(until) = cooldown_until(agent) {
569                earliest = Some(earliest.map_or(until, |current| current.min(until)));
570            }
571
572            if let Some(comp) = complexity
573                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
574            {
575                for model in pool {
576                    let key = format!("{agent}:{model}");
577                    if let Some(until) = cooldown_until(&key) {
578                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
579                    }
580                }
581            }
582        }
583
584        earliest
585    }
586
587    /// Find the earliest cooldown expiration among the router LLM pool entries.
588    ///
589    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
590    /// or `None` if no pool model has a cooldown.
591    fn earliest_pool_cooldown(&self) -> Option<i64> {
592        let mut earliest: Option<i64> = None;
593        for (agent, model) in &self.router_pool {
594            let key = format!("{agent}:{model}");
595            if let Some(until) = cooldown_until(&key) {
596                earliest = Some(earliest.map_or(until, |current| current.min(until)));
597            }
598        }
599        earliest
600    }
601
602    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
603        // If any cooldowns are present, return an error immediately so the
604        // caller (the tick loop) can skip this task and retry on the next tick.
605        // Sleeping inside the router blocks the entire tick loop for the
606        // cooldown duration which starves other tasks. The engine tick already
607        // retries every `tick_interval` (default 10s), so allow the tick loop
608        // to drive retries instead of sleeping here.
609        let now = chrono::Utc::now().timestamp();
610        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
611            // All agents are degraded but none have a timed cooldown entry
612            // (e.g. over-threshold rate limiting without explicit cooldown duration).
613            // Log at warn and return an error so the tick skips this task and retries after backoff.
614            tracing::warn!(
615                scope = complexity.unwrap_or("all agents"),
616                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
617            );
618            // Return an error similar to the cooldown case to trigger backoff in the tick loop
619            return Err(AllAgentsCooledError::new(
620                complexity.unwrap_or("all agents").to_string(),
621                None,
622            )
623            .into());
624        };
625
626        let remaining = earliest.saturating_sub(now);
627        if remaining > 0 {
628            let scope_str = complexity.unwrap_or("all agents").to_string();
629            tracing::warn!(
630                remaining_secs = remaining,
631                scope = %scope_str,
632                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
633            );
634            // Return a domain error indicating all candidates are cooled so the
635            // caller can handle it (tick will skip this task and retry later).
636            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
637        }
638
639        Ok(())
640    }
641
642    /// Get the first available agent.
643    /// Pick next agent via round-robin (for review or other non-task routing).
644    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
645    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
646    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
647        // Use available_agents_for_complexity which already filters using agent_is_routable,
648        // eliminating duplicate cooldown/degraded/model checks.
649        let candidates = self.available_agents_for_complexity(complexity);
650        if candidates.is_empty() {
651            return None;
652        }
653        let idx = self.review_rr_index;
654        let n = candidates.len();
655
656        // Try to find an agent that isn't excluded
657        let agent = (0..n)
658            .map(|offset| &candidates[(idx + offset) % n])
659            .find(|a| !exclude.contains(&a.as_str()))
660            .cloned()
661            // Fallback: use the first available candidate (even if excluded)
662            .or_else(|| candidates.first().cloned())?;
663
664        self.review_rr_index = (idx + 1) % candidates.len().max(1);
665        Some(agent)
666    }
667
668    /// If the selected agent matches `no_code_last_agent` for this task, filter it
669    /// out of the candidates and return the filtered list. Used to prevent the
670    /// same agent from being selected twice in a row after a no-code failure (#2410).
671    async fn skip_no_code_agent(
672        &self,
673        store: &Arc<TaskStore>,
674        repo: &str,
675        task_id: &str,
676        candidates: &[String],
677    ) -> Option<Vec<String>> {
678        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
679            .await
680            .unwrap_or_else(|| {
681                tracing::warn!(
682                    task_id,
683                    "no_code_last_agent DB read returned None — loop detection may be impaired"
684                );
685                String::new()
686            });
687        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
688            return None;
689        }
690        let filtered: Vec<String> = candidates
691            .iter()
692            .filter(|a| a.as_str() != no_code_agent)
693            .cloned()
694            .collect();
695        if filtered.is_empty() {
696            tracing::warn!(
697                task_id,
698                no_code_agent = %no_code_agent,
699                "all available agents are the no-code agent — cannot skip"
700            );
701            return None;
702        }
703        tracing::info!(
704            task_id,
705            skipped = %no_code_agent,
706            remaining = ?filtered,
707            "skipping no-code agent, routing to alternative"
708        );
709        Some(filtered)
710    }
711
712    /// Route a task to the best agent.
713    ///
714    /// Routing logic (in priority order):
715    /// 1. Check for `agent:*` label — use that agent directly
716    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
717    /// 3. If round_robin mode, cycle through agents (stateful)
718    /// 4. Call LLM classifier for intelligent routing
719    /// 5. After max_route_attempts LLM failures, fall back to round-robin
720    pub async fn route(
721        &mut self,
722        task: &ExternalTask,
723        store: &Arc<TaskStore>,
724        repo: &str,
725    ) -> anyhow::Result<RouteResult> {
726        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
727        // before the routing loop, not per-task. Do not re-add it here.
728
729        // Hold the task before any classification, label override, or fallback.
730        // A waiting task must not increment route_attempts.
731        match store.resolve_id_by_external(repo, &task.id.0).await {
732            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
733                Ok(Some(blockers)) => {
734                    return Err(WaitingOnDependency { blockers }.into());
735                }
736                Ok(None) => {}
737                Err(err) => return Err(err),
738            },
739            Ok(None) => {}
740            Err(err) => return Err(err),
741        }
742
743        loop {
744            // 1. Check store agent field first (set by failover, authoritative over labels)
745            // Use a lightweight lookup to avoid deserializing the full Task row when
746            // only the `agent` column is needed. This avoids two full-row loads
747            // per routing pass (resolve_task_id + get) and significantly reduces
748            // CPU work on busy instances.
749            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
750                Ok(Some(agent)) => {
751                    // Router enforces that the agent string matches a configured agent
752                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
753                        Some(agent)
754                    } else {
755                        None
756                    }
757                }
758                Ok(None) => None,
759                Err(e) => {
760                    tracing::warn!(
761                        repo,
762                        task_id = %task.id.0,
763                        err = %e,
764                        "failed to fetch stored agent, falling back to label/LLM routing"
765                    );
766                    None
767                }
768            };
769
770            // 2. Fall back to explicit agent label
771            let resolved_agent = store_agent.or_else(|| {
772                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
773            });
774
775            if let Some(agent) = resolved_agent {
776                let complexity = strategies::extract_complexity_from_labels(&task.labels);
777                // If agent appears routable (not in agent-wide cooldown and has
778                // available models or no model_map configured), check whether
779                // we actually have a concrete model to dispatch. If the
780                // resolved label maps to no model (None), we fall through to
781                // the standard routing logic so the router LLM can pick a
782                // proper agent/model, instead of dispatching with an empty
783                // model which causes some agents (e.g. opencode) to exit
784                // silently.
785                if self.agent_is_routable(&agent, &complexity) {
786                    let (model, tier_warning) =
787                        RouteResult::resolve_model(&self.config, &agent, &complexity, &task.id.0);
788
789                    // Only dispatch label overrides when we have a concrete
790                    // model to send to the executor. Dispatching with an
791                    // empty model led to silent exits for some agents
792                    // (notably opencode). If no concrete model is available
793                    // for the labeled agent, fall through to the standard
794                    // routing logic so the LLM or round-robin can pick an
795                    // appropriate agent/model.
796                    if model.is_some() {
797                        tracing::debug!(
798                            task_id = %task.id.0,
799                            agent = %agent,
800                            complexity = %complexity,
801                            "routed via label"
802                        );
803                        let mut result = RouteResult::from_config(
804                            agent.clone(),
805                            complexity.clone(),
806                            format!("label agent:{agent}"),
807                            &self.config,
808                            &task.id.0,
809                        );
810                        // Label routing uses agent-specific role and the
811                        // resolved model (may differ from from_config's default).
812                        result.profile.role = format!("{agent} specialist");
813                        result.model = model.clone();
814                        result.warning = tier_warning;
815                        self.log_route_activity(store, repo, &task.id.0, &result, None)
816                            .await;
817                        return Ok(result);
818                    }
819                    // No concrete model found for the labeled agent — do
820                    // not honor the label override. Fall through to the
821                    // standard routing logic so another agent or the LLM
822                    // can be chosen with a valid model.
823                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
824                }
825
826                let candidates = self.available_agents_for_complexity(&complexity);
827                if candidates.is_empty() {
828                    self.wait_for_cooldown(Some(&complexity))?;
829                    continue;
830                }
831                // Label target is cooled or lacked model; fall through to standard routing.
832            }
833
834            let complexity = strategies::extract_complexity_from_labels(&task.labels);
835
836            // Check if any agents have available models for this complexity
837            let mut candidates = self.available_agents_for_complexity(&complexity);
838            if candidates.is_empty() {
839                self.wait_for_cooldown(Some(&complexity))?;
840                continue;
841            }
842
843            // Skip the no-code agent if one is recorded for this task (#2410).
844            // If all available agents are filtered out, we fall through to
845            // LLM routing with no constraints (LLM will pick the same agent but
846            // the response_handler will block it immediately via same-agent check).
847            if let Some(filtered) = self
848                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
849                .await
850            {
851                candidates = filtered;
852            }
853
854            // 2. Weighted round-robin — capacity-based selection
855            if self.config.weighted_round_robin {
856                let result = strategies::route_via_weighted_round_robin(
857                    &candidates,
858                    &self.weights,
859                    &self.config,
860                    task,
861                    &mut self.last_agent,
862                )?;
863                self.log_route_activity(store, repo, &task.id.0, &result, None)
864                    .await;
865                return Ok(result);
866            }
867
868            // 3. Round-robin mode — use stateful round-robin
869            if self.config.mode == "round_robin" {
870                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
871                let result = strategies::route_via_round_robin_stateful(
872                    &candidates,
873                    &self.config,
874                    task,
875                    &mut self.rr_index,
876                    &mut self.last_agent,
877                )?;
878                self.log_route_activity(store, repo, &task.id.0, &result, None)
879                    .await;
880                return Ok(result);
881            }
882
883            // 4. Local Ollama routing
884            if self.config.mode == "local" {
885                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
886                let ollama_router = self
887                    .ollama_router
888                    .as_ref()
889                    .expect("ollama_router must be initialized in Router::new when mode=local");
890                match ollama_router.route(task, &self.config).await {
891                    Ok(result) => {
892                        self.log_route_activity(store, repo, &task.id.0, &result, None)
893                            .await;
894                        return Ok(result);
895                    }
896                    Err(e) => {
897                        tracing::warn!(
898                            task_id = %task.id.0,
899                            error = %e,
900                            "Ollama routing failed, falling back to round-robin"
901                        );
902                        let result = strategies::route_via_round_robin_stateful(
903                            &candidates,
904                            &self.config,
905                            task,
906                            &mut self.rr_index,
907                            &mut self.last_agent,
908                        )?;
909                        self.log_route_activity(store, repo, &task.id.0, &result, None)
910                            .await;
911                        return Ok(result);
912                    }
913                }
914            }
915
916            // 5. LLM-based routing with retry tracking
917            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
918
919            if route_attempts >= self.config.max_route_attempts {
920                tracing::warn!(
921                    task_id = %task.id.0,
922                    attempts = route_attempts,
923                    max = self.config.max_route_attempts,
924                    "max LLM route attempts reached, falling back to round-robin"
925                );
926                let candidates = self.available_agents_for_complexity(&complexity);
927                if candidates.is_empty() {
928                    self.wait_for_cooldown(Some(&complexity))?;
929                    continue;
930                }
931                let result = strategies::route_via_round_robin_stateful(
932                    &candidates,
933                    &self.config,
934                    task,
935                    &mut self.rr_index,
936                    &mut self.last_agent,
937                )?;
938                self.log_route_activity(store, repo, &task.id.0, &result, None)
939                    .await;
940                return Ok(result);
941            }
942
943            // Log routing start (before await)
944            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
945
946            match self.route_with_llm(task, repo).await {
947                Ok(result) => {
948                    let candidates = self.available_agents_for_complexity(&result.complexity);
949                    if candidates.is_empty() {
950                        self.wait_for_cooldown(Some(&result.complexity))?;
951                        continue;
952                    }
953
954                    // If the LLM selected the no-code agent, filter it out and
955                    // pick an alternative (#2410). Re-use the existing fallback logic.
956                    let no_code_agent =
957                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
958                             .await
959                             .unwrap_or_else(|| {
960                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
961                                 String::new()
962                             });
963                    if result.agent == no_code_agent {
964                        // Pick the first agent that isn't the no-code agent.
965                        let fallback_agent = candidates
966                            .iter()
967                            .find(|a| a.as_str() != no_code_agent)
968                            .cloned()
969                            .unwrap_or_else(|| candidates[0].clone());
970                        if fallback_agent.as_str() == no_code_agent {
971                            tracing::warn!(
972                                task_id = %task.id.0,
973                                no_code_agent = %no_code_agent,
974                                complexity = %result.complexity,
975                                "all available agents are the no-code agent — waiting for another agent"
976                            );
977                            self.wait_for_cooldown(Some(&result.complexity))?;
978                            continue;
979                        }
980                        tracing::info!(
981                            task_id = %task.id.0,
982                            llm_selected = %result.agent,
983                            skipped = %no_code_agent,
984                            fallback = %fallback_agent,
985                            "LLM selected the no-code agent — routing to alternative"
986                        );
987                        let (fallback_model, _) = RouteResult::resolve_model(
988                            &self.config,
989                            &fallback_agent,
990                            &result.complexity,
991                            &task.id.0,
992                        );
993
994                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
995                        let result = RouteResult {
996                            agent: fallback_agent.clone(),
997                            model: fallback_model,
998                            complexity: result.complexity.clone(),
999                            estimate: result.estimate,
1000                            reason: format!(
1001                                "LLM selected no-code agent {}; routed to {}",
1002                                result.agent, fallback_agent
1003                            ),
1004                            profile: result.profile.clone(),
1005                            warning: Some(format!(
1006                                "LLM selected no-code agent {}; used fallback {}",
1007                                result.agent, fallback_agent
1008                            )),
1009                        };
1010                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1011                            .await;
1012                        return Ok(result);
1013                    }
1014
1015                    let model_cooled = result
1016                        .model
1017                        .as_deref()
1018                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1019                    if !candidates.contains(&result.agent) || model_cooled {
1020                        let fallback_agent = candidates
1021                            .iter()
1022                            .find(|agent| *agent != &result.agent)
1023                            .cloned()
1024                            .unwrap_or_else(|| candidates[0].clone());
1025                        let (fallback_model, _) = RouteResult::resolve_model(
1026                            &self.config,
1027                            &fallback_agent,
1028                            &result.complexity,
1029                            &task.id.0,
1030                        );
1031
1032                        // If the fallback model is None and the fallback agent is the
1033                        // same as the LLM-selected agent, it means all models for the
1034                        // only available agent are cooled. In that case, wait for
1035                        // cooldowns instead of returning a RouteResult with
1036                        // model: None which would be dispatched incorrectly.
1037                        if fallback_model.is_none() && fallback_agent == result.agent {
1038                            tracing::warn!(
1039                                task_id = %task.id.0,
1040                                agent = %result.agent,
1041                                complexity = %result.complexity,
1042                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1043                            );
1044                            self.wait_for_cooldown(Some(&result.complexity))?;
1045                            continue;
1046                        }
1047
1048                        // Distinguish logging and reasons when we actually switch
1049                        // agents vs when we keep the same agent but pick a
1050                        // different (non-cooled) model.
1051                        if fallback_agent == result.agent {
1052                            tracing::warn!(
1053                                task_id = %task.id.0,
1054                                agent = %result.agent,
1055                                complexity = %result.complexity,
1056                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1057                            );
1058                        } else {
1059                            tracing::warn!(
1060                                task_id = %task.id.0,
1061                                agent = %result.agent,
1062                                fallback = %fallback_agent,
1063                                complexity = %result.complexity,
1064                                "LLM selected cooled agent/model; rerouting to available agent"
1065                            );
1066                        }
1067
1068                        // Reset attempts on success
1069                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1070
1071                        let reason = if fallback_agent == result.agent {
1072                            format!(
1073                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1074                            )
1075                        } else {
1076                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1077                        };
1078
1079                        let result = RouteResult {
1080                            agent: fallback_agent.clone(),
1081                            model: fallback_model,
1082                            complexity: result.complexity.clone(),
1083                            estimate: result.estimate,
1084                            reason: reason.clone(),
1085                            profile: result.profile.clone(),
1086                            warning: Some(if fallback_agent == result.agent {
1087                                "LLM-selected model was cooled; selected alternative model for same agent"
1088                                    .to_string()
1089                            } else {
1090                                "LLM-selected agent/model was cooled; rerouted to available agent"
1091                                    .to_string()
1092                            }),
1093                        };
1094                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1095                            .await;
1096                        return Ok(result);
1097                    }
1098
1099                    // Reset attempts on success
1100                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1101                    tracing::info!(
1102                        task_id = %task.id.0,
1103                        agent = %result.agent,
1104                        complexity = %result.complexity,
1105                        "routed via LLM"
1106                    );
1107                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1108                        .await;
1109                    return Ok(result);
1110                }
1111                Err(e) => {
1112                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1113                        let scope = err.scope();
1114                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1115                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1116                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1117                        // (no complexity-specific pool lookup, which would fail for "router pool").
1118                        let scope_opt = match scope {
1119                            "all agents" | "router pool" => None,
1120                            _ => Some(scope),
1121                        };
1122                        self.wait_for_cooldown(scope_opt)?;
1123                        continue;
1124                    }
1125
1126                    let new_attempts = route_attempts + 1;
1127                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1128                        .await;
1129                    tracing::warn!(
1130                        task_id = %task.id.0,
1131                        error = %e,
1132                        error_chain = ?e,
1133                        attempt = new_attempts,
1134                        max = self.config.max_route_attempts,
1135                        "LLM routing failed"
1136                    );
1137
1138                    if new_attempts >= self.config.max_route_attempts {
1139                        tracing::info!(
1140                            task_id = %task.id.0,
1141                            "falling back to round-robin after {} failed attempts",
1142                            new_attempts
1143                        );
1144                        let candidates = self.available_agents_for_complexity(&complexity);
1145                        if candidates.is_empty() {
1146                            self.wait_for_cooldown(Some(&complexity))?;
1147                            continue;
1148                        }
1149                        let result = strategies::route_via_round_robin_stateful(
1150                            &candidates,
1151                            &self.config,
1152                            task,
1153                            &mut self.rr_index,
1154                            &mut self.last_agent,
1155                        )?;
1156                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1157                            .await;
1158                        return Ok(result);
1159                    }
1160
1161                    let candidates = self.available_agents_for_complexity(&complexity);
1162                    if candidates.is_empty() {
1163                        self.wait_for_cooldown(Some(&complexity))?;
1164                        continue;
1165                    }
1166                    let result = strategies::route_via_fallback(
1167                        &candidates,
1168                        &self.config,
1169                        task,
1170                        Some(&self.weights),
1171                    )?;
1172                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1173                        .await;
1174                    return Ok(result);
1175                }
1176            }
1177        }
1178    }
1179
1180    /// Get the number of LLM routing attempts for a task from the store.
1181    async fn get_route_attempts(&self, task_id: &str, store: &Arc<TaskStore>, repo: &str) -> u32 {
1182        match store.resolve_task_id(repo, task_id).await {
1183            Ok(Some(store_id)) => store
1184                .get(store_id)
1185                .await
1186                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1187            _ => 0,
1188        }
1189    }
1190
1191    /// Set the number of LLM routing attempts for a task in the store.
1192    async fn set_route_attempts(
1193        &self,
1194        task_id: &str,
1195        attempts: u32,
1196        store: &Arc<TaskStore>,
1197        repo: &str,
1198    ) {
1199        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1200            && let Err(e) = store
1201                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1202                .await
1203        {
1204            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1205        }
1206    }
1207
1208    /// Log a route event to task activity timeline.
1209    async fn log_route_activity(
1210        &self,
1211        store: &Arc<TaskStore>,
1212        repo: &str,
1213        task_id: &str,
1214        result: &RouteResult,
1215        from_agent: Option<&str>,
1216    ) {
1217        let details = serde_json::json!({
1218            "reason": result.reason,
1219            "complexity": result.complexity,
1220            "role": result.profile.role,
1221            "warning": result.warning,
1222        });
1223        let event_type = if from_agent.is_some() {
1224            "rerouted"
1225        } else {
1226            "routed"
1227        };
1228        store_log_activity(
1229            &Some(Arc::clone(store)),
1230            repo,
1231            task_id,
1232            event_type,
1233            None,
1234            Some("routed"),
1235            Some(&result.agent),
1236            result.model.as_deref(),
1237            Some(&details),
1238        )
1239        .await;
1240    }
1241
1242    /// Route using LLM classification with pool round-robin.
1243    ///
1244    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1245    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1246    /// returns the last error (the caller falls back to round-robin agent selection).
1247    async fn route_with_llm(
1248        &mut self,
1249        task: &ExternalTask,
1250        repo: &str,
1251    ) -> anyhow::Result<RouteResult> {
1252        // Filter out cooled and degraded agents, and agents without an available
1253        // model for the coding tiers (`medium`/`complex`) that real tasks are
1254        // classified into, so the LLM only sees candidates that could actually
1255        // be dispatched for whatever complexity it picks. An agent whose only
1256        // healthy tier is `simple` is excluded too: the classifier can't be
1257        // constrained to `simple`, so leaving such an agent in would waste the
1258        // call whenever it picks `medium`/`complex` (issue #3521).
1259        // Fall back to the full list if all agents are cooled/degraded.
1260        // Cloned to avoid borrow conflict with &mut self in the loop.
1261        let uncooled_agents: Vec<String> = self
1262            .available_agents
1263            .iter()
1264            .filter(|a| {
1265                !is_agent_in_cooldown(a)
1266                    && !is_agent_degraded(a)
1267                    && self.agent_has_available_model_for_coding_tiers(a)
1268            })
1269            .cloned()
1270            .collect();
1271        if uncooled_agents.is_empty() {
1272            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1273        }
1274        let llm_agents = uncooled_agents;
1275
1276        let pool = self.router_pool.clone();
1277        let n = pool.len();
1278        let start = self.pool_index;
1279        let mut last_err: Option<anyhow::Error> = None;
1280        let mut attempted_any = false;
1281
1282        // Try pool entries in round-robin order, skipping cooled ones
1283        for i in 0..n {
1284            let idx = (start + i) % n;
1285            let (agent, model) = &pool[idx];
1286            let model_str = model.as_str();
1287
1288            if response::is_model_in_cooldown(agent, model_str) {
1289                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1290                continue;
1291            }
1292
1293            // At least one pool entry was available to attempt
1294            attempted_any = true;
1295
1296            let model_opt = if model_str.is_empty() {
1297                None
1298            } else {
1299                Some(model_str)
1300            };
1301
1302            match self
1303                .llm_router
1304                .route_with_llm_using(
1305                    task,
1306                    &llm_agents,
1307                    &self.config,
1308                    &mut self.last_agent,
1309                    repo,
1310                    agent,
1311                    model_opt,
1312                )
1313                .await
1314            {
1315                Ok(result) => {
1316                    // Advance index so the next call starts at the next pool entry
1317                    self.advance_pool_index_after_attempt(idx, n);
1318                    tracing::debug!(
1319                        agent,
1320                        model = model_str,
1321                        pool_idx = idx,
1322                        "router LLM pool entry succeeded"
1323                    );
1324                    record_agent_success(agent, model_str).await;
1325                    return Ok(result);
1326                }
1327                Err(e) => {
1328                    let err_msg = e.to_string();
1329                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1330                    if is_timeout {
1331                        // Advance pool index so the next tick starts at the next entry,
1332                        // then return immediately. Continuing to try further pool entries
1333                        // in the same tick would accumulate multiple timeout windows and
1334                        // stall the tick loop for minutes (issue #3187).
1335                        //
1336                        // Also record a standard cooldown, same as the non-timeout branch,
1337                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1338                        // on future ticks instead of burning the full timeout window again
1339                        // every time round-robin reaches it (issue #3422).
1340                        self.advance_pool_index_after_attempt(idx, n);
1341                        tracing::warn!(
1342                            agent,
1343                            model = model_str,
1344                            next_pool_idx = self.pool_index,
1345                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1346                        );
1347                        response::record_model_failure(agent, model_str).await;
1348                        return Err(e);
1349                    }
1350                    if is_max_turns_error(&err_msg) {
1351                        tracing::warn!(
1352                            agent,
1353                            model = model_str,
1354                            error = %e,
1355                            "pool entry hit max turns — trying next without cooldown"
1356                        );
1357                    } else {
1358                        tracing::warn!(
1359                            agent,
1360                            model = model_str,
1361                            error = %e,
1362                            "pool entry failed, recording cooldown and trying next"
1363                        );
1364                        if !cooldown::record_credit_exhaustion_from_message(
1365                            agent,
1366                            Some(model_str),
1367                            &err_msg,
1368                        )
1369                        .await
1370                        {
1371                            response::record_model_failure(agent, model_str).await;
1372                        }
1373                    }
1374                    last_err = Some(e);
1375                    self.advance_pool_index_after_attempt(idx, n);
1376                }
1377            }
1378        }
1379
1380        // If every pool entry was pre-cooled (nothing was attempted), do not
1381        // immediately exhaust the pool again — fail fast and let the tick loop
1382        // retry on the next cycle instead of cascading failures.
1383        // However, we should still try the fallback if configured.
1384        if !attempted_any {
1385            let now = chrono::Utc::now().timestamp();
1386            let earliest = self.earliest_pool_cooldown();
1387            if let Some(until) = earliest {
1388                // Pool entries exist but are all cooled - we'll try fallback below
1389                let remaining = until.saturating_sub(now);
1390                tracing::warn!(
1391                    remaining_secs = remaining,
1392                    pool = ?pool,
1393                    "all router LLM pool entries on cooldown — will try fallback"
1394                );
1395                // Continue to fallback logic instead of returning early
1396            } else {
1397                // No pool cooldowns found either — this means the pool was empty
1398                // or every entry was filtered out for some other reason.
1399                return Err(last_err
1400                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1401            }
1402        }
1403
1404        // All pool entries failed or were cooled — try the configured fallback
1405        let fallback = self.config.effective_fallback();
1406        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1407        let fb_model_str = fb_model.as_str();
1408
1409        // Only try fallback if it wasn't already in the pool (avoid double-try)
1410        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1411
1412        if !fallback_already_tried && !response::is_model_in_cooldown(&fb_agent, fb_model_str) {
1413            tracing::info!(
1414                agent = %fb_agent,
1415                model = %fb_model_str,
1416                "all pool entries exhausted — trying fallback router LLM"
1417            );
1418            let fb_model_opt = if fb_model_str.is_empty() {
1419                None
1420            } else {
1421                Some(fb_model_str)
1422            };
1423            match self
1424                .llm_router
1425                .route_with_llm_using(
1426                    task,
1427                    &llm_agents,
1428                    &self.config,
1429                    &mut self.last_agent,
1430                    repo,
1431                    &fb_agent,
1432                    fb_model_opt,
1433                )
1434                .await
1435            {
1436                Ok(result) => {
1437                    record_agent_success(&fb_agent, fb_model_str).await;
1438                    return Ok(result);
1439                }
1440                Err(e) => {
1441                    let err_msg = e.to_string();
1442                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1443                    if is_timeout {
1444                        // Cool down on timeout like pool entries (#3422), else a
1445                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1446                        response::record_model_failure(&fb_agent, fb_model_str).await;
1447                        tracing::warn!(
1448                            agent = %fb_agent,
1449                            model = %fb_model_str,
1450                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1451                        );
1452                        return Err(e);
1453                    }
1454                    tracing::warn!(
1455                        agent = %fb_agent,
1456                        model = %fb_model_str,
1457                        error = %e,
1458                        "fallback router LLM also failed"
1459                    );
1460                    if !is_max_turns_error(&err_msg) {
1461                        response::record_model_failure(&fb_agent, fb_model_str).await;
1462                    }
1463                    last_err = Some(e);
1464                }
1465            }
1466        }
1467
1468        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1469    }
1470
1471    /// Record a rate limit event for an agent, reducing its routing weight.
1472    ///
1473    /// Called by the engine when an agent returns a 429/rate limit error.
1474    pub fn record_rate_limit(&mut self, agent: &str) {
1475        self.weights.record_rate_limit(agent);
1476    }
1477
1478    /// Record a successful task completion, restoring agent weight.
1479    pub fn record_success(&mut self, agent: &str) {
1480        self.weights.record_success(agent);
1481    }
1482
1483    /// Tick weight recovery for all agents.
1484    ///
1485    /// Called periodically by the engine to gradually restore weights
1486    /// as rate limit windows expire.
1487    pub fn tick_weight_recovery(&mut self) {
1488        self.weights.tick_recovery();
1489    }
1490
1491    /// Store routing result in the task store.
1492    pub async fn store_route_result(
1493        &self,
1494        task_id: &str,
1495        result: &RouteResult,
1496        store: &Arc<TaskStore>,
1497        repo: &str,
1498    ) -> anyhow::Result<()> {
1499        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1500            store
1501                .set_fields(
1502                    store_id,
1503                    &[
1504                        ("agent", serde_json::json!(result.agent)),
1505                        ("complexity", serde_json::json!(result.complexity)),
1506                        ("route_reason", serde_json::json!(result.reason)),
1507                        (
1508                            "model",
1509                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1510                        ),
1511                    ],
1512                )
1513                .await?;
1514        }
1515        Ok(())
1516    }
1517}
1518
1519/// Retrieve routing result from the task store.
1520pub async fn get_route_result(
1521    store: &Arc<TaskStore>,
1522    repo: &str,
1523    task_id: &str,
1524) -> Result<RouteResult, RouteResultError> {
1525    let store_id = store
1526        .resolve_task_id(repo, task_id)
1527        .await
1528        .map_err(RouteResultError::Other)?
1529        .ok_or_else(|| {
1530            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1531        })?;
1532    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1533
1534    let agent = task
1535        .agent
1536        .clone()
1537        .filter(|a| !a.is_empty())
1538        .ok_or_else(|| RouteResultError::NoAgent {
1539            task_id: task_id.to_string(),
1540        })?;
1541
1542    let complexity = if task.complexity.is_empty() {
1543        "medium".to_string()
1544    } else {
1545        task.complexity.clone()
1546    };
1547
1548    let reason = task.route_reason.clone();
1549    let model = task.model.clone().filter(|m| !m.is_empty());
1550
1551    // Validate that the stored model is valid for the resolved agent.
1552    // Agent-specific aliases (e.g. "opus") can leak across agents during
1553    // failover, so we verify against the router's model pools (#1604).
1554    let config = RouterConfig::from_config();
1555    let model = model.filter(|m| {
1556        let complexities = ["simple", "medium", "complex", "review"];
1557        // Collect all model pools configured for this agent across complexity tiers.
1558        let all_pools: Vec<Vec<String>> = complexities
1559            .iter()
1560            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1561            .collect();
1562        // If the agent has no model pools configured, we can't validate — pass through.
1563        if all_pools.is_empty() {
1564            return true;
1565        }
1566        let valid = all_pools.iter().any(|pool| pool.contains(m));
1567        if !valid {
1568            tracing::warn!(
1569                task_id,
1570                agent,
1571                model = %m,
1572                "discarding stale model: not in agent's model pools"
1573            );
1574        }
1575        valid
1576    });
1577
1578    // A stale model is replaced by the agent's model for the task's complexity, an empty
1579    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1580    let original_model = task.model.clone().filter(|m| !m.is_empty());
1581    let mut model = model;
1582    if original_model.is_some() && model.is_none() {
1583        model = config.model_for_complexity(&agent, &complexity, task_id);
1584        let stale_model = original_model.as_deref().unwrap_or("unknown");
1585        let stale_err = format!(
1586            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1587            model.as_deref().unwrap_or("the agent default")
1588        );
1589        if let Err(e) = store
1590            .set_fields(
1591                store_id,
1592                &[
1593                    (
1594                        "model",
1595                        serde_json::json!(model.clone().unwrap_or_default()),
1596                    ),
1597                    ("last_error", serde_json::json!(stale_err)),
1598                ],
1599            )
1600            .await
1601        {
1602            // Log the error so transient DB failures are visible and diagnosable.
1603            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1604        }
1605    }
1606
1607    Ok(RouteResult {
1608        agent,
1609        model,
1610        complexity,
1611        estimate: 0,
1612        reason,
1613        profile: AgentProfile::default(),
1614        warning: None,
1615    })
1616}
1617
1618#[cfg(test)]
1619#[expect(
1620    clippy::float_cmp,
1621    reason = "the weights are stored constants, so they compare exactly"
1622)]
1623mod tests {
1624    use super::config::DEFAULT_AGENTS;
1625    use super::llm::LlmRouteResponse;
1626    use super::strategies;
1627    use super::weights::default_missing_weight;
1628    use super::weights::{
1629        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1630    };
1631    use super::*;
1632    use crate::backends::{ExternalId, ExternalTask};
1633    use crate::engine::cooldown::{
1634        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1635        record_model_failure, set_model_cooldown,
1636    };
1637    use crate::store::NewTask;
1638    use crate::store::TaskStore;
1639    use serial_test::serial;
1640    use std::collections::HashMap;
1641    use std::env;
1642    #[cfg(unix)]
1643    use std::ffi::OsString;
1644    use std::fs;
1645    #[cfg(unix)]
1646    use std::fs::Permissions;
1647    #[cfg(unix)]
1648    use std::os::unix::fs::PermissionsExt;
1649    #[cfg(unix)]
1650    use std::process;
1651    #[cfg(unix)]
1652    use std::process::Command;
1653    #[cfg(unix)]
1654    use std::thread;
1655    use std::time::{Duration, Instant};
1656
1657    #[test]
1658    fn max_turns_error_matches_grok_phrases() {
1659        assert!(is_max_turns_error(
1660            "router LLM returned error: Reached the maximum number of turns"
1661        ));
1662        assert!(is_max_turns_error("Error: max turns reached"));
1663        assert!(is_max_turns_error(
1664            "router LLM failed: router LLM returned error: Reached the maximum number of turns"
1665        ));
1666        assert!(!is_max_turns_error("router LLM timed out after 30s"));
1667        assert!(!is_max_turns_error("rate limited"));
1668    }
1669
1670    #[tokio::test]
1671    #[serial_test::serial(orch_home)]
1672    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1673        let home = tempfile::tempdir().unwrap();
1674        fs::write(
1675            home.path().join("config.yml"),
1676            "model_map:\n  medium:\n    claude: sonnet\n",
1677        )
1678        .unwrap();
1679        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1680        unsafe { env::set_var("ORCH_HOME", home.path()) };
1681
1682        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1683        let store_id = store
1684            .create(&NewTask {
1685                external_id: Some("7".to_string()),
1686                repo: "owner/repo".to_string(),
1687                origin: "github".to_string(),
1688                title: "stale model".to_string(),
1689                ..Default::default()
1690            })
1691            .await
1692            .unwrap();
1693        store
1694            .set_fields(
1695                store_id,
1696                &[
1697                    ("agent", serde_json::json!("claude")),
1698                    ("model", serde_json::json!("claude-removed-model")),
1699                    ("complexity", serde_json::json!("medium")),
1700                ],
1701            )
1702            .await
1703            .unwrap();
1704
1705        let route = get_route_result(&store, "owner/repo", "7").await;
1706        let stored = store.get(store_id).await.unwrap().model;
1707        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1708        unsafe { env::remove_var("ORCH_HOME") };
1709
1710        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1711        assert_eq!(stored.as_deref(), Some("sonnet"));
1712    }
1713
1714    // Test-only delegate so tests can call router.parse_llm_response() directly
1715    // without referencing llm_router.
1716    impl Router {
1717        #[expect(
1718            clippy::unused_self,
1719            reason = "the test helper keeps the router method syntax"
1720        )]
1721        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1722            LlmRouter::parse_llm_response("claude", response)
1723        }
1724    }
1725
1726    async fn test_store() -> Arc<TaskStore> {
1727        Arc::new(TaskStore::open_memory().await.unwrap())
1728    }
1729
1730    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1731        ExternalTask {
1732            id: ExternalId(id.to_string()),
1733            title: title.to_string(),
1734            body: "Test body".to_string(),
1735            state: "open".to_string(),
1736            labels,
1737            author: "test".to_string(),
1738            created_at: "2026-01-01T00:00:00Z".to_string(),
1739            updated_at: "2026-01-01T00:00:00Z".to_string(),
1740            url: format!("https://github.com/test/test/issues/{id}"),
1741        }
1742    }
1743
1744    #[serial(cooldown_state)]
1745    #[test]
1746    fn extract_agent_from_labels() {
1747        let mut config = RouterConfig::default();
1748        // Ensure a concrete model exists for the labeled agent so the label
1749        // override is honored by the router (routing requires a model).
1750        config
1751            .model_map
1752            .entry("medium".to_string())
1753            .or_default()
1754            .insert("claude".to_string(), vec!["haiku".to_string()]);
1755
1756        assert_eq!(
1757            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1758            Some("claude".to_string())
1759        );
1760        assert_eq!(
1761            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1762            Some("codex".to_string())
1763        );
1764        assert_eq!(
1765            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1766            Some("opencode".to_string())
1767        );
1768        assert_eq!(
1769            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1770            None
1771        );
1772        // Verify kimi and minimax are recognized from labels
1773        assert_eq!(
1774            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1775            Some("kimi".to_string())
1776        );
1777        assert_eq!(
1778            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1779            Some("minimax".to_string())
1780        );
1781    }
1782
1783    #[serial(cooldown_state)]
1784    #[tokio::test]
1785    async fn all_agents_cooled_error_names_degraded_agents() {
1786        cooldown::reset_global_state().await;
1787        mark_agent_degraded("claude");
1788        mark_agent_degraded("grok");
1789        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1790        let msg = err.to_string();
1791        assert!(
1792            msg.contains("degraded: claude, grok"),
1793            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1794        );
1795        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1796        clear_agent_degraded("claude");
1797        clear_agent_degraded("grok");
1798    }
1799
1800    #[serial(cooldown_state)]
1801    #[test]
1802    fn default_agents_constant() {
1803        assert_eq!(DEFAULT_AGENTS.len(), 7);
1804        assert!(DEFAULT_AGENTS.contains(&"claude"));
1805        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1806        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1807        assert!(DEFAULT_AGENTS.contains(&"glm"));
1808        assert!(DEFAULT_AGENTS.contains(&"grok"));
1809    }
1810
1811    #[serial(cooldown_state)]
1812    #[test]
1813    fn extract_complexity_from_labels() {
1814        assert_eq!(
1815            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1816            "simple"
1817        );
1818        assert_eq!(
1819            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1820            "medium"
1821        );
1822        assert_eq!(
1823            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1824            "complex"
1825        );
1826        assert_eq!(
1827            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1828            "medium"
1829        );
1830    }
1831
1832    #[serial(cooldown_state)]
1833    #[test]
1834    fn route_result_serialization() {
1835        let result = RouteResult {
1836            agent: "claude".to_string(),
1837            model: Some("sonnet".to_string()),
1838            complexity: "medium".to_string(),
1839            estimate: 5,
1840            reason: "test".to_string(),
1841            profile: AgentProfile {
1842                role: "backend".to_string(),
1843                tools: vec!["git".to_string()],
1844                constraints: vec![],
1845            },
1846            warning: None,
1847        };
1848
1849        let json = serde_json::to_string(&result).unwrap();
1850        assert!(json.contains("claude"));
1851        assert!(json.contains("sonnet"));
1852
1853        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1854        assert_eq!(deserialized.agent, "claude");
1855        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1856    }
1857
1858    #[serial(cooldown_state)]
1859    #[test]
1860    fn router_config_default() {
1861        let mut config = RouterConfig::default();
1862        // Ensure a concrete model exists for the labeled agent so the label
1863        // override is honored by the router (routing requires a model).
1864        config
1865            .model_map
1866            .entry("medium".to_string())
1867            .or_default()
1868            .insert("claude".to_string(), vec!["haiku".to_string()]);
1869
1870        assert_eq!(config.mode, "round_robin");
1871        assert_eq!(config.router_agent, "claude");
1872        assert_eq!(
1873            config.router_model, "",
1874            "no router.model means the router agent's CLI default"
1875        );
1876        assert_eq!(config.fallback_executor, "codex");
1877        assert_eq!(config.max_route_attempts, 3);
1878        assert_ne!(config.allowed_tools, [] as [String; 0]);
1879
1880        // Verify configurable agents list includes all 7 agents
1881        assert_eq!(config.agents.len(), 7);
1882        assert!(config.agents.contains(&"claude".to_string()));
1883        assert!(config.agents.contains(&"codex".to_string()));
1884        assert!(config.agents.contains(&"opencode".to_string()));
1885        assert!(config.agents.contains(&"kimi".to_string()));
1886        assert!(config.agents.contains(&"minimax".to_string()));
1887        assert!(config.agents.contains(&"glm".to_string()));
1888        assert!(config.agents.contains(&"grok".to_string()));
1889    }
1890
1891    #[serial(cooldown_state)]
1892    #[test]
1893    fn model_map_lookup_returns_none_without_config() {
1894        // Default config has no hardcoded models — config.yml is the source of truth.
1895        let config = RouterConfig::default();
1896        assert!(
1897            config
1898                .model_for_complexity("claude", "simple", "")
1899                .is_none()
1900        );
1901        assert!(
1902            config
1903                .model_for_complexity("claude", "medium", "")
1904                .is_none()
1905        );
1906        assert!(
1907            config
1908                .model_for_complexity("claude", "complex", "")
1909                .is_none()
1910        );
1911        assert!(
1912            config
1913                .model_for_complexity("opencode", "review", "")
1914                .is_none()
1915        );
1916    }
1917
1918    #[serial(cooldown_state)]
1919    #[test]
1920    fn model_map_lookup_returns_configured_value() {
1921        // When model_map is populated (from config), it is returned correctly.
1922        let mut config = RouterConfig::default();
1923        config
1924            .model_map
1925            .entry("simple".to_string())
1926            .or_default()
1927            .insert("claude".to_string(), vec!["haiku".to_string()]);
1928        assert_eq!(
1929            config.model_for_complexity("claude", "simple", ""),
1930            Some("haiku".to_string())
1931        );
1932    }
1933
1934    #[serial(cooldown_state)]
1935    #[tokio::test]
1936    async fn model_pool_selection_skips_cooled() {
1937        cooldown::reset_global_state().await;
1938
1939        let mut config = RouterConfig::default();
1940        // Set up a two-model pool for opencode/simple
1941        config
1942            .model_map
1943            .entry("simple".to_string())
1944            .or_default()
1945            .insert(
1946                "opencode".to_string(),
1947                vec!["model-a".to_string(), "model-b".to_string()],
1948            );
1949
1950        // Before any cooldown, both models are candidates
1951        let m = config.model_for_complexity("opencode", "simple", "task-1");
1952        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1953
1954        // Cool model-a
1955        record_model_failure("opencode", "model-a").await;
1956        assert!(is_model_in_cooldown("opencode", "model-a"));
1957
1958        // Now only model-b should be returned
1959        // (try many task_ids to rule out hash coincidence)
1960        for i in 0..20 {
1961            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1962            assert_eq!(
1963                result,
1964                Some("model-b".to_string()),
1965                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1966            );
1967        }
1968
1969        // Cool model-b too — now all models are cooled
1970        record_model_failure("opencode", "model-b").await;
1971        // All cooled → return None so the caller can fall back to a different agent
1972        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1973        assert_eq!(fallback, None);
1974    }
1975
1976    #[serial(cooldown_state)]
1977    #[test]
1978    fn model_pool_single_string_backward_compat() {
1979        // Single-item pools behave identically to the old string format.
1980        // Manually insert a single-item pool and verify it is returned correctly.
1981        let mut config = RouterConfig::default();
1982        config
1983            .model_map
1984            .entry("simple".to_string())
1985            .or_default()
1986            .insert("claude".to_string(), vec!["haiku".to_string()]);
1987        let m = config.model_for_complexity("claude", "simple", "any-task");
1988        assert_eq!(m, Some("haiku".to_string()));
1989    }
1990
1991    #[serial(cooldown_state)]
1992    #[test]
1993    fn parse_llm_response_direct_json() {
1994        let config = RouterConfig::default();
1995        let router = Router::new(config);
1996
1997        let response = r#"{
1998            "executor": "claude",
1999            "complexity": "complex",
2000            "reason": "requires architecture analysis",
2001            "profile": {
2002                "role": "architect",
2003                "skills": ["rust", "design"],
2004                "tools": ["git", "rg"],
2005                "constraints": []
2006            },
2007            "selected_skills": ["gh"]
2008        }"#;
2009
2010        let parsed = router.parse_llm_response(response).unwrap();
2011        assert_eq!(parsed.executor, "claude");
2012        assert_eq!(parsed.complexity, "complex");
2013        assert_eq!(parsed.reason, "requires architecture analysis");
2014        assert_eq!(parsed.profile.role, "architect");
2015    }
2016
2017    #[serial(cooldown_state)]
2018    #[test]
2019    fn parse_llm_response_markdown_fenced() {
2020        let config = RouterConfig::default();
2021        let router = Router::new(config);
2022
2023        let response = r#"Here's my analysis:
2024
2025```json
2026{
2027    "executor": "codex",
2028    "complexity": "medium",
2029    "reason": "coding task",
2030    "profile": {
2031        "role": "developer",
2032        "skills": ["coding"],
2033        "tools": [],
2034        "constraints": []
2035    },
2036    "selected_skills": []
2037}
2038```
2039
2040Hope that helps!"#;
2041
2042        let parsed = router.parse_llm_response(response).unwrap();
2043        assert_eq!(parsed.executor, "codex");
2044        assert_eq!(parsed.complexity, "medium");
2045    }
2046
2047    #[serial(cooldown_state)]
2048    #[test]
2049    fn parse_llm_response_claude_envelope() {
2050        let config = RouterConfig::default();
2051        let router = Router::new(config);
2052
2053        // Claude --output-format json wraps the response in an envelope
2054        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2055
2056        let parsed = router.parse_llm_response(response).unwrap();
2057        assert_eq!(parsed.executor, "claude");
2058        assert_eq!(parsed.complexity, "complex");
2059        assert_eq!(parsed.reason, "requires deep analysis");
2060    }
2061
2062    #[serial(cooldown_state)]
2063    #[test]
2064    fn parse_llm_response_claude_envelope_with_code_block() {
2065        let config = RouterConfig::default();
2066        let router = Router::new(config);
2067
2068        // Claude may return the JSON inside a code block within the envelope
2069        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2070        let envelope = serde_json::json!({
2071            "type": "result",
2072            "subtype": "success",
2073            "is_error": false,
2074            "result": inner,
2075            "usage": {"input_tokens": 50, "output_tokens": 30}
2076        });
2077
2078        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2079        assert_eq!(parsed.executor, "codex");
2080        assert_eq!(parsed.complexity, "simple");
2081    }
2082
2083    #[serial(cooldown_state)]
2084    #[test]
2085    fn parse_llm_response_claude_envelope_error() {
2086        let config = RouterConfig::default();
2087        let router = Router::new(config);
2088
2089        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2090
2091        let err = router.parse_llm_response(response).unwrap_err();
2092        assert!(err.to_string().contains("error"), "got: {err}");
2093    }
2094
2095    #[serial(cooldown_state)]
2096    #[test]
2097    fn parse_llm_response_empty() {
2098        let config = RouterConfig::default();
2099        let router = Router::new(config);
2100
2101        let err = router.parse_llm_response("").unwrap_err();
2102        assert!(err.to_string().contains("empty"), "got: {err}");
2103    }
2104
2105    /// Claude envelope where result is a JSON object, not a string.
2106    /// Some Claude versions may return the result as a nested object.
2107    #[serial(cooldown_state)]
2108    #[test]
2109    fn parse_llm_response_claude_envelope_result_as_object() {
2110        let config = RouterConfig::default();
2111        let router = Router::new(config);
2112
2113        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2114
2115        let parsed = router.parse_llm_response(response);
2116        assert!(
2117            parsed.is_ok(),
2118            "should handle result-as-object: {}",
2119            parsed.unwrap_err()
2120        );
2121        assert_eq!(parsed.unwrap().executor, "claude");
2122    }
2123
2124    /// LLM returns "agent" instead of "executor" — common hallucination.
2125    #[serial(cooldown_state)]
2126    #[test]
2127    fn parse_llm_response_agent_instead_of_executor() {
2128        let config = RouterConfig::default();
2129        let router = Router::new(config);
2130
2131        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2132
2133        let parsed = router.parse_llm_response(response);
2134        assert!(
2135            parsed.is_ok(),
2136            "should accept 'agent' alias: {}",
2137            parsed.unwrap_err()
2138        );
2139        assert_eq!(parsed.unwrap().executor, "claude");
2140    }
2141
2142    /// LLM returns minimal JSON without profile or reason.
2143    #[serial(cooldown_state)]
2144    #[test]
2145    fn parse_llm_response_minimal_json() {
2146        let config = RouterConfig::default();
2147        let router = Router::new(config);
2148
2149        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2150
2151        let parsed = router.parse_llm_response(response);
2152        assert!(
2153            parsed.is_ok(),
2154            "should accept minimal JSON: {}",
2155            parsed.unwrap_err()
2156        );
2157        assert_eq!(parsed.unwrap().executor, "codex");
2158    }
2159
2160    // ── Real fixture tests: route prompt → response → parsed result ──
2161
2162    /// Real Claude envelope with result as escaped JSON string.
2163    #[serial(cooldown_state)]
2164    #[test]
2165    fn fixture_route_response_string() {
2166        let config = RouterConfig::default();
2167        let router = Router::new(config);
2168
2169        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2170        let parsed = router.parse_llm_response(response).unwrap();
2171        assert_eq!(parsed.executor, "codex");
2172        assert_eq!(parsed.complexity, "medium");
2173        assert_ne!(parsed.reason, "");
2174        assert_ne!(parsed.profile.role, "");
2175    }
2176
2177    /// Real Claude envelope with result as JSON object (not string).
2178    /// This was the root cause of all router failures on 2026-02-27.
2179    #[serial(cooldown_state)]
2180    #[test]
2181    fn fixture_route_response_object() {
2182        let config = RouterConfig::default();
2183        let router = Router::new(config);
2184
2185        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2186        let parsed = router.parse_llm_response(response).unwrap();
2187        assert_eq!(parsed.executor, "claude");
2188        assert_eq!(parsed.complexity, "complex");
2189    }
2190
2191    /// Real Claude envelope with result as string containing markdown + JSON.
2192    #[serial(cooldown_state)]
2193    #[test]
2194    fn fixture_route_response_markdown() {
2195        let config = RouterConfig::default();
2196        let router = Router::new(config);
2197
2198        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2199        let parsed = router.parse_llm_response(response).unwrap();
2200        assert_eq!(parsed.executor, "codex");
2201        assert_eq!(parsed.complexity, "medium");
2202    }
2203
2204    /// Claude envelope with escaped JSON string containing newlines and markdown.
2205    #[serial(cooldown_state)]
2206    #[test]
2207    fn parse_llm_response_claude_envelope_with_prose() {
2208        let config = RouterConfig::default();
2209        let router = Router::new(config);
2210
2211        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2212        let envelope = serde_json::json!({
2213            "type": "result",
2214            "subtype": "success",
2215            "is_error": false,
2216            "result": inner,
2217            "usage": {"input_tokens": 100, "output_tokens": 80}
2218        });
2219
2220        let parsed = router.parse_llm_response(&envelope.to_string());
2221        assert!(
2222            parsed.is_ok(),
2223            "should handle prose + code block in envelope: {}",
2224            parsed.unwrap_err()
2225        );
2226        assert_eq!(parsed.unwrap().executor, "codex");
2227    }
2228
2229    #[serial(cooldown_state)]
2230    #[tokio::test]
2231    async fn route_round_robin_basic() {
2232        cooldown::reset_global_state().await;
2233        // Force at least one agent to be available for testing
2234        // In real usage, discover_agents finds installed CLIs
2235        let config = RouterConfig {
2236            mode: "round_robin".to_string(),
2237            ..Default::default()
2238        };
2239
2240        // Create router with mock available agents
2241        let agents = vec!["claude".to_string(), "codex".to_string()];
2242        let mut weights = AgentWeights::default();
2243        weights.ensure_agents(&agents);
2244        let router = Router {
2245            config,
2246            available_agents: agents,
2247            weights,
2248            llm_router: Arc::new(LlmRouter::new()),
2249            ollama_router: None,
2250            rr_index: 0,
2251            last_agent: None,
2252            review_rr_index: 0,
2253            router_pool: vec![],
2254            pool_index: 0,
2255        };
2256
2257        let task = create_test_task("1", "Test task", vec![]);
2258        let result =
2259            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2260                .unwrap();
2261
2262        // Task 1 % 2 agents = agent at index 1 = codex
2263        assert_eq!(result.agent, "codex");
2264        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2265    }
2266
2267    #[serial(cooldown_state)]
2268    #[tokio::test]
2269    async fn route_uses_label_override() {
2270        cooldown::reset_global_state().await;
2271        let mut config = RouterConfig::default();
2272        // Ensure a concrete model exists so the label override can be honored
2273        // in the test environment where no external config.yml is present.
2274        config
2275            .model_map
2276            .entry("medium".to_string())
2277            .or_default()
2278            .insert("claude".to_string(), vec!["haiku".to_string()]);
2279        let agents = vec!["claude".to_string(), "codex".to_string()];
2280        let mut weights = AgentWeights::default();
2281        weights.ensure_agents(&agents);
2282        let mut router = Router {
2283            config,
2284            available_agents: agents,
2285            weights,
2286            llm_router: Arc::new(LlmRouter::new()),
2287            ollama_router: None,
2288            rr_index: 0,
2289            last_agent: None,
2290            review_rr_index: 0,
2291            router_pool: vec![],
2292            pool_index: 0,
2293        };
2294
2295        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2296        let store = test_store().await;
2297
2298        // Should use label override, not LLM
2299        let result = router.route(&task, &store, "test/repo").await.unwrap();
2300        assert_eq!(result.agent, "claude");
2301        assert!(result.reason.contains("label"));
2302    }
2303
2304    #[serial(cooldown_state)]
2305    #[tokio::test]
2306    async fn route_uses_stored_agent_over_label() {
2307        cooldown::reset_global_state().await;
2308        let mut config = RouterConfig::default();
2309        let medium = config.model_map.entry("medium".to_string()).or_default();
2310        medium.insert("claude".to_string(), vec!["haiku".to_string()]);
2311        medium.insert("codex".to_string(), vec!["gpt-test".to_string()]);
2312        let agents = vec!["claude".to_string(), "codex".to_string()];
2313        let mut router = Router::new_for_test(config, agents);
2314
2315        let store = test_store().await;
2316        let store_id = store
2317            .create(&NewTask {
2318                external_id: Some("1".to_string()),
2319                repo: "test/repo".to_string(),
2320                origin: "github".to_string(),
2321                title: "Test".to_string(),
2322                ..Default::default()
2323            })
2324            .await
2325            .unwrap();
2326        store
2327            .set_fields(store_id, &[("agent", serde_json::json!("codex"))])
2328            .await
2329            .unwrap();
2330
2331        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2332        let result = router.route(&task, &store, "test/repo").await.unwrap();
2333        assert_eq!(result.agent, "codex");
2334    }
2335
2336    #[serial(cooldown_state)]
2337    #[test]
2338    fn check_routing_sanity_warnings() {
2339        // Backend task routed to claude should warn
2340        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2341        let profile = AgentProfile {
2342            role: "general".to_string(),
2343            tools: vec![],
2344            constraints: vec![],
2345        };
2346        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2347        assert!(warning.is_some());
2348        assert!(warning.unwrap().contains("backend"));
2349
2350        // Docs task routed to codex should warn
2351        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2352        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2353        assert!(warning.is_some());
2354        assert!(warning.unwrap().contains("docs"));
2355
2356        // Normal routing should not warn
2357        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2358        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2359        assert!(warning.is_none());
2360    }
2361
2362    #[serial(cooldown_state)]
2363    #[tokio::test]
2364    async fn router_reload_preserves_structure() {
2365        cooldown::reset_global_state().await;
2366        let config = RouterConfig::default();
2367        let agents = vec!["claude".to_string()];
2368        let mut weights = AgentWeights::default();
2369        weights.ensure_agents(&agents);
2370        let mut router = Router {
2371            config,
2372            available_agents: agents,
2373            weights,
2374            llm_router: Arc::new(LlmRouter::new()),
2375            ollama_router: None,
2376            rr_index: 0,
2377            last_agent: None,
2378            review_rr_index: 0,
2379            router_pool: vec![],
2380            pool_index: 0,
2381        };
2382
2383        // Reload — should re-read config and remain valid
2384        router.reload_async().await;
2385
2386        // After reload, mode should be a valid value (llm, local, or round_robin)
2387        assert!(
2388            router.config.mode == "llm"
2389                || router.config.mode == "round_robin"
2390                || router.config.mode == "local",
2391            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2392            router.config.mode
2393        );
2394        // Fallback executor should always be set
2395        assert_ne!(router.config.fallback_executor, "");
2396        // Tools should always be populated
2397        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2398    }
2399
2400    // --- Weighted round-robin tests ---
2401
2402    #[serial(cooldown_state)]
2403    #[test]
2404    fn rate_limit_state_defaults_to_full_weight() {
2405        let state = RateLimitState::default();
2406        assert_eq!(state.weight, DEFAULT_WEIGHT);
2407        assert_eq!(state.consecutive_hits, 0);
2408        assert!(state.last_limited_at.is_none());
2409        assert!(!state.is_limited());
2410    }
2411
2412    #[serial(cooldown_state)]
2413    #[test]
2414    fn rate_limit_state_decays_on_hit() {
2415        let mut state = RateLimitState::default();
2416        state.record_rate_limit();
2417
2418        assert!(state.weight < DEFAULT_WEIGHT);
2419        assert_eq!(state.consecutive_hits, 1);
2420        assert!(state.last_limited_at.is_some());
2421        assert!(state.is_limited());
2422
2423        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2424        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2425        assert!((state.weight - expected).abs() < 1e-10);
2426    }
2427
2428    #[serial(cooldown_state)]
2429    #[test]
2430    fn rate_limit_state_never_drops_below_min() {
2431        let mut state = RateLimitState::default();
2432
2433        // Hit many times
2434        for _ in 0..100 {
2435            state.record_rate_limit();
2436        }
2437
2438        assert!(state.weight >= MIN_WEIGHT);
2439        assert_eq!(state.consecutive_hits, 100);
2440    }
2441
2442    #[serial(cooldown_state)]
2443    #[test]
2444    fn rate_limit_state_recovers_on_success() {
2445        let mut state = RateLimitState::default();
2446
2447        // Decay first
2448        state.record_rate_limit();
2449        state.record_rate_limit();
2450        let after_decay = state.weight;
2451
2452        // Record success
2453        state.record_success();
2454        assert!(state.weight > after_decay);
2455        assert_eq!(state.consecutive_hits, 0);
2456    }
2457
2458    #[serial(cooldown_state)]
2459    #[test]
2460    fn rate_limit_state_success_caps_at_default() {
2461        let mut state = RateLimitState::default();
2462
2463        // Already at full weight, success shouldn't exceed it
2464        state.record_success();
2465        assert_eq!(state.weight, DEFAULT_WEIGHT);
2466    }
2467
2468    #[serial(cooldown_state)]
2469    #[test]
2470    fn agent_weights_ensure_agents() {
2471        let mut weights = AgentWeights::default();
2472        let agents = vec!["claude".to_string(), "codex".to_string()];
2473        weights.ensure_agents(&agents);
2474
2475        assert_eq!(weights.states.len(), 2);
2476        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2477        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2478    }
2479
2480    #[serial(cooldown_state)]
2481    #[test]
2482    fn agent_weights_ensure_agents_preserves_existing() {
2483        let mut weights = AgentWeights::default();
2484        let agents = vec!["claude".to_string(), "codex".to_string()];
2485        weights.ensure_agents(&agents);
2486
2487        // Reduce claude's weight
2488        weights.record_rate_limit("claude");
2489        let claude_weight = weights.get_weight("claude");
2490        assert!(claude_weight < DEFAULT_WEIGHT);
2491
2492        // Ensure agents again — claude's weight should be preserved
2493        let agents2 = vec![
2494            "claude".to_string(),
2495            "codex".to_string(),
2496            "opencode".to_string(),
2497        ];
2498        weights.ensure_agents(&agents2);
2499
2500        assert_eq!(weights.get_weight("claude"), claude_weight);
2501        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2502    }
2503
2504    #[test]
2505    fn default_missing_weight_uses_minimum_of_configured() {
2506        let mut configured = HashMap::new();
2507        configured.insert("claude".to_string(), 0.2);
2508        configured.insert("codex".to_string(), 0.2);
2509        configured.insert("opencode".to_string(), 0.2);
2510        configured.insert("kimi".to_string(), 0.2);
2511
2512        // minimax omitted from router.weights — must not silently outrank
2513        // every explicitly configured agent (issue #3596).
2514        assert_eq!(default_missing_weight(&configured), 0.2);
2515    }
2516
2517    #[test]
2518    fn default_missing_weight_falls_back_when_none_configured() {
2519        let configured = HashMap::new();
2520        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2521    }
2522
2523    #[serial(cooldown_state)]
2524    #[test]
2525    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2526        let mut weights = AgentWeights::default();
2527        let mut configured = HashMap::new();
2528        configured.insert("claude".to_string(), 0.2);
2529        configured.insert("codex".to_string(), 0.2);
2530        configured.insert("opencode".to_string(), 0.2);
2531        configured.insert("kimi".to_string(), 0.2);
2532
2533        let agents = vec![
2534            "claude".to_string(),
2535            "codex".to_string(),
2536            "opencode".to_string(),
2537            "kimi".to_string(),
2538            "minimax".to_string(),
2539        ];
2540        weights.ensure_agents_with_weights(&agents, &configured);
2541
2542        // minimax was omitted from router.weights but must not get the old
2543        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2544        assert_eq!(weights.get_weight("minimax"), 0.2);
2545        assert_eq!(weights.get_weight("claude"), 0.2);
2546    }
2547
2548    #[serial(cooldown_state)]
2549    #[test]
2550    fn agent_weights_record_rate_limit() {
2551        let mut weights = AgentWeights::default();
2552        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2553
2554        weights.record_rate_limit("claude");
2555
2556        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2557        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2558    }
2559
2560    #[serial(cooldown_state)]
2561    #[test]
2562    fn agent_weights_record_success() {
2563        let mut weights = AgentWeights::default();
2564        weights.ensure_agents(&["claude".to_string()]);
2565
2566        weights.record_rate_limit("claude");
2567        let after_limit = weights.get_weight("claude");
2568
2569        weights.record_success("claude");
2570        assert!(weights.get_weight("claude") > after_limit);
2571    }
2572
2573    #[serial(cooldown_state)]
2574    #[test]
2575    fn agent_weights_snapshot() {
2576        let mut weights = AgentWeights::default();
2577        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2578        weights.record_rate_limit("claude");
2579
2580        let snap = weights.snapshot();
2581        assert_eq!(snap.len(), 2);
2582
2583        // Snapshot is sorted alphabetically
2584        assert_eq!(snap[0].0, "claude");
2585        assert!(snap[0].1 < DEFAULT_WEIGHT);
2586        assert_eq!(snap[0].2, 1); // 1 hit
2587
2588        assert_eq!(snap[1].0, "codex");
2589        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2590        assert_eq!(snap[1].2, 0); // 0 hits
2591    }
2592
2593    #[serial(cooldown_state)]
2594    #[test]
2595    fn agent_weights_weighted_select_favors_higher_weight() {
2596        let mut weights = AgentWeights::default();
2597        let agents = vec!["claude".to_string(), "codex".to_string()];
2598        weights.ensure_agents(&agents);
2599
2600        // Heavily penalize claude
2601        for _ in 0..10 {
2602            weights.record_rate_limit("claude");
2603        }
2604
2605        // Run many selections and count
2606        let mut claude_count = 0;
2607        let mut codex_count = 0;
2608        for _ in 0..100 {
2609            match weights.weighted_select(&agents, "task-1") {
2610                Some(ref a) if a == "claude" => claude_count += 1,
2611                Some(ref a) if a == "codex" => codex_count += 1,
2612                _ => {}
2613            }
2614        }
2615
2616        // Codex should get significantly more selections
2617        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2618        assert!(
2619            codex_count > claude_count,
2620            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2621        );
2622    }
2623
2624    #[serial(cooldown_state)]
2625    #[test]
2626    fn agent_weights_weighted_select_empty_returns_none() {
2627        let weights = AgentWeights::default();
2628        assert!(weights.weighted_select(&[], "task-1").is_none());
2629    }
2630
2631    #[serial(cooldown_state)]
2632    #[test]
2633    fn agent_weights_batch_routing_produces_varied_selections() {
2634        // Regression test: when routing a batch of tasks in the same tick,
2635        // different task IDs should not all select the same agent.
2636        let mut weights = AgentWeights::default();
2637        let agents = vec!["claude".to_string(), "codex".to_string()];
2638        weights.ensure_agents(&agents);
2639
2640        // Simulate batch routing: 20 different task IDs called in rapid succession
2641        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2642        let selections: Vec<String> = task_ids
2643            .iter()
2644            .filter_map(|id| weights.weighted_select(&agents, id))
2645            .collect();
2646
2647        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2648        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2649        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2650        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2651        assert!(
2652            claude_count > 0 && codex_count > 0,
2653            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2654        );
2655    }
2656
2657    #[serial(cooldown_state)]
2658    #[test]
2659    fn agent_weights_weighted_select_single_agent() {
2660        let mut weights = AgentWeights::default();
2661        let agents = vec!["claude".to_string()];
2662        weights.ensure_agents(&agents);
2663
2664        let selected = weights.weighted_select(&agents, "task-1");
2665        assert_eq!(selected, Some("claude".to_string()));
2666    }
2667
2668    #[serial(cooldown_state)]
2669    #[test]
2670    fn agent_weights_tick_recovery_restores_weight() {
2671        let mut weights = AgentWeights::default();
2672        weights.ensure_agents(&["claude".to_string()]);
2673
2674        // Set up a rate limit that looks like it happened long ago
2675        let state = weights.states.get_mut("claude").unwrap();
2676        state.weight = 0.5;
2677        state.last_limited_at = Some(
2678            Instant::now()
2679                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2680                .expect("the clock is past the recovery delay"),
2681        );
2682        state.consecutive_hits = 2;
2683
2684        let before = weights.get_weight("claude");
2685        weights.tick_recovery();
2686        let after = weights.get_weight("claude");
2687
2688        assert!(after > before, "weight should increase after recovery tick");
2689    }
2690
2691    #[serial(cooldown_state)]
2692    #[test]
2693    fn agent_weights_tick_recovery_clears_on_full_restore() {
2694        let mut weights = AgentWeights::default();
2695        weights.ensure_agents(&["claude".to_string()]);
2696
2697        // Set weight just below full with an old limit
2698        let state = weights.states.get_mut("claude").unwrap();
2699        state.weight = DEFAULT_WEIGHT - 0.01;
2700        state.last_limited_at = Some(
2701            Instant::now()
2702                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2703                .expect("the clock is past the recovery delay"),
2704        );
2705        state.consecutive_hits = 1;
2706
2707        weights.tick_recovery();
2708
2709        let state = weights.states.get("claude").unwrap();
2710        assert_eq!(state.weight, DEFAULT_WEIGHT);
2711        assert!(state.last_limited_at.is_none());
2712        assert_eq!(state.consecutive_hits, 0);
2713    }
2714
2715    #[serial(cooldown_state)]
2716    #[tokio::test]
2717    async fn route_weighted_round_robin_basic() {
2718        cooldown::reset_global_state().await;
2719        let mut config = RouterConfig {
2720            weighted_round_robin: true,
2721            ..Default::default()
2722        };
2723        // Provide a concrete model for codex so label override can be used.
2724        config
2725            .model_map
2726            .entry("medium".to_string())
2727            .or_default()
2728            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2729
2730        let agents = vec!["claude".to_string(), "codex".to_string()];
2731        let mut weights = AgentWeights::default();
2732        weights.ensure_agents(&agents);
2733        let mut router = Router {
2734            config,
2735            available_agents: agents,
2736            weights,
2737            llm_router: Arc::new(LlmRouter::new()),
2738            ollama_router: None,
2739            rr_index: 0,
2740            last_agent: None,
2741            review_rr_index: 0,
2742            router_pool: vec![],
2743            pool_index: 0,
2744        };
2745
2746        let task = create_test_task("1", "Test task", vec![]);
2747        let store = test_store().await;
2748        let result = router.route(&task, &store, "test/repo").await.unwrap();
2749
2750        // Should use weighted_round_robin
2751        assert!(result.reason.contains("weighted_round_robin"));
2752        assert!(
2753            result.agent == "claude" || result.agent == "codex",
2754            "agent should be claude or codex, got '{}'",
2755            result.agent
2756        );
2757    }
2758
2759    #[serial(cooldown_state)]
2760    #[tokio::test]
2761    async fn route_weighted_round_robin_respects_label_override() {
2762        cooldown::reset_global_state().await;
2763        let mut config = RouterConfig {
2764            weighted_round_robin: true,
2765            ..Default::default()
2766        };
2767        // Provide a concrete model for codex so label override can be used.
2768        config
2769            .model_map
2770            .entry("medium".to_string())
2771            .or_default()
2772            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2773
2774        let agents = vec!["claude".to_string(), "codex".to_string()];
2775        let mut weights = AgentWeights::default();
2776        weights.ensure_agents(&agents);
2777        let mut router = Router {
2778            config,
2779            available_agents: agents,
2780            weights,
2781            llm_router: Arc::new(LlmRouter::new()),
2782            ollama_router: None,
2783            rr_index: 0,
2784            last_agent: None,
2785            review_rr_index: 0,
2786            router_pool: vec![],
2787            pool_index: 0,
2788        };
2789
2790        // Label override should take precedence over weighted routing
2791        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2792        let store = test_store().await;
2793        let result = router.route(&task, &store, "test/repo").await.unwrap();
2794        assert_eq!(result.agent, "codex");
2795        assert!(result.reason.contains("label"));
2796    }
2797
2798    #[serial(cooldown_state)]
2799    #[tokio::test]
2800    async fn route_takes_the_adjacent_tier_model_when_the_own_tier_is_cooled() {
2801        cooldown::reset_global_state().await;
2802        let agent = "tierroute_agent";
2803        let mut config = RouterConfig {
2804            agents: vec![agent.to_string()],
2805            ..Default::default()
2806        };
2807        config.model_map.clear();
2808        for (tier, model) in [("medium", "tr-m"), ("complex", "tr-c")] {
2809            config
2810                .model_map
2811                .entry(tier.to_string())
2812                .or_default()
2813                .insert(agent.to_string(), vec![model.to_string()]);
2814        }
2815        cooldown::record_model_failure(agent, "tr-m").await;
2816        let mut router = Router::new_for_test(config, vec![agent.to_string()]);
2817
2818        let task = create_test_task(
2819            "1",
2820            "Test task",
2821            vec![format!("agent:{agent}"), "complexity:medium".to_string()],
2822        );
2823        let store = test_store().await;
2824        let result = router.route(&task, &store, "test/repo").await.unwrap();
2825        assert_eq!(result.agent, agent);
2826        assert_eq!(result.model.as_deref(), Some("tr-c"));
2827        assert_eq!(result.complexity, "medium");
2828        assert!(result.warning.unwrap().contains("complex tier"));
2829    }
2830
2831    #[serial(cooldown_state)]
2832    #[test]
2833    fn router_config_weighted_round_robin_default_false() {
2834        let config = RouterConfig::default();
2835        assert!(!config.weighted_round_robin);
2836    }
2837
2838    #[serial(cooldown_state)]
2839    #[test]
2840    fn router_record_and_recover_weights() {
2841        let config = RouterConfig {
2842            weighted_round_robin: true,
2843            ..Default::default()
2844        };
2845        let mut router = Router::new(config);
2846        // Override discovered agents for test
2847        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2848        router.weights.ensure_agents(&router.available_agents);
2849
2850        // Record rate limit
2851        router.record_rate_limit("claude");
2852        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2853        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2854
2855        // Record success for claude
2856        router.record_success("claude");
2857        let after_success = router.weights.get_weight("claude");
2858        assert!(after_success > MIN_WEIGHT);
2859    }
2860
2861    #[serial(cooldown_state)]
2862    #[test]
2863    fn review_rr_index_advances() {
2864        let mut config = RouterConfig::default();
2865        // Set up model_map so has_available_model_for_complexity returns true for test agents
2866        for agent in &["test_a", "test_b", "test_c"] {
2867            config
2868                .model_map
2869                .entry("medium".to_string())
2870                .or_default()
2871                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2872        }
2873        let agents = vec![
2874            "test_a".to_string(),
2875            "test_b".to_string(),
2876            "test_c".to_string(),
2877        ];
2878        let mut weights = AgentWeights::default();
2879        weights.ensure_agents(&agents);
2880        let mut router = Router {
2881            config,
2882            available_agents: agents,
2883            weights,
2884            llm_router: Arc::new(LlmRouter::new()),
2885            ollama_router: None,
2886            rr_index: 0,
2887            last_agent: None,
2888            review_rr_index: 0,
2889            router_pool: vec![],
2890            pool_index: 0,
2891        };
2892
2893        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2894        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2895        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2896
2897        // All three agents should appear (order depends on start index)
2898        let mut seen = vec![a1, a2, a3];
2899        seen.sort();
2900        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2901    }
2902
2903    #[serial(cooldown_state)]
2904    #[test]
2905    fn review_rr_fallback_with_cooled_agent() {
2906        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2907        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2908        // candidates.get(idx) to return None even though candidates is non-empty.
2909        let mut config = RouterConfig::default();
2910        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2911        for agent in &["test_a", "test_b", "test_c"] {
2912            config
2913                .model_map
2914                .entry("medium".to_string())
2915                .or_default()
2916                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2917        }
2918        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2919        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2920        let mut weights = AgentWeights::default();
2921        weights.ensure_agents(&agents);
2922        let mut router = Router {
2923            config,
2924            available_agents: agents,
2925            weights,
2926            llm_router: Arc::new(LlmRouter::new()),
2927            ollama_router: None,
2928            rr_index: 0,
2929            last_agent: None,
2930            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2931            review_rr_index: 2,
2932            router_pool: vec![],
2933            pool_index: 0,
2934        };
2935
2936        // All candidates excluded — should fall back to first candidate, not None
2937        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2938        assert!(
2939            result.is_some(),
2940            "next_round_robin_agent must not return None when candidates is non-empty"
2941        );
2942    }
2943
2944    #[serial(cooldown_state)]
2945    #[test]
2946    fn review_rr_index_uses_candidates_len_not_available_len() {
2947        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2948        // not available_agents.len(). When an agent has no model for the complexity,
2949        // candidates.len() < available_agents.len() and the old code produced a drifting
2950        // index that caused non-uniform distribution.
2951        let mut config = RouterConfig::default();
2952        // Only register models for test_a and test_b — test_c has no model, so it is
2953        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2954        for agent in &["test_a", "test_b"] {
2955            config
2956                .model_map
2957                .entry("medium".to_string())
2958                .or_default()
2959                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2960        }
2961        let agents = vec![
2962            "test_a".to_string(),
2963            "test_b".to_string(),
2964            "test_c".to_string(), // available but no model → filtered out of candidates
2965        ];
2966        let mut weights = AgentWeights::default();
2967        weights.ensure_agents(&agents);
2968        let mut router = Router {
2969            config,
2970            available_agents: agents,
2971            weights,
2972            llm_router: Arc::new(LlmRouter::new()),
2973            ollama_router: None,
2974            rr_index: 0,
2975            last_agent: None,
2976            review_rr_index: 0,
2977            router_pool: vec![],
2978            pool_index: 0,
2979        };
2980
2981        // With the bug: index advances modulo 3 (available_agents.len()), so the
2982        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2983        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2984        // looks uniform by accident, but adding a 4th available agent breaks it.
2985        //
2986        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2987        // cycle. Verify this by checking review_rr_index after each call.
2988
2989        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2990        assert_eq!(
2991            router.review_rr_index, 1,
2992            "index should be 1 after first call"
2993        );
2994
2995        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2996        assert_eq!(
2997            router.review_rr_index, 0,
2998            "index should wrap to 0 after second call"
2999        );
3000
3001        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
3002        assert_eq!(
3003            router.review_rr_index, 1,
3004            "index should be 1 after third call"
3005        );
3006
3007        // Agents must alternate uniformly
3008        assert_ne!(r1, r2, "consecutive calls should select different agents");
3009        assert_eq!(r1, r3, "third call should match first (clean cycle)");
3010    }
3011
3012    #[serial(cooldown_state)]
3013    #[tokio::test]
3014    async fn last_agent_tracks_routing() {
3015        cooldown::reset_global_state().await;
3016        let mut config = RouterConfig {
3017            mode: "round_robin".to_string(),
3018            ..Default::default()
3019        };
3020        config
3021            .model_map
3022            .entry("medium".to_string())
3023            .or_default()
3024            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
3025        config
3026            .model_map
3027            .entry("medium".to_string())
3028            .or_default()
3029            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
3030        let agents = vec!["test_x".to_string(), "test_y".to_string()];
3031        let mut weights = AgentWeights::default();
3032        weights.ensure_agents(&agents);
3033        let mut router = Router {
3034            config,
3035            available_agents: agents,
3036            weights,
3037            llm_router: Arc::new(LlmRouter::new()),
3038            ollama_router: None,
3039            rr_index: 0,
3040            last_agent: None,
3041            review_rr_index: 0,
3042            router_pool: vec![],
3043            pool_index: 0,
3044        };
3045
3046        assert!(router.last_agent.is_none());
3047
3048        let task = create_test_task("1", "Test routing", vec![]);
3049        let store = test_store().await;
3050        let result = router.route(&task, &store, "test/repo").await.unwrap();
3051
3052        assert!(router.last_agent.is_some());
3053        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
3054    }
3055
3056    // ── pool config ───────────────────────────────────────────────────────────
3057
3058    #[serial(cooldown_state)]
3059    #[test]
3060    fn parse_pool_entry_splits_on_first_colon() {
3061        let (agent, model) = super::parse_pool_entry("claude:haiku");
3062        assert_eq!(agent, "claude");
3063        assert_eq!(model, "haiku");
3064    }
3065
3066    #[serial(cooldown_state)]
3067    #[test]
3068    fn parse_pool_entry_model_with_slash() {
3069        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
3070        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
3071        assert_eq!(agent, "opencode");
3072        assert_eq!(model, "github-copilot/gpt-5-mini");
3073    }
3074
3075    #[serial(cooldown_state)]
3076    #[test]
3077    fn parse_pool_entry_no_colon() {
3078        let (agent, model) = super::parse_pool_entry("claude");
3079        assert_eq!(agent, "claude");
3080        assert_eq!(model, "");
3081    }
3082
3083    #[serial(cooldown_state)]
3084    #[test]
3085    fn effective_pool_uses_configured_pool() {
3086        let config = RouterConfig {
3087            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3088            ..RouterConfig::default()
3089        };
3090        let pool = config.effective_pool();
3091        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3092    }
3093
3094    #[serial(cooldown_state)]
3095    #[test]
3096    fn effective_pool_falls_back_to_router_agent_model() {
3097        let config = RouterConfig::default();
3098        // Default has no pool → derives from router_agent:router_model
3099        let pool = config.effective_pool();
3100        assert_eq!(pool.len(), 1);
3101        assert!(pool[0].starts_with(&config.router_agent));
3102        assert!(pool[0].contains(&config.router_model));
3103    }
3104
3105    #[serial(cooldown_state)]
3106    #[test]
3107    fn effective_fallback_uses_configured_fallback() {
3108        let config = RouterConfig {
3109            fallback: "claude:haiku".to_string(),
3110            ..RouterConfig::default()
3111        };
3112        assert_eq!(config.effective_fallback(), "claude:haiku");
3113    }
3114
3115    #[serial(cooldown_state)]
3116    #[test]
3117    fn effective_fallback_derives_from_router_agent_model() {
3118        let config = RouterConfig::default();
3119        let fallback = config.effective_fallback();
3120        assert!(fallback.starts_with(&config.router_agent));
3121        assert!(fallback.contains(&config.router_model));
3122    }
3123
3124    // ── pool expansion ────────────────────────────────────────────────────────
3125
3126    #[serial(cooldown_state)]
3127    #[test]
3128    fn expand_pool_basic_entries() {
3129        let config = RouterConfig {
3130            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3131            ..RouterConfig::default()
3132        };
3133        let expanded = Router::expand_pool(&config);
3134        assert_eq!(expanded.len(), 2);
3135        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3136        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3137    }
3138
3139    #[serial(cooldown_state)]
3140    #[test]
3141    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3142        let config = RouterConfig {
3143            pool: vec![],
3144            router_agent: "claude".to_string(),
3145            router_model: "haiku".to_string(),
3146            ..RouterConfig::default()
3147        };
3148        let expanded = Router::expand_pool(&config);
3149        assert_eq!(expanded.len(), 1);
3150        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3151    }
3152
3153    #[serial(cooldown_state)]
3154    #[test]
3155    fn expand_pool_opencode_free_skipped_when_not_installed() {
3156        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3157        // to nothing and the pool falls back to effective_fallback
3158        let config = RouterConfig {
3159            pool: vec!["opencode:free".to_string()],
3160            fallback: "claude:haiku".to_string(),
3161            ..RouterConfig::default()
3162        };
3163        let expanded = Router::expand_pool(&config);
3164        assert_eq!(expanded.len(), 1);
3165        assert_eq!(expanded[0].0, "claude");
3166        assert_eq!(expanded[0].1, "haiku");
3167    }
3168
3169    #[serial(cooldown_state)]
3170    #[test]
3171    fn expand_pool_single_entry_from_default() {
3172        let config = RouterConfig::default();
3173        let expanded = Router::expand_pool(&config);
3174        assert!(
3175            !expanded.is_empty(),
3176            "pool must never be empty after expansion"
3177        );
3178    }
3179
3180    // ── pool round-robin index ────────────────────────────────────────────────
3181
3182    #[serial(cooldown_state)]
3183    #[test]
3184    fn pool_index_initializes_to_zero() {
3185        let config = RouterConfig::default();
3186        let router = Router::new(config);
3187        assert_eq!(router.pool_index, 0);
3188    }
3189
3190    #[serial(cooldown_state)]
3191    #[test]
3192    fn router_pool_non_empty_after_construction() {
3193        let config = RouterConfig::default();
3194        let router = Router::new(config);
3195        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3196    }
3197
3198    /**
3199     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3200     * `opencode serve --service` child behind, so a suite run leaked one server per test
3201     * process. The stub on PATH records any exec, so this fails if discovery comes back
3202     **/
3203    #[cfg(unix)]
3204    #[serial(cooldown_state)]
3205    #[test]
3206    fn building_a_router_never_runs_opencode() {
3207        struct PathRestore(Option<OsString>);
3208        impl Drop for PathRestore {
3209            fn drop(&mut self) {
3210                match &self.0 {
3211                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3212                    Some(p) => unsafe { env::set_var("PATH", p) },
3213                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3214                    None => unsafe { env::remove_var("PATH") },
3215                }
3216                cmd_cache::reset_for_test();
3217            }
3218        }
3219
3220        let dir = tempfile::tempdir().unwrap();
3221        let marker = dir.path().join("spawned");
3222        let bin_dir = dir.path().join("bin");
3223        fs::create_dir(&bin_dir).unwrap();
3224        let stub = bin_dir.join("opencode");
3225        fs::write(
3226            &stub,
3227            format!(
3228                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3229                marker.display()
3230            ),
3231        )
3232        .unwrap();
3233        fs::set_permissions(&stub, Permissions::from_mode(0o755)).unwrap();
3234
3235        let prev_path = env::var_os("PATH");
3236        let mut new_path = bin_dir.as_os_str().to_os_string();
3237        new_path.push(":");
3238        if let Some(ref p) = prev_path {
3239            new_path.push(p);
3240        }
3241        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3242        unsafe { env::set_var("PATH", &new_path) };
3243        cmd_cache::reset_for_test();
3244        let _restore = PathRestore(prev_path);
3245
3246        // A config that names opencode everywhere discovery reads from
3247        let mut config = RouterConfig {
3248            pool: vec!["opencode:free".to_string()],
3249            fallback: "opencode:free".to_string(),
3250            ..RouterConfig::default()
3251        };
3252        config
3253            .model_map
3254            .entry("simple".to_string())
3255            .or_default()
3256            .insert(
3257                "opencode".to_string(),
3258                vec!["canary-model-free".to_string()],
3259            );
3260
3261        let _router = Router::new(config.clone());
3262        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3263        let _pool = Router::expand_pool(&config);
3264        // Discovery refreshes on a background thread, give it room to exec
3265        thread::sleep(Duration::from_millis(300));
3266
3267        assert!(
3268            !marker.exists(),
3269            "building a router ran opencode: {}",
3270            fs::read_to_string(&marker).unwrap_or_default().trim()
3271        );
3272        assert!(
3273            opencode_children().is_empty(),
3274            "building a router left opencode processes: {:?}",
3275            opencode_children()
3276        );
3277    }
3278
3279    /// Commands of this process's direct children that mention opencode
3280    #[cfg(unix)]
3281    fn opencode_children() -> Vec<String> {
3282        let me = process::id().to_string();
3283        let output = Command::new("ps")
3284            .args(["-axo", "ppid=,command="])
3285            .output()
3286            .expect("ps");
3287        String::from_utf8_lossy(&output.stdout)
3288            .lines()
3289            .filter_map(|line| {
3290                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3291                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3292            })
3293            .collect()
3294    }
3295
3296    #[serial(cooldown_state)]
3297    #[tokio::test]
3298    async fn reload_resets_pool_index() {
3299        cooldown::reset_global_state().await;
3300        let config = RouterConfig::default();
3301        let mut router = Router::new(config);
3302        router.pool_index = 5;
3303        router.reload_async().await;
3304        assert_eq!(router.pool_index, 0);
3305    }
3306
3307    #[serial(cooldown_state)]
3308    #[test]
3309    fn pool_index_advances_after_failed_attempt() {
3310        let mut router = Router::new(RouterConfig::default());
3311        router.router_pool = vec![
3312            ("claude".to_string(), "haiku".to_string()),
3313            ("codex".to_string(), "gpt-5.2".to_string()),
3314            (
3315                "opencode".to_string(),
3316                "github-copilot/gpt-5-mini".to_string(),
3317            ),
3318        ];
3319        router.pool_index = 0;
3320
3321        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3322
3323        assert_eq!(router.pool_index, 1);
3324    }
3325
3326    #[serial(cooldown_state)]
3327    #[test]
3328    fn pool_index_wraps_after_last_attempt() {
3329        let mut router = Router::new(RouterConfig::default());
3330        router.router_pool = vec![
3331            ("claude".to_string(), "haiku".to_string()),
3332            ("codex".to_string(), "gpt-5.2".to_string()),
3333            (
3334                "opencode".to_string(),
3335                "github-copilot/gpt-5-mini".to_string(),
3336            ),
3337        ];
3338        router.pool_index = 2;
3339
3340        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3341
3342        assert_eq!(router.pool_index, 0);
3343    }
3344
3345    #[serial(cooldown_state)]
3346    #[test]
3347    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3348        let router = Router::new(RouterConfig::default());
3349        assert!(router.earliest_pool_cooldown().is_none());
3350    }
3351
3352    #[serial(cooldown_state)]
3353    #[tokio::test]
3354    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3355        cooldown::reset_global_state().await;
3356        // Use explicit pool with predictable model names to avoid opencode discovery.
3357        let config = RouterConfig {
3358            pool: vec![
3359                "claude:haiku-4-5-20251001".to_string(),
3360                "opencode:haiku-4-5-20251001".to_string(),
3361            ],
3362            ..Default::default()
3363        };
3364        let router = Router::new(config);
3365        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3366        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3367
3368        let earliest = router.earliest_pool_cooldown();
3369        assert!(earliest.is_some());
3370        let now = chrono::Utc::now().timestamp();
3371        let remaining = earliest.unwrap() - now;
3372        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3373        assert!((295..=300).contains(&remaining));
3374    }
3375
3376    #[serial(cooldown_state)]
3377    #[tokio::test]
3378    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3379        cooldown::reset_global_state().await;
3380        // Use explicit pool so we can predict what entries exist.
3381        let config = RouterConfig {
3382            pool: vec![
3383                "claude:pool-model-a".to_string(),
3384                "opencode:pool-model-b".to_string(),
3385            ],
3386            ..Default::default()
3387        };
3388        let router = Router::new(config);
3389        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3390        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3391        // Set cooldown on a pool entry (opencode:pool-model-b)
3392        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3393
3394        let earliest = router.earliest_pool_cooldown();
3395        assert!(earliest.is_some());
3396        let now = chrono::Utc::now().timestamp();
3397        let remaining = earliest.unwrap() - now;
3398        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3399        assert!((295..=300).contains(&remaining));
3400    }
3401
3402    // ---- Pre-emptive health check: degraded agent exclusion ----
3403
3404    #[serial(cooldown_state)]
3405    #[test]
3406    fn degraded_agent_excluded_from_routing() {
3407        let mut config = RouterConfig::default();
3408        config
3409            .model_map
3410            .entry("medium".to_string())
3411            .or_default()
3412            .insert(
3413                "test_degraded_routing".to_string(),
3414                vec!["model-a".to_string()],
3415            );
3416
3417        let mut router = Router::new(config);
3418        router.available_agents = vec!["test_degraded_routing".to_string()];
3419
3420        // Agent is routable before degradation
3421        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3422
3423        // Mark as degraded
3424        mark_agent_degraded("test_degraded_routing");
3425
3426        // Now excluded from routing
3427        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3428
3429        // available_agents_for_complexity returns empty
3430        assert!(
3431            router.available_agents_for_complexity("medium").is_empty(),
3432            "degraded agent should be excluded from available agents"
3433        );
3434
3435        // healthy_agent_count reflects exclusion
3436        assert_eq!(router.healthy_agent_count("medium"), 0);
3437
3438        // Cleanup
3439        clear_agent_degraded("test_degraded_routing");
3440    }
3441
3442    #[serial(cooldown_state)]
3443    #[test]
3444    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3445        let mut config = RouterConfig::default();
3446        for agent in &["agent_healthy", "agent_degraded"] {
3447            config
3448                .model_map
3449                .entry("medium".to_string())
3450                .or_default()
3451                .insert(agent.to_string(), vec!["model-x".to_string()]);
3452        }
3453
3454        let mut router = Router::new(config);
3455        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3456
3457        mark_agent_degraded("agent_degraded");
3458
3459        let candidates = router.available_agents_for_complexity("medium");
3460        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3461        assert_eq!(router.healthy_agent_count("medium"), 1);
3462
3463        // Cleanup
3464        clear_agent_degraded("agent_degraded");
3465    }
3466
3467    #[serial(cooldown_state)]
3468    #[tokio::test]
3469    async fn refresh_health_marks_degraded_from_rate_limits() {
3470        cooldown::reset_global_state().await;
3471        let store = test_store().await;
3472        let agent = "test_refresh_health_agent";
3473
3474        let mut config = RouterConfig::default();
3475        config
3476            .model_map
3477            .entry("medium".to_string())
3478            .or_default()
3479            .insert(agent.to_string(), vec!["model-z".to_string()]);
3480
3481        let mut router = Router::new(config);
3482        router.available_agents = vec![agent.to_string()];
3483
3484        // Insert rate limit events exceeding default threshold (3)
3485        for _ in 0..4 {
3486            store.record_rate_limit(agent, "rate", None).await.unwrap();
3487        }
3488
3489        router.refresh_health(&store).await;
3490
3491        assert!(
3492            is_agent_degraded(agent),
3493            "agent should be degraded after exceeding rate limit threshold"
3494        );
3495        assert!(
3496            !router.agent_is_routable(agent, "medium"),
3497            "degraded agent should not be routable"
3498        );
3499
3500        // Cleanup
3501        clear_agent_degraded(agent);
3502    }
3503
3504    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3505
3506    #[serial(cooldown_state)]
3507    #[tokio::test]
3508    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3509        cooldown::reset_global_state().await;
3510        let mut config = RouterConfig::default();
3511        // Agent's only model, shared across every complexity tier it resolves to.
3512        for tier in ["simple", "medium", "complex", "review"] {
3513            config
3514                .model_map
3515                .entry(tier.to_string())
3516                .or_default()
3517                .insert(
3518                    "test_all_cooled".to_string(),
3519                    vec!["only-model".to_string()],
3520                );
3521        }
3522
3523        let mut router = Router::new(config);
3524        router.available_agents = vec!["test_all_cooled".to_string()];
3525
3526        assert!(router.agent_has_any_available_model("test_all_cooled"));
3527
3528        record_model_failure("test_all_cooled", "only-model").await;
3529        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3530
3531        assert!(
3532            !router.agent_has_any_available_model("test_all_cooled"),
3533            "agent with its only model cooled across every tier should have no available model"
3534        );
3535    }
3536
3537    #[serial(cooldown_state)]
3538    #[tokio::test]
3539    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3540        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3541        // agent that has no agent-level cooldown but whose only resolvable model (across
3542        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3543        cooldown::reset_global_state().await;
3544        let mut config = RouterConfig::default();
3545        for tier in ["simple", "medium", "complex", "review"] {
3546            config
3547                .model_map
3548                .entry(tier.to_string())
3549                .or_default()
3550                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3551            config
3552                .model_map
3553                .get_mut(tier)
3554                .unwrap()
3555                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3556        }
3557
3558        let mut router = Router::new(config);
3559        router.available_agents = vec![
3560            "agent_cooled_model".to_string(),
3561            "agent_healthy".to_string(),
3562        ];
3563
3564        record_model_failure("agent_cooled_model", "opus").await;
3565        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3566        // No agent-level cooldown — the old agent-only filter would still offer it.
3567        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3568
3569        let uncooled_agents: Vec<String> = router
3570            .available_agents
3571            .iter()
3572            .filter(|a| {
3573                !is_agent_in_cooldown(a)
3574                    && !is_agent_degraded(a)
3575                    && router.agent_has_available_model_for_coding_tiers(a)
3576            })
3577            .cloned()
3578            .collect();
3579
3580        assert_eq!(
3581            uncooled_agents,
3582            vec!["agent_healthy".to_string()],
3583            "agent whose only model is cooled must be excluded from the LLM candidate list"
3584        );
3585    }
3586
3587    #[serial(cooldown_state)]
3588    #[tokio::test]
3589    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3590        // Regression test for #3521: an agent whose only healthy tier is `simple`
3591        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3592        // excluded from the LLM classification candidate list, otherwise the
3593        // classifier can pick it for a `medium`/`complex` task it can't serve,
3594        // wasting the classification call. Mirrors the real-world shape where
3595        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3596        cooldown::reset_global_state().await;
3597        let mut config = RouterConfig::default();
3598        config
3599            .model_map
3600            .entry("simple".to_string())
3601            .or_default()
3602            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3603        config
3604            .model_map
3605            .entry("medium".to_string())
3606            .or_default()
3607            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3608        config
3609            .model_map
3610            .entry("complex".to_string())
3611            .or_default()
3612            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3613        // review must be irrelevant to the coding-tier filter
3614        config
3615            .model_map
3616            .entry("review".to_string())
3617            .or_default()
3618            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3619        for tier in ["simple", "medium", "complex", "review"] {
3620            config
3621                .model_map
3622                .get_mut(tier)
3623                .unwrap()
3624                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3625        }
3626
3627        let mut router = Router::new(config);
3628        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3629
3630        record_model_failure("minimax_like", "opus").await;
3631        assert!(is_model_in_cooldown("minimax_like", "opus"));
3632        // No agent-level cooldown, and the simple tier is healthy, so the loose
3633        // "any tier" check still passes — this is exactly the leak #3511 missed.
3634        assert!(!is_agent_in_cooldown("minimax_like"));
3635        assert!(router.agent_has_any_available_model("minimax_like"));
3636        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3637
3638        let uncooled_agents: Vec<String> = router
3639            .available_agents
3640            .iter()
3641            .filter(|a| {
3642                !is_agent_in_cooldown(a)
3643                    && !is_agent_degraded(a)
3644                    && router.agent_has_available_model_for_coding_tiers(a)
3645            })
3646            .cloned()
3647            .collect();
3648
3649        assert_eq!(
3650            uncooled_agents,
3651            vec!["agent_healthy".to_string()],
3652            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3653        );
3654    }
3655}