Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22use crate::backends::ExternalTask;
23use crate::cmd_cache;
24use crate::engine::cooldown;
25use crate::engine::cooldown::{
26    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
27    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
28};
29use crate::engine::runner::agents::opencode;
30use crate::engine::runner::response;
31use crate::store::WaitingOnDependency;
32use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
33pub use config::{RouterConfig, parse_pool_entry};
34use llm::{LlmRouter, TIMEOUT_PREFIX};
35use serde::{Deserialize, Serialize};
36use std::fmt;
37use std::fmt::Debug;
38use std::fmt::Formatter;
39use std::sync::Arc;
40use tokio::task;
41pub use weights::AgentWeights;
42
43fn fmt_degraded(degraded: &[String]) -> String {
44    if degraded.is_empty() {
45        String::new()
46    } else {
47        format!(" (degraded: {})", degraded.join(", "))
48    }
49}
50
51/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
52///
53/// This is a transient error that should not count against `max_route_attempts` since
54/// no agent was actually invoked. The caller should back off and retry later.
55#[derive(Debug, thiserror::Error)]
56pub enum AllAgentsCooledError {
57    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
58    AllCooled {
59        scope: String,
60        remaining_secs: Option<u64>,
61        degraded: Vec<String>,
62    },
63}
64
65impl AllAgentsCooledError {
66    /// Create a new error for the given scope, naming currently degraded agents
67    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
68        Self::AllCooled {
69            scope,
70            remaining_secs,
71            degraded: list_degraded_agents()
72                .into_iter()
73                .map(|(agent, _reason)| agent)
74                .collect(),
75        }
76    }
77
78    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
79
80    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
81    pub fn scope(&self) -> &str {
82        match self {
83            Self::AllCooled { scope, .. } => scope,
84        }
85    }
86
87    /// Remaining cooldown duration in seconds when available.
88    pub fn remaining_secs(&self) -> Option<u64> {
89        match self {
90            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
91        }
92    }
93
94    /// Agents currently marked degraded, shown by `orch cooldown list`
95    pub fn degraded(&self) -> &[String] {
96        match self {
97            Self::AllCooled { degraded, .. } => degraded,
98        }
99    }
100}
101
102/// Typed error returned by [`get_route_result`].
103#[derive(Debug, thiserror::Error)]
104pub enum RouteResultError {
105    /// The task has no agent field — it needs to be re-routed.
106    #[error("no agent configured for task {task_id}")]
107    NoAgent { task_id: String },
108    /// Any other failure (store lookup, SQL, etc.).
109    #[error(transparent)]
110    Other(#[from] anyhow::Error),
111}
112
113/// Result of routing a task to an agent.
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
115pub struct RouteResult {
116    /// The selected agent: "claude", "codex", or "opencode"
117    pub agent: String,
118    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
119    pub model: Option<String>,
120    /// Complexity level: "simple", "medium", or "complex"
121    pub complexity: String,
122    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
123    pub estimate: u8,
124    /// Why this agent was selected
125    pub reason: String,
126    /// Specialized agent profile (skills, tools, constraints)
127    pub profile: AgentProfile,
128    /// Selected skill IDs from the catalog
129    pub selected_skills: Vec<String>,
130    /// Optional warning about routing decision
131    pub warning: Option<String>,
132}
133
134/// Specialized agent profile for a task.
135#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
136pub struct AgentProfile {
137    /// Role name, e.g., "backend specialist"
138    pub role: String,
139    /// Focus skills for this task
140    pub skills: Vec<String>,
141    /// Tools allowed for this task
142    pub tools: Vec<String>,
143    /// Constraints for this task
144    pub constraints: Vec<String>,
145}
146
147impl AgentProfile {
148    /// Build the default "general" profile from router config.
149    pub fn default_from_config(config: &RouterConfig) -> Self {
150        Self {
151            role: "general".to_string(),
152            skills: vec![],
153            tools: config.allowed_tools.clone(),
154            constraints: vec![],
155        }
156    }
157}
158
159impl RouteResult {
160    /// Build a standard route result from the three task-specific values.
161    ///
162    /// Fills `profile`, `selected_skills`, and `warning` with router-config
163    /// defaults so call sites only need to supply what varies per task.
164    pub fn from_config(
165        agent: String,
166        complexity: String,
167        reason: String,
168        config: &RouterConfig,
169        task_id: &str,
170    ) -> Self {
171        let model = config.model_for_complexity(&agent, &complexity, task_id);
172        Self {
173            agent,
174            model,
175            complexity,
176            estimate: 0,
177            reason,
178            profile: AgentProfile::default_from_config(config),
179            selected_skills: config.default_skills.clone(),
180            warning: None,
181        }
182    }
183}
184
185/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
186///
187/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
188/// **not** require holding the router's `RwLock`, so async operations can
189/// safely be called without starving write-lock waiters.
190pub struct LlmRouterHandle(Arc<LlmRouter>);
191
192impl Debug for LlmRouterHandle {
193    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
194        f.debug_struct("LlmRouterHandle").finish_non_exhaustive()
195    }
196}
197
198impl LlmRouterHandle {
199    /// Invalidate the skills catalog cache so the next routing call reloads
200    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
201    pub async fn invalidate_skills_catalog(&self) {
202        self.0.invalidate_skills_catalog().await;
203    }
204}
205
206/// The agent router.
207#[expect(
208    clippy::struct_field_names,
209    reason = "the router holds sub-routers whose names say which backend they use"
210)]
211pub struct Router {
212    /// Router configuration
213    pub config: RouterConfig,
214    /// Available agents discovered at runtime
215    pub available_agents: Vec<String>,
216    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
217    pub weights: AgentWeights,
218    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
219    /// without holding the router RwLock across async operations)
220    llm_router: Arc<LlmRouter>,
221    /// Ollama router for local routing (contains reusable reqwest::Client)
222    ollama_router: Option<ollama::OllamaRouter>,
223    /// Round-robin index for task routing
224    pub(crate) rr_index: usize,
225    /// Last agent routed to (for distribution tracking)
226    pub(crate) last_agent: Option<String>,
227    /// Round-robin index for review agent selection
228    pub(crate) review_rr_index: usize,
229    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
230    /// `opencode:free` entries are expanded at construction time.
231    pub(crate) router_pool: Vec<(String, String)>,
232    /// Current round-robin index into router_pool
233    pub(crate) pool_index: usize,
234}
235
236impl Debug for Router {
237    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
238        f.debug_struct("Router")
239            .field("config", &self.config)
240            .field("available_agents", &self.available_agents)
241            .field("weights", &self.weights)
242            .field("ollama_router", &self.ollama_router)
243            .field("rr_index", &self.rr_index)
244            .field("last_agent", &self.last_agent)
245            .field("review_rr_index", &self.review_rr_index)
246            .field("router_pool", &self.router_pool)
247            .field("pool_index", &self.pool_index)
248            .finish_non_exhaustive()
249    }
250}
251
252impl Router {
253    /// Create a new router with the given configuration.
254    pub fn new(config: RouterConfig) -> Self {
255        let available_agents = Self::discover_agents(&config.agents);
256        let mut weights = AgentWeights {
257            base_weights: config.weights.clone(),
258            ..Default::default()
259        };
260        weights.ensure_agents_with_weights(&available_agents, &config.weights);
261        if !config.weights.is_empty() {
262            tracing::info!(
263                weights = ?config.weights,
264                "configured agent routing weights"
265            );
266        }
267        let router_pool = Self::expand_pool(&config);
268        tracing::info!(
269            pool = ?router_pool,
270            fallback = %config.effective_fallback(),
271            "router LLM pool initialized"
272        );
273        // Prime only when this config names opencode, so a default or claude-only
274        // router never runs `opencode models`
275        if config.uses_opencode() {
276            opencode::prime_free_model_cache();
277        }
278        let ollama_router = if config.mode == "local" {
279            Some(ollama::OllamaRouter::new(&config))
280        } else {
281            None
282        };
283        Self {
284            config,
285            available_agents,
286            weights,
287            llm_router: Arc::new(LlmRouter::new()),
288            ollama_router,
289            rr_index: 0,
290            last_agent: None,
291            review_rr_index: 0,
292            router_pool,
293            pool_index: 0,
294        }
295    }
296
297    /// Create a router with default configuration loaded from files.
298    pub fn from_config() -> Self {
299        Self::new(RouterConfig::from_config())
300    }
301
302    /// Create a router with a forced agent list, bypassing CLI discovery.
303    /// Only for use in unit tests.
304    #[cfg(test)]
305    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
306        let mut weights = AgentWeights {
307            base_weights: config.weights.clone(),
308            ..Default::default()
309        };
310        weights.ensure_agents(&available_agents);
311        let router_pool = Self::expand_pool(&config);
312        Self {
313            config,
314            available_agents,
315            weights,
316            llm_router: Arc::new(LlmRouter::new()),
317            ollama_router: None,
318            rr_index: 0,
319            last_agent: None,
320            review_rr_index: 0,
321            router_pool,
322            pool_index: 0,
323        }
324    }
325
326    /// Return a cloned handle to the LLM router subsystem.
327    ///
328    /// Callers that need to invoke async methods on the underlying `LlmRouter`
329    /// should clone this handle *before* releasing any `RwLock` read guard on
330    /// `Router`, then call the async method after the guard is dropped.  This
331    /// prevents holding the read lock across await points (which starves
332    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
333    pub fn llm_router_handle(&self) -> LlmRouterHandle {
334        LlmRouterHandle(Arc::clone(&self.llm_router))
335    }
336
337    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
338        debug_assert!(pool_len > 0, "router pool must not be empty");
339        self.pool_index = (idx + 1) % pool_len;
340    }
341
342    /// Discover available agent CLIs in PATH.
343    /// Checks all agents from the configured list.
344    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
345        let mut agents = Vec::new();
346        for agent in configured_agents {
347            if cmd_cache::command_exists(agent) {
348                agents.push(agent.clone());
349            }
350        }
351        agents
352    }
353
354    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
355    ///
356    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
357    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
358    /// - If `opencode` is not installed or the command fails, the entry is skipped.
359    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
360    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
361        let raw_pool = config.effective_pool();
362        let mut expanded: Vec<(String, String)> = Vec::new();
363
364        for entry in &raw_pool {
365            if entry == "opencode:free" {
366                let free_models = Self::discover_free_opencode_models();
367                if free_models.is_empty() {
368                    tracing::debug!("opencode:free expanded to nothing — skipping");
369                } else {
370                    tracing::debug!(models = ?free_models, "opencode:free expanded");
371                    for model in free_models {
372                        expanded.push(("opencode".to_string(), model));
373                    }
374                }
375            } else {
376                let (agent, model) = parse_pool_entry(entry);
377                expanded.push((agent, model));
378            }
379        }
380
381        if expanded.is_empty() {
382            // All entries were opencode:free and opencode isn't installed — use fallback
383            let (agent, model) = parse_pool_entry(&config.effective_fallback());
384            tracing::debug!(
385                agent = %agent, model = %model,
386                "pool empty after expansion, using fallback as sole pool entry"
387            );
388            expanded.push((agent, model));
389        }
390
391        expanded
392    }
393
394    /// Async version of expand_pool which may query the opencode CLI without
395    /// blocking the Tokio runtime.
396    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
397        let raw_pool = config.effective_pool();
398        let mut expanded: Vec<(String, String)> = Vec::new();
399
400        for entry in &raw_pool {
401            if entry == "opencode:free" {
402                let free_models = Self::discover_free_opencode_models_async().await;
403                if free_models.is_empty() {
404                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
405                } else {
406                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
407                    for model in free_models {
408                        expanded.push(("opencode".to_string(), model));
409                    }
410                }
411            } else {
412                let (agent, model) = parse_pool_entry(entry);
413                expanded.push((agent, model));
414            }
415        }
416
417        if expanded.is_empty() {
418            // All entries were opencode:free and opencode isn't installed — use fallback
419            let (agent, model) = parse_pool_entry(&config.effective_fallback());
420            tracing::debug!(
421                agent = %agent, model = %model,
422                "pool empty after async expansion, using fallback as sole pool entry"
423            );
424            expanded.push((agent, model));
425        }
426
427        expanded
428    }
429
430    /// Async reload that uses async pool expansion to avoid blocking the runtime
431    /// when discovering opencode free models.
432    pub async fn reload_async(&mut self) {
433        let new_config = RouterConfig::from_config();
434        let new_agents = Self::discover_agents(&new_config.agents);
435        let new_pool = Self::expand_pool_async(&new_config).await;
436        tracing::info!(
437            mode = %new_config.mode,
438            agents = ?new_agents,
439            fallback = %new_config.fallback_executor,
440            weighted_rr = new_config.weighted_round_robin,
441            pool = ?new_pool,
442            "router reloaded (async)"
443        );
444        self.config = new_config;
445        self.available_agents.clone_from(&new_agents);
446        // Ensure new agents have weight entries (preserves existing weights)
447        self.weights.ensure_agents(&new_agents);
448        self.router_pool = new_pool;
449        self.pool_index = 0;
450        // Reinitialize Ollama router if mode changed to local
451        self.ollama_router = if self.config.mode == "local" {
452            Some(ollama::OllamaRouter::new(&self.config))
453        } else {
454            None
455        };
456    }
457
458    fn discover_free_opencode_models() -> Vec<String> {
459        opencode::discover_free_opencode_models()
460    }
461
462    async fn discover_free_opencode_models_async() -> Vec<String> {
463        task::spawn_blocking(Self::discover_free_opencode_models)
464            .await
465            .unwrap_or_default()
466    }
467
468    /// Run the pre-emptive health check, refreshing the degraded-agent set.
469    ///
470    /// Queries the `rate_limits` table for recent events and combines with
471    /// cooldown state to mark agents as degraded before routing attempts them.
472    pub async fn refresh_health(&self, store: &Arc<TaskStore>) {
473        // KV is the source of truth for cooldowns. Reconcile the in-memory map
474        // from KV every tick so external mutations (e.g. `orch cooldown clear`
475        // from a separate CLI process) actually take effect on the running
476        // service. Without this, the engine keeps stale entries until they
477        // expire on their own, producing the "all agents cooled" log with
478        // `orch cooldown list` simultaneously reporting "No active cooldowns".
479        sync_from_kv(store).await;
480
481        let agents = self.available_agents.clone();
482        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
483        refresh_degraded_agents(
484            store,
485            &agents,
486            &model_checker,
487            config::health_check_window_hours(),
488            config::degraded_rate_limit_threshold(),
489        )
490        .await;
491    }
492
493    /// Check if an agent is available.
494    pub fn is_agent_available(&self, agent: &str) -> bool {
495        self.available_agents.contains(&agent.to_string())
496    }
497
498    /// True if the agent has at least one uncooled model across any complexity
499    /// tier. Used where the eventual complexity isn't known yet (e.g. before
500    /// LLM classification), so we can't check a single tier.
501    fn agent_has_any_available_model(&self, agent: &str) -> bool {
502        ["simple", "medium", "complex", "review"]
503            .iter()
504            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
505    }
506
507    /// True if the agent has an available (uncooled) model for the `medium`
508    /// and `complex` tiers — the tiers real coding tasks are classified into
509    /// by the LLM classifier. Used to pre-filter the candidate list handed to
510    /// that classifier so an agent whose healthy tiers are only `simple` isn't
511    /// offered: the classifier can't be constrained to `simple`, so offering
512    /// such an agent produces a near-certain wasted classification call whenever
513    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
514    /// outcome of initial classification and is deliberately excluded.
515    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
516        ["medium", "complex"]
517            .iter()
518            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
519    }
520
521    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
522        if !self.is_agent_available(agent) {
523            return false;
524        }
525        if is_agent_in_cooldown(agent) {
526            tracing::debug!(agent, "agent skipped: in cooldown");
527            return false;
528        }
529        if is_agent_degraded(agent) {
530            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
531            return false;
532        }
533        // If weighted routing is used, consider the agent degraded when its
534        // weight has decayed below the threshold. This avoids
535        // proactively routing to agents that recently hit many rate limits.
536        if self.config.weighted_round_robin {
537            let weight = self.weights.get_weight(agent);
538            if weight < config::SKIP_LIMITED_THRESHOLD {
539                tracing::debug!(
540                    agent,
541                    weight,
542                    "agent considered degraded by weight threshold, skipping"
543                );
544                return false;
545            }
546        }
547        if !self
548            .config
549            .has_available_model_for_complexity(agent, complexity)
550        {
551            tracing::debug!(agent, complexity, "agent skipped: no available model");
552            return false;
553        }
554        true
555    }
556
557    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
558        self.available_agents
559            .iter()
560            .filter(|agent| self.agent_is_routable(agent, complexity))
561            .cloned()
562            .collect()
563    }
564
565    /// Count of healthy (routable) agents for a given complexity.
566    /// Returns the number of agents that are available, not in cooldown, and have an available model.
567    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
568        self.available_agents_for_complexity(complexity).len()
569    }
570
571    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
572    /// Uses "simple" complexity as the baseline since it has the widest model support.
573    pub fn is_degraded(&self, threshold: usize) -> bool {
574        self.healthy_agent_count("simple") < threshold
575    }
576
577    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
578        let mut earliest: Option<i64> = None;
579
580        for agent in &self.available_agents {
581            if let Some(until) = cooldown_until(agent) {
582                earliest = Some(earliest.map_or(until, |current| current.min(until)));
583            }
584
585            if let Some(comp) = complexity
586                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
587            {
588                for model in pool {
589                    let key = format!("{agent}:{model}");
590                    if let Some(until) = cooldown_until(&key) {
591                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
592                    }
593                }
594            }
595        }
596
597        earliest
598    }
599
600    /// Find the earliest cooldown expiration among the router LLM pool entries.
601    ///
602    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
603    /// or `None` if no pool model has a cooldown.
604    fn earliest_pool_cooldown(&self) -> Option<i64> {
605        let mut earliest: Option<i64> = None;
606        for (agent, model) in &self.router_pool {
607            let key = format!("{agent}:{model}");
608            if let Some(until) = cooldown_until(&key) {
609                earliest = Some(earliest.map_or(until, |current| current.min(until)));
610            }
611        }
612        earliest
613    }
614
615    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
616        // If any cooldowns are present, return an error immediately so the
617        // caller (the tick loop) can skip this task and retry on the next tick.
618        // Sleeping inside the router blocks the entire tick loop for the
619        // cooldown duration which starves other tasks. The engine tick already
620        // retries every `tick_interval` (default 10s), so allow the tick loop
621        // to drive retries instead of sleeping here.
622        let now = chrono::Utc::now().timestamp();
623        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
624            // All agents are degraded but none have a timed cooldown entry
625            // (e.g. over-threshold rate limiting without explicit cooldown duration).
626            // Log at warn and return an error so the tick skips this task and retries after backoff.
627            tracing::warn!(
628                scope = complexity.unwrap_or("all agents"),
629                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
630            );
631            // Return an error similar to the cooldown case to trigger backoff in the tick loop
632            return Err(AllAgentsCooledError::new(
633                complexity.unwrap_or("all agents").to_string(),
634                None,
635            )
636            .into());
637        };
638
639        let remaining = earliest.saturating_sub(now);
640        if remaining > 0 {
641            let scope_str = complexity.unwrap_or("all agents").to_string();
642            tracing::warn!(
643                remaining_secs = remaining,
644                scope = %scope_str,
645                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
646            );
647            // Return a domain error indicating all candidates are cooled so the
648            // caller can handle it (tick will skip this task and retry later).
649            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
650        }
651
652        Ok(())
653    }
654
655    /// Get the first available agent.
656    /// Pick next agent via round-robin (for review or other non-task routing).
657    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
658    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
659    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
660        // Use available_agents_for_complexity which already filters using agent_is_routable,
661        // eliminating duplicate cooldown/degraded/model checks.
662        let candidates = self.available_agents_for_complexity(complexity);
663        if candidates.is_empty() {
664            return None;
665        }
666        let idx = self.review_rr_index;
667        let n = candidates.len();
668
669        // Try to find an agent that isn't excluded
670        let agent = (0..n)
671            .map(|offset| &candidates[(idx + offset) % n])
672            .find(|a| !exclude.contains(&a.as_str()))
673            .cloned()
674            // Fallback: use the first available candidate (even if excluded)
675            .or_else(|| candidates.first().cloned())?;
676
677        self.review_rr_index = (idx + 1) % candidates.len().max(1);
678        Some(agent)
679    }
680
681    /// If the selected agent matches `no_code_last_agent` for this task, filter it
682    /// out of the candidates and return the filtered list. Used to prevent the
683    /// same agent from being selected twice in a row after a no-code failure (#2410).
684    async fn skip_no_code_agent(
685        &self,
686        store: &Arc<TaskStore>,
687        repo: &str,
688        task_id: &str,
689        candidates: &[String],
690    ) -> Option<Vec<String>> {
691        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
692            .await
693            .unwrap_or_else(|| {
694                tracing::warn!(
695                    task_id,
696                    "no_code_last_agent DB read returned None — loop detection may be impaired"
697                );
698                String::new()
699            });
700        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
701            return None;
702        }
703        let filtered: Vec<String> = candidates
704            .iter()
705            .filter(|a| a.as_str() != no_code_agent)
706            .cloned()
707            .collect();
708        if filtered.is_empty() {
709            tracing::warn!(
710                task_id,
711                no_code_agent = %no_code_agent,
712                "all available agents are the no-code agent — cannot skip"
713            );
714            return None;
715        }
716        tracing::info!(
717            task_id,
718            skipped = %no_code_agent,
719            remaining = ?filtered,
720            "skipping no-code agent, routing to alternative"
721        );
722        Some(filtered)
723    }
724
725    /// Route a task to the best agent.
726    ///
727    /// Routing logic (in priority order):
728    /// 1. Check for `agent:*` label — use that agent directly
729    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
730    /// 3. If round_robin mode, cycle through agents (stateful)
731    /// 4. Call LLM classifier for intelligent routing
732    /// 5. After max_route_attempts LLM failures, fall back to round-robin
733    pub async fn route(
734        &mut self,
735        task: &ExternalTask,
736        store: &Arc<TaskStore>,
737        repo: &str,
738    ) -> anyhow::Result<RouteResult> {
739        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
740        // before the routing loop, not per-task. Do not re-add it here.
741
742        // Hold the task before any classification, label override, or fallback.
743        // A waiting task must not increment route_attempts.
744        match store.resolve_id_by_external(repo, &task.id.0).await {
745            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
746                Ok(Some(blockers)) => {
747                    return Err(WaitingOnDependency { blockers }.into());
748                }
749                Ok(None) => {}
750                Err(err) => return Err(err),
751            },
752            Ok(None) => {}
753            Err(err) => return Err(err),
754        }
755
756        loop {
757            // 1. Check store agent field first (set by failover, authoritative over labels)
758            // Use a lightweight lookup to avoid deserializing the full Task row when
759            // only the `agent` column is needed. This avoids two full-row loads
760            // per routing pass (resolve_task_id + get) and significantly reduces
761            // CPU work on busy instances.
762            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
763                Ok(Some(agent)) => {
764                    // Router enforces that the agent string matches a configured agent
765                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
766                        Some(agent)
767                    } else {
768                        None
769                    }
770                }
771                Ok(None) => None,
772                Err(e) => {
773                    tracing::warn!(
774                        repo,
775                        task_id = %task.id.0,
776                        err = %e,
777                        "failed to fetch stored agent, falling back to label/LLM routing"
778                    );
779                    None
780                }
781            };
782
783            // 2. Fall back to explicit agent label
784            let resolved_agent = store_agent.or_else(|| {
785                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
786            });
787
788            if let Some(agent) = resolved_agent {
789                let complexity = strategies::extract_complexity_from_labels(&task.labels);
790                // If agent appears routable (not in agent-wide cooldown and has
791                // available models or no model_map configured), check whether
792                // we actually have a concrete model to dispatch. If the
793                // resolved label maps to no model (None), we fall through to
794                // the standard routing logic so the router LLM can pick a
795                // proper agent/model, instead of dispatching with an empty
796                // model which causes some agents (e.g. opencode) to exit
797                // silently.
798                if self.agent_is_routable(&agent, &complexity) {
799                    let model = self
800                        .config
801                        .model_for_complexity(&agent, &complexity, &task.id.0);
802
803                    // Only dispatch label overrides when we have a concrete
804                    // model to send to the executor. Dispatching with an
805                    // empty model led to silent exits for some agents
806                    // (notably opencode). If no concrete model is available
807                    // for the labeled agent, fall through to the standard
808                    // routing logic so the LLM or round-robin can pick an
809                    // appropriate agent/model.
810                    if model.is_some() {
811                        tracing::debug!(
812                            task_id = %task.id.0,
813                            agent = %agent,
814                            complexity = %complexity,
815                            "routed via label"
816                        );
817                        let mut result = RouteResult::from_config(
818                            agent.clone(),
819                            complexity.clone(),
820                            format!("label agent:{agent}"),
821                            &self.config,
822                            &task.id.0,
823                        );
824                        // Label routing uses agent-specific role and the
825                        // resolved model (may differ from from_config's default).
826                        result.profile.role = format!("{agent} specialist");
827                        result.model = model.clone();
828                        self.log_route_activity(store, repo, &task.id.0, &result, None)
829                            .await;
830                        return Ok(result);
831                    }
832                    // No concrete model found for the labeled agent — do
833                    // not honor the label override. Fall through to the
834                    // standard routing logic so another agent or the LLM
835                    // can be chosen with a valid model.
836                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
837                }
838
839                let candidates = self.available_agents_for_complexity(&complexity);
840                if candidates.is_empty() {
841                    self.wait_for_cooldown(Some(&complexity))?;
842                    continue;
843                }
844                // Label target is cooled or lacked model; fall through to standard routing.
845            }
846
847            let complexity = strategies::extract_complexity_from_labels(&task.labels);
848
849            // Check if any agents have available models for this complexity
850            let mut candidates = self.available_agents_for_complexity(&complexity);
851            if candidates.is_empty() {
852                self.wait_for_cooldown(Some(&complexity))?;
853                continue;
854            }
855
856            // Skip the no-code agent if one is recorded for this task (#2410).
857            // If all available agents are filtered out, we fall through to
858            // LLM routing with no constraints (LLM will pick the same agent but
859            // the response_handler will block it immediately via same-agent check).
860            if let Some(filtered) = self
861                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
862                .await
863            {
864                candidates = filtered;
865            }
866
867            // 2. Weighted round-robin — capacity-based selection
868            if self.config.weighted_round_robin {
869                let result = strategies::route_via_weighted_round_robin(
870                    &candidates,
871                    &self.weights,
872                    &self.config,
873                    task,
874                    &mut self.last_agent,
875                )?;
876                self.log_route_activity(store, repo, &task.id.0, &result, None)
877                    .await;
878                return Ok(result);
879            }
880
881            // 3. Round-robin mode — use stateful round-robin
882            if self.config.mode == "round_robin" {
883                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
884                let result = strategies::route_via_round_robin_stateful(
885                    &candidates,
886                    &self.config,
887                    task,
888                    &mut self.rr_index,
889                    &mut self.last_agent,
890                )?;
891                self.log_route_activity(store, repo, &task.id.0, &result, None)
892                    .await;
893                return Ok(result);
894            }
895
896            // 4. Local Ollama routing
897            if self.config.mode == "local" {
898                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
899                let ollama_router = self
900                    .ollama_router
901                    .as_ref()
902                    .expect("ollama_router must be initialized in Router::new when mode=local");
903                match ollama_router.route(task, &self.config).await {
904                    Ok(result) => {
905                        self.log_route_activity(store, repo, &task.id.0, &result, None)
906                            .await;
907                        return Ok(result);
908                    }
909                    Err(e) => {
910                        tracing::warn!(
911                            task_id = %task.id.0,
912                            error = %e,
913                            "Ollama routing failed, falling back to round-robin"
914                        );
915                        let result = strategies::route_via_round_robin_stateful(
916                            &candidates,
917                            &self.config,
918                            task,
919                            &mut self.rr_index,
920                            &mut self.last_agent,
921                        )?;
922                        self.log_route_activity(store, repo, &task.id.0, &result, None)
923                            .await;
924                        return Ok(result);
925                    }
926                }
927            }
928
929            // 5. LLM-based routing with retry tracking
930            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
931
932            if route_attempts >= self.config.max_route_attempts {
933                tracing::warn!(
934                    task_id = %task.id.0,
935                    attempts = route_attempts,
936                    max = self.config.max_route_attempts,
937                    "max LLM route attempts reached, falling back to round-robin"
938                );
939                let candidates = self.available_agents_for_complexity(&complexity);
940                if candidates.is_empty() {
941                    self.wait_for_cooldown(Some(&complexity))?;
942                    continue;
943                }
944                let result = strategies::route_via_round_robin_stateful(
945                    &candidates,
946                    &self.config,
947                    task,
948                    &mut self.rr_index,
949                    &mut self.last_agent,
950                )?;
951                self.log_route_activity(store, repo, &task.id.0, &result, None)
952                    .await;
953                return Ok(result);
954            }
955
956            // Log routing start (before await)
957            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
958
959            match self.route_with_llm(task, repo).await {
960                Ok(result) => {
961                    let candidates = self.available_agents_for_complexity(&result.complexity);
962                    if candidates.is_empty() {
963                        self.wait_for_cooldown(Some(&result.complexity))?;
964                        continue;
965                    }
966
967                    // If the LLM selected the no-code agent, filter it out and
968                    // pick an alternative (#2410). Re-use the existing fallback logic.
969                    let no_code_agent =
970                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
971                             .await
972                             .unwrap_or_else(|| {
973                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
974                                 String::new()
975                             });
976                    if result.agent == no_code_agent {
977                        // Pick the first agent that isn't the no-code agent.
978                        let fallback_agent = candidates
979                            .iter()
980                            .find(|a| a.as_str() != no_code_agent)
981                            .cloned()
982                            .unwrap_or_else(|| candidates[0].clone());
983                        if fallback_agent.as_str() == no_code_agent {
984                            tracing::warn!(
985                                task_id = %task.id.0,
986                                no_code_agent = %no_code_agent,
987                                complexity = %result.complexity,
988                                "all available agents are the no-code agent — waiting for another agent"
989                            );
990                            self.wait_for_cooldown(Some(&result.complexity))?;
991                            continue;
992                        }
993                        tracing::info!(
994                            task_id = %task.id.0,
995                            llm_selected = %result.agent,
996                            skipped = %no_code_agent,
997                            fallback = %fallback_agent,
998                            "LLM selected the no-code agent — routing to alternative"
999                        );
1000                        let fallback_model = self.config.model_for_complexity(
1001                            &fallback_agent,
1002                            &result.complexity,
1003                            &task.id.0,
1004                        );
1005
1006                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1007                        let result = RouteResult {
1008                            agent: fallback_agent.clone(),
1009                            model: fallback_model,
1010                            complexity: result.complexity.clone(),
1011                            estimate: result.estimate,
1012                            reason: format!(
1013                                "LLM selected no-code agent {}; routed to {}",
1014                                result.agent, fallback_agent
1015                            ),
1016                            profile: result.profile.clone(),
1017                            selected_skills: result.selected_skills.clone(),
1018                            warning: Some(format!(
1019                                "LLM selected no-code agent {}; used fallback {}",
1020                                result.agent, fallback_agent
1021                            )),
1022                        };
1023                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1024                            .await;
1025                        return Ok(result);
1026                    }
1027
1028                    let model_cooled = result
1029                        .model
1030                        .as_deref()
1031                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1032                    if !candidates.contains(&result.agent) || model_cooled {
1033                        let fallback_agent = candidates
1034                            .iter()
1035                            .find(|agent| *agent != &result.agent)
1036                            .cloned()
1037                            .unwrap_or_else(|| candidates[0].clone());
1038                        let fallback_model = self.config.model_for_complexity(
1039                            &fallback_agent,
1040                            &result.complexity,
1041                            &task.id.0,
1042                        );
1043
1044                        // If the fallback model is None and the fallback agent is the
1045                        // same as the LLM-selected agent, it means all models for the
1046                        // only available agent are cooled. In that case, wait for
1047                        // cooldowns instead of returning a RouteResult with
1048                        // model: None which would be dispatched incorrectly.
1049                        if fallback_model.is_none() && fallback_agent == result.agent {
1050                            tracing::warn!(
1051                                task_id = %task.id.0,
1052                                agent = %result.agent,
1053                                complexity = %result.complexity,
1054                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1055                            );
1056                            self.wait_for_cooldown(Some(&result.complexity))?;
1057                            continue;
1058                        }
1059
1060                        // Distinguish logging and reasons when we actually switch
1061                        // agents vs when we keep the same agent but pick a
1062                        // different (non-cooled) model.
1063                        if fallback_agent == result.agent {
1064                            tracing::warn!(
1065                                task_id = %task.id.0,
1066                                agent = %result.agent,
1067                                complexity = %result.complexity,
1068                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1069                            );
1070                        } else {
1071                            tracing::warn!(
1072                                task_id = %task.id.0,
1073                                agent = %result.agent,
1074                                fallback = %fallback_agent,
1075                                complexity = %result.complexity,
1076                                "LLM selected cooled agent/model; rerouting to available agent"
1077                            );
1078                        }
1079
1080                        // Reset attempts on success
1081                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1082
1083                        let reason = if fallback_agent == result.agent {
1084                            format!(
1085                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1086                            )
1087                        } else {
1088                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1089                        };
1090
1091                        let result = RouteResult {
1092                            agent: fallback_agent.clone(),
1093                            model: fallback_model,
1094                            complexity: result.complexity.clone(),
1095                            estimate: result.estimate,
1096                            reason: reason.clone(),
1097                            profile: result.profile.clone(),
1098                            selected_skills: result.selected_skills.clone(),
1099                            warning: Some(if fallback_agent == result.agent {
1100                                "LLM-selected model was cooled; selected alternative model for same agent"
1101                                    .to_string()
1102                            } else {
1103                                "LLM-selected agent/model was cooled; rerouted to available agent"
1104                                    .to_string()
1105                            }),
1106                        };
1107                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1108                            .await;
1109                        return Ok(result);
1110                    }
1111
1112                    // Reset attempts on success
1113                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1114                    tracing::info!(
1115                        task_id = %task.id.0,
1116                        agent = %result.agent,
1117                        complexity = %result.complexity,
1118                        "routed via LLM"
1119                    );
1120                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1121                        .await;
1122                    return Ok(result);
1123                }
1124                Err(e) => {
1125                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1126                        let scope = err.scope();
1127                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1128                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1129                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1130                        // (no complexity-specific pool lookup, which would fail for "router pool").
1131                        let scope_opt = match scope {
1132                            "all agents" | "router pool" => None,
1133                            _ => Some(scope),
1134                        };
1135                        self.wait_for_cooldown(scope_opt)?;
1136                        continue;
1137                    }
1138
1139                    let new_attempts = route_attempts + 1;
1140                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1141                        .await;
1142                    tracing::warn!(
1143                        task_id = %task.id.0,
1144                        error = %e,
1145                        error_chain = ?e,
1146                        attempt = new_attempts,
1147                        max = self.config.max_route_attempts,
1148                        "LLM routing failed"
1149                    );
1150
1151                    if new_attempts >= self.config.max_route_attempts {
1152                        tracing::info!(
1153                            task_id = %task.id.0,
1154                            "falling back to round-robin after {} failed attempts",
1155                            new_attempts
1156                        );
1157                        let candidates = self.available_agents_for_complexity(&complexity);
1158                        if candidates.is_empty() {
1159                            self.wait_for_cooldown(Some(&complexity))?;
1160                            continue;
1161                        }
1162                        let result = strategies::route_via_round_robin_stateful(
1163                            &candidates,
1164                            &self.config,
1165                            task,
1166                            &mut self.rr_index,
1167                            &mut self.last_agent,
1168                        )?;
1169                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1170                            .await;
1171                        return Ok(result);
1172                    }
1173
1174                    let candidates = self.available_agents_for_complexity(&complexity);
1175                    if candidates.is_empty() {
1176                        self.wait_for_cooldown(Some(&complexity))?;
1177                        continue;
1178                    }
1179                    let result = strategies::route_via_fallback(
1180                        &candidates,
1181                        &self.config,
1182                        task,
1183                        Some(&self.weights),
1184                    )?;
1185                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1186                        .await;
1187                    return Ok(result);
1188                }
1189            }
1190        }
1191    }
1192
1193    /// Get the number of LLM routing attempts for a task from the store.
1194    async fn get_route_attempts(&self, task_id: &str, store: &Arc<TaskStore>, repo: &str) -> u32 {
1195        match store.resolve_task_id(repo, task_id).await {
1196            Ok(Some(store_id)) => store
1197                .get(store_id)
1198                .await
1199                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1200            _ => 0,
1201        }
1202    }
1203
1204    /// Set the number of LLM routing attempts for a task in the store.
1205    async fn set_route_attempts(
1206        &self,
1207        task_id: &str,
1208        attempts: u32,
1209        store: &Arc<TaskStore>,
1210        repo: &str,
1211    ) {
1212        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1213            && let Err(e) = store
1214                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1215                .await
1216        {
1217            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1218        }
1219    }
1220
1221    /// Log a route event to task activity timeline.
1222    async fn log_route_activity(
1223        &self,
1224        store: &Arc<TaskStore>,
1225        repo: &str,
1226        task_id: &str,
1227        result: &RouteResult,
1228        from_agent: Option<&str>,
1229    ) {
1230        let details = serde_json::json!({
1231            "reason": result.reason,
1232            "complexity": result.complexity,
1233            "skills": result.selected_skills,
1234            "role": result.profile.role,
1235            "warning": result.warning,
1236        });
1237        let event_type = if from_agent.is_some() {
1238            "rerouted"
1239        } else {
1240            "routed"
1241        };
1242        store_log_activity(
1243            &Some(Arc::clone(store)),
1244            repo,
1245            task_id,
1246            event_type,
1247            None,
1248            Some("routed"),
1249            Some(&result.agent),
1250            result.model.as_deref(),
1251            Some(&details),
1252        )
1253        .await;
1254    }
1255
1256    /// Route using LLM classification with pool round-robin.
1257    ///
1258    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1259    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1260    /// returns the last error (the caller falls back to round-robin agent selection).
1261    async fn route_with_llm(
1262        &mut self,
1263        task: &ExternalTask,
1264        repo: &str,
1265    ) -> anyhow::Result<RouteResult> {
1266        // Filter out cooled and degraded agents, and agents without an available
1267        // model for the coding tiers (`medium`/`complex`) that real tasks are
1268        // classified into, so the LLM only sees candidates that could actually
1269        // be dispatched for whatever complexity it picks. An agent whose only
1270        // healthy tier is `simple` is excluded too: the classifier can't be
1271        // constrained to `simple`, so leaving such an agent in would waste the
1272        // call whenever it picks `medium`/`complex` (issue #3521).
1273        // Fall back to the full list if all agents are cooled/degraded.
1274        // Cloned to avoid borrow conflict with &mut self in the loop.
1275        let uncooled_agents: Vec<String> = self
1276            .available_agents
1277            .iter()
1278            .filter(|a| {
1279                !is_agent_in_cooldown(a)
1280                    && !is_agent_degraded(a)
1281                    && self.agent_has_available_model_for_coding_tiers(a)
1282            })
1283            .cloned()
1284            .collect();
1285        if uncooled_agents.is_empty() {
1286            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1287        }
1288        let llm_agents = uncooled_agents;
1289
1290        let pool = self.router_pool.clone();
1291        let n = pool.len();
1292        let start = self.pool_index;
1293        let mut last_err: Option<anyhow::Error> = None;
1294        let mut attempted_any = false;
1295
1296        // Try pool entries in round-robin order, skipping cooled ones
1297        for i in 0..n {
1298            let idx = (start + i) % n;
1299            let (agent, model) = &pool[idx];
1300            let model_str = model.as_str();
1301
1302            if response::is_model_in_cooldown(agent, model_str) {
1303                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1304                continue;
1305            }
1306
1307            // At least one pool entry was available to attempt
1308            attempted_any = true;
1309
1310            let model_opt = if model_str.is_empty() {
1311                None
1312            } else {
1313                Some(model_str)
1314            };
1315
1316            match self
1317                .llm_router
1318                .route_with_llm_using(
1319                    task,
1320                    &llm_agents,
1321                    &self.config,
1322                    &mut self.last_agent,
1323                    repo,
1324                    agent,
1325                    model_opt,
1326                )
1327                .await
1328            {
1329                Ok(result) => {
1330                    // Advance index so the next call starts at the next pool entry
1331                    self.advance_pool_index_after_attempt(idx, n);
1332                    tracing::debug!(
1333                        agent,
1334                        model = model_str,
1335                        pool_idx = idx,
1336                        "router LLM pool entry succeeded"
1337                    );
1338                    record_agent_success(agent, model_str).await;
1339                    return Ok(result);
1340                }
1341                Err(e) => {
1342                    let err_msg = e.to_string();
1343                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1344                    if is_timeout {
1345                        // Advance pool index so the next tick starts at the next entry,
1346                        // then return immediately. Continuing to try further pool entries
1347                        // in the same tick would accumulate multiple timeout windows and
1348                        // stall the tick loop for minutes (issue #3187).
1349                        //
1350                        // Also record a standard cooldown, same as the non-timeout branch,
1351                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1352                        // on future ticks instead of burning the full timeout window again
1353                        // every time round-robin reaches it (issue #3422).
1354                        self.advance_pool_index_after_attempt(idx, n);
1355                        tracing::warn!(
1356                            agent,
1357                            model = model_str,
1358                            next_pool_idx = self.pool_index,
1359                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1360                        );
1361                        response::record_model_failure(agent, model_str).await;
1362                        return Err(e);
1363                    }
1364                    tracing::warn!(
1365                        agent,
1366                        model = model_str,
1367                        error = %e,
1368                        "pool entry failed, recording cooldown and trying next"
1369                    );
1370                    if !cooldown::record_credit_exhaustion_from_message(
1371                        agent,
1372                        Some(model_str),
1373                        &err_msg,
1374                    )
1375                    .await
1376                    {
1377                        response::record_model_failure(agent, model_str).await;
1378                    }
1379                    last_err = Some(e);
1380                    self.advance_pool_index_after_attempt(idx, n);
1381                }
1382            }
1383        }
1384
1385        // If every pool entry was pre-cooled (nothing was attempted), do not
1386        // immediately exhaust the pool again — fail fast and let the tick loop
1387        // retry on the next cycle instead of cascading failures.
1388        // However, we should still try the fallback if configured.
1389        if !attempted_any {
1390            let now = chrono::Utc::now().timestamp();
1391            let earliest = self.earliest_pool_cooldown();
1392            if let Some(until) = earliest {
1393                // Pool entries exist but are all cooled - we'll try fallback below
1394                let remaining = until.saturating_sub(now);
1395                tracing::warn!(
1396                    remaining_secs = remaining,
1397                    pool = ?pool,
1398                    "all router LLM pool entries on cooldown — will try fallback"
1399                );
1400                // Continue to fallback logic instead of returning early
1401            } else {
1402                // No pool cooldowns found either — this means the pool was empty
1403                // or every entry was filtered out for some other reason.
1404                return Err(last_err
1405                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1406            }
1407        }
1408
1409        // All pool entries failed or were cooled — try the configured fallback
1410        let fallback = self.config.effective_fallback();
1411        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1412        let fb_model_str = fb_model.as_str();
1413
1414        // Only try fallback if it wasn't already in the pool (avoid double-try)
1415        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1416
1417        if !fallback_already_tried && !response::is_model_in_cooldown(&fb_agent, fb_model_str) {
1418            tracing::info!(
1419                agent = %fb_agent,
1420                model = %fb_model_str,
1421                "all pool entries exhausted — trying fallback router LLM"
1422            );
1423            let fb_model_opt = if fb_model_str.is_empty() {
1424                None
1425            } else {
1426                Some(fb_model_str)
1427            };
1428            match self
1429                .llm_router
1430                .route_with_llm_using(
1431                    task,
1432                    &llm_agents,
1433                    &self.config,
1434                    &mut self.last_agent,
1435                    repo,
1436                    &fb_agent,
1437                    fb_model_opt,
1438                )
1439                .await
1440            {
1441                Ok(result) => {
1442                    record_agent_success(&fb_agent, fb_model_str).await;
1443                    return Ok(result);
1444                }
1445                Err(e) => {
1446                    let err_msg = e.to_string();
1447                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1448                    if is_timeout {
1449                        // Cool down on timeout like pool entries (#3422), else a
1450                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1451                        response::record_model_failure(&fb_agent, fb_model_str).await;
1452                        tracing::warn!(
1453                            agent = %fb_agent,
1454                            model = %fb_model_str,
1455                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1456                        );
1457                        return Err(e);
1458                    }
1459                    tracing::warn!(
1460                        agent = %fb_agent,
1461                        model = %fb_model_str,
1462                        error = %e,
1463                        "fallback router LLM also failed"
1464                    );
1465                    response::record_model_failure(&fb_agent, fb_model_str).await;
1466                    last_err = Some(e);
1467                }
1468            }
1469        }
1470
1471        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1472    }
1473
1474    /// Record a rate limit event for an agent, reducing its routing weight.
1475    ///
1476    /// Called by the engine when an agent returns a 429/rate limit error.
1477    pub fn record_rate_limit(&mut self, agent: &str) {
1478        self.weights.record_rate_limit(agent);
1479    }
1480
1481    /// Record a successful task completion, restoring agent weight.
1482    pub fn record_success(&mut self, agent: &str) {
1483        self.weights.record_success(agent);
1484    }
1485
1486    /// Tick weight recovery for all agents.
1487    ///
1488    /// Called periodically by the engine to gradually restore weights
1489    /// as rate limit windows expire.
1490    pub fn tick_weight_recovery(&mut self) {
1491        self.weights.tick_recovery();
1492    }
1493
1494    /// Store routing result in the task store.
1495    pub async fn store_route_result(
1496        &self,
1497        task_id: &str,
1498        result: &RouteResult,
1499        store: &Arc<TaskStore>,
1500        repo: &str,
1501    ) -> anyhow::Result<()> {
1502        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1503            store
1504                .set_fields(
1505                    store_id,
1506                    &[
1507                        ("agent", serde_json::json!(result.agent)),
1508                        ("complexity", serde_json::json!(result.complexity)),
1509                        ("route_reason", serde_json::json!(result.reason)),
1510                        (
1511                            "model",
1512                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1513                        ),
1514                    ],
1515                )
1516                .await?;
1517        }
1518        Ok(())
1519    }
1520}
1521
1522/// Retrieve routing result from the task store.
1523pub async fn get_route_result(
1524    store: &Arc<TaskStore>,
1525    repo: &str,
1526    task_id: &str,
1527) -> Result<RouteResult, RouteResultError> {
1528    let store_id = store
1529        .resolve_task_id(repo, task_id)
1530        .await
1531        .map_err(RouteResultError::Other)?
1532        .ok_or_else(|| {
1533            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1534        })?;
1535    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1536
1537    let agent = task
1538        .agent
1539        .clone()
1540        .filter(|a| !a.is_empty())
1541        .ok_or_else(|| RouteResultError::NoAgent {
1542            task_id: task_id.to_string(),
1543        })?;
1544
1545    let complexity = if task.complexity.is_empty() {
1546        "medium".to_string()
1547    } else {
1548        task.complexity.clone()
1549    };
1550
1551    let reason = task.route_reason.clone();
1552    let model = task.model.clone().filter(|m| !m.is_empty());
1553
1554    // Validate that the stored model is valid for the resolved agent.
1555    // Agent-specific aliases (e.g. "opus") can leak across agents during
1556    // failover, so we verify against the router's model pools (#1604).
1557    let config = RouterConfig::from_config();
1558    let model = model.filter(|m| {
1559        let complexities = ["simple", "medium", "complex", "review"];
1560        // Collect all model pools configured for this agent across complexity tiers.
1561        let all_pools: Vec<Vec<String>> = complexities
1562            .iter()
1563            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1564            .collect();
1565        // If the agent has no model pools configured, we can't validate — pass through.
1566        if all_pools.is_empty() {
1567            return true;
1568        }
1569        let valid = all_pools.iter().any(|pool| pool.contains(m));
1570        if !valid {
1571            tracing::warn!(
1572                task_id,
1573                agent,
1574                model = %m,
1575                "discarding stale model: not in agent's model pools"
1576            );
1577        }
1578        valid
1579    });
1580
1581    // A stale model is replaced by the agent's model for the task's complexity, an empty
1582    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1583    let original_model = task.model.clone().filter(|m| !m.is_empty());
1584    let mut model = model;
1585    if original_model.is_some() && model.is_none() {
1586        model = config.model_for_complexity(&agent, &complexity, task_id);
1587        let stale_model = original_model.as_deref().unwrap_or("unknown");
1588        let stale_err = format!(
1589            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1590            model.as_deref().unwrap_or("the agent default")
1591        );
1592        if let Err(e) = store
1593            .set_fields(
1594                store_id,
1595                &[
1596                    (
1597                        "model",
1598                        serde_json::json!(model.clone().unwrap_or_default()),
1599                    ),
1600                    ("last_error", serde_json::json!(stale_err)),
1601                ],
1602            )
1603            .await
1604        {
1605            // Log the error so transient DB failures are visible and diagnosable.
1606            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1607        }
1608    }
1609
1610    Ok(RouteResult {
1611        agent,
1612        model,
1613        complexity,
1614        estimate: 0,
1615        reason,
1616        profile: AgentProfile::default(),
1617        selected_skills: vec![],
1618        warning: None,
1619    })
1620}
1621
1622#[cfg(test)]
1623#[expect(
1624    clippy::float_cmp,
1625    reason = "the weights are stored constants, so they compare exactly"
1626)]
1627mod tests {
1628    use super::config::DEFAULT_AGENTS;
1629    use super::llm::LlmRouteResponse;
1630    use super::strategies;
1631    use super::weights::default_missing_weight;
1632    use super::weights::{
1633        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1634    };
1635    use super::*;
1636    use crate::backends::{ExternalId, ExternalTask};
1637    use crate::engine::cooldown::{
1638        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1639        record_model_failure, set_model_cooldown,
1640    };
1641    use crate::store::NewTask;
1642    use crate::store::TaskStore;
1643    use serial_test::serial;
1644    use std::collections::HashMap;
1645    use std::env;
1646    #[cfg(unix)]
1647    use std::ffi::OsString;
1648    use std::fs;
1649    #[cfg(unix)]
1650    use std::fs::Permissions;
1651    #[cfg(unix)]
1652    use std::os::unix::fs::PermissionsExt;
1653    #[cfg(unix)]
1654    use std::process;
1655    #[cfg(unix)]
1656    use std::process::Command;
1657    #[cfg(unix)]
1658    use std::thread;
1659    use std::time::{Duration, Instant};
1660
1661    #[tokio::test]
1662    #[serial_test::serial(orch_home)]
1663    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1664        let home = tempfile::tempdir().unwrap();
1665        fs::write(
1666            home.path().join("config.yml"),
1667            "model_map:\n  medium:\n    claude: sonnet\n",
1668        )
1669        .unwrap();
1670        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1671        unsafe { env::set_var("ORCH_HOME", home.path()) };
1672
1673        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1674        let store_id = store
1675            .create(&NewTask {
1676                external_id: Some("7".to_string()),
1677                repo: "owner/repo".to_string(),
1678                origin: "github".to_string(),
1679                title: "stale model".to_string(),
1680                ..Default::default()
1681            })
1682            .await
1683            .unwrap();
1684        store
1685            .set_fields(
1686                store_id,
1687                &[
1688                    ("agent", serde_json::json!("claude")),
1689                    ("model", serde_json::json!("claude-removed-model")),
1690                    ("complexity", serde_json::json!("medium")),
1691                ],
1692            )
1693            .await
1694            .unwrap();
1695
1696        let route = get_route_result(&store, "owner/repo", "7").await;
1697        let stored = store.get(store_id).await.unwrap().model;
1698        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1699        unsafe { env::remove_var("ORCH_HOME") };
1700
1701        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1702        assert_eq!(stored.as_deref(), Some("sonnet"));
1703    }
1704
1705    // Test-only delegate so tests can call router.parse_llm_response() directly
1706    // without referencing llm_router.
1707    impl Router {
1708        #[expect(
1709            clippy::unused_self,
1710            reason = "the test helper keeps the router method syntax"
1711        )]
1712        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1713            LlmRouter::parse_llm_response("claude", response)
1714        }
1715    }
1716
1717    async fn test_store() -> Arc<TaskStore> {
1718        Arc::new(TaskStore::open_memory().await.unwrap())
1719    }
1720
1721    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1722        ExternalTask {
1723            id: ExternalId(id.to_string()),
1724            title: title.to_string(),
1725            body: "Test body".to_string(),
1726            state: "open".to_string(),
1727            labels,
1728            author: "test".to_string(),
1729            created_at: "2026-01-01T00:00:00Z".to_string(),
1730            updated_at: "2026-01-01T00:00:00Z".to_string(),
1731            url: format!("https://github.com/test/test/issues/{id}"),
1732        }
1733    }
1734
1735    #[serial(cooldown_state)]
1736    #[test]
1737    fn extract_agent_from_labels() {
1738        let mut config = RouterConfig::default();
1739        // Ensure a concrete model exists for the labeled agent so the label
1740        // override is honored by the router (routing requires a model).
1741        config
1742            .model_map
1743            .entry("medium".to_string())
1744            .or_default()
1745            .insert("claude".to_string(), vec!["haiku".to_string()]);
1746
1747        assert_eq!(
1748            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1749            Some("claude".to_string())
1750        );
1751        assert_eq!(
1752            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1753            Some("codex".to_string())
1754        );
1755        assert_eq!(
1756            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1757            Some("opencode".to_string())
1758        );
1759        assert_eq!(
1760            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1761            None
1762        );
1763        // Verify kimi and minimax are recognized from labels
1764        assert_eq!(
1765            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1766            Some("kimi".to_string())
1767        );
1768        assert_eq!(
1769            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1770            Some("minimax".to_string())
1771        );
1772    }
1773
1774    #[serial(cooldown_state)]
1775    #[tokio::test]
1776    async fn all_agents_cooled_error_names_degraded_agents() {
1777        cooldown::reset_global_state().await;
1778        mark_agent_degraded("claude");
1779        mark_agent_degraded("grok");
1780        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1781        let msg = err.to_string();
1782        assert!(
1783            msg.contains("degraded: claude, grok"),
1784            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1785        );
1786        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1787        clear_agent_degraded("claude");
1788        clear_agent_degraded("grok");
1789    }
1790
1791    #[serial(cooldown_state)]
1792    #[test]
1793    fn default_agents_constant() {
1794        assert_eq!(DEFAULT_AGENTS.len(), 7);
1795        assert!(DEFAULT_AGENTS.contains(&"claude"));
1796        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1797        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1798        assert!(DEFAULT_AGENTS.contains(&"glm"));
1799        assert!(DEFAULT_AGENTS.contains(&"grok"));
1800    }
1801
1802    #[serial(cooldown_state)]
1803    #[test]
1804    fn extract_complexity_from_labels() {
1805        assert_eq!(
1806            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1807            "simple"
1808        );
1809        assert_eq!(
1810            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1811            "medium"
1812        );
1813        assert_eq!(
1814            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1815            "complex"
1816        );
1817        assert_eq!(
1818            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1819            "medium"
1820        );
1821    }
1822
1823    #[serial(cooldown_state)]
1824    #[test]
1825    fn route_result_serialization() {
1826        let result = RouteResult {
1827            agent: "claude".to_string(),
1828            model: Some("sonnet".to_string()),
1829            complexity: "medium".to_string(),
1830            estimate: 5,
1831            reason: "test".to_string(),
1832            profile: AgentProfile {
1833                role: "backend".to_string(),
1834                skills: vec!["rust".to_string()],
1835                tools: vec!["git".to_string()],
1836                constraints: vec![],
1837            },
1838            selected_skills: vec!["gh".to_string()],
1839            warning: None,
1840        };
1841
1842        let json = serde_json::to_string(&result).unwrap();
1843        assert!(json.contains("claude"));
1844        assert!(json.contains("sonnet"));
1845
1846        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1847        assert_eq!(deserialized.agent, "claude");
1848        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1849    }
1850
1851    #[serial(cooldown_state)]
1852    #[test]
1853    fn router_config_default() {
1854        let mut config = RouterConfig::default();
1855        // Ensure a concrete model exists for the labeled agent so the label
1856        // override is honored by the router (routing requires a model).
1857        config
1858            .model_map
1859            .entry("medium".to_string())
1860            .or_default()
1861            .insert("claude".to_string(), vec!["haiku".to_string()]);
1862
1863        assert_eq!(config.mode, "round_robin");
1864        assert_eq!(config.router_agent, "claude");
1865        assert_eq!(config.router_model, "haiku");
1866        assert_eq!(config.fallback_executor, "codex");
1867        assert_eq!(config.max_route_attempts, 3);
1868        assert_ne!(config.allowed_tools, [] as [String; 0]);
1869        assert_ne!(config.default_skills, [] as [String; 0]);
1870
1871        // Verify configurable agents list includes all 7 agents
1872        assert_eq!(config.agents.len(), 7);
1873        assert!(config.agents.contains(&"claude".to_string()));
1874        assert!(config.agents.contains(&"codex".to_string()));
1875        assert!(config.agents.contains(&"opencode".to_string()));
1876        assert!(config.agents.contains(&"kimi".to_string()));
1877        assert!(config.agents.contains(&"minimax".to_string()));
1878        assert!(config.agents.contains(&"glm".to_string()));
1879        assert!(config.agents.contains(&"grok".to_string()));
1880    }
1881
1882    #[serial(cooldown_state)]
1883    #[test]
1884    fn model_map_lookup_returns_none_without_config() {
1885        // Default config has no hardcoded models — config.yml is the source of truth.
1886        let config = RouterConfig::default();
1887        assert!(
1888            config
1889                .model_for_complexity("claude", "simple", "")
1890                .is_none()
1891        );
1892        assert!(
1893            config
1894                .model_for_complexity("claude", "medium", "")
1895                .is_none()
1896        );
1897        assert!(
1898            config
1899                .model_for_complexity("claude", "complex", "")
1900                .is_none()
1901        );
1902        assert!(
1903            config
1904                .model_for_complexity("opencode", "review", "")
1905                .is_none()
1906        );
1907    }
1908
1909    #[serial(cooldown_state)]
1910    #[test]
1911    fn model_map_lookup_returns_configured_value() {
1912        // When model_map is populated (from config), it is returned correctly.
1913        let mut config = RouterConfig::default();
1914        config
1915            .model_map
1916            .entry("simple".to_string())
1917            .or_default()
1918            .insert("claude".to_string(), vec!["haiku".to_string()]);
1919        assert_eq!(
1920            config.model_for_complexity("claude", "simple", ""),
1921            Some("haiku".to_string())
1922        );
1923    }
1924
1925    #[serial(cooldown_state)]
1926    #[tokio::test]
1927    async fn model_pool_selection_skips_cooled() {
1928        cooldown::reset_global_state().await;
1929
1930        let mut config = RouterConfig::default();
1931        // Set up a two-model pool for opencode/simple
1932        config
1933            .model_map
1934            .entry("simple".to_string())
1935            .or_default()
1936            .insert(
1937                "opencode".to_string(),
1938                vec!["model-a".to_string(), "model-b".to_string()],
1939            );
1940
1941        // Before any cooldown, both models are candidates
1942        let m = config.model_for_complexity("opencode", "simple", "task-1");
1943        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1944
1945        // Cool model-a
1946        record_model_failure("opencode", "model-a").await;
1947        assert!(is_model_in_cooldown("opencode", "model-a"));
1948
1949        // Now only model-b should be returned
1950        // (try many task_ids to rule out hash coincidence)
1951        for i in 0..20 {
1952            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1953            assert_eq!(
1954                result,
1955                Some("model-b".to_string()),
1956                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1957            );
1958        }
1959
1960        // Cool model-b too — now all models are cooled
1961        record_model_failure("opencode", "model-b").await;
1962        // All cooled → return None so the caller can fall back to a different agent
1963        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1964        assert_eq!(fallback, None);
1965    }
1966
1967    #[serial(cooldown_state)]
1968    #[test]
1969    fn model_pool_single_string_backward_compat() {
1970        // Single-item pools behave identically to the old string format.
1971        // Manually insert a single-item pool and verify it is returned correctly.
1972        let mut config = RouterConfig::default();
1973        config
1974            .model_map
1975            .entry("simple".to_string())
1976            .or_default()
1977            .insert("claude".to_string(), vec!["haiku".to_string()]);
1978        let m = config.model_for_complexity("claude", "simple", "any-task");
1979        assert_eq!(m, Some("haiku".to_string()));
1980    }
1981
1982    #[serial(cooldown_state)]
1983    #[test]
1984    fn parse_llm_response_direct_json() {
1985        let config = RouterConfig::default();
1986        let router = Router::new(config);
1987
1988        let response = r#"{
1989            "executor": "claude",
1990            "complexity": "complex",
1991            "reason": "requires architecture analysis",
1992            "profile": {
1993                "role": "architect",
1994                "skills": ["rust", "design"],
1995                "tools": ["git", "rg"],
1996                "constraints": []
1997            },
1998            "selected_skills": ["gh"]
1999        }"#;
2000
2001        let parsed = router.parse_llm_response(response).unwrap();
2002        assert_eq!(parsed.executor, "claude");
2003        assert_eq!(parsed.complexity, "complex");
2004        assert_eq!(parsed.reason, "requires architecture analysis");
2005        assert_eq!(parsed.profile.role, "architect");
2006    }
2007
2008    #[serial(cooldown_state)]
2009    #[test]
2010    fn parse_llm_response_markdown_fenced() {
2011        let config = RouterConfig::default();
2012        let router = Router::new(config);
2013
2014        let response = r#"Here's my analysis:
2015
2016```json
2017{
2018    "executor": "codex",
2019    "complexity": "medium",
2020    "reason": "coding task",
2021    "profile": {
2022        "role": "developer",
2023        "skills": ["coding"],
2024        "tools": [],
2025        "constraints": []
2026    },
2027    "selected_skills": []
2028}
2029```
2030
2031Hope that helps!"#;
2032
2033        let parsed = router.parse_llm_response(response).unwrap();
2034        assert_eq!(parsed.executor, "codex");
2035        assert_eq!(parsed.complexity, "medium");
2036    }
2037
2038    #[serial(cooldown_state)]
2039    #[test]
2040    fn parse_llm_response_claude_envelope() {
2041        let config = RouterConfig::default();
2042        let router = Router::new(config);
2043
2044        // Claude --output-format json wraps the response in an envelope
2045        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2046
2047        let parsed = router.parse_llm_response(response).unwrap();
2048        assert_eq!(parsed.executor, "claude");
2049        assert_eq!(parsed.complexity, "complex");
2050        assert_eq!(parsed.reason, "requires deep analysis");
2051    }
2052
2053    #[serial(cooldown_state)]
2054    #[test]
2055    fn parse_llm_response_claude_envelope_with_code_block() {
2056        let config = RouterConfig::default();
2057        let router = Router::new(config);
2058
2059        // Claude may return the JSON inside a code block within the envelope
2060        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2061        let envelope = serde_json::json!({
2062            "type": "result",
2063            "subtype": "success",
2064            "is_error": false,
2065            "result": inner,
2066            "usage": {"input_tokens": 50, "output_tokens": 30}
2067        });
2068
2069        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2070        assert_eq!(parsed.executor, "codex");
2071        assert_eq!(parsed.complexity, "simple");
2072    }
2073
2074    #[serial(cooldown_state)]
2075    #[test]
2076    fn parse_llm_response_claude_envelope_error() {
2077        let config = RouterConfig::default();
2078        let router = Router::new(config);
2079
2080        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2081
2082        let err = router.parse_llm_response(response).unwrap_err();
2083        assert!(err.to_string().contains("error"), "got: {err}");
2084    }
2085
2086    #[serial(cooldown_state)]
2087    #[test]
2088    fn parse_llm_response_empty() {
2089        let config = RouterConfig::default();
2090        let router = Router::new(config);
2091
2092        let err = router.parse_llm_response("").unwrap_err();
2093        assert!(err.to_string().contains("empty"), "got: {err}");
2094    }
2095
2096    /// Claude envelope where result is a JSON object, not a string.
2097    /// Some Claude versions may return the result as a nested object.
2098    #[serial(cooldown_state)]
2099    #[test]
2100    fn parse_llm_response_claude_envelope_result_as_object() {
2101        let config = RouterConfig::default();
2102        let router = Router::new(config);
2103
2104        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2105
2106        let parsed = router.parse_llm_response(response);
2107        assert!(
2108            parsed.is_ok(),
2109            "should handle result-as-object: {}",
2110            parsed.unwrap_err()
2111        );
2112        assert_eq!(parsed.unwrap().executor, "claude");
2113    }
2114
2115    /// LLM returns "agent" instead of "executor" — common hallucination.
2116    #[serial(cooldown_state)]
2117    #[test]
2118    fn parse_llm_response_agent_instead_of_executor() {
2119        let config = RouterConfig::default();
2120        let router = Router::new(config);
2121
2122        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2123
2124        let parsed = router.parse_llm_response(response);
2125        assert!(
2126            parsed.is_ok(),
2127            "should accept 'agent' alias: {}",
2128            parsed.unwrap_err()
2129        );
2130        assert_eq!(parsed.unwrap().executor, "claude");
2131    }
2132
2133    /// LLM returns minimal JSON without profile or reason.
2134    #[serial(cooldown_state)]
2135    #[test]
2136    fn parse_llm_response_minimal_json() {
2137        let config = RouterConfig::default();
2138        let router = Router::new(config);
2139
2140        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2141
2142        let parsed = router.parse_llm_response(response);
2143        assert!(
2144            parsed.is_ok(),
2145            "should accept minimal JSON: {}",
2146            parsed.unwrap_err()
2147        );
2148        assert_eq!(parsed.unwrap().executor, "codex");
2149    }
2150
2151    // ── Real fixture tests: route prompt → response → parsed result ──
2152
2153    /// Real Claude envelope with result as escaped JSON string.
2154    #[serial(cooldown_state)]
2155    #[test]
2156    fn fixture_route_response_string() {
2157        let config = RouterConfig::default();
2158        let router = Router::new(config);
2159
2160        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2161        let parsed = router.parse_llm_response(response).unwrap();
2162        assert_eq!(parsed.executor, "codex");
2163        assert_eq!(parsed.complexity, "medium");
2164        assert_ne!(parsed.reason, "");
2165        assert_ne!(parsed.profile.role, "");
2166    }
2167
2168    /// Real Claude envelope with result as JSON object (not string).
2169    /// This was the root cause of all router failures on 2026-02-27.
2170    #[serial(cooldown_state)]
2171    #[test]
2172    fn fixture_route_response_object() {
2173        let config = RouterConfig::default();
2174        let router = Router::new(config);
2175
2176        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2177        let parsed = router.parse_llm_response(response).unwrap();
2178        assert_eq!(parsed.executor, "claude");
2179        assert_eq!(parsed.complexity, "complex");
2180    }
2181
2182    /// Real Claude envelope with result as string containing markdown + JSON.
2183    #[serial(cooldown_state)]
2184    #[test]
2185    fn fixture_route_response_markdown() {
2186        let config = RouterConfig::default();
2187        let router = Router::new(config);
2188
2189        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2190        let parsed = router.parse_llm_response(response).unwrap();
2191        assert_eq!(parsed.executor, "codex");
2192        assert_eq!(parsed.complexity, "medium");
2193    }
2194
2195    /// Claude envelope with escaped JSON string containing newlines and markdown.
2196    #[serial(cooldown_state)]
2197    #[test]
2198    fn parse_llm_response_claude_envelope_with_prose() {
2199        let config = RouterConfig::default();
2200        let router = Router::new(config);
2201
2202        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2203        let envelope = serde_json::json!({
2204            "type": "result",
2205            "subtype": "success",
2206            "is_error": false,
2207            "result": inner,
2208            "usage": {"input_tokens": 100, "output_tokens": 80}
2209        });
2210
2211        let parsed = router.parse_llm_response(&envelope.to_string());
2212        assert!(
2213            parsed.is_ok(),
2214            "should handle prose + code block in envelope: {}",
2215            parsed.unwrap_err()
2216        );
2217        assert_eq!(parsed.unwrap().executor, "codex");
2218    }
2219
2220    #[serial(cooldown_state)]
2221    #[tokio::test]
2222    async fn route_round_robin_basic() {
2223        cooldown::reset_global_state().await;
2224        // Force at least one agent to be available for testing
2225        // In real usage, discover_agents finds installed CLIs
2226        let config = RouterConfig {
2227            mode: "round_robin".to_string(),
2228            ..Default::default()
2229        };
2230
2231        // Create router with mock available agents
2232        let agents = vec!["claude".to_string(), "codex".to_string()];
2233        let mut weights = AgentWeights::default();
2234        weights.ensure_agents(&agents);
2235        let router = Router {
2236            config,
2237            available_agents: agents,
2238            weights,
2239            llm_router: Arc::new(LlmRouter::new()),
2240            ollama_router: None,
2241            rr_index: 0,
2242            last_agent: None,
2243            review_rr_index: 0,
2244            router_pool: vec![],
2245            pool_index: 0,
2246        };
2247
2248        let task = create_test_task("1", "Test task", vec![]);
2249        let result =
2250            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2251                .unwrap();
2252
2253        // Task 1 % 2 agents = agent at index 1 = codex
2254        assert_eq!(result.agent, "codex");
2255        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2256    }
2257
2258    #[serial(cooldown_state)]
2259    #[tokio::test]
2260    async fn route_uses_label_override() {
2261        cooldown::reset_global_state().await;
2262        let mut config = RouterConfig::default();
2263        // Ensure a concrete model exists so the label override can be honored
2264        // in the test environment where no external config.yml is present.
2265        config
2266            .model_map
2267            .entry("medium".to_string())
2268            .or_default()
2269            .insert("claude".to_string(), vec!["haiku".to_string()]);
2270        let agents = vec!["claude".to_string(), "codex".to_string()];
2271        let mut weights = AgentWeights::default();
2272        weights.ensure_agents(&agents);
2273        let mut router = Router {
2274            config,
2275            available_agents: agents,
2276            weights,
2277            llm_router: Arc::new(LlmRouter::new()),
2278            ollama_router: None,
2279            rr_index: 0,
2280            last_agent: None,
2281            review_rr_index: 0,
2282            router_pool: vec![],
2283            pool_index: 0,
2284        };
2285
2286        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2287        let store = test_store().await;
2288
2289        // Should use label override, not LLM
2290        let result = router.route(&task, &store, "test/repo").await.unwrap();
2291        assert_eq!(result.agent, "claude");
2292        assert!(result.reason.contains("label"));
2293    }
2294
2295    #[serial(cooldown_state)]
2296    #[test]
2297    fn check_routing_sanity_warnings() {
2298        // Backend task routed to claude should warn
2299        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2300        let profile = AgentProfile {
2301            role: "general".to_string(),
2302            skills: vec!["api".to_string()],
2303            tools: vec![],
2304            constraints: vec![],
2305        };
2306        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2307        assert!(warning.is_some());
2308        assert!(warning.unwrap().contains("backend"));
2309
2310        // Docs task routed to codex should warn
2311        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2312        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2313        assert!(warning.is_some());
2314        assert!(warning.unwrap().contains("docs"));
2315
2316        // Normal routing should not warn
2317        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2318        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2319        assert!(warning.is_none());
2320    }
2321
2322    #[serial(cooldown_state)]
2323    #[tokio::test]
2324    async fn router_reload_preserves_structure() {
2325        cooldown::reset_global_state().await;
2326        let config = RouterConfig::default();
2327        let agents = vec!["claude".to_string()];
2328        let mut weights = AgentWeights::default();
2329        weights.ensure_agents(&agents);
2330        let mut router = Router {
2331            config,
2332            available_agents: agents,
2333            weights,
2334            llm_router: Arc::new(LlmRouter::new()),
2335            ollama_router: None,
2336            rr_index: 0,
2337            last_agent: None,
2338            review_rr_index: 0,
2339            router_pool: vec![],
2340            pool_index: 0,
2341        };
2342
2343        // Reload — should re-read config and remain valid
2344        router.reload_async().await;
2345
2346        // After reload, mode should be a valid value (llm, local, or round_robin)
2347        assert!(
2348            router.config.mode == "llm"
2349                || router.config.mode == "round_robin"
2350                || router.config.mode == "local",
2351            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2352            router.config.mode
2353        );
2354        // Fallback executor should always be set
2355        assert_ne!(router.config.fallback_executor, "");
2356        // Tools should always be populated
2357        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2358    }
2359
2360    // --- Weighted round-robin tests ---
2361
2362    #[serial(cooldown_state)]
2363    #[test]
2364    fn rate_limit_state_defaults_to_full_weight() {
2365        let state = RateLimitState::default();
2366        assert_eq!(state.weight, DEFAULT_WEIGHT);
2367        assert_eq!(state.consecutive_hits, 0);
2368        assert!(state.last_limited_at.is_none());
2369        assert!(!state.is_limited());
2370    }
2371
2372    #[serial(cooldown_state)]
2373    #[test]
2374    fn rate_limit_state_decays_on_hit() {
2375        let mut state = RateLimitState::default();
2376        state.record_rate_limit();
2377
2378        assert!(state.weight < DEFAULT_WEIGHT);
2379        assert_eq!(state.consecutive_hits, 1);
2380        assert!(state.last_limited_at.is_some());
2381        assert!(state.is_limited());
2382
2383        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2384        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2385        assert!((state.weight - expected).abs() < 1e-10);
2386    }
2387
2388    #[serial(cooldown_state)]
2389    #[test]
2390    fn rate_limit_state_never_drops_below_min() {
2391        let mut state = RateLimitState::default();
2392
2393        // Hit many times
2394        for _ in 0..100 {
2395            state.record_rate_limit();
2396        }
2397
2398        assert!(state.weight >= MIN_WEIGHT);
2399        assert_eq!(state.consecutive_hits, 100);
2400    }
2401
2402    #[serial(cooldown_state)]
2403    #[test]
2404    fn rate_limit_state_recovers_on_success() {
2405        let mut state = RateLimitState::default();
2406
2407        // Decay first
2408        state.record_rate_limit();
2409        state.record_rate_limit();
2410        let after_decay = state.weight;
2411
2412        // Record success
2413        state.record_success();
2414        assert!(state.weight > after_decay);
2415        assert_eq!(state.consecutive_hits, 0);
2416    }
2417
2418    #[serial(cooldown_state)]
2419    #[test]
2420    fn rate_limit_state_success_caps_at_default() {
2421        let mut state = RateLimitState::default();
2422
2423        // Already at full weight, success shouldn't exceed it
2424        state.record_success();
2425        assert_eq!(state.weight, DEFAULT_WEIGHT);
2426    }
2427
2428    #[serial(cooldown_state)]
2429    #[test]
2430    fn agent_weights_ensure_agents() {
2431        let mut weights = AgentWeights::default();
2432        let agents = vec!["claude".to_string(), "codex".to_string()];
2433        weights.ensure_agents(&agents);
2434
2435        assert_eq!(weights.states.len(), 2);
2436        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2437        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2438    }
2439
2440    #[serial(cooldown_state)]
2441    #[test]
2442    fn agent_weights_ensure_agents_preserves_existing() {
2443        let mut weights = AgentWeights::default();
2444        let agents = vec!["claude".to_string(), "codex".to_string()];
2445        weights.ensure_agents(&agents);
2446
2447        // Reduce claude's weight
2448        weights.record_rate_limit("claude");
2449        let claude_weight = weights.get_weight("claude");
2450        assert!(claude_weight < DEFAULT_WEIGHT);
2451
2452        // Ensure agents again — claude's weight should be preserved
2453        let agents2 = vec![
2454            "claude".to_string(),
2455            "codex".to_string(),
2456            "opencode".to_string(),
2457        ];
2458        weights.ensure_agents(&agents2);
2459
2460        assert_eq!(weights.get_weight("claude"), claude_weight);
2461        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2462    }
2463
2464    #[test]
2465    fn default_missing_weight_uses_minimum_of_configured() {
2466        let mut configured = HashMap::new();
2467        configured.insert("claude".to_string(), 0.2);
2468        configured.insert("codex".to_string(), 0.2);
2469        configured.insert("opencode".to_string(), 0.2);
2470        configured.insert("kimi".to_string(), 0.2);
2471
2472        // minimax omitted from router.weights — must not silently outrank
2473        // every explicitly configured agent (issue #3596).
2474        assert_eq!(default_missing_weight(&configured), 0.2);
2475    }
2476
2477    #[test]
2478    fn default_missing_weight_falls_back_when_none_configured() {
2479        let configured = HashMap::new();
2480        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2481    }
2482
2483    #[serial(cooldown_state)]
2484    #[test]
2485    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2486        let mut weights = AgentWeights::default();
2487        let mut configured = HashMap::new();
2488        configured.insert("claude".to_string(), 0.2);
2489        configured.insert("codex".to_string(), 0.2);
2490        configured.insert("opencode".to_string(), 0.2);
2491        configured.insert("kimi".to_string(), 0.2);
2492
2493        let agents = vec![
2494            "claude".to_string(),
2495            "codex".to_string(),
2496            "opencode".to_string(),
2497            "kimi".to_string(),
2498            "minimax".to_string(),
2499        ];
2500        weights.ensure_agents_with_weights(&agents, &configured);
2501
2502        // minimax was omitted from router.weights but must not get the old
2503        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2504        assert_eq!(weights.get_weight("minimax"), 0.2);
2505        assert_eq!(weights.get_weight("claude"), 0.2);
2506    }
2507
2508    #[serial(cooldown_state)]
2509    #[test]
2510    fn agent_weights_record_rate_limit() {
2511        let mut weights = AgentWeights::default();
2512        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2513
2514        weights.record_rate_limit("claude");
2515
2516        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2517        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2518    }
2519
2520    #[serial(cooldown_state)]
2521    #[test]
2522    fn agent_weights_record_success() {
2523        let mut weights = AgentWeights::default();
2524        weights.ensure_agents(&["claude".to_string()]);
2525
2526        weights.record_rate_limit("claude");
2527        let after_limit = weights.get_weight("claude");
2528
2529        weights.record_success("claude");
2530        assert!(weights.get_weight("claude") > after_limit);
2531    }
2532
2533    #[serial(cooldown_state)]
2534    #[test]
2535    fn agent_weights_snapshot() {
2536        let mut weights = AgentWeights::default();
2537        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2538        weights.record_rate_limit("claude");
2539
2540        let snap = weights.snapshot();
2541        assert_eq!(snap.len(), 2);
2542
2543        // Snapshot is sorted alphabetically
2544        assert_eq!(snap[0].0, "claude");
2545        assert!(snap[0].1 < DEFAULT_WEIGHT);
2546        assert_eq!(snap[0].2, 1); // 1 hit
2547
2548        assert_eq!(snap[1].0, "codex");
2549        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2550        assert_eq!(snap[1].2, 0); // 0 hits
2551    }
2552
2553    #[serial(cooldown_state)]
2554    #[test]
2555    fn agent_weights_weighted_select_favors_higher_weight() {
2556        let mut weights = AgentWeights::default();
2557        let agents = vec!["claude".to_string(), "codex".to_string()];
2558        weights.ensure_agents(&agents);
2559
2560        // Heavily penalize claude
2561        for _ in 0..10 {
2562            weights.record_rate_limit("claude");
2563        }
2564
2565        // Run many selections and count
2566        let mut claude_count = 0;
2567        let mut codex_count = 0;
2568        for _ in 0..100 {
2569            match weights.weighted_select(&agents, "task-1") {
2570                Some(ref a) if a == "claude" => claude_count += 1,
2571                Some(ref a) if a == "codex" => codex_count += 1,
2572                _ => {}
2573            }
2574        }
2575
2576        // Codex should get significantly more selections
2577        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2578        assert!(
2579            codex_count > claude_count,
2580            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2581        );
2582    }
2583
2584    #[serial(cooldown_state)]
2585    #[test]
2586    fn agent_weights_weighted_select_empty_returns_none() {
2587        let weights = AgentWeights::default();
2588        assert!(weights.weighted_select(&[], "task-1").is_none());
2589    }
2590
2591    #[serial(cooldown_state)]
2592    #[test]
2593    fn agent_weights_batch_routing_produces_varied_selections() {
2594        // Regression test: when routing a batch of tasks in the same tick,
2595        // different task IDs should not all select the same agent.
2596        let mut weights = AgentWeights::default();
2597        let agents = vec!["claude".to_string(), "codex".to_string()];
2598        weights.ensure_agents(&agents);
2599
2600        // Simulate batch routing: 20 different task IDs called in rapid succession
2601        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2602        let selections: Vec<String> = task_ids
2603            .iter()
2604            .filter_map(|id| weights.weighted_select(&agents, id))
2605            .collect();
2606
2607        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2608        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2609        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2610        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2611        assert!(
2612            claude_count > 0 && codex_count > 0,
2613            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2614        );
2615    }
2616
2617    #[serial(cooldown_state)]
2618    #[test]
2619    fn agent_weights_weighted_select_single_agent() {
2620        let mut weights = AgentWeights::default();
2621        let agents = vec!["claude".to_string()];
2622        weights.ensure_agents(&agents);
2623
2624        let selected = weights.weighted_select(&agents, "task-1");
2625        assert_eq!(selected, Some("claude".to_string()));
2626    }
2627
2628    #[serial(cooldown_state)]
2629    #[test]
2630    fn agent_weights_tick_recovery_restores_weight() {
2631        let mut weights = AgentWeights::default();
2632        weights.ensure_agents(&["claude".to_string()]);
2633
2634        // Set up a rate limit that looks like it happened long ago
2635        let state = weights.states.get_mut("claude").unwrap();
2636        state.weight = 0.5;
2637        state.last_limited_at = Some(
2638            Instant::now()
2639                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2640                .expect("the clock is past the recovery delay"),
2641        );
2642        state.consecutive_hits = 2;
2643
2644        let before = weights.get_weight("claude");
2645        weights.tick_recovery();
2646        let after = weights.get_weight("claude");
2647
2648        assert!(after > before, "weight should increase after recovery tick");
2649    }
2650
2651    #[serial(cooldown_state)]
2652    #[test]
2653    fn agent_weights_tick_recovery_clears_on_full_restore() {
2654        let mut weights = AgentWeights::default();
2655        weights.ensure_agents(&["claude".to_string()]);
2656
2657        // Set weight just below full with an old limit
2658        let state = weights.states.get_mut("claude").unwrap();
2659        state.weight = DEFAULT_WEIGHT - 0.01;
2660        state.last_limited_at = Some(
2661            Instant::now()
2662                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2663                .expect("the clock is past the recovery delay"),
2664        );
2665        state.consecutive_hits = 1;
2666
2667        weights.tick_recovery();
2668
2669        let state = weights.states.get("claude").unwrap();
2670        assert_eq!(state.weight, DEFAULT_WEIGHT);
2671        assert!(state.last_limited_at.is_none());
2672        assert_eq!(state.consecutive_hits, 0);
2673    }
2674
2675    #[serial(cooldown_state)]
2676    #[tokio::test]
2677    async fn route_weighted_round_robin_basic() {
2678        cooldown::reset_global_state().await;
2679        let mut config = RouterConfig {
2680            weighted_round_robin: true,
2681            ..Default::default()
2682        };
2683        // Provide a concrete model for codex so label override can be used.
2684        config
2685            .model_map
2686            .entry("medium".to_string())
2687            .or_default()
2688            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2689
2690        let agents = vec!["claude".to_string(), "codex".to_string()];
2691        let mut weights = AgentWeights::default();
2692        weights.ensure_agents(&agents);
2693        let mut router = Router {
2694            config,
2695            available_agents: agents,
2696            weights,
2697            llm_router: Arc::new(LlmRouter::new()),
2698            ollama_router: None,
2699            rr_index: 0,
2700            last_agent: None,
2701            review_rr_index: 0,
2702            router_pool: vec![],
2703            pool_index: 0,
2704        };
2705
2706        let task = create_test_task("1", "Test task", vec![]);
2707        let store = test_store().await;
2708        let result = router.route(&task, &store, "test/repo").await.unwrap();
2709
2710        // Should use weighted_round_robin
2711        assert!(result.reason.contains("weighted_round_robin"));
2712        assert!(
2713            result.agent == "claude" || result.agent == "codex",
2714            "agent should be claude or codex, got '{}'",
2715            result.agent
2716        );
2717    }
2718
2719    #[serial(cooldown_state)]
2720    #[tokio::test]
2721    async fn route_weighted_round_robin_respects_label_override() {
2722        cooldown::reset_global_state().await;
2723        let mut config = RouterConfig {
2724            weighted_round_robin: true,
2725            ..Default::default()
2726        };
2727        // Provide a concrete model for codex so label override can be used.
2728        config
2729            .model_map
2730            .entry("medium".to_string())
2731            .or_default()
2732            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2733
2734        let agents = vec!["claude".to_string(), "codex".to_string()];
2735        let mut weights = AgentWeights::default();
2736        weights.ensure_agents(&agents);
2737        let mut router = Router {
2738            config,
2739            available_agents: agents,
2740            weights,
2741            llm_router: Arc::new(LlmRouter::new()),
2742            ollama_router: None,
2743            rr_index: 0,
2744            last_agent: None,
2745            review_rr_index: 0,
2746            router_pool: vec![],
2747            pool_index: 0,
2748        };
2749
2750        // Label override should take precedence over weighted routing
2751        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2752        let store = test_store().await;
2753        let result = router.route(&task, &store, "test/repo").await.unwrap();
2754        assert_eq!(result.agent, "codex");
2755        assert!(result.reason.contains("label"));
2756    }
2757
2758    #[serial(cooldown_state)]
2759    #[test]
2760    fn router_config_weighted_round_robin_default_false() {
2761        let config = RouterConfig::default();
2762        assert!(!config.weighted_round_robin);
2763    }
2764
2765    #[serial(cooldown_state)]
2766    #[test]
2767    fn router_record_and_recover_weights() {
2768        let config = RouterConfig {
2769            weighted_round_robin: true,
2770            ..Default::default()
2771        };
2772        let mut router = Router::new(config);
2773        // Override discovered agents for test
2774        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2775        router.weights.ensure_agents(&router.available_agents);
2776
2777        // Record rate limit
2778        router.record_rate_limit("claude");
2779        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2780        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2781
2782        // Record success for claude
2783        router.record_success("claude");
2784        let after_success = router.weights.get_weight("claude");
2785        assert!(after_success > MIN_WEIGHT);
2786    }
2787
2788    #[serial(cooldown_state)]
2789    #[test]
2790    fn review_rr_index_advances() {
2791        let mut config = RouterConfig::default();
2792        // Set up model_map so has_available_model_for_complexity returns true for test agents
2793        for agent in &["test_a", "test_b", "test_c"] {
2794            config
2795                .model_map
2796                .entry("medium".to_string())
2797                .or_default()
2798                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2799        }
2800        let agents = vec![
2801            "test_a".to_string(),
2802            "test_b".to_string(),
2803            "test_c".to_string(),
2804        ];
2805        let mut weights = AgentWeights::default();
2806        weights.ensure_agents(&agents);
2807        let mut router = Router {
2808            config,
2809            available_agents: agents,
2810            weights,
2811            llm_router: Arc::new(LlmRouter::new()),
2812            ollama_router: None,
2813            rr_index: 0,
2814            last_agent: None,
2815            review_rr_index: 0,
2816            router_pool: vec![],
2817            pool_index: 0,
2818        };
2819
2820        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2821        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2822        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2823
2824        // All three agents should appear (order depends on start index)
2825        let mut seen = vec![a1, a2, a3];
2826        seen.sort();
2827        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2828    }
2829
2830    #[serial(cooldown_state)]
2831    #[test]
2832    fn review_rr_fallback_with_cooled_agent() {
2833        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2834        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2835        // candidates.get(idx) to return None even though candidates is non-empty.
2836        let mut config = RouterConfig::default();
2837        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2838        for agent in &["test_a", "test_b", "test_c"] {
2839            config
2840                .model_map
2841                .entry("medium".to_string())
2842                .or_default()
2843                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2844        }
2845        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2846        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2847        let mut weights = AgentWeights::default();
2848        weights.ensure_agents(&agents);
2849        let mut router = Router {
2850            config,
2851            available_agents: agents,
2852            weights,
2853            llm_router: Arc::new(LlmRouter::new()),
2854            ollama_router: None,
2855            rr_index: 0,
2856            last_agent: None,
2857            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2858            review_rr_index: 2,
2859            router_pool: vec![],
2860            pool_index: 0,
2861        };
2862
2863        // All candidates excluded — should fall back to first candidate, not None
2864        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2865        assert!(
2866            result.is_some(),
2867            "next_round_robin_agent must not return None when candidates is non-empty"
2868        );
2869    }
2870
2871    #[serial(cooldown_state)]
2872    #[test]
2873    fn review_rr_index_uses_candidates_len_not_available_len() {
2874        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2875        // not available_agents.len(). When an agent has no model for the complexity,
2876        // candidates.len() < available_agents.len() and the old code produced a drifting
2877        // index that caused non-uniform distribution.
2878        let mut config = RouterConfig::default();
2879        // Only register models for test_a and test_b — test_c has no model, so it is
2880        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2881        for agent in &["test_a", "test_b"] {
2882            config
2883                .model_map
2884                .entry("medium".to_string())
2885                .or_default()
2886                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2887        }
2888        let agents = vec![
2889            "test_a".to_string(),
2890            "test_b".to_string(),
2891            "test_c".to_string(), // available but no model → filtered out of candidates
2892        ];
2893        let mut weights = AgentWeights::default();
2894        weights.ensure_agents(&agents);
2895        let mut router = Router {
2896            config,
2897            available_agents: agents,
2898            weights,
2899            llm_router: Arc::new(LlmRouter::new()),
2900            ollama_router: None,
2901            rr_index: 0,
2902            last_agent: None,
2903            review_rr_index: 0,
2904            router_pool: vec![],
2905            pool_index: 0,
2906        };
2907
2908        // With the bug: index advances modulo 3 (available_agents.len()), so the
2909        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2910        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2911        // looks uniform by accident, but adding a 4th available agent breaks it.
2912        //
2913        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2914        // cycle. Verify this by checking review_rr_index after each call.
2915
2916        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2917        assert_eq!(
2918            router.review_rr_index, 1,
2919            "index should be 1 after first call"
2920        );
2921
2922        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2923        assert_eq!(
2924            router.review_rr_index, 0,
2925            "index should wrap to 0 after second call"
2926        );
2927
2928        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2929        assert_eq!(
2930            router.review_rr_index, 1,
2931            "index should be 1 after third call"
2932        );
2933
2934        // Agents must alternate uniformly
2935        assert_ne!(r1, r2, "consecutive calls should select different agents");
2936        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2937    }
2938
2939    #[serial(cooldown_state)]
2940    #[tokio::test]
2941    async fn last_agent_tracks_routing() {
2942        cooldown::reset_global_state().await;
2943        let mut config = RouterConfig {
2944            mode: "round_robin".to_string(),
2945            ..Default::default()
2946        };
2947        config
2948            .model_map
2949            .entry("medium".to_string())
2950            .or_default()
2951            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
2952        config
2953            .model_map
2954            .entry("medium".to_string())
2955            .or_default()
2956            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
2957        let agents = vec!["test_x".to_string(), "test_y".to_string()];
2958        let mut weights = AgentWeights::default();
2959        weights.ensure_agents(&agents);
2960        let mut router = Router {
2961            config,
2962            available_agents: agents,
2963            weights,
2964            llm_router: Arc::new(LlmRouter::new()),
2965            ollama_router: None,
2966            rr_index: 0,
2967            last_agent: None,
2968            review_rr_index: 0,
2969            router_pool: vec![],
2970            pool_index: 0,
2971        };
2972
2973        assert!(router.last_agent.is_none());
2974
2975        let task = create_test_task("1", "Test routing", vec![]);
2976        let store = test_store().await;
2977        let result = router.route(&task, &store, "test/repo").await.unwrap();
2978
2979        assert!(router.last_agent.is_some());
2980        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
2981    }
2982
2983    // ── pool config ───────────────────────────────────────────────────────────
2984
2985    #[serial(cooldown_state)]
2986    #[test]
2987    fn parse_pool_entry_splits_on_first_colon() {
2988        let (agent, model) = super::parse_pool_entry("claude:haiku");
2989        assert_eq!(agent, "claude");
2990        assert_eq!(model, "haiku");
2991    }
2992
2993    #[serial(cooldown_state)]
2994    #[test]
2995    fn parse_pool_entry_model_with_slash() {
2996        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
2997        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
2998        assert_eq!(agent, "opencode");
2999        assert_eq!(model, "github-copilot/gpt-5-mini");
3000    }
3001
3002    #[serial(cooldown_state)]
3003    #[test]
3004    fn parse_pool_entry_no_colon() {
3005        let (agent, model) = super::parse_pool_entry("claude");
3006        assert_eq!(agent, "claude");
3007        assert_eq!(model, "");
3008    }
3009
3010    #[serial(cooldown_state)]
3011    #[test]
3012    fn effective_pool_uses_configured_pool() {
3013        let config = RouterConfig {
3014            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3015            ..RouterConfig::default()
3016        };
3017        let pool = config.effective_pool();
3018        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3019    }
3020
3021    #[serial(cooldown_state)]
3022    #[test]
3023    fn effective_pool_falls_back_to_router_agent_model() {
3024        let config = RouterConfig::default();
3025        // Default has no pool → derives from router_agent:router_model
3026        let pool = config.effective_pool();
3027        assert_eq!(pool.len(), 1);
3028        assert!(pool[0].starts_with(&config.router_agent));
3029        assert!(pool[0].contains(&config.router_model));
3030    }
3031
3032    #[serial(cooldown_state)]
3033    #[test]
3034    fn effective_fallback_uses_configured_fallback() {
3035        let config = RouterConfig {
3036            fallback: "claude:haiku".to_string(),
3037            ..RouterConfig::default()
3038        };
3039        assert_eq!(config.effective_fallback(), "claude:haiku");
3040    }
3041
3042    #[serial(cooldown_state)]
3043    #[test]
3044    fn effective_fallback_derives_from_router_agent_model() {
3045        let config = RouterConfig::default();
3046        let fallback = config.effective_fallback();
3047        assert!(fallback.starts_with(&config.router_agent));
3048        assert!(fallback.contains(&config.router_model));
3049    }
3050
3051    // ── pool expansion ────────────────────────────────────────────────────────
3052
3053    #[serial(cooldown_state)]
3054    #[test]
3055    fn expand_pool_basic_entries() {
3056        let config = RouterConfig {
3057            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3058            ..RouterConfig::default()
3059        };
3060        let expanded = Router::expand_pool(&config);
3061        assert_eq!(expanded.len(), 2);
3062        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3063        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3064    }
3065
3066    #[serial(cooldown_state)]
3067    #[test]
3068    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3069        let config = RouterConfig {
3070            pool: vec![],
3071            router_agent: "claude".to_string(),
3072            router_model: "haiku".to_string(),
3073            ..RouterConfig::default()
3074        };
3075        let expanded = Router::expand_pool(&config);
3076        assert_eq!(expanded.len(), 1);
3077        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3078    }
3079
3080    #[serial(cooldown_state)]
3081    #[test]
3082    fn expand_pool_opencode_free_skipped_when_not_installed() {
3083        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3084        // to nothing and the pool falls back to effective_fallback
3085        let config = RouterConfig {
3086            pool: vec!["opencode:free".to_string()],
3087            fallback: "claude:haiku".to_string(),
3088            ..RouterConfig::default()
3089        };
3090        let expanded = Router::expand_pool(&config);
3091        assert_eq!(expanded.len(), 1);
3092        assert_eq!(expanded[0].0, "claude");
3093        assert_eq!(expanded[0].1, "haiku");
3094    }
3095
3096    #[serial(cooldown_state)]
3097    #[test]
3098    fn expand_pool_single_entry_from_default() {
3099        let config = RouterConfig::default();
3100        let expanded = Router::expand_pool(&config);
3101        assert!(
3102            !expanded.is_empty(),
3103            "pool must never be empty after expansion"
3104        );
3105    }
3106
3107    // ── pool round-robin index ────────────────────────────────────────────────
3108
3109    #[serial(cooldown_state)]
3110    #[test]
3111    fn pool_index_initializes_to_zero() {
3112        let config = RouterConfig::default();
3113        let router = Router::new(config);
3114        assert_eq!(router.pool_index, 0);
3115    }
3116
3117    #[serial(cooldown_state)]
3118    #[test]
3119    fn router_pool_non_empty_after_construction() {
3120        let config = RouterConfig::default();
3121        let router = Router::new(config);
3122        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3123    }
3124
3125    /**
3126     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3127     * `opencode serve --service` child behind, so a suite run leaked one server per test
3128     * process. The stub on PATH records any exec, so this fails if discovery comes back
3129     **/
3130    #[cfg(unix)]
3131    #[serial(cooldown_state)]
3132    #[test]
3133    fn building_a_router_never_runs_opencode() {
3134        struct PathRestore(Option<OsString>);
3135        impl Drop for PathRestore {
3136            fn drop(&mut self) {
3137                match &self.0 {
3138                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3139                    Some(p) => unsafe { env::set_var("PATH", p) },
3140                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3141                    None => unsafe { env::remove_var("PATH") },
3142                }
3143                cmd_cache::reset_for_test();
3144            }
3145        }
3146
3147        let dir = tempfile::tempdir().unwrap();
3148        let marker = dir.path().join("spawned");
3149        let bin_dir = dir.path().join("bin");
3150        fs::create_dir(&bin_dir).unwrap();
3151        let stub = bin_dir.join("opencode");
3152        fs::write(
3153            &stub,
3154            format!(
3155                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3156                marker.display()
3157            ),
3158        )
3159        .unwrap();
3160        fs::set_permissions(&stub, Permissions::from_mode(0o755)).unwrap();
3161
3162        let prev_path = env::var_os("PATH");
3163        let mut new_path = bin_dir.as_os_str().to_os_string();
3164        new_path.push(":");
3165        if let Some(ref p) = prev_path {
3166            new_path.push(p);
3167        }
3168        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3169        unsafe { env::set_var("PATH", &new_path) };
3170        cmd_cache::reset_for_test();
3171        let _restore = PathRestore(prev_path);
3172
3173        // A config that names opencode everywhere discovery reads from
3174        let mut config = RouterConfig {
3175            pool: vec!["opencode:free".to_string()],
3176            fallback: "opencode:free".to_string(),
3177            ..RouterConfig::default()
3178        };
3179        config
3180            .model_map
3181            .entry("simple".to_string())
3182            .or_default()
3183            .insert(
3184                "opencode".to_string(),
3185                vec!["canary-model-free".to_string()],
3186            );
3187
3188        let _router = Router::new(config.clone());
3189        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3190        let _pool = Router::expand_pool(&config);
3191        // Discovery refreshes on a background thread, give it room to exec
3192        thread::sleep(Duration::from_millis(300));
3193
3194        assert!(
3195            !marker.exists(),
3196            "building a router ran opencode: {}",
3197            fs::read_to_string(&marker).unwrap_or_default().trim()
3198        );
3199        assert!(
3200            opencode_children().is_empty(),
3201            "building a router left opencode processes: {:?}",
3202            opencode_children()
3203        );
3204    }
3205
3206    /// Commands of this process's direct children that mention opencode
3207    #[cfg(unix)]
3208    fn opencode_children() -> Vec<String> {
3209        let me = process::id().to_string();
3210        let output = Command::new("ps")
3211            .args(["-axo", "ppid=,command="])
3212            .output()
3213            .expect("ps");
3214        String::from_utf8_lossy(&output.stdout)
3215            .lines()
3216            .filter_map(|line| {
3217                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3218                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3219            })
3220            .collect()
3221    }
3222
3223    #[serial(cooldown_state)]
3224    #[tokio::test]
3225    async fn reload_resets_pool_index() {
3226        cooldown::reset_global_state().await;
3227        let config = RouterConfig::default();
3228        let mut router = Router::new(config);
3229        router.pool_index = 5;
3230        router.reload_async().await;
3231        assert_eq!(router.pool_index, 0);
3232    }
3233
3234    #[serial(cooldown_state)]
3235    #[test]
3236    fn pool_index_advances_after_failed_attempt() {
3237        let mut router = Router::new(RouterConfig::default());
3238        router.router_pool = vec![
3239            ("claude".to_string(), "haiku".to_string()),
3240            ("codex".to_string(), "gpt-5.2".to_string()),
3241            (
3242                "opencode".to_string(),
3243                "github-copilot/gpt-5-mini".to_string(),
3244            ),
3245        ];
3246        router.pool_index = 0;
3247
3248        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3249
3250        assert_eq!(router.pool_index, 1);
3251    }
3252
3253    #[serial(cooldown_state)]
3254    #[test]
3255    fn pool_index_wraps_after_last_attempt() {
3256        let mut router = Router::new(RouterConfig::default());
3257        router.router_pool = vec![
3258            ("claude".to_string(), "haiku".to_string()),
3259            ("codex".to_string(), "gpt-5.2".to_string()),
3260            (
3261                "opencode".to_string(),
3262                "github-copilot/gpt-5-mini".to_string(),
3263            ),
3264        ];
3265        router.pool_index = 2;
3266
3267        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3268
3269        assert_eq!(router.pool_index, 0);
3270    }
3271
3272    #[serial(cooldown_state)]
3273    #[test]
3274    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3275        let router = Router::new(RouterConfig::default());
3276        assert!(router.earliest_pool_cooldown().is_none());
3277    }
3278
3279    #[serial(cooldown_state)]
3280    #[tokio::test]
3281    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3282        cooldown::reset_global_state().await;
3283        // Use explicit pool with predictable model names to avoid opencode discovery.
3284        let config = RouterConfig {
3285            pool: vec![
3286                "claude:haiku-4-5-20251001".to_string(),
3287                "opencode:haiku-4-5-20251001".to_string(),
3288            ],
3289            ..Default::default()
3290        };
3291        let router = Router::new(config);
3292        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3293        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3294
3295        let earliest = router.earliest_pool_cooldown();
3296        assert!(earliest.is_some());
3297        let now = chrono::Utc::now().timestamp();
3298        let remaining = earliest.unwrap() - now;
3299        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3300        assert!((295..=300).contains(&remaining));
3301    }
3302
3303    #[serial(cooldown_state)]
3304    #[tokio::test]
3305    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3306        cooldown::reset_global_state().await;
3307        // Use explicit pool so we can predict what entries exist.
3308        let config = RouterConfig {
3309            pool: vec![
3310                "claude:pool-model-a".to_string(),
3311                "opencode:pool-model-b".to_string(),
3312            ],
3313            ..Default::default()
3314        };
3315        let router = Router::new(config);
3316        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3317        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3318        // Set cooldown on a pool entry (opencode:pool-model-b)
3319        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3320
3321        let earliest = router.earliest_pool_cooldown();
3322        assert!(earliest.is_some());
3323        let now = chrono::Utc::now().timestamp();
3324        let remaining = earliest.unwrap() - now;
3325        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3326        assert!((295..=300).contains(&remaining));
3327    }
3328
3329    // ---- Pre-emptive health check: degraded agent exclusion ----
3330
3331    #[serial(cooldown_state)]
3332    #[test]
3333    fn degraded_agent_excluded_from_routing() {
3334        let mut config = RouterConfig::default();
3335        config
3336            .model_map
3337            .entry("medium".to_string())
3338            .or_default()
3339            .insert(
3340                "test_degraded_routing".to_string(),
3341                vec!["model-a".to_string()],
3342            );
3343
3344        let mut router = Router::new(config);
3345        router.available_agents = vec!["test_degraded_routing".to_string()];
3346
3347        // Agent is routable before degradation
3348        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3349
3350        // Mark as degraded
3351        mark_agent_degraded("test_degraded_routing");
3352
3353        // Now excluded from routing
3354        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3355
3356        // available_agents_for_complexity returns empty
3357        assert!(
3358            router.available_agents_for_complexity("medium").is_empty(),
3359            "degraded agent should be excluded from available agents"
3360        );
3361
3362        // healthy_agent_count reflects exclusion
3363        assert_eq!(router.healthy_agent_count("medium"), 0);
3364
3365        // Cleanup
3366        clear_agent_degraded("test_degraded_routing");
3367    }
3368
3369    #[serial(cooldown_state)]
3370    #[test]
3371    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3372        let mut config = RouterConfig::default();
3373        for agent in &["agent_healthy", "agent_degraded"] {
3374            config
3375                .model_map
3376                .entry("medium".to_string())
3377                .or_default()
3378                .insert(agent.to_string(), vec!["model-x".to_string()]);
3379        }
3380
3381        let mut router = Router::new(config);
3382        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3383
3384        mark_agent_degraded("agent_degraded");
3385
3386        let candidates = router.available_agents_for_complexity("medium");
3387        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3388        assert_eq!(router.healthy_agent_count("medium"), 1);
3389
3390        // Cleanup
3391        clear_agent_degraded("agent_degraded");
3392    }
3393
3394    #[serial(cooldown_state)]
3395    #[tokio::test]
3396    async fn refresh_health_marks_degraded_from_rate_limits() {
3397        cooldown::reset_global_state().await;
3398        let store = test_store().await;
3399        let agent = "test_refresh_health_agent";
3400
3401        let mut config = RouterConfig::default();
3402        config
3403            .model_map
3404            .entry("medium".to_string())
3405            .or_default()
3406            .insert(agent.to_string(), vec!["model-z".to_string()]);
3407
3408        let mut router = Router::new(config);
3409        router.available_agents = vec![agent.to_string()];
3410
3411        // Insert rate limit events exceeding default threshold (3)
3412        for _ in 0..4 {
3413            store.record_rate_limit(agent, "rate", None).await.unwrap();
3414        }
3415
3416        router.refresh_health(&store).await;
3417
3418        assert!(
3419            is_agent_degraded(agent),
3420            "agent should be degraded after exceeding rate limit threshold"
3421        );
3422        assert!(
3423            !router.agent_is_routable(agent, "medium"),
3424            "degraded agent should not be routable"
3425        );
3426
3427        // Cleanup
3428        clear_agent_degraded(agent);
3429    }
3430
3431    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3432
3433    #[serial(cooldown_state)]
3434    #[tokio::test]
3435    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3436        cooldown::reset_global_state().await;
3437        let mut config = RouterConfig::default();
3438        // Agent's only model, shared across every complexity tier it resolves to.
3439        for tier in ["simple", "medium", "complex", "review"] {
3440            config
3441                .model_map
3442                .entry(tier.to_string())
3443                .or_default()
3444                .insert(
3445                    "test_all_cooled".to_string(),
3446                    vec!["only-model".to_string()],
3447                );
3448        }
3449
3450        let mut router = Router::new(config);
3451        router.available_agents = vec!["test_all_cooled".to_string()];
3452
3453        assert!(router.agent_has_any_available_model("test_all_cooled"));
3454
3455        record_model_failure("test_all_cooled", "only-model").await;
3456        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3457
3458        assert!(
3459            !router.agent_has_any_available_model("test_all_cooled"),
3460            "agent with its only model cooled across every tier should have no available model"
3461        );
3462    }
3463
3464    #[serial(cooldown_state)]
3465    #[tokio::test]
3466    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3467        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3468        // agent that has no agent-level cooldown but whose only resolvable model (across
3469        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3470        cooldown::reset_global_state().await;
3471        let mut config = RouterConfig::default();
3472        for tier in ["simple", "medium", "complex", "review"] {
3473            config
3474                .model_map
3475                .entry(tier.to_string())
3476                .or_default()
3477                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3478            config
3479                .model_map
3480                .get_mut(tier)
3481                .unwrap()
3482                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3483        }
3484
3485        let mut router = Router::new(config);
3486        router.available_agents = vec![
3487            "agent_cooled_model".to_string(),
3488            "agent_healthy".to_string(),
3489        ];
3490
3491        record_model_failure("agent_cooled_model", "opus").await;
3492        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3493        // No agent-level cooldown — the old agent-only filter would still offer it.
3494        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3495
3496        let uncooled_agents: Vec<String> = router
3497            .available_agents
3498            .iter()
3499            .filter(|a| {
3500                !is_agent_in_cooldown(a)
3501                    && !is_agent_degraded(a)
3502                    && router.agent_has_available_model_for_coding_tiers(a)
3503            })
3504            .cloned()
3505            .collect();
3506
3507        assert_eq!(
3508            uncooled_agents,
3509            vec!["agent_healthy".to_string()],
3510            "agent whose only model is cooled must be excluded from the LLM candidate list"
3511        );
3512    }
3513
3514    #[serial(cooldown_state)]
3515    #[tokio::test]
3516    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3517        // Regression test for #3521: an agent whose only healthy tier is `simple`
3518        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3519        // excluded from the LLM classification candidate list, otherwise the
3520        // classifier can pick it for a `medium`/`complex` task it can't serve,
3521        // wasting the classification call. Mirrors the real-world shape where
3522        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3523        cooldown::reset_global_state().await;
3524        let mut config = RouterConfig::default();
3525        config
3526            .model_map
3527            .entry("simple".to_string())
3528            .or_default()
3529            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3530        config
3531            .model_map
3532            .entry("medium".to_string())
3533            .or_default()
3534            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3535        config
3536            .model_map
3537            .entry("complex".to_string())
3538            .or_default()
3539            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3540        // review must be irrelevant to the coding-tier filter
3541        config
3542            .model_map
3543            .entry("review".to_string())
3544            .or_default()
3545            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3546        for tier in ["simple", "medium", "complex", "review"] {
3547            config
3548                .model_map
3549                .get_mut(tier)
3550                .unwrap()
3551                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3552        }
3553
3554        let mut router = Router::new(config);
3555        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3556
3557        record_model_failure("minimax_like", "opus").await;
3558        assert!(is_model_in_cooldown("minimax_like", "opus"));
3559        // No agent-level cooldown, and the simple tier is healthy, so the loose
3560        // "any tier" check still passes — this is exactly the leak #3511 missed.
3561        assert!(!is_agent_in_cooldown("minimax_like"));
3562        assert!(router.agent_has_any_available_model("minimax_like"));
3563        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3564
3565        let uncooled_agents: Vec<String> = router
3566            .available_agents
3567            .iter()
3568            .filter(|a| {
3569                !is_agent_in_cooldown(a)
3570                    && !is_agent_degraded(a)
3571                    && router.agent_has_available_model_for_coding_tiers(a)
3572            })
3573            .cloned()
3574            .collect();
3575
3576        assert_eq!(
3577            uncooled_agents,
3578            vec!["agent_healthy".to_string()],
3579            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3580        );
3581    }
3582}