Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22pub use config::{RouterConfig, parse_pool_entry};
23pub use weights::AgentWeights;
24
25fn fmt_degraded(degraded: &[String]) -> String {
26    if degraded.is_empty() {
27        String::new()
28    } else {
29        format!(" (degraded: {})", degraded.join(", "))
30    }
31}
32
33/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
34///
35/// This is a transient error that should not count against `max_route_attempts` since
36/// no agent was actually invoked. The caller should back off and retry later.
37#[derive(Debug, thiserror::Error)]
38pub enum AllAgentsCooledError {
39    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
40    AllCooled {
41        scope: String,
42        remaining_secs: Option<u64>,
43        degraded: Vec<String>,
44    },
45}
46
47impl AllAgentsCooledError {
48    /// Create a new error for the given scope, naming currently degraded agents
49    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
50        Self::AllCooled {
51            scope,
52            remaining_secs,
53            degraded: list_degraded_agents()
54                .into_iter()
55                .map(|(agent, _reason)| agent)
56                .collect(),
57        }
58    }
59
60    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
61
62    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
63    pub fn scope(&self) -> &str {
64        match self {
65            Self::AllCooled { scope, .. } => scope,
66        }
67    }
68
69    /// Remaining cooldown duration in seconds when available.
70    pub fn remaining_secs(&self) -> Option<u64> {
71        match self {
72            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
73        }
74    }
75
76    /// Agents currently marked degraded, shown by `orch cooldown list`
77    pub fn degraded(&self) -> &[String] {
78        match self {
79            Self::AllCooled { degraded, .. } => degraded,
80        }
81    }
82}
83
84/// Typed error returned by [`get_route_result`].
85#[derive(Debug, thiserror::Error)]
86pub enum RouteResultError {
87    /// The task has no agent field — it needs to be re-routed.
88    #[error("no agent configured for task {task_id}")]
89    NoAgent { task_id: String },
90    /// Any other failure (store lookup, SQL, etc.).
91    #[error(transparent)]
92    Other(#[from] anyhow::Error),
93}
94
95use crate::backends::ExternalTask;
96use crate::engine::cooldown::{
97    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
98    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
99};
100use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
101use serde::{Deserialize, Serialize};
102use std::sync::Arc;
103
104use llm::{LlmRouter, TIMEOUT_PREFIX};
105
106/// Result of routing a task to an agent.
107#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
108pub struct RouteResult {
109    /// The selected agent: "claude", "codex", or "opencode"
110    pub agent: String,
111    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
112    pub model: Option<String>,
113    /// Complexity level: "simple", "medium", or "complex"
114    pub complexity: String,
115    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
116    pub estimate: u8,
117    /// Why this agent was selected
118    pub reason: String,
119    /// Specialized agent profile (skills, tools, constraints)
120    pub profile: AgentProfile,
121    /// Selected skill IDs from the catalog
122    pub selected_skills: Vec<String>,
123    /// Optional warning about routing decision
124    pub warning: Option<String>,
125}
126
127/// Specialized agent profile for a task.
128#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
129pub struct AgentProfile {
130    /// Role name, e.g., "backend specialist"
131    pub role: String,
132    /// Focus skills for this task
133    pub skills: Vec<String>,
134    /// Tools allowed for this task
135    pub tools: Vec<String>,
136    /// Constraints for this task
137    pub constraints: Vec<String>,
138}
139
140impl AgentProfile {
141    /// Build the default "general" profile from router config.
142    pub fn default_from_config(config: &RouterConfig) -> Self {
143        Self {
144            role: "general".to_string(),
145            skills: vec![],
146            tools: config.allowed_tools.clone(),
147            constraints: vec![],
148        }
149    }
150}
151
152impl RouteResult {
153    /// Build a standard route result from the three task-specific values.
154    ///
155    /// Fills `profile`, `selected_skills`, and `warning` with router-config
156    /// defaults so call sites only need to supply what varies per task.
157    pub fn from_config(
158        agent: String,
159        complexity: String,
160        reason: String,
161        config: &RouterConfig,
162        task_id: &str,
163    ) -> Self {
164        let model = config.model_for_complexity(&agent, &complexity, task_id);
165        Self {
166            agent,
167            model,
168            complexity,
169            estimate: 0,
170            reason,
171            profile: AgentProfile::default_from_config(config),
172            selected_skills: config.default_skills.clone(),
173            warning: None,
174        }
175    }
176}
177
178/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
179///
180/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
181/// **not** require holding the router's `RwLock`, so async operations can
182/// safely be called without starving write-lock waiters.
183pub struct LlmRouterHandle(std::sync::Arc<LlmRouter>);
184
185impl std::fmt::Debug for LlmRouterHandle {
186    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187        f.debug_struct("LlmRouterHandle").finish_non_exhaustive()
188    }
189}
190
191impl LlmRouterHandle {
192    /// Invalidate the skills catalog cache so the next routing call reloads
193    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
194    pub async fn invalidate_skills_catalog(&self) {
195        self.0.invalidate_skills_catalog().await;
196    }
197}
198
199/// The agent router.
200#[expect(
201    clippy::struct_field_names,
202    reason = "the router holds sub-routers whose names say which backend they use"
203)]
204pub struct Router {
205    /// Router configuration
206    pub config: RouterConfig,
207    /// Available agents discovered at runtime
208    pub available_agents: Vec<String>,
209    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
210    pub weights: AgentWeights,
211    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
212    /// without holding the router RwLock across async operations)
213    llm_router: std::sync::Arc<LlmRouter>,
214    /// Ollama router for local routing (contains reusable reqwest::Client)
215    ollama_router: Option<ollama::OllamaRouter>,
216    /// Round-robin index for task routing
217    pub(crate) rr_index: usize,
218    /// Last agent routed to (for distribution tracking)
219    pub(crate) last_agent: Option<String>,
220    /// Round-robin index for review agent selection
221    pub(crate) review_rr_index: usize,
222    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
223    /// `opencode:free` entries are expanded at construction time.
224    pub(crate) router_pool: Vec<(String, String)>,
225    /// Current round-robin index into router_pool
226    pub(crate) pool_index: usize,
227}
228
229impl std::fmt::Debug for Router {
230    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231        f.debug_struct("Router")
232            .field("config", &self.config)
233            .field("available_agents", &self.available_agents)
234            .field("weights", &self.weights)
235            .field("ollama_router", &self.ollama_router)
236            .field("rr_index", &self.rr_index)
237            .field("last_agent", &self.last_agent)
238            .field("review_rr_index", &self.review_rr_index)
239            .field("router_pool", &self.router_pool)
240            .field("pool_index", &self.pool_index)
241            .finish_non_exhaustive()
242    }
243}
244
245impl Router {
246    /// Create a new router with the given configuration.
247    pub fn new(config: RouterConfig) -> Self {
248        let available_agents = Self::discover_agents(&config.agents);
249        let mut weights = AgentWeights {
250            base_weights: config.weights.clone(),
251            ..Default::default()
252        };
253        weights.ensure_agents_with_weights(&available_agents, &config.weights);
254        if !config.weights.is_empty() {
255            tracing::info!(
256                weights = ?config.weights,
257                "configured agent routing weights"
258            );
259        }
260        let router_pool = Self::expand_pool(&config);
261        tracing::info!(
262            pool = ?router_pool,
263            fallback = %config.effective_fallback(),
264            "router LLM pool initialized"
265        );
266        // Prime only when this config names opencode, so a default or claude-only
267        // router never runs `opencode models`
268        if config.uses_opencode() {
269            crate::engine::runner::agents::opencode::prime_free_model_cache();
270        }
271        let ollama_router = if config.mode == "local" {
272            Some(ollama::OllamaRouter::new(&config))
273        } else {
274            None
275        };
276        Self {
277            config,
278            available_agents,
279            weights,
280            llm_router: std::sync::Arc::new(LlmRouter::new()),
281            ollama_router,
282            rr_index: 0,
283            last_agent: None,
284            review_rr_index: 0,
285            router_pool,
286            pool_index: 0,
287        }
288    }
289
290    /// Create a router with default configuration loaded from files.
291    pub fn from_config() -> Self {
292        Self::new(RouterConfig::from_config())
293    }
294
295    /// Create a router with a forced agent list, bypassing CLI discovery.
296    /// Only for use in unit tests.
297    #[cfg(test)]
298    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
299        let mut weights = AgentWeights {
300            base_weights: config.weights.clone(),
301            ..Default::default()
302        };
303        weights.ensure_agents(&available_agents);
304        let router_pool = Self::expand_pool(&config);
305        Self {
306            config,
307            available_agents,
308            weights,
309            llm_router: std::sync::Arc::new(LlmRouter::new()),
310            ollama_router: None,
311            rr_index: 0,
312            last_agent: None,
313            review_rr_index: 0,
314            router_pool,
315            pool_index: 0,
316        }
317    }
318
319    /// Return a cloned handle to the LLM router subsystem.
320    ///
321    /// Callers that need to invoke async methods on the underlying `LlmRouter`
322    /// should clone this handle *before* releasing any `RwLock` read guard on
323    /// `Router`, then call the async method after the guard is dropped.  This
324    /// prevents holding the read lock across await points (which starves
325    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
326    pub fn llm_router_handle(&self) -> LlmRouterHandle {
327        LlmRouterHandle(std::sync::Arc::clone(&self.llm_router))
328    }
329
330    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
331        debug_assert!(pool_len > 0, "router pool must not be empty");
332        self.pool_index = (idx + 1) % pool_len;
333    }
334
335    /// Discover available agent CLIs in PATH.
336    /// Checks all agents from the configured list.
337    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
338        let mut agents = Vec::new();
339        for agent in configured_agents {
340            if crate::cmd_cache::command_exists(agent) {
341                agents.push(agent.clone());
342            }
343        }
344        agents
345    }
346
347    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
348    ///
349    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
350    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
351    /// - If `opencode` is not installed or the command fails, the entry is skipped.
352    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
353    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
354        let raw_pool = config.effective_pool();
355        let mut expanded: Vec<(String, String)> = Vec::new();
356
357        for entry in &raw_pool {
358            if entry == "opencode:free" {
359                let free_models = Self::discover_free_opencode_models();
360                if free_models.is_empty() {
361                    tracing::debug!("opencode:free expanded to nothing — skipping");
362                } else {
363                    tracing::debug!(models = ?free_models, "opencode:free expanded");
364                    for model in free_models {
365                        expanded.push(("opencode".to_string(), model));
366                    }
367                }
368            } else {
369                let (agent, model) = parse_pool_entry(entry);
370                expanded.push((agent, model));
371            }
372        }
373
374        if expanded.is_empty() {
375            // All entries were opencode:free and opencode isn't installed — use fallback
376            let (agent, model) = parse_pool_entry(&config.effective_fallback());
377            tracing::debug!(
378                agent = %agent, model = %model,
379                "pool empty after expansion, using fallback as sole pool entry"
380            );
381            expanded.push((agent, model));
382        }
383
384        expanded
385    }
386
387    /// Async version of expand_pool which may query the opencode CLI without
388    /// blocking the Tokio runtime.
389    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
390        let raw_pool = config.effective_pool();
391        let mut expanded: Vec<(String, String)> = Vec::new();
392
393        for entry in &raw_pool {
394            if entry == "opencode:free" {
395                let free_models = Self::discover_free_opencode_models_async().await;
396                if free_models.is_empty() {
397                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
398                } else {
399                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
400                    for model in free_models {
401                        expanded.push(("opencode".to_string(), model));
402                    }
403                }
404            } else {
405                let (agent, model) = parse_pool_entry(entry);
406                expanded.push((agent, model));
407            }
408        }
409
410        if expanded.is_empty() {
411            // All entries were opencode:free and opencode isn't installed — use fallback
412            let (agent, model) = parse_pool_entry(&config.effective_fallback());
413            tracing::debug!(
414                agent = %agent, model = %model,
415                "pool empty after async expansion, using fallback as sole pool entry"
416            );
417            expanded.push((agent, model));
418        }
419
420        expanded
421    }
422
423    /// Async reload that uses async pool expansion to avoid blocking the runtime
424    /// when discovering opencode free models.
425    pub async fn reload_async(&mut self) {
426        let new_config = RouterConfig::from_config();
427        let new_agents = Self::discover_agents(&new_config.agents);
428        let new_pool = Self::expand_pool_async(&new_config).await;
429        tracing::info!(
430            mode = %new_config.mode,
431            agents = ?new_agents,
432            fallback = %new_config.fallback_executor,
433            weighted_rr = new_config.weighted_round_robin,
434            pool = ?new_pool,
435            "router reloaded (async)"
436        );
437        self.config = new_config;
438        self.available_agents.clone_from(&new_agents);
439        // Ensure new agents have weight entries (preserves existing weights)
440        self.weights.ensure_agents(&new_agents);
441        self.router_pool = new_pool;
442        self.pool_index = 0;
443        // Reinitialize Ollama router if mode changed to local
444        self.ollama_router = if self.config.mode == "local" {
445            Some(ollama::OllamaRouter::new(&self.config))
446        } else {
447            None
448        };
449    }
450
451    fn discover_free_opencode_models() -> Vec<String> {
452        crate::engine::runner::agents::opencode::discover_free_opencode_models()
453    }
454
455    async fn discover_free_opencode_models_async() -> Vec<String> {
456        tokio::task::spawn_blocking(Self::discover_free_opencode_models)
457            .await
458            .unwrap_or_default()
459    }
460
461    /// Run the pre-emptive health check, refreshing the degraded-agent set.
462    ///
463    /// Queries the `rate_limits` table for recent events and combines with
464    /// cooldown state to mark agents as degraded before routing attempts them.
465    pub async fn refresh_health(&self, store: &std::sync::Arc<TaskStore>) {
466        // KV is the source of truth for cooldowns. Reconcile the in-memory map
467        // from KV every tick so external mutations (e.g. `orch cooldown clear`
468        // from a separate CLI process) actually take effect on the running
469        // service. Without this, the engine keeps stale entries until they
470        // expire on their own, producing the "all agents cooled" log with
471        // `orch cooldown list` simultaneously reporting "No active cooldowns".
472        sync_from_kv(store).await;
473
474        let agents = self.available_agents.clone();
475        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
476        refresh_degraded_agents(
477            store,
478            &agents,
479            &model_checker,
480            config::health_check_window_hours(),
481            config::degraded_rate_limit_threshold(),
482        )
483        .await;
484    }
485
486    /// Check if an agent is available.
487    pub fn is_agent_available(&self, agent: &str) -> bool {
488        self.available_agents.contains(&agent.to_string())
489    }
490
491    /// True if the agent has at least one uncooled model across any complexity
492    /// tier. Used where the eventual complexity isn't known yet (e.g. before
493    /// LLM classification), so we can't check a single tier.
494    fn agent_has_any_available_model(&self, agent: &str) -> bool {
495        ["simple", "medium", "complex", "review"]
496            .iter()
497            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
498    }
499
500    /// True if the agent has an available (uncooled) model for the `medium`
501    /// and `complex` tiers — the tiers real coding tasks are classified into
502    /// by the LLM classifier. Used to pre-filter the candidate list handed to
503    /// that classifier so an agent whose healthy tiers are only `simple` isn't
504    /// offered: the classifier can't be constrained to `simple`, so offering
505    /// such an agent produces a near-certain wasted classification call whenever
506    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
507    /// outcome of initial classification and is deliberately excluded.
508    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
509        ["medium", "complex"]
510            .iter()
511            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
512    }
513
514    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
515        if !self.is_agent_available(agent) {
516            return false;
517        }
518        if is_agent_in_cooldown(agent) {
519            tracing::debug!(agent, "agent skipped: in cooldown");
520            return false;
521        }
522        if is_agent_degraded(agent) {
523            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
524            return false;
525        }
526        // If weighted routing is used, consider the agent degraded when its
527        // weight has decayed below the threshold. This avoids
528        // proactively routing to agents that recently hit many rate limits.
529        if self.config.weighted_round_robin {
530            let weight = self.weights.get_weight(agent);
531            if weight < config::SKIP_LIMITED_THRESHOLD {
532                tracing::debug!(
533                    agent,
534                    weight,
535                    "agent considered degraded by weight threshold, skipping"
536                );
537                return false;
538            }
539        }
540        if !self
541            .config
542            .has_available_model_for_complexity(agent, complexity)
543        {
544            tracing::debug!(agent, complexity, "agent skipped: no available model");
545            return false;
546        }
547        true
548    }
549
550    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
551        self.available_agents
552            .iter()
553            .filter(|agent| self.agent_is_routable(agent, complexity))
554            .cloned()
555            .collect()
556    }
557
558    /// Count of healthy (routable) agents for a given complexity.
559    /// Returns the number of agents that are available, not in cooldown, and have an available model.
560    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
561        self.available_agents_for_complexity(complexity).len()
562    }
563
564    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
565    /// Uses "simple" complexity as the baseline since it has the widest model support.
566    pub fn is_degraded(&self, threshold: usize) -> bool {
567        self.healthy_agent_count("simple") < threshold
568    }
569
570    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
571        let mut earliest: Option<i64> = None;
572
573        for agent in &self.available_agents {
574            if let Some(until) = cooldown_until(agent) {
575                earliest = Some(earliest.map_or(until, |current| current.min(until)));
576            }
577
578            if let Some(comp) = complexity
579                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
580            {
581                for model in pool {
582                    let key = format!("{agent}:{model}");
583                    if let Some(until) = cooldown_until(&key) {
584                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
585                    }
586                }
587            }
588        }
589
590        earliest
591    }
592
593    /// Find the earliest cooldown expiration among the router LLM pool entries.
594    ///
595    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
596    /// or `None` if no pool model has a cooldown.
597    fn earliest_pool_cooldown(&self) -> Option<i64> {
598        let mut earliest: Option<i64> = None;
599        for (agent, model) in &self.router_pool {
600            let key = format!("{agent}:{model}");
601            if let Some(until) = cooldown_until(&key) {
602                earliest = Some(earliest.map_or(until, |current| current.min(until)));
603            }
604        }
605        earliest
606    }
607
608    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
609        // If any cooldowns are present, return an error immediately so the
610        // caller (the tick loop) can skip this task and retry on the next tick.
611        // Sleeping inside the router blocks the entire tick loop for the
612        // cooldown duration which starves other tasks. The engine tick already
613        // retries every `tick_interval` (default 10s), so allow the tick loop
614        // to drive retries instead of sleeping here.
615        let now = chrono::Utc::now().timestamp();
616        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
617            // All agents are degraded but none have a timed cooldown entry
618            // (e.g. over-threshold rate limiting without explicit cooldown duration).
619            // Log at warn and return an error so the tick skips this task and retries after backoff.
620            tracing::warn!(
621                scope = complexity.unwrap_or("all agents"),
622                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
623            );
624            // Return an error similar to the cooldown case to trigger backoff in the tick loop
625            return Err(AllAgentsCooledError::new(
626                complexity.unwrap_or("all agents").to_string(),
627                None,
628            )
629            .into());
630        };
631
632        let remaining = earliest.saturating_sub(now);
633        if remaining > 0 {
634            let scope_str = complexity.unwrap_or("all agents").to_string();
635            tracing::warn!(
636                remaining_secs = remaining,
637                scope = %scope_str,
638                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
639            );
640            // Return a domain error indicating all candidates are cooled so the
641            // caller can handle it (tick will skip this task and retry later).
642            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
643        }
644
645        Ok(())
646    }
647
648    /// Get the first available agent.
649    /// Pick next agent via round-robin (for review or other non-task routing).
650    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
651    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
652    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
653        // Use available_agents_for_complexity which already filters using agent_is_routable,
654        // eliminating duplicate cooldown/degraded/model checks.
655        let candidates = self.available_agents_for_complexity(complexity);
656        if candidates.is_empty() {
657            return None;
658        }
659        let idx = self.review_rr_index;
660        let n = candidates.len();
661
662        // Try to find an agent that isn't excluded
663        let agent = (0..n)
664            .map(|offset| &candidates[(idx + offset) % n])
665            .find(|a| !exclude.contains(&a.as_str()))
666            .cloned()
667            // Fallback: use the first available candidate (even if excluded)
668            .or_else(|| candidates.first().cloned())?;
669
670        self.review_rr_index = (idx + 1) % candidates.len().max(1);
671        Some(agent)
672    }
673
674    /// If the selected agent matches `no_code_last_agent` for this task, filter it
675    /// out of the candidates and return the filtered list. Used to prevent the
676    /// same agent from being selected twice in a row after a no-code failure (#2410).
677    async fn skip_no_code_agent(
678        &self,
679        store: &std::sync::Arc<TaskStore>,
680        repo: &str,
681        task_id: &str,
682        candidates: &[String],
683    ) -> Option<Vec<String>> {
684        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
685            .await
686            .unwrap_or_else(|| {
687                tracing::warn!(
688                    task_id,
689                    "no_code_last_agent DB read returned None — loop detection may be impaired"
690                );
691                String::new()
692            });
693        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
694            return None;
695        }
696        let filtered: Vec<String> = candidates
697            .iter()
698            .filter(|a| a.as_str() != no_code_agent)
699            .cloned()
700            .collect();
701        if filtered.is_empty() {
702            tracing::warn!(
703                task_id,
704                no_code_agent = %no_code_agent,
705                "all available agents are the no-code agent — cannot skip"
706            );
707            return None;
708        }
709        tracing::info!(
710            task_id,
711            skipped = %no_code_agent,
712            remaining = ?filtered,
713            "skipping no-code agent, routing to alternative"
714        );
715        Some(filtered)
716    }
717
718    /// Route a task to the best agent.
719    ///
720    /// Routing logic (in priority order):
721    /// 1. Check for `agent:*` label — use that agent directly
722    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
723    /// 3. If round_robin mode, cycle through agents (stateful)
724    /// 4. Call LLM classifier for intelligent routing
725    /// 5. After max_route_attempts LLM failures, fall back to round-robin
726    pub async fn route(
727        &mut self,
728        task: &ExternalTask,
729        store: &std::sync::Arc<TaskStore>,
730        repo: &str,
731    ) -> anyhow::Result<RouteResult> {
732        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
733        // before the routing loop, not per-task. Do not re-add it here.
734
735        // Hold the task before any classification, label override, or fallback.
736        // A waiting task must not increment route_attempts.
737        match store.resolve_id_by_external(repo, &task.id.0).await {
738            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
739                Ok(Some(blockers)) => {
740                    return Err(crate::store::WaitingOnDependency { blockers }.into());
741                }
742                Ok(None) => {}
743                Err(err) => return Err(err),
744            },
745            Ok(None) => {}
746            Err(err) => return Err(err),
747        }
748
749        loop {
750            // 1. Check store agent field first (set by failover, authoritative over labels)
751            // Use a lightweight lookup to avoid deserializing the full Task row when
752            // only the `agent` column is needed. This avoids two full-row loads
753            // per routing pass (resolve_task_id + get) and significantly reduces
754            // CPU work on busy instances.
755            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
756                Ok(Some(agent)) => {
757                    // Router enforces that the agent string matches a configured agent
758                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
759                        Some(agent)
760                    } else {
761                        None
762                    }
763                }
764                Ok(None) => None,
765                Err(e) => {
766                    tracing::warn!(
767                        repo,
768                        task_id = %task.id.0,
769                        err = %e,
770                        "failed to fetch stored agent, falling back to label/LLM routing"
771                    );
772                    None
773                }
774            };
775
776            // 2. Fall back to explicit agent label
777            let resolved_agent = store_agent.or_else(|| {
778                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
779            });
780
781            if let Some(agent) = resolved_agent {
782                let complexity = strategies::extract_complexity_from_labels(&task.labels);
783                // If agent appears routable (not in agent-wide cooldown and has
784                // available models or no model_map configured), check whether
785                // we actually have a concrete model to dispatch. If the
786                // resolved label maps to no model (None), we fall through to
787                // the standard routing logic so the router LLM can pick a
788                // proper agent/model, instead of dispatching with an empty
789                // model which causes some agents (e.g. opencode) to exit
790                // silently.
791                if self.agent_is_routable(&agent, &complexity) {
792                    let model = self
793                        .config
794                        .model_for_complexity(&agent, &complexity, &task.id.0);
795
796                    // Only dispatch label overrides when we have a concrete
797                    // model to send to the executor. Dispatching with an
798                    // empty model led to silent exits for some agents
799                    // (notably opencode). If no concrete model is available
800                    // for the labeled agent, fall through to the standard
801                    // routing logic so the LLM or round-robin can pick an
802                    // appropriate agent/model.
803                    if model.is_some() {
804                        tracing::debug!(
805                            task_id = %task.id.0,
806                            agent = %agent,
807                            complexity = %complexity,
808                            "routed via label"
809                        );
810                        let mut result = RouteResult::from_config(
811                            agent.clone(),
812                            complexity.clone(),
813                            format!("label agent:{agent}"),
814                            &self.config,
815                            &task.id.0,
816                        );
817                        // Label routing uses agent-specific role and the
818                        // resolved model (may differ from from_config's default).
819                        result.profile.role = format!("{agent} specialist");
820                        result.model = model.clone();
821                        self.log_route_activity(store, repo, &task.id.0, &result, None)
822                            .await;
823                        return Ok(result);
824                    }
825                    // No concrete model found for the labeled agent — do
826                    // not honor the label override. Fall through to the
827                    // standard routing logic so another agent or the LLM
828                    // can be chosen with a valid model.
829                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
830                }
831
832                let candidates = self.available_agents_for_complexity(&complexity);
833                if candidates.is_empty() {
834                    self.wait_for_cooldown(Some(&complexity))?;
835                    continue;
836                }
837                // Label target is cooled or lacked model; fall through to standard routing.
838            }
839
840            let complexity = strategies::extract_complexity_from_labels(&task.labels);
841
842            // Check if any agents have available models for this complexity
843            let mut candidates = self.available_agents_for_complexity(&complexity);
844            if candidates.is_empty() {
845                self.wait_for_cooldown(Some(&complexity))?;
846                continue;
847            }
848
849            // Skip the no-code agent if one is recorded for this task (#2410).
850            // If all available agents are filtered out, we fall through to
851            // LLM routing with no constraints (LLM will pick the same agent but
852            // the response_handler will block it immediately via same-agent check).
853            if let Some(filtered) = self
854                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
855                .await
856            {
857                candidates = filtered;
858            }
859
860            // 2. Weighted round-robin — capacity-based selection
861            if self.config.weighted_round_robin {
862                let result = strategies::route_via_weighted_round_robin(
863                    &candidates,
864                    &self.weights,
865                    &self.config,
866                    task,
867                    &mut self.last_agent,
868                )?;
869                self.log_route_activity(store, repo, &task.id.0, &result, None)
870                    .await;
871                return Ok(result);
872            }
873
874            // 3. Round-robin mode — use stateful round-robin
875            if self.config.mode == "round_robin" {
876                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
877                let result = strategies::route_via_round_robin_stateful(
878                    &candidates,
879                    &self.config,
880                    task,
881                    &mut self.rr_index,
882                    &mut self.last_agent,
883                )?;
884                self.log_route_activity(store, repo, &task.id.0, &result, None)
885                    .await;
886                return Ok(result);
887            }
888
889            // 4. Local Ollama routing
890            if self.config.mode == "local" {
891                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
892                let ollama_router = self
893                    .ollama_router
894                    .as_ref()
895                    .expect("ollama_router must be initialized in Router::new when mode=local");
896                match ollama_router.route(task, &self.config).await {
897                    Ok(result) => {
898                        self.log_route_activity(store, repo, &task.id.0, &result, None)
899                            .await;
900                        return Ok(result);
901                    }
902                    Err(e) => {
903                        tracing::warn!(
904                            task_id = %task.id.0,
905                            error = %e,
906                            "Ollama routing failed, falling back to round-robin"
907                        );
908                        let result = strategies::route_via_round_robin_stateful(
909                            &candidates,
910                            &self.config,
911                            task,
912                            &mut self.rr_index,
913                            &mut self.last_agent,
914                        )?;
915                        self.log_route_activity(store, repo, &task.id.0, &result, None)
916                            .await;
917                        return Ok(result);
918                    }
919                }
920            }
921
922            // 5. LLM-based routing with retry tracking
923            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
924
925            if route_attempts >= self.config.max_route_attempts {
926                tracing::warn!(
927                    task_id = %task.id.0,
928                    attempts = route_attempts,
929                    max = self.config.max_route_attempts,
930                    "max LLM route attempts reached, falling back to round-robin"
931                );
932                let candidates = self.available_agents_for_complexity(&complexity);
933                if candidates.is_empty() {
934                    self.wait_for_cooldown(Some(&complexity))?;
935                    continue;
936                }
937                let result = strategies::route_via_round_robin_stateful(
938                    &candidates,
939                    &self.config,
940                    task,
941                    &mut self.rr_index,
942                    &mut self.last_agent,
943                )?;
944                self.log_route_activity(store, repo, &task.id.0, &result, None)
945                    .await;
946                return Ok(result);
947            }
948
949            // Log routing start (before await)
950            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
951
952            match self.route_with_llm(task, repo).await {
953                Ok(result) => {
954                    let candidates = self.available_agents_for_complexity(&result.complexity);
955                    if candidates.is_empty() {
956                        self.wait_for_cooldown(Some(&result.complexity))?;
957                        continue;
958                    }
959
960                    // If the LLM selected the no-code agent, filter it out and
961                    // pick an alternative (#2410). Re-use the existing fallback logic.
962                    let no_code_agent =
963                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
964                             .await
965                             .unwrap_or_else(|| {
966                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
967                                 String::new()
968                             });
969                    if result.agent == no_code_agent {
970                        // Pick the first agent that isn't the no-code agent.
971                        let fallback_agent = candidates
972                            .iter()
973                            .find(|a| a.as_str() != no_code_agent)
974                            .cloned()
975                            .unwrap_or_else(|| candidates[0].clone());
976                        if fallback_agent.as_str() == no_code_agent {
977                            tracing::warn!(
978                                task_id = %task.id.0,
979                                no_code_agent = %no_code_agent,
980                                complexity = %result.complexity,
981                                "all available agents are the no-code agent — waiting for another agent"
982                            );
983                            self.wait_for_cooldown(Some(&result.complexity))?;
984                            continue;
985                        }
986                        tracing::info!(
987                            task_id = %task.id.0,
988                            llm_selected = %result.agent,
989                            skipped = %no_code_agent,
990                            fallback = %fallback_agent,
991                            "LLM selected the no-code agent — routing to alternative"
992                        );
993                        let fallback_model = self.config.model_for_complexity(
994                            &fallback_agent,
995                            &result.complexity,
996                            &task.id.0,
997                        );
998
999                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1000                        let result = RouteResult {
1001                            agent: fallback_agent.clone(),
1002                            model: fallback_model,
1003                            complexity: result.complexity.clone(),
1004                            estimate: result.estimate,
1005                            reason: format!(
1006                                "LLM selected no-code agent {}; routed to {}",
1007                                result.agent, fallback_agent
1008                            ),
1009                            profile: result.profile.clone(),
1010                            selected_skills: result.selected_skills.clone(),
1011                            warning: Some(format!(
1012                                "LLM selected no-code agent {}; used fallback {}",
1013                                result.agent, fallback_agent
1014                            )),
1015                        };
1016                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1017                            .await;
1018                        return Ok(result);
1019                    }
1020
1021                    let model_cooled = result
1022                        .model
1023                        .as_deref()
1024                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1025                    if !candidates.contains(&result.agent) || model_cooled {
1026                        let fallback_agent = candidates
1027                            .iter()
1028                            .find(|agent| *agent != &result.agent)
1029                            .cloned()
1030                            .unwrap_or_else(|| candidates[0].clone());
1031                        let fallback_model = self.config.model_for_complexity(
1032                            &fallback_agent,
1033                            &result.complexity,
1034                            &task.id.0,
1035                        );
1036
1037                        // If the fallback model is None and the fallback agent is the
1038                        // same as the LLM-selected agent, it means all models for the
1039                        // only available agent are cooled. In that case, wait for
1040                        // cooldowns instead of returning a RouteResult with
1041                        // model: None which would be dispatched incorrectly.
1042                        if fallback_model.is_none() && fallback_agent == result.agent {
1043                            tracing::warn!(
1044                                task_id = %task.id.0,
1045                                agent = %result.agent,
1046                                complexity = %result.complexity,
1047                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1048                            );
1049                            self.wait_for_cooldown(Some(&result.complexity))?;
1050                            continue;
1051                        }
1052
1053                        // Distinguish logging and reasons when we actually switch
1054                        // agents vs when we keep the same agent but pick a
1055                        // different (non-cooled) model.
1056                        if fallback_agent == result.agent {
1057                            tracing::warn!(
1058                                task_id = %task.id.0,
1059                                agent = %result.agent,
1060                                complexity = %result.complexity,
1061                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1062                            );
1063                        } else {
1064                            tracing::warn!(
1065                                task_id = %task.id.0,
1066                                agent = %result.agent,
1067                                fallback = %fallback_agent,
1068                                complexity = %result.complexity,
1069                                "LLM selected cooled agent/model; rerouting to available agent"
1070                            );
1071                        }
1072
1073                        // Reset attempts on success
1074                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1075
1076                        let reason = if fallback_agent == result.agent {
1077                            format!(
1078                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1079                            )
1080                        } else {
1081                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1082                        };
1083
1084                        let result = RouteResult {
1085                            agent: fallback_agent.clone(),
1086                            model: fallback_model,
1087                            complexity: result.complexity.clone(),
1088                            estimate: result.estimate,
1089                            reason: reason.clone(),
1090                            profile: result.profile.clone(),
1091                            selected_skills: result.selected_skills.clone(),
1092                            warning: Some(if fallback_agent == result.agent {
1093                                "LLM-selected model was cooled; selected alternative model for same agent"
1094                                    .to_string()
1095                            } else {
1096                                "LLM-selected agent/model was cooled; rerouted to available agent"
1097                                    .to_string()
1098                            }),
1099                        };
1100                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1101                            .await;
1102                        return Ok(result);
1103                    }
1104
1105                    // Reset attempts on success
1106                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1107                    tracing::info!(
1108                        task_id = %task.id.0,
1109                        agent = %result.agent,
1110                        complexity = %result.complexity,
1111                        "routed via LLM"
1112                    );
1113                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1114                        .await;
1115                    return Ok(result);
1116                }
1117                Err(e) => {
1118                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1119                        let scope = err.scope();
1120                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1121                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1122                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1123                        // (no complexity-specific pool lookup, which would fail for "router pool").
1124                        let scope_opt = match scope {
1125                            "all agents" | "router pool" => None,
1126                            _ => Some(scope),
1127                        };
1128                        self.wait_for_cooldown(scope_opt)?;
1129                        continue;
1130                    }
1131
1132                    let new_attempts = route_attempts + 1;
1133                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1134                        .await;
1135                    tracing::warn!(
1136                        task_id = %task.id.0,
1137                        error = %e,
1138                        error_chain = ?e,
1139                        attempt = new_attempts,
1140                        max = self.config.max_route_attempts,
1141                        "LLM routing failed"
1142                    );
1143
1144                    if new_attempts >= self.config.max_route_attempts {
1145                        tracing::info!(
1146                            task_id = %task.id.0,
1147                            "falling back to round-robin after {} failed attempts",
1148                            new_attempts
1149                        );
1150                        let candidates = self.available_agents_for_complexity(&complexity);
1151                        if candidates.is_empty() {
1152                            self.wait_for_cooldown(Some(&complexity))?;
1153                            continue;
1154                        }
1155                        let result = strategies::route_via_round_robin_stateful(
1156                            &candidates,
1157                            &self.config,
1158                            task,
1159                            &mut self.rr_index,
1160                            &mut self.last_agent,
1161                        )?;
1162                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1163                            .await;
1164                        return Ok(result);
1165                    }
1166
1167                    let candidates = self.available_agents_for_complexity(&complexity);
1168                    if candidates.is_empty() {
1169                        self.wait_for_cooldown(Some(&complexity))?;
1170                        continue;
1171                    }
1172                    let result = strategies::route_via_fallback(
1173                        &candidates,
1174                        &self.config,
1175                        task,
1176                        Some(&self.weights),
1177                    )?;
1178                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1179                        .await;
1180                    return Ok(result);
1181                }
1182            }
1183        }
1184    }
1185
1186    /// Get the number of LLM routing attempts for a task from the store.
1187    async fn get_route_attempts(
1188        &self,
1189        task_id: &str,
1190        store: &std::sync::Arc<TaskStore>,
1191        repo: &str,
1192    ) -> u32 {
1193        match store.resolve_task_id(repo, task_id).await {
1194            Ok(Some(store_id)) => store
1195                .get(store_id)
1196                .await
1197                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1198            _ => 0,
1199        }
1200    }
1201
1202    /// Set the number of LLM routing attempts for a task in the store.
1203    async fn set_route_attempts(
1204        &self,
1205        task_id: &str,
1206        attempts: u32,
1207        store: &std::sync::Arc<TaskStore>,
1208        repo: &str,
1209    ) {
1210        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1211            && let Err(e) = store
1212                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1213                .await
1214        {
1215            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1216        }
1217    }
1218
1219    /// Log a route event to task activity timeline.
1220    async fn log_route_activity(
1221        &self,
1222        store: &std::sync::Arc<TaskStore>,
1223        repo: &str,
1224        task_id: &str,
1225        result: &RouteResult,
1226        from_agent: Option<&str>,
1227    ) {
1228        let details = serde_json::json!({
1229            "reason": result.reason,
1230            "complexity": result.complexity,
1231            "skills": result.selected_skills,
1232            "role": result.profile.role,
1233            "warning": result.warning,
1234        });
1235        let event_type = if from_agent.is_some() {
1236            "rerouted"
1237        } else {
1238            "routed"
1239        };
1240        store_log_activity(
1241            &Some(Arc::clone(store)),
1242            repo,
1243            task_id,
1244            event_type,
1245            None,
1246            Some("routed"),
1247            Some(&result.agent),
1248            result.model.as_deref(),
1249            Some(&details),
1250        )
1251        .await;
1252    }
1253
1254    /// Route using LLM classification with pool round-robin.
1255    ///
1256    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1257    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1258    /// returns the last error (the caller falls back to round-robin agent selection).
1259    async fn route_with_llm(
1260        &mut self,
1261        task: &ExternalTask,
1262        repo: &str,
1263    ) -> anyhow::Result<RouteResult> {
1264        // Filter out cooled and degraded agents, and agents without an available
1265        // model for the coding tiers (`medium`/`complex`) that real tasks are
1266        // classified into, so the LLM only sees candidates that could actually
1267        // be dispatched for whatever complexity it picks. An agent whose only
1268        // healthy tier is `simple` is excluded too: the classifier can't be
1269        // constrained to `simple`, so leaving such an agent in would waste the
1270        // call whenever it picks `medium`/`complex` (issue #3521).
1271        // Fall back to the full list if all agents are cooled/degraded.
1272        // Cloned to avoid borrow conflict with &mut self in the loop.
1273        let uncooled_agents: Vec<String> = self
1274            .available_agents
1275            .iter()
1276            .filter(|a| {
1277                !is_agent_in_cooldown(a)
1278                    && !is_agent_degraded(a)
1279                    && self.agent_has_available_model_for_coding_tiers(a)
1280            })
1281            .cloned()
1282            .collect();
1283        if uncooled_agents.is_empty() {
1284            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1285        }
1286        let llm_agents = uncooled_agents;
1287
1288        let pool = self.router_pool.clone();
1289        let n = pool.len();
1290        let start = self.pool_index;
1291        let mut last_err: Option<anyhow::Error> = None;
1292        let mut attempted_any = false;
1293
1294        // Try pool entries in round-robin order, skipping cooled ones
1295        for i in 0..n {
1296            let idx = (start + i) % n;
1297            let (agent, model) = &pool[idx];
1298            let model_str = model.as_str();
1299
1300            if crate::engine::runner::response::is_model_in_cooldown(agent, model_str) {
1301                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1302                continue;
1303            }
1304
1305            // At least one pool entry was available to attempt
1306            attempted_any = true;
1307
1308            let model_opt = if model_str.is_empty() {
1309                None
1310            } else {
1311                Some(model_str)
1312            };
1313
1314            match self
1315                .llm_router
1316                .route_with_llm_using(
1317                    task,
1318                    &llm_agents,
1319                    &self.config,
1320                    &mut self.last_agent,
1321                    repo,
1322                    agent,
1323                    model_opt,
1324                )
1325                .await
1326            {
1327                Ok(result) => {
1328                    // Advance index so the next call starts at the next pool entry
1329                    self.advance_pool_index_after_attempt(idx, n);
1330                    tracing::debug!(
1331                        agent,
1332                        model = model_str,
1333                        pool_idx = idx,
1334                        "router LLM pool entry succeeded"
1335                    );
1336                    record_agent_success(agent, model_str).await;
1337                    return Ok(result);
1338                }
1339                Err(e) => {
1340                    let err_msg = e.to_string();
1341                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1342                    if is_timeout {
1343                        // Advance pool index so the next tick starts at the next entry,
1344                        // then return immediately. Continuing to try further pool entries
1345                        // in the same tick would accumulate multiple timeout windows and
1346                        // stall the tick loop for minutes (issue #3187).
1347                        //
1348                        // Also record a standard cooldown, same as the non-timeout branch,
1349                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1350                        // on future ticks instead of burning the full timeout window again
1351                        // every time round-robin reaches it (issue #3422).
1352                        self.advance_pool_index_after_attempt(idx, n);
1353                        tracing::warn!(
1354                            agent,
1355                            model = model_str,
1356                            next_pool_idx = self.pool_index,
1357                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1358                        );
1359                        crate::engine::runner::response::record_model_failure(agent, model_str)
1360                            .await;
1361                        return Err(e);
1362                    }
1363                    tracing::warn!(
1364                        agent,
1365                        model = model_str,
1366                        error = %e,
1367                        "pool entry failed, recording cooldown and trying next"
1368                    );
1369                    if !crate::engine::cooldown::record_credit_exhaustion_from_message(
1370                        agent,
1371                        Some(model_str),
1372                        &err_msg,
1373                    )
1374                    .await
1375                    {
1376                        crate::engine::runner::response::record_model_failure(agent, model_str)
1377                            .await;
1378                    }
1379                    last_err = Some(e);
1380                    self.advance_pool_index_after_attempt(idx, n);
1381                }
1382            }
1383        }
1384
1385        // If every pool entry was pre-cooled (nothing was attempted), do not
1386        // immediately exhaust the pool again — fail fast and let the tick loop
1387        // retry on the next cycle instead of cascading failures.
1388        // However, we should still try the fallback if configured.
1389        if !attempted_any {
1390            let now = chrono::Utc::now().timestamp();
1391            let earliest = self.earliest_pool_cooldown();
1392            if let Some(until) = earliest {
1393                // Pool entries exist but are all cooled - we'll try fallback below
1394                let remaining = until.saturating_sub(now);
1395                tracing::warn!(
1396                    remaining_secs = remaining,
1397                    pool = ?pool,
1398                    "all router LLM pool entries on cooldown — will try fallback"
1399                );
1400                // Continue to fallback logic instead of returning early
1401            } else {
1402                // No pool cooldowns found either — this means the pool was empty
1403                // or every entry was filtered out for some other reason.
1404                return Err(last_err
1405                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1406            }
1407        }
1408
1409        // All pool entries failed or were cooled — try the configured fallback
1410        let fallback = self.config.effective_fallback();
1411        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1412        let fb_model_str = fb_model.as_str();
1413
1414        // Only try fallback if it wasn't already in the pool (avoid double-try)
1415        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1416
1417        if !fallback_already_tried
1418            && !crate::engine::runner::response::is_model_in_cooldown(&fb_agent, fb_model_str)
1419        {
1420            tracing::info!(
1421                agent = %fb_agent,
1422                model = %fb_model_str,
1423                "all pool entries exhausted — trying fallback router LLM"
1424            );
1425            let fb_model_opt = if fb_model_str.is_empty() {
1426                None
1427            } else {
1428                Some(fb_model_str)
1429            };
1430            match self
1431                .llm_router
1432                .route_with_llm_using(
1433                    task,
1434                    &llm_agents,
1435                    &self.config,
1436                    &mut self.last_agent,
1437                    repo,
1438                    &fb_agent,
1439                    fb_model_opt,
1440                )
1441                .await
1442            {
1443                Ok(result) => {
1444                    record_agent_success(&fb_agent, fb_model_str).await;
1445                    return Ok(result);
1446                }
1447                Err(e) => {
1448                    let err_msg = e.to_string();
1449                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1450                    if is_timeout {
1451                        // Cool down on timeout like pool entries (#3422), else a
1452                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1453                        crate::engine::runner::response::record_model_failure(
1454                            &fb_agent,
1455                            fb_model_str,
1456                        )
1457                        .await;
1458                        tracing::warn!(
1459                            agent = %fb_agent,
1460                            model = %fb_model_str,
1461                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1462                        );
1463                        return Err(e);
1464                    }
1465                    tracing::warn!(
1466                        agent = %fb_agent,
1467                        model = %fb_model_str,
1468                        error = %e,
1469                        "fallback router LLM also failed"
1470                    );
1471                    crate::engine::runner::response::record_model_failure(&fb_agent, fb_model_str)
1472                        .await;
1473                    last_err = Some(e);
1474                }
1475            }
1476        }
1477
1478        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1479    }
1480
1481    /// Record a rate limit event for an agent, reducing its routing weight.
1482    ///
1483    /// Called by the engine when an agent returns a 429/rate limit error.
1484    pub fn record_rate_limit(&mut self, agent: &str) {
1485        self.weights.record_rate_limit(agent);
1486    }
1487
1488    /// Record a successful task completion, restoring agent weight.
1489    pub fn record_success(&mut self, agent: &str) {
1490        self.weights.record_success(agent);
1491    }
1492
1493    /// Tick weight recovery for all agents.
1494    ///
1495    /// Called periodically by the engine to gradually restore weights
1496    /// as rate limit windows expire.
1497    pub fn tick_weight_recovery(&mut self) {
1498        self.weights.tick_recovery();
1499    }
1500
1501    /// Store routing result in the task store.
1502    pub async fn store_route_result(
1503        &self,
1504        task_id: &str,
1505        result: &RouteResult,
1506        store: &std::sync::Arc<TaskStore>,
1507        repo: &str,
1508    ) -> anyhow::Result<()> {
1509        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1510            store
1511                .set_fields(
1512                    store_id,
1513                    &[
1514                        ("agent", serde_json::json!(result.agent)),
1515                        ("complexity", serde_json::json!(result.complexity)),
1516                        ("route_reason", serde_json::json!(result.reason)),
1517                        (
1518                            "model",
1519                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1520                        ),
1521                    ],
1522                )
1523                .await?;
1524        }
1525        Ok(())
1526    }
1527}
1528
1529/// Retrieve routing result from the task store.
1530pub async fn get_route_result(
1531    store: &std::sync::Arc<TaskStore>,
1532    repo: &str,
1533    task_id: &str,
1534) -> Result<RouteResult, RouteResultError> {
1535    let store_id = store
1536        .resolve_task_id(repo, task_id)
1537        .await
1538        .map_err(RouteResultError::Other)?
1539        .ok_or_else(|| {
1540            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1541        })?;
1542    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1543
1544    let agent = task
1545        .agent
1546        .clone()
1547        .filter(|a| !a.is_empty())
1548        .ok_or_else(|| RouteResultError::NoAgent {
1549            task_id: task_id.to_string(),
1550        })?;
1551
1552    let complexity = if task.complexity.is_empty() {
1553        "medium".to_string()
1554    } else {
1555        task.complexity.clone()
1556    };
1557
1558    let reason = task.route_reason.clone();
1559    let model = task.model.clone().filter(|m| !m.is_empty());
1560
1561    // Validate that the stored model is valid for the resolved agent.
1562    // Agent-specific aliases (e.g. "opus") can leak across agents during
1563    // failover, so we verify against the router's model pools (#1604).
1564    let config = RouterConfig::from_config();
1565    let model = model.filter(|m| {
1566        let complexities = ["simple", "medium", "complex", "review"];
1567        // Collect all model pools configured for this agent across complexity tiers.
1568        let all_pools: Vec<Vec<String>> = complexities
1569            .iter()
1570            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1571            .collect();
1572        // If the agent has no model pools configured, we can't validate — pass through.
1573        if all_pools.is_empty() {
1574            return true;
1575        }
1576        let valid = all_pools.iter().any(|pool| pool.contains(m));
1577        if !valid {
1578            tracing::warn!(
1579                task_id,
1580                agent,
1581                model = %m,
1582                "discarding stale model: not in agent's model pools"
1583            );
1584        }
1585        valid
1586    });
1587
1588    // A stale model is replaced by the agent's model for the task's complexity, an empty
1589    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1590    let original_model = task.model.clone().filter(|m| !m.is_empty());
1591    let mut model = model;
1592    if original_model.is_some() && model.is_none() {
1593        model = config.model_for_complexity(&agent, &complexity, task_id);
1594        let stale_model = original_model.as_deref().unwrap_or("unknown");
1595        let stale_err = format!(
1596            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1597            model.as_deref().unwrap_or("the agent default")
1598        );
1599        if let Err(e) = store
1600            .set_fields(
1601                store_id,
1602                &[
1603                    (
1604                        "model",
1605                        serde_json::json!(model.clone().unwrap_or_default()),
1606                    ),
1607                    ("last_error", serde_json::json!(stale_err)),
1608                ],
1609            )
1610            .await
1611        {
1612            // Log the error so transient DB failures are visible and diagnosable.
1613            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1614        }
1615    }
1616
1617    Ok(RouteResult {
1618        agent,
1619        model,
1620        complexity,
1621        estimate: 0,
1622        reason,
1623        profile: AgentProfile::default(),
1624        selected_skills: vec![],
1625        warning: None,
1626    })
1627}
1628
1629#[cfg(test)]
1630#[expect(
1631    clippy::float_cmp,
1632    reason = "the weights are stored constants, so they compare exactly"
1633)]
1634mod tests {
1635    use super::config::DEFAULT_AGENTS;
1636    use super::llm::LlmRouteResponse;
1637    use super::strategies;
1638    use super::weights::{
1639        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1640    };
1641    use super::*;
1642    use crate::backends::{ExternalId, ExternalTask};
1643    use crate::engine::cooldown::{
1644        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1645        record_model_failure, set_model_cooldown,
1646    };
1647    use crate::store::TaskStore;
1648    use serial_test::serial;
1649    use std::time::{Duration, Instant};
1650
1651    #[tokio::test]
1652    #[serial_test::serial(orch_home)]
1653    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1654        let home = tempfile::tempdir().unwrap();
1655        std::fs::write(
1656            home.path().join("config.yml"),
1657            "model_map:\n  medium:\n    claude: sonnet\n",
1658        )
1659        .unwrap();
1660        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1661        unsafe { std::env::set_var("ORCH_HOME", home.path()) };
1662
1663        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1664        let store_id = store
1665            .create(&crate::store::NewTask {
1666                external_id: Some("7".to_string()),
1667                repo: "owner/repo".to_string(),
1668                origin: "github".to_string(),
1669                title: "stale model".to_string(),
1670                ..Default::default()
1671            })
1672            .await
1673            .unwrap();
1674        store
1675            .set_fields(
1676                store_id,
1677                &[
1678                    ("agent", serde_json::json!("claude")),
1679                    ("model", serde_json::json!("claude-removed-model")),
1680                    ("complexity", serde_json::json!("medium")),
1681                ],
1682            )
1683            .await
1684            .unwrap();
1685
1686        let route = get_route_result(&store, "owner/repo", "7").await;
1687        let stored = store.get(store_id).await.unwrap().model;
1688        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1689        unsafe { std::env::remove_var("ORCH_HOME") };
1690
1691        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1692        assert_eq!(stored.as_deref(), Some("sonnet"));
1693    }
1694
1695    // Test-only delegate so tests can call router.parse_llm_response() directly
1696    // without referencing llm_router.
1697    impl Router {
1698        #[expect(
1699            clippy::unused_self,
1700            reason = "the test helper keeps the router method syntax"
1701        )]
1702        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1703            LlmRouter::parse_llm_response("claude", response)
1704        }
1705    }
1706
1707    async fn test_store() -> std::sync::Arc<TaskStore> {
1708        std::sync::Arc::new(TaskStore::open_memory().await.unwrap())
1709    }
1710
1711    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1712        ExternalTask {
1713            id: ExternalId(id.to_string()),
1714            title: title.to_string(),
1715            body: "Test body".to_string(),
1716            state: "open".to_string(),
1717            labels,
1718            author: "test".to_string(),
1719            created_at: "2026-01-01T00:00:00Z".to_string(),
1720            updated_at: "2026-01-01T00:00:00Z".to_string(),
1721            url: format!("https://github.com/test/test/issues/{id}"),
1722        }
1723    }
1724
1725    #[serial(cooldown_state)]
1726    #[test]
1727    fn extract_agent_from_labels() {
1728        let mut config = RouterConfig::default();
1729        // Ensure a concrete model exists for the labeled agent so the label
1730        // override is honored by the router (routing requires a model).
1731        config
1732            .model_map
1733            .entry("medium".to_string())
1734            .or_default()
1735            .insert("claude".to_string(), vec!["haiku".to_string()]);
1736
1737        assert_eq!(
1738            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1739            Some("claude".to_string())
1740        );
1741        assert_eq!(
1742            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1743            Some("codex".to_string())
1744        );
1745        assert_eq!(
1746            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1747            Some("opencode".to_string())
1748        );
1749        assert_eq!(
1750            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1751            None
1752        );
1753        // Verify kimi and minimax are recognized from labels
1754        assert_eq!(
1755            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1756            Some("kimi".to_string())
1757        );
1758        assert_eq!(
1759            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1760            Some("minimax".to_string())
1761        );
1762    }
1763
1764    #[serial(cooldown_state)]
1765    #[tokio::test]
1766    async fn all_agents_cooled_error_names_degraded_agents() {
1767        crate::engine::cooldown::reset_global_state().await;
1768        mark_agent_degraded("claude");
1769        mark_agent_degraded("grok");
1770        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1771        let msg = err.to_string();
1772        assert!(
1773            msg.contains("degraded: claude, grok"),
1774            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1775        );
1776        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1777        clear_agent_degraded("claude");
1778        clear_agent_degraded("grok");
1779    }
1780
1781    #[serial(cooldown_state)]
1782    #[test]
1783    fn default_agents_constant() {
1784        assert_eq!(DEFAULT_AGENTS.len(), 7);
1785        assert!(DEFAULT_AGENTS.contains(&"claude"));
1786        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1787        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1788        assert!(DEFAULT_AGENTS.contains(&"glm"));
1789        assert!(DEFAULT_AGENTS.contains(&"grok"));
1790    }
1791
1792    #[serial(cooldown_state)]
1793    #[test]
1794    fn extract_complexity_from_labels() {
1795        assert_eq!(
1796            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1797            "simple"
1798        );
1799        assert_eq!(
1800            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1801            "medium"
1802        );
1803        assert_eq!(
1804            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1805            "complex"
1806        );
1807        assert_eq!(
1808            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1809            "medium"
1810        );
1811    }
1812
1813    #[serial(cooldown_state)]
1814    #[test]
1815    fn route_result_serialization() {
1816        let result = RouteResult {
1817            agent: "claude".to_string(),
1818            model: Some("sonnet".to_string()),
1819            complexity: "medium".to_string(),
1820            estimate: 5,
1821            reason: "test".to_string(),
1822            profile: AgentProfile {
1823                role: "backend".to_string(),
1824                skills: vec!["rust".to_string()],
1825                tools: vec!["git".to_string()],
1826                constraints: vec![],
1827            },
1828            selected_skills: vec!["gh".to_string()],
1829            warning: None,
1830        };
1831
1832        let json = serde_json::to_string(&result).unwrap();
1833        assert!(json.contains("claude"));
1834        assert!(json.contains("sonnet"));
1835
1836        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1837        assert_eq!(deserialized.agent, "claude");
1838        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1839    }
1840
1841    #[serial(cooldown_state)]
1842    #[test]
1843    fn router_config_default() {
1844        let mut config = RouterConfig::default();
1845        // Ensure a concrete model exists for the labeled agent so the label
1846        // override is honored by the router (routing requires a model).
1847        config
1848            .model_map
1849            .entry("medium".to_string())
1850            .or_default()
1851            .insert("claude".to_string(), vec!["haiku".to_string()]);
1852
1853        assert_eq!(config.mode, "round_robin");
1854        assert_eq!(config.router_agent, "claude");
1855        assert_eq!(config.router_model, "haiku");
1856        assert_eq!(config.fallback_executor, "codex");
1857        assert_eq!(config.max_route_attempts, 3);
1858        assert_ne!(config.allowed_tools, [] as [String; 0]);
1859        assert_ne!(config.default_skills, [] as [String; 0]);
1860
1861        // Verify configurable agents list includes all 7 agents
1862        assert_eq!(config.agents.len(), 7);
1863        assert!(config.agents.contains(&"claude".to_string()));
1864        assert!(config.agents.contains(&"codex".to_string()));
1865        assert!(config.agents.contains(&"opencode".to_string()));
1866        assert!(config.agents.contains(&"kimi".to_string()));
1867        assert!(config.agents.contains(&"minimax".to_string()));
1868        assert!(config.agents.contains(&"glm".to_string()));
1869        assert!(config.agents.contains(&"grok".to_string()));
1870    }
1871
1872    #[serial(cooldown_state)]
1873    #[test]
1874    fn model_map_lookup_returns_none_without_config() {
1875        // Default config has no hardcoded models — config.yml is the source of truth.
1876        let config = RouterConfig::default();
1877        assert!(
1878            config
1879                .model_for_complexity("claude", "simple", "")
1880                .is_none()
1881        );
1882        assert!(
1883            config
1884                .model_for_complexity("claude", "medium", "")
1885                .is_none()
1886        );
1887        assert!(
1888            config
1889                .model_for_complexity("claude", "complex", "")
1890                .is_none()
1891        );
1892        assert!(
1893            config
1894                .model_for_complexity("opencode", "review", "")
1895                .is_none()
1896        );
1897    }
1898
1899    #[serial(cooldown_state)]
1900    #[test]
1901    fn model_map_lookup_returns_configured_value() {
1902        // When model_map is populated (from config), it is returned correctly.
1903        let mut config = RouterConfig::default();
1904        config
1905            .model_map
1906            .entry("simple".to_string())
1907            .or_default()
1908            .insert("claude".to_string(), vec!["haiku".to_string()]);
1909        assert_eq!(
1910            config.model_for_complexity("claude", "simple", ""),
1911            Some("haiku".to_string())
1912        );
1913    }
1914
1915    #[serial(cooldown_state)]
1916    #[tokio::test]
1917    async fn model_pool_selection_skips_cooled() {
1918        crate::engine::cooldown::reset_global_state().await;
1919
1920        let mut config = RouterConfig::default();
1921        // Set up a two-model pool for opencode/simple
1922        config
1923            .model_map
1924            .entry("simple".to_string())
1925            .or_default()
1926            .insert(
1927                "opencode".to_string(),
1928                vec!["model-a".to_string(), "model-b".to_string()],
1929            );
1930
1931        // Before any cooldown, both models are candidates
1932        let m = config.model_for_complexity("opencode", "simple", "task-1");
1933        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1934
1935        // Cool model-a
1936        record_model_failure("opencode", "model-a").await;
1937        assert!(is_model_in_cooldown("opencode", "model-a"));
1938
1939        // Now only model-b should be returned
1940        // (try many task_ids to rule out hash coincidence)
1941        for i in 0..20 {
1942            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1943            assert_eq!(
1944                result,
1945                Some("model-b".to_string()),
1946                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1947            );
1948        }
1949
1950        // Cool model-b too — now all models are cooled
1951        record_model_failure("opencode", "model-b").await;
1952        // All cooled → return None so the caller can fall back to a different agent
1953        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1954        assert_eq!(fallback, None);
1955    }
1956
1957    #[serial(cooldown_state)]
1958    #[test]
1959    fn model_pool_single_string_backward_compat() {
1960        // Single-item pools behave identically to the old string format.
1961        // Manually insert a single-item pool and verify it is returned correctly.
1962        let mut config = RouterConfig::default();
1963        config
1964            .model_map
1965            .entry("simple".to_string())
1966            .or_default()
1967            .insert("claude".to_string(), vec!["haiku".to_string()]);
1968        let m = config.model_for_complexity("claude", "simple", "any-task");
1969        assert_eq!(m, Some("haiku".to_string()));
1970    }
1971
1972    #[serial(cooldown_state)]
1973    #[test]
1974    fn parse_llm_response_direct_json() {
1975        let config = RouterConfig::default();
1976        let router = Router::new(config);
1977
1978        let response = r#"{
1979            "executor": "claude",
1980            "complexity": "complex",
1981            "reason": "requires architecture analysis",
1982            "profile": {
1983                "role": "architect",
1984                "skills": ["rust", "design"],
1985                "tools": ["git", "rg"],
1986                "constraints": []
1987            },
1988            "selected_skills": ["gh"]
1989        }"#;
1990
1991        let parsed = router.parse_llm_response(response).unwrap();
1992        assert_eq!(parsed.executor, "claude");
1993        assert_eq!(parsed.complexity, "complex");
1994        assert_eq!(parsed.reason, "requires architecture analysis");
1995        assert_eq!(parsed.profile.role, "architect");
1996    }
1997
1998    #[serial(cooldown_state)]
1999    #[test]
2000    fn parse_llm_response_markdown_fenced() {
2001        let config = RouterConfig::default();
2002        let router = Router::new(config);
2003
2004        let response = r#"Here's my analysis:
2005
2006```json
2007{
2008    "executor": "codex",
2009    "complexity": "medium",
2010    "reason": "coding task",
2011    "profile": {
2012        "role": "developer",
2013        "skills": ["coding"],
2014        "tools": [],
2015        "constraints": []
2016    },
2017    "selected_skills": []
2018}
2019```
2020
2021Hope that helps!"#;
2022
2023        let parsed = router.parse_llm_response(response).unwrap();
2024        assert_eq!(parsed.executor, "codex");
2025        assert_eq!(parsed.complexity, "medium");
2026    }
2027
2028    #[serial(cooldown_state)]
2029    #[test]
2030    fn parse_llm_response_claude_envelope() {
2031        let config = RouterConfig::default();
2032        let router = Router::new(config);
2033
2034        // Claude --output-format json wraps the response in an envelope
2035        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2036
2037        let parsed = router.parse_llm_response(response).unwrap();
2038        assert_eq!(parsed.executor, "claude");
2039        assert_eq!(parsed.complexity, "complex");
2040        assert_eq!(parsed.reason, "requires deep analysis");
2041    }
2042
2043    #[serial(cooldown_state)]
2044    #[test]
2045    fn parse_llm_response_claude_envelope_with_code_block() {
2046        let config = RouterConfig::default();
2047        let router = Router::new(config);
2048
2049        // Claude may return the JSON inside a code block within the envelope
2050        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2051        let envelope = serde_json::json!({
2052            "type": "result",
2053            "subtype": "success",
2054            "is_error": false,
2055            "result": inner,
2056            "usage": {"input_tokens": 50, "output_tokens": 30}
2057        });
2058
2059        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2060        assert_eq!(parsed.executor, "codex");
2061        assert_eq!(parsed.complexity, "simple");
2062    }
2063
2064    #[serial(cooldown_state)]
2065    #[test]
2066    fn parse_llm_response_claude_envelope_error() {
2067        let config = RouterConfig::default();
2068        let router = Router::new(config);
2069
2070        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2071
2072        let err = router.parse_llm_response(response).unwrap_err();
2073        assert!(err.to_string().contains("error"), "got: {err}");
2074    }
2075
2076    #[serial(cooldown_state)]
2077    #[test]
2078    fn parse_llm_response_empty() {
2079        let config = RouterConfig::default();
2080        let router = Router::new(config);
2081
2082        let err = router.parse_llm_response("").unwrap_err();
2083        assert!(err.to_string().contains("empty"), "got: {err}");
2084    }
2085
2086    /// Claude envelope where result is a JSON object, not a string.
2087    /// Some Claude versions may return the result as a nested object.
2088    #[serial(cooldown_state)]
2089    #[test]
2090    fn parse_llm_response_claude_envelope_result_as_object() {
2091        let config = RouterConfig::default();
2092        let router = Router::new(config);
2093
2094        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2095
2096        let parsed = router.parse_llm_response(response);
2097        assert!(
2098            parsed.is_ok(),
2099            "should handle result-as-object: {}",
2100            parsed.unwrap_err()
2101        );
2102        assert_eq!(parsed.unwrap().executor, "claude");
2103    }
2104
2105    /// LLM returns "agent" instead of "executor" — common hallucination.
2106    #[serial(cooldown_state)]
2107    #[test]
2108    fn parse_llm_response_agent_instead_of_executor() {
2109        let config = RouterConfig::default();
2110        let router = Router::new(config);
2111
2112        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2113
2114        let parsed = router.parse_llm_response(response);
2115        assert!(
2116            parsed.is_ok(),
2117            "should accept 'agent' alias: {}",
2118            parsed.unwrap_err()
2119        );
2120        assert_eq!(parsed.unwrap().executor, "claude");
2121    }
2122
2123    /// LLM returns minimal JSON without profile or reason.
2124    #[serial(cooldown_state)]
2125    #[test]
2126    fn parse_llm_response_minimal_json() {
2127        let config = RouterConfig::default();
2128        let router = Router::new(config);
2129
2130        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2131
2132        let parsed = router.parse_llm_response(response);
2133        assert!(
2134            parsed.is_ok(),
2135            "should accept minimal JSON: {}",
2136            parsed.unwrap_err()
2137        );
2138        assert_eq!(parsed.unwrap().executor, "codex");
2139    }
2140
2141    // ── Real fixture tests: route prompt → response → parsed result ──
2142
2143    /// Real Claude envelope with result as escaped JSON string.
2144    #[serial(cooldown_state)]
2145    #[test]
2146    fn fixture_route_response_string() {
2147        let config = RouterConfig::default();
2148        let router = Router::new(config);
2149
2150        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2151        let parsed = router.parse_llm_response(response).unwrap();
2152        assert_eq!(parsed.executor, "codex");
2153        assert_eq!(parsed.complexity, "medium");
2154        assert_ne!(parsed.reason, "");
2155        assert_ne!(parsed.profile.role, "");
2156    }
2157
2158    /// Real Claude envelope with result as JSON object (not string).
2159    /// This was the root cause of all router failures on 2026-02-27.
2160    #[serial(cooldown_state)]
2161    #[test]
2162    fn fixture_route_response_object() {
2163        let config = RouterConfig::default();
2164        let router = Router::new(config);
2165
2166        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2167        let parsed = router.parse_llm_response(response).unwrap();
2168        assert_eq!(parsed.executor, "claude");
2169        assert_eq!(parsed.complexity, "complex");
2170    }
2171
2172    /// Real Claude envelope with result as string containing markdown + JSON.
2173    #[serial(cooldown_state)]
2174    #[test]
2175    fn fixture_route_response_markdown() {
2176        let config = RouterConfig::default();
2177        let router = Router::new(config);
2178
2179        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2180        let parsed = router.parse_llm_response(response).unwrap();
2181        assert_eq!(parsed.executor, "codex");
2182        assert_eq!(parsed.complexity, "medium");
2183    }
2184
2185    /// Claude envelope with escaped JSON string containing newlines and markdown.
2186    #[serial(cooldown_state)]
2187    #[test]
2188    fn parse_llm_response_claude_envelope_with_prose() {
2189        let config = RouterConfig::default();
2190        let router = Router::new(config);
2191
2192        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2193        let envelope = serde_json::json!({
2194            "type": "result",
2195            "subtype": "success",
2196            "is_error": false,
2197            "result": inner,
2198            "usage": {"input_tokens": 100, "output_tokens": 80}
2199        });
2200
2201        let parsed = router.parse_llm_response(&envelope.to_string());
2202        assert!(
2203            parsed.is_ok(),
2204            "should handle prose + code block in envelope: {}",
2205            parsed.unwrap_err()
2206        );
2207        assert_eq!(parsed.unwrap().executor, "codex");
2208    }
2209
2210    #[serial(cooldown_state)]
2211    #[tokio::test]
2212    async fn route_round_robin_basic() {
2213        crate::engine::cooldown::reset_global_state().await;
2214        // Force at least one agent to be available for testing
2215        // In real usage, discover_agents finds installed CLIs
2216        let config = RouterConfig {
2217            mode: "round_robin".to_string(),
2218            ..Default::default()
2219        };
2220
2221        // Create router with mock available agents
2222        let agents = vec!["claude".to_string(), "codex".to_string()];
2223        let mut weights = AgentWeights::default();
2224        weights.ensure_agents(&agents);
2225        let router = Router {
2226            config,
2227            available_agents: agents,
2228            weights,
2229            llm_router: std::sync::Arc::new(LlmRouter::new()),
2230            ollama_router: None,
2231            rr_index: 0,
2232            last_agent: None,
2233            review_rr_index: 0,
2234            router_pool: vec![],
2235            pool_index: 0,
2236        };
2237
2238        let task = create_test_task("1", "Test task", vec![]);
2239        let result =
2240            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2241                .unwrap();
2242
2243        // Task 1 % 2 agents = agent at index 1 = codex
2244        assert_eq!(result.agent, "codex");
2245        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2246    }
2247
2248    #[serial(cooldown_state)]
2249    #[tokio::test]
2250    async fn route_uses_label_override() {
2251        crate::engine::cooldown::reset_global_state().await;
2252        let mut config = RouterConfig::default();
2253        // Ensure a concrete model exists so the label override can be honored
2254        // in the test environment where no external config.yml is present.
2255        config
2256            .model_map
2257            .entry("medium".to_string())
2258            .or_default()
2259            .insert("claude".to_string(), vec!["haiku".to_string()]);
2260        let agents = vec!["claude".to_string(), "codex".to_string()];
2261        let mut weights = AgentWeights::default();
2262        weights.ensure_agents(&agents);
2263        let mut router = Router {
2264            config,
2265            available_agents: agents,
2266            weights,
2267            llm_router: std::sync::Arc::new(LlmRouter::new()),
2268            ollama_router: None,
2269            rr_index: 0,
2270            last_agent: None,
2271            review_rr_index: 0,
2272            router_pool: vec![],
2273            pool_index: 0,
2274        };
2275
2276        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2277        let store = test_store().await;
2278
2279        // Should use label override, not LLM
2280        let result = router.route(&task, &store, "test/repo").await.unwrap();
2281        assert_eq!(result.agent, "claude");
2282        assert!(result.reason.contains("label"));
2283    }
2284
2285    #[serial(cooldown_state)]
2286    #[test]
2287    fn check_routing_sanity_warnings() {
2288        // Backend task routed to claude should warn
2289        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2290        let profile = AgentProfile {
2291            role: "general".to_string(),
2292            skills: vec!["api".to_string()],
2293            tools: vec![],
2294            constraints: vec![],
2295        };
2296        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2297        assert!(warning.is_some());
2298        assert!(warning.unwrap().contains("backend"));
2299
2300        // Docs task routed to codex should warn
2301        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2302        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2303        assert!(warning.is_some());
2304        assert!(warning.unwrap().contains("docs"));
2305
2306        // Normal routing should not warn
2307        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2308        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2309        assert!(warning.is_none());
2310    }
2311
2312    #[serial(cooldown_state)]
2313    #[tokio::test]
2314    async fn router_reload_preserves_structure() {
2315        crate::engine::cooldown::reset_global_state().await;
2316        let config = RouterConfig::default();
2317        let agents = vec!["claude".to_string()];
2318        let mut weights = AgentWeights::default();
2319        weights.ensure_agents(&agents);
2320        let mut router = Router {
2321            config,
2322            available_agents: agents,
2323            weights,
2324            llm_router: std::sync::Arc::new(LlmRouter::new()),
2325            ollama_router: None,
2326            rr_index: 0,
2327            last_agent: None,
2328            review_rr_index: 0,
2329            router_pool: vec![],
2330            pool_index: 0,
2331        };
2332
2333        // Reload — should re-read config and remain valid
2334        router.reload_async().await;
2335
2336        // After reload, mode should be a valid value (llm, local, or round_robin)
2337        assert!(
2338            router.config.mode == "llm"
2339                || router.config.mode == "round_robin"
2340                || router.config.mode == "local",
2341            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2342            router.config.mode
2343        );
2344        // Fallback executor should always be set
2345        assert_ne!(router.config.fallback_executor, "");
2346        // Tools should always be populated
2347        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2348    }
2349
2350    // --- Weighted round-robin tests ---
2351
2352    #[serial(cooldown_state)]
2353    #[test]
2354    fn rate_limit_state_defaults_to_full_weight() {
2355        let state = RateLimitState::default();
2356        assert_eq!(state.weight, DEFAULT_WEIGHT);
2357        assert_eq!(state.consecutive_hits, 0);
2358        assert!(state.last_limited_at.is_none());
2359        assert!(!state.is_limited());
2360    }
2361
2362    #[serial(cooldown_state)]
2363    #[test]
2364    fn rate_limit_state_decays_on_hit() {
2365        let mut state = RateLimitState::default();
2366        state.record_rate_limit();
2367
2368        assert!(state.weight < DEFAULT_WEIGHT);
2369        assert_eq!(state.consecutive_hits, 1);
2370        assert!(state.last_limited_at.is_some());
2371        assert!(state.is_limited());
2372
2373        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2374        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2375        assert!((state.weight - expected).abs() < 1e-10);
2376    }
2377
2378    #[serial(cooldown_state)]
2379    #[test]
2380    fn rate_limit_state_never_drops_below_min() {
2381        let mut state = RateLimitState::default();
2382
2383        // Hit many times
2384        for _ in 0..100 {
2385            state.record_rate_limit();
2386        }
2387
2388        assert!(state.weight >= MIN_WEIGHT);
2389        assert_eq!(state.consecutive_hits, 100);
2390    }
2391
2392    #[serial(cooldown_state)]
2393    #[test]
2394    fn rate_limit_state_recovers_on_success() {
2395        let mut state = RateLimitState::default();
2396
2397        // Decay first
2398        state.record_rate_limit();
2399        state.record_rate_limit();
2400        let after_decay = state.weight;
2401
2402        // Record success
2403        state.record_success();
2404        assert!(state.weight > after_decay);
2405        assert_eq!(state.consecutive_hits, 0);
2406    }
2407
2408    #[serial(cooldown_state)]
2409    #[test]
2410    fn rate_limit_state_success_caps_at_default() {
2411        let mut state = RateLimitState::default();
2412
2413        // Already at full weight, success shouldn't exceed it
2414        state.record_success();
2415        assert_eq!(state.weight, DEFAULT_WEIGHT);
2416    }
2417
2418    #[serial(cooldown_state)]
2419    #[test]
2420    fn agent_weights_ensure_agents() {
2421        let mut weights = AgentWeights::default();
2422        let agents = vec!["claude".to_string(), "codex".to_string()];
2423        weights.ensure_agents(&agents);
2424
2425        assert_eq!(weights.states.len(), 2);
2426        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2427        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2428    }
2429
2430    #[serial(cooldown_state)]
2431    #[test]
2432    fn agent_weights_ensure_agents_preserves_existing() {
2433        let mut weights = AgentWeights::default();
2434        let agents = vec!["claude".to_string(), "codex".to_string()];
2435        weights.ensure_agents(&agents);
2436
2437        // Reduce claude's weight
2438        weights.record_rate_limit("claude");
2439        let claude_weight = weights.get_weight("claude");
2440        assert!(claude_weight < DEFAULT_WEIGHT);
2441
2442        // Ensure agents again — claude's weight should be preserved
2443        let agents2 = vec![
2444            "claude".to_string(),
2445            "codex".to_string(),
2446            "opencode".to_string(),
2447        ];
2448        weights.ensure_agents(&agents2);
2449
2450        assert_eq!(weights.get_weight("claude"), claude_weight);
2451        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2452    }
2453
2454    #[test]
2455    fn default_missing_weight_uses_minimum_of_configured() {
2456        use super::weights::default_missing_weight;
2457
2458        let mut configured = std::collections::HashMap::new();
2459        configured.insert("claude".to_string(), 0.2);
2460        configured.insert("codex".to_string(), 0.2);
2461        configured.insert("opencode".to_string(), 0.2);
2462        configured.insert("kimi".to_string(), 0.2);
2463
2464        // minimax omitted from router.weights — must not silently outrank
2465        // every explicitly configured agent (issue #3596).
2466        assert_eq!(default_missing_weight(&configured), 0.2);
2467    }
2468
2469    #[test]
2470    fn default_missing_weight_falls_back_when_none_configured() {
2471        use super::weights::default_missing_weight;
2472
2473        let configured = std::collections::HashMap::new();
2474        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2475    }
2476
2477    #[serial(cooldown_state)]
2478    #[test]
2479    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2480        let mut weights = AgentWeights::default();
2481        let mut configured = std::collections::HashMap::new();
2482        configured.insert("claude".to_string(), 0.2);
2483        configured.insert("codex".to_string(), 0.2);
2484        configured.insert("opencode".to_string(), 0.2);
2485        configured.insert("kimi".to_string(), 0.2);
2486
2487        let agents = vec![
2488            "claude".to_string(),
2489            "codex".to_string(),
2490            "opencode".to_string(),
2491            "kimi".to_string(),
2492            "minimax".to_string(),
2493        ];
2494        weights.ensure_agents_with_weights(&agents, &configured);
2495
2496        // minimax was omitted from router.weights but must not get the old
2497        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2498        assert_eq!(weights.get_weight("minimax"), 0.2);
2499        assert_eq!(weights.get_weight("claude"), 0.2);
2500    }
2501
2502    #[serial(cooldown_state)]
2503    #[test]
2504    fn agent_weights_record_rate_limit() {
2505        let mut weights = AgentWeights::default();
2506        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2507
2508        weights.record_rate_limit("claude");
2509
2510        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2511        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2512    }
2513
2514    #[serial(cooldown_state)]
2515    #[test]
2516    fn agent_weights_record_success() {
2517        let mut weights = AgentWeights::default();
2518        weights.ensure_agents(&["claude".to_string()]);
2519
2520        weights.record_rate_limit("claude");
2521        let after_limit = weights.get_weight("claude");
2522
2523        weights.record_success("claude");
2524        assert!(weights.get_weight("claude") > after_limit);
2525    }
2526
2527    #[serial(cooldown_state)]
2528    #[test]
2529    fn agent_weights_snapshot() {
2530        let mut weights = AgentWeights::default();
2531        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2532        weights.record_rate_limit("claude");
2533
2534        let snap = weights.snapshot();
2535        assert_eq!(snap.len(), 2);
2536
2537        // Snapshot is sorted alphabetically
2538        assert_eq!(snap[0].0, "claude");
2539        assert!(snap[0].1 < DEFAULT_WEIGHT);
2540        assert_eq!(snap[0].2, 1); // 1 hit
2541
2542        assert_eq!(snap[1].0, "codex");
2543        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2544        assert_eq!(snap[1].2, 0); // 0 hits
2545    }
2546
2547    #[serial(cooldown_state)]
2548    #[test]
2549    fn agent_weights_weighted_select_favors_higher_weight() {
2550        let mut weights = AgentWeights::default();
2551        let agents = vec!["claude".to_string(), "codex".to_string()];
2552        weights.ensure_agents(&agents);
2553
2554        // Heavily penalize claude
2555        for _ in 0..10 {
2556            weights.record_rate_limit("claude");
2557        }
2558
2559        // Run many selections and count
2560        let mut claude_count = 0;
2561        let mut codex_count = 0;
2562        for _ in 0..100 {
2563            match weights.weighted_select(&agents, "task-1") {
2564                Some(ref a) if a == "claude" => claude_count += 1,
2565                Some(ref a) if a == "codex" => codex_count += 1,
2566                _ => {}
2567            }
2568        }
2569
2570        // Codex should get significantly more selections
2571        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2572        assert!(
2573            codex_count > claude_count,
2574            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2575        );
2576    }
2577
2578    #[serial(cooldown_state)]
2579    #[test]
2580    fn agent_weights_weighted_select_empty_returns_none() {
2581        let weights = AgentWeights::default();
2582        assert!(weights.weighted_select(&[], "task-1").is_none());
2583    }
2584
2585    #[serial(cooldown_state)]
2586    #[test]
2587    fn agent_weights_batch_routing_produces_varied_selections() {
2588        // Regression test: when routing a batch of tasks in the same tick,
2589        // different task IDs should not all select the same agent.
2590        let mut weights = AgentWeights::default();
2591        let agents = vec!["claude".to_string(), "codex".to_string()];
2592        weights.ensure_agents(&agents);
2593
2594        // Simulate batch routing: 20 different task IDs called in rapid succession
2595        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2596        let selections: Vec<String> = task_ids
2597            .iter()
2598            .filter_map(|id| weights.weighted_select(&agents, id))
2599            .collect();
2600
2601        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2602        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2603        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2604        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2605        assert!(
2606            claude_count > 0 && codex_count > 0,
2607            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2608        );
2609    }
2610
2611    #[serial(cooldown_state)]
2612    #[test]
2613    fn agent_weights_weighted_select_single_agent() {
2614        let mut weights = AgentWeights::default();
2615        let agents = vec!["claude".to_string()];
2616        weights.ensure_agents(&agents);
2617
2618        let selected = weights.weighted_select(&agents, "task-1");
2619        assert_eq!(selected, Some("claude".to_string()));
2620    }
2621
2622    #[serial(cooldown_state)]
2623    #[test]
2624    fn agent_weights_tick_recovery_restores_weight() {
2625        let mut weights = AgentWeights::default();
2626        weights.ensure_agents(&["claude".to_string()]);
2627
2628        // Set up a rate limit that looks like it happened long ago
2629        let state = weights.states.get_mut("claude").unwrap();
2630        state.weight = 0.5;
2631        state.last_limited_at = Some(
2632            Instant::now()
2633                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2634                .expect("the clock is past the recovery delay"),
2635        );
2636        state.consecutive_hits = 2;
2637
2638        let before = weights.get_weight("claude");
2639        weights.tick_recovery();
2640        let after = weights.get_weight("claude");
2641
2642        assert!(after > before, "weight should increase after recovery tick");
2643    }
2644
2645    #[serial(cooldown_state)]
2646    #[test]
2647    fn agent_weights_tick_recovery_clears_on_full_restore() {
2648        let mut weights = AgentWeights::default();
2649        weights.ensure_agents(&["claude".to_string()]);
2650
2651        // Set weight just below full with an old limit
2652        let state = weights.states.get_mut("claude").unwrap();
2653        state.weight = DEFAULT_WEIGHT - 0.01;
2654        state.last_limited_at = Some(
2655            Instant::now()
2656                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2657                .expect("the clock is past the recovery delay"),
2658        );
2659        state.consecutive_hits = 1;
2660
2661        weights.tick_recovery();
2662
2663        let state = weights.states.get("claude").unwrap();
2664        assert_eq!(state.weight, DEFAULT_WEIGHT);
2665        assert!(state.last_limited_at.is_none());
2666        assert_eq!(state.consecutive_hits, 0);
2667    }
2668
2669    #[serial(cooldown_state)]
2670    #[tokio::test]
2671    async fn route_weighted_round_robin_basic() {
2672        crate::engine::cooldown::reset_global_state().await;
2673        let mut config = RouterConfig {
2674            weighted_round_robin: true,
2675            ..Default::default()
2676        };
2677        // Provide a concrete model for codex so label override can be used.
2678        config
2679            .model_map
2680            .entry("medium".to_string())
2681            .or_default()
2682            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2683
2684        let agents = vec!["claude".to_string(), "codex".to_string()];
2685        let mut weights = AgentWeights::default();
2686        weights.ensure_agents(&agents);
2687        let mut router = Router {
2688            config,
2689            available_agents: agents,
2690            weights,
2691            llm_router: std::sync::Arc::new(LlmRouter::new()),
2692            ollama_router: None,
2693            rr_index: 0,
2694            last_agent: None,
2695            review_rr_index: 0,
2696            router_pool: vec![],
2697            pool_index: 0,
2698        };
2699
2700        let task = create_test_task("1", "Test task", vec![]);
2701        let store = test_store().await;
2702        let result = router.route(&task, &store, "test/repo").await.unwrap();
2703
2704        // Should use weighted_round_robin
2705        assert!(result.reason.contains("weighted_round_robin"));
2706        assert!(
2707            result.agent == "claude" || result.agent == "codex",
2708            "agent should be claude or codex, got '{}'",
2709            result.agent
2710        );
2711    }
2712
2713    #[serial(cooldown_state)]
2714    #[tokio::test]
2715    async fn route_weighted_round_robin_respects_label_override() {
2716        crate::engine::cooldown::reset_global_state().await;
2717        let mut config = RouterConfig {
2718            weighted_round_robin: true,
2719            ..Default::default()
2720        };
2721        // Provide a concrete model for codex so label override can be used.
2722        config
2723            .model_map
2724            .entry("medium".to_string())
2725            .or_default()
2726            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2727
2728        let agents = vec!["claude".to_string(), "codex".to_string()];
2729        let mut weights = AgentWeights::default();
2730        weights.ensure_agents(&agents);
2731        let mut router = Router {
2732            config,
2733            available_agents: agents,
2734            weights,
2735            llm_router: std::sync::Arc::new(LlmRouter::new()),
2736            ollama_router: None,
2737            rr_index: 0,
2738            last_agent: None,
2739            review_rr_index: 0,
2740            router_pool: vec![],
2741            pool_index: 0,
2742        };
2743
2744        // Label override should take precedence over weighted routing
2745        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2746        let store = test_store().await;
2747        let result = router.route(&task, &store, "test/repo").await.unwrap();
2748        assert_eq!(result.agent, "codex");
2749        assert!(result.reason.contains("label"));
2750    }
2751
2752    #[serial(cooldown_state)]
2753    #[test]
2754    fn router_config_weighted_round_robin_default_false() {
2755        let config = RouterConfig::default();
2756        assert!(!config.weighted_round_robin);
2757    }
2758
2759    #[serial(cooldown_state)]
2760    #[test]
2761    fn router_record_and_recover_weights() {
2762        let config = RouterConfig {
2763            weighted_round_robin: true,
2764            ..Default::default()
2765        };
2766        let mut router = Router::new(config);
2767        // Override discovered agents for test
2768        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2769        router.weights.ensure_agents(&router.available_agents);
2770
2771        // Record rate limit
2772        router.record_rate_limit("claude");
2773        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2774        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2775
2776        // Record success for claude
2777        router.record_success("claude");
2778        let after_success = router.weights.get_weight("claude");
2779        assert!(after_success > MIN_WEIGHT);
2780    }
2781
2782    #[serial(cooldown_state)]
2783    #[test]
2784    fn review_rr_index_advances() {
2785        let mut config = RouterConfig::default();
2786        // Set up model_map so has_available_model_for_complexity returns true for test agents
2787        for agent in &["test_a", "test_b", "test_c"] {
2788            config
2789                .model_map
2790                .entry("medium".to_string())
2791                .or_default()
2792                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2793        }
2794        let agents = vec![
2795            "test_a".to_string(),
2796            "test_b".to_string(),
2797            "test_c".to_string(),
2798        ];
2799        let mut weights = AgentWeights::default();
2800        weights.ensure_agents(&agents);
2801        let mut router = Router {
2802            config,
2803            available_agents: agents,
2804            weights,
2805            llm_router: std::sync::Arc::new(LlmRouter::new()),
2806            ollama_router: None,
2807            rr_index: 0,
2808            last_agent: None,
2809            review_rr_index: 0,
2810            router_pool: vec![],
2811            pool_index: 0,
2812        };
2813
2814        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2815        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2816        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2817
2818        // All three agents should appear (order depends on start index)
2819        let mut seen = vec![a1, a2, a3];
2820        seen.sort();
2821        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2822    }
2823
2824    #[serial(cooldown_state)]
2825    #[test]
2826    fn review_rr_fallback_with_cooled_agent() {
2827        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2828        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2829        // candidates.get(idx) to return None even though candidates is non-empty.
2830        let mut config = RouterConfig::default();
2831        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2832        for agent in &["test_a", "test_b", "test_c"] {
2833            config
2834                .model_map
2835                .entry("medium".to_string())
2836                .or_default()
2837                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2838        }
2839        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2840        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2841        let mut weights = AgentWeights::default();
2842        weights.ensure_agents(&agents);
2843        let mut router = Router {
2844            config,
2845            available_agents: agents,
2846            weights,
2847            llm_router: std::sync::Arc::new(LlmRouter::new()),
2848            ollama_router: None,
2849            rr_index: 0,
2850            last_agent: None,
2851            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2852            review_rr_index: 2,
2853            router_pool: vec![],
2854            pool_index: 0,
2855        };
2856
2857        // All candidates excluded — should fall back to first candidate, not None
2858        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2859        assert!(
2860            result.is_some(),
2861            "next_round_robin_agent must not return None when candidates is non-empty"
2862        );
2863    }
2864
2865    #[serial(cooldown_state)]
2866    #[test]
2867    fn review_rr_index_uses_candidates_len_not_available_len() {
2868        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2869        // not available_agents.len(). When an agent has no model for the complexity,
2870        // candidates.len() < available_agents.len() and the old code produced a drifting
2871        // index that caused non-uniform distribution.
2872        let mut config = RouterConfig::default();
2873        // Only register models for test_a and test_b — test_c has no model, so it is
2874        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2875        for agent in &["test_a", "test_b"] {
2876            config
2877                .model_map
2878                .entry("medium".to_string())
2879                .or_default()
2880                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2881        }
2882        let agents = vec![
2883            "test_a".to_string(),
2884            "test_b".to_string(),
2885            "test_c".to_string(), // available but no model → filtered out of candidates
2886        ];
2887        let mut weights = AgentWeights::default();
2888        weights.ensure_agents(&agents);
2889        let mut router = Router {
2890            config,
2891            available_agents: agents,
2892            weights,
2893            llm_router: std::sync::Arc::new(LlmRouter::new()),
2894            ollama_router: None,
2895            rr_index: 0,
2896            last_agent: None,
2897            review_rr_index: 0,
2898            router_pool: vec![],
2899            pool_index: 0,
2900        };
2901
2902        // With the bug: index advances modulo 3 (available_agents.len()), so the
2903        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2904        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2905        // looks uniform by accident, but adding a 4th available agent breaks it.
2906        //
2907        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2908        // cycle. Verify this by checking review_rr_index after each call.
2909
2910        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2911        assert_eq!(
2912            router.review_rr_index, 1,
2913            "index should be 1 after first call"
2914        );
2915
2916        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2917        assert_eq!(
2918            router.review_rr_index, 0,
2919            "index should wrap to 0 after second call"
2920        );
2921
2922        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2923        assert_eq!(
2924            router.review_rr_index, 1,
2925            "index should be 1 after third call"
2926        );
2927
2928        // Agents must alternate uniformly
2929        assert_ne!(r1, r2, "consecutive calls should select different agents");
2930        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2931    }
2932
2933    #[serial(cooldown_state)]
2934    #[tokio::test]
2935    async fn last_agent_tracks_routing() {
2936        crate::engine::cooldown::reset_global_state().await;
2937        let mut config = RouterConfig {
2938            mode: "round_robin".to_string(),
2939            ..Default::default()
2940        };
2941        config
2942            .model_map
2943            .entry("medium".to_string())
2944            .or_default()
2945            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
2946        config
2947            .model_map
2948            .entry("medium".to_string())
2949            .or_default()
2950            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
2951        let agents = vec!["test_x".to_string(), "test_y".to_string()];
2952        let mut weights = AgentWeights::default();
2953        weights.ensure_agents(&agents);
2954        let mut router = Router {
2955            config,
2956            available_agents: agents,
2957            weights,
2958            llm_router: std::sync::Arc::new(LlmRouter::new()),
2959            ollama_router: None,
2960            rr_index: 0,
2961            last_agent: None,
2962            review_rr_index: 0,
2963            router_pool: vec![],
2964            pool_index: 0,
2965        };
2966
2967        assert!(router.last_agent.is_none());
2968
2969        let task = create_test_task("1", "Test routing", vec![]);
2970        let store = test_store().await;
2971        let result = router.route(&task, &store, "test/repo").await.unwrap();
2972
2973        assert!(router.last_agent.is_some());
2974        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
2975    }
2976
2977    // ── pool config ───────────────────────────────────────────────────────────
2978
2979    #[serial(cooldown_state)]
2980    #[test]
2981    fn parse_pool_entry_splits_on_first_colon() {
2982        let (agent, model) = super::parse_pool_entry("claude:haiku");
2983        assert_eq!(agent, "claude");
2984        assert_eq!(model, "haiku");
2985    }
2986
2987    #[serial(cooldown_state)]
2988    #[test]
2989    fn parse_pool_entry_model_with_slash() {
2990        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
2991        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
2992        assert_eq!(agent, "opencode");
2993        assert_eq!(model, "github-copilot/gpt-5-mini");
2994    }
2995
2996    #[serial(cooldown_state)]
2997    #[test]
2998    fn parse_pool_entry_no_colon() {
2999        let (agent, model) = super::parse_pool_entry("claude");
3000        assert_eq!(agent, "claude");
3001        assert_eq!(model, "");
3002    }
3003
3004    #[serial(cooldown_state)]
3005    #[test]
3006    fn effective_pool_uses_configured_pool() {
3007        let config = RouterConfig {
3008            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3009            ..RouterConfig::default()
3010        };
3011        let pool = config.effective_pool();
3012        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3013    }
3014
3015    #[serial(cooldown_state)]
3016    #[test]
3017    fn effective_pool_falls_back_to_router_agent_model() {
3018        let config = RouterConfig::default();
3019        // Default has no pool → derives from router_agent:router_model
3020        let pool = config.effective_pool();
3021        assert_eq!(pool.len(), 1);
3022        assert!(pool[0].starts_with(&config.router_agent));
3023        assert!(pool[0].contains(&config.router_model));
3024    }
3025
3026    #[serial(cooldown_state)]
3027    #[test]
3028    fn effective_fallback_uses_configured_fallback() {
3029        let config = RouterConfig {
3030            fallback: "claude:haiku".to_string(),
3031            ..RouterConfig::default()
3032        };
3033        assert_eq!(config.effective_fallback(), "claude:haiku");
3034    }
3035
3036    #[serial(cooldown_state)]
3037    #[test]
3038    fn effective_fallback_derives_from_router_agent_model() {
3039        let config = RouterConfig::default();
3040        let fallback = config.effective_fallback();
3041        assert!(fallback.starts_with(&config.router_agent));
3042        assert!(fallback.contains(&config.router_model));
3043    }
3044
3045    // ── pool expansion ────────────────────────────────────────────────────────
3046
3047    #[serial(cooldown_state)]
3048    #[test]
3049    fn expand_pool_basic_entries() {
3050        let config = RouterConfig {
3051            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3052            ..RouterConfig::default()
3053        };
3054        let expanded = Router::expand_pool(&config);
3055        assert_eq!(expanded.len(), 2);
3056        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3057        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3058    }
3059
3060    #[serial(cooldown_state)]
3061    #[test]
3062    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3063        let config = RouterConfig {
3064            pool: vec![],
3065            router_agent: "claude".to_string(),
3066            router_model: "haiku".to_string(),
3067            ..RouterConfig::default()
3068        };
3069        let expanded = Router::expand_pool(&config);
3070        assert_eq!(expanded.len(), 1);
3071        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3072    }
3073
3074    #[serial(cooldown_state)]
3075    #[test]
3076    fn expand_pool_opencode_free_skipped_when_not_installed() {
3077        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3078        // to nothing and the pool falls back to effective_fallback
3079        let config = RouterConfig {
3080            pool: vec!["opencode:free".to_string()],
3081            fallback: "claude:haiku".to_string(),
3082            ..RouterConfig::default()
3083        };
3084        let expanded = Router::expand_pool(&config);
3085        assert_eq!(expanded.len(), 1);
3086        assert_eq!(expanded[0].0, "claude");
3087        assert_eq!(expanded[0].1, "haiku");
3088    }
3089
3090    #[serial(cooldown_state)]
3091    #[test]
3092    fn expand_pool_single_entry_from_default() {
3093        let config = RouterConfig::default();
3094        let expanded = Router::expand_pool(&config);
3095        assert!(
3096            !expanded.is_empty(),
3097            "pool must never be empty after expansion"
3098        );
3099    }
3100
3101    // ── pool round-robin index ────────────────────────────────────────────────
3102
3103    #[serial(cooldown_state)]
3104    #[test]
3105    fn pool_index_initializes_to_zero() {
3106        let config = RouterConfig::default();
3107        let router = Router::new(config);
3108        assert_eq!(router.pool_index, 0);
3109    }
3110
3111    #[serial(cooldown_state)]
3112    #[test]
3113    fn router_pool_non_empty_after_construction() {
3114        let config = RouterConfig::default();
3115        let router = Router::new(config);
3116        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3117    }
3118
3119    /**
3120     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3121     * `opencode serve --service` child behind, so a suite run leaked one server per test
3122     * process. The stub on PATH records any exec, so this fails if discovery comes back
3123     **/
3124    #[cfg(unix)]
3125    #[serial(cooldown_state)]
3126    #[test]
3127    fn building_a_router_never_runs_opencode() {
3128        use std::os::unix::fs::PermissionsExt;
3129
3130        struct PathRestore(Option<std::ffi::OsString>);
3131        impl Drop for PathRestore {
3132            fn drop(&mut self) {
3133                match &self.0 {
3134                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3135                    Some(p) => unsafe { std::env::set_var("PATH", p) },
3136                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3137                    None => unsafe { std::env::remove_var("PATH") },
3138                }
3139                crate::cmd_cache::reset_for_test();
3140            }
3141        }
3142
3143        let dir = tempfile::tempdir().unwrap();
3144        let marker = dir.path().join("spawned");
3145        let bin_dir = dir.path().join("bin");
3146        std::fs::create_dir(&bin_dir).unwrap();
3147        let stub = bin_dir.join("opencode");
3148        std::fs::write(
3149            &stub,
3150            format!(
3151                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3152                marker.display()
3153            ),
3154        )
3155        .unwrap();
3156        std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap();
3157
3158        let prev_path = std::env::var_os("PATH");
3159        let mut new_path = bin_dir.as_os_str().to_os_string();
3160        new_path.push(":");
3161        if let Some(ref p) = prev_path {
3162            new_path.push(p);
3163        }
3164        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3165        unsafe { std::env::set_var("PATH", &new_path) };
3166        crate::cmd_cache::reset_for_test();
3167        let _restore = PathRestore(prev_path);
3168
3169        // A config that names opencode everywhere discovery reads from
3170        let mut config = RouterConfig {
3171            pool: vec!["opencode:free".to_string()],
3172            fallback: "opencode:free".to_string(),
3173            ..RouterConfig::default()
3174        };
3175        config
3176            .model_map
3177            .entry("simple".to_string())
3178            .or_default()
3179            .insert(
3180                "opencode".to_string(),
3181                vec!["canary-model-free".to_string()],
3182            );
3183
3184        let _router = Router::new(config.clone());
3185        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3186        let _pool = Router::expand_pool(&config);
3187        // Discovery refreshes on a background thread, give it room to exec
3188        std::thread::sleep(Duration::from_millis(300));
3189
3190        assert!(
3191            !marker.exists(),
3192            "building a router ran opencode: {}",
3193            std::fs::read_to_string(&marker).unwrap_or_default().trim()
3194        );
3195        assert!(
3196            opencode_children().is_empty(),
3197            "building a router left opencode processes: {:?}",
3198            opencode_children()
3199        );
3200    }
3201
3202    /// Commands of this process's direct children that mention opencode
3203    #[cfg(unix)]
3204    fn opencode_children() -> Vec<String> {
3205        let me = std::process::id().to_string();
3206        let output = std::process::Command::new("ps")
3207            .args(["-axo", "ppid=,command="])
3208            .output()
3209            .expect("ps");
3210        String::from_utf8_lossy(&output.stdout)
3211            .lines()
3212            .filter_map(|line| {
3213                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3214                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3215            })
3216            .collect()
3217    }
3218
3219    #[serial(cooldown_state)]
3220    #[tokio::test]
3221    async fn reload_resets_pool_index() {
3222        crate::engine::cooldown::reset_global_state().await;
3223        let config = RouterConfig::default();
3224        let mut router = Router::new(config);
3225        router.pool_index = 5;
3226        router.reload_async().await;
3227        assert_eq!(router.pool_index, 0);
3228    }
3229
3230    #[serial(cooldown_state)]
3231    #[test]
3232    fn pool_index_advances_after_failed_attempt() {
3233        let mut router = Router::new(RouterConfig::default());
3234        router.router_pool = vec![
3235            ("claude".to_string(), "haiku".to_string()),
3236            ("codex".to_string(), "gpt-5.2".to_string()),
3237            (
3238                "opencode".to_string(),
3239                "github-copilot/gpt-5-mini".to_string(),
3240            ),
3241        ];
3242        router.pool_index = 0;
3243
3244        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3245
3246        assert_eq!(router.pool_index, 1);
3247    }
3248
3249    #[serial(cooldown_state)]
3250    #[test]
3251    fn pool_index_wraps_after_last_attempt() {
3252        let mut router = Router::new(RouterConfig::default());
3253        router.router_pool = vec![
3254            ("claude".to_string(), "haiku".to_string()),
3255            ("codex".to_string(), "gpt-5.2".to_string()),
3256            (
3257                "opencode".to_string(),
3258                "github-copilot/gpt-5-mini".to_string(),
3259            ),
3260        ];
3261        router.pool_index = 2;
3262
3263        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3264
3265        assert_eq!(router.pool_index, 0);
3266    }
3267
3268    #[serial(cooldown_state)]
3269    #[test]
3270    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3271        let router = Router::new(RouterConfig::default());
3272        assert!(router.earliest_pool_cooldown().is_none());
3273    }
3274
3275    #[serial(cooldown_state)]
3276    #[tokio::test]
3277    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3278        crate::engine::cooldown::reset_global_state().await;
3279        // Use explicit pool with predictable model names to avoid opencode discovery.
3280        let config = RouterConfig {
3281            pool: vec![
3282                "claude:haiku-4-5-20251001".to_string(),
3283                "opencode:haiku-4-5-20251001".to_string(),
3284            ],
3285            ..Default::default()
3286        };
3287        let router = Router::new(config);
3288        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3289        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3290
3291        let earliest = router.earliest_pool_cooldown();
3292        assert!(earliest.is_some());
3293        let now = chrono::Utc::now().timestamp();
3294        let remaining = earliest.unwrap() - now;
3295        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3296        assert!((295..=300).contains(&remaining));
3297    }
3298
3299    #[serial(cooldown_state)]
3300    #[tokio::test]
3301    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3302        crate::engine::cooldown::reset_global_state().await;
3303        // Use explicit pool so we can predict what entries exist.
3304        let config = RouterConfig {
3305            pool: vec![
3306                "claude:pool-model-a".to_string(),
3307                "opencode:pool-model-b".to_string(),
3308            ],
3309            ..Default::default()
3310        };
3311        let router = Router::new(config);
3312        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3313        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3314        // Set cooldown on a pool entry (opencode:pool-model-b)
3315        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3316
3317        let earliest = router.earliest_pool_cooldown();
3318        assert!(earliest.is_some());
3319        let now = chrono::Utc::now().timestamp();
3320        let remaining = earliest.unwrap() - now;
3321        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3322        assert!((295..=300).contains(&remaining));
3323    }
3324
3325    // ---- Pre-emptive health check: degraded agent exclusion ----
3326
3327    #[serial(cooldown_state)]
3328    #[test]
3329    fn degraded_agent_excluded_from_routing() {
3330        let mut config = RouterConfig::default();
3331        config
3332            .model_map
3333            .entry("medium".to_string())
3334            .or_default()
3335            .insert(
3336                "test_degraded_routing".to_string(),
3337                vec!["model-a".to_string()],
3338            );
3339
3340        let mut router = Router::new(config);
3341        router.available_agents = vec!["test_degraded_routing".to_string()];
3342
3343        // Agent is routable before degradation
3344        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3345
3346        // Mark as degraded
3347        mark_agent_degraded("test_degraded_routing");
3348
3349        // Now excluded from routing
3350        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3351
3352        // available_agents_for_complexity returns empty
3353        assert!(
3354            router.available_agents_for_complexity("medium").is_empty(),
3355            "degraded agent should be excluded from available agents"
3356        );
3357
3358        // healthy_agent_count reflects exclusion
3359        assert_eq!(router.healthy_agent_count("medium"), 0);
3360
3361        // Cleanup
3362        clear_agent_degraded("test_degraded_routing");
3363    }
3364
3365    #[serial(cooldown_state)]
3366    #[test]
3367    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3368        let mut config = RouterConfig::default();
3369        for agent in &["agent_healthy", "agent_degraded"] {
3370            config
3371                .model_map
3372                .entry("medium".to_string())
3373                .or_default()
3374                .insert(agent.to_string(), vec!["model-x".to_string()]);
3375        }
3376
3377        let mut router = Router::new(config);
3378        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3379
3380        mark_agent_degraded("agent_degraded");
3381
3382        let candidates = router.available_agents_for_complexity("medium");
3383        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3384        assert_eq!(router.healthy_agent_count("medium"), 1);
3385
3386        // Cleanup
3387        clear_agent_degraded("agent_degraded");
3388    }
3389
3390    #[serial(cooldown_state)]
3391    #[tokio::test]
3392    async fn refresh_health_marks_degraded_from_rate_limits() {
3393        crate::engine::cooldown::reset_global_state().await;
3394        let store = test_store().await;
3395        let agent = "test_refresh_health_agent";
3396
3397        let mut config = RouterConfig::default();
3398        config
3399            .model_map
3400            .entry("medium".to_string())
3401            .or_default()
3402            .insert(agent.to_string(), vec!["model-z".to_string()]);
3403
3404        let mut router = Router::new(config);
3405        router.available_agents = vec![agent.to_string()];
3406
3407        // Insert rate limit events exceeding default threshold (3)
3408        for _ in 0..4 {
3409            store.record_rate_limit(agent, "rate", None).await.unwrap();
3410        }
3411
3412        router.refresh_health(&store).await;
3413
3414        assert!(
3415            is_agent_degraded(agent),
3416            "agent should be degraded after exceeding rate limit threshold"
3417        );
3418        assert!(
3419            !router.agent_is_routable(agent, "medium"),
3420            "degraded agent should not be routable"
3421        );
3422
3423        // Cleanup
3424        clear_agent_degraded(agent);
3425    }
3426
3427    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3428
3429    #[serial(cooldown_state)]
3430    #[tokio::test]
3431    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3432        crate::engine::cooldown::reset_global_state().await;
3433        let mut config = RouterConfig::default();
3434        // Agent's only model, shared across every complexity tier it resolves to.
3435        for tier in ["simple", "medium", "complex", "review"] {
3436            config
3437                .model_map
3438                .entry(tier.to_string())
3439                .or_default()
3440                .insert(
3441                    "test_all_cooled".to_string(),
3442                    vec!["only-model".to_string()],
3443                );
3444        }
3445
3446        let mut router = Router::new(config);
3447        router.available_agents = vec!["test_all_cooled".to_string()];
3448
3449        assert!(router.agent_has_any_available_model("test_all_cooled"));
3450
3451        record_model_failure("test_all_cooled", "only-model").await;
3452        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3453
3454        assert!(
3455            !router.agent_has_any_available_model("test_all_cooled"),
3456            "agent with its only model cooled across every tier should have no available model"
3457        );
3458    }
3459
3460    #[serial(cooldown_state)]
3461    #[tokio::test]
3462    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3463        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3464        // agent that has no agent-level cooldown but whose only resolvable model (across
3465        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3466        crate::engine::cooldown::reset_global_state().await;
3467        let mut config = RouterConfig::default();
3468        for tier in ["simple", "medium", "complex", "review"] {
3469            config
3470                .model_map
3471                .entry(tier.to_string())
3472                .or_default()
3473                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3474            config
3475                .model_map
3476                .get_mut(tier)
3477                .unwrap()
3478                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3479        }
3480
3481        let mut router = Router::new(config);
3482        router.available_agents = vec![
3483            "agent_cooled_model".to_string(),
3484            "agent_healthy".to_string(),
3485        ];
3486
3487        record_model_failure("agent_cooled_model", "opus").await;
3488        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3489        // No agent-level cooldown — the old agent-only filter would still offer it.
3490        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3491
3492        let uncooled_agents: Vec<String> = router
3493            .available_agents
3494            .iter()
3495            .filter(|a| {
3496                !is_agent_in_cooldown(a)
3497                    && !is_agent_degraded(a)
3498                    && router.agent_has_available_model_for_coding_tiers(a)
3499            })
3500            .cloned()
3501            .collect();
3502
3503        assert_eq!(
3504            uncooled_agents,
3505            vec!["agent_healthy".to_string()],
3506            "agent whose only model is cooled must be excluded from the LLM candidate list"
3507        );
3508    }
3509
3510    #[serial(cooldown_state)]
3511    #[tokio::test]
3512    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3513        // Regression test for #3521: an agent whose only healthy tier is `simple`
3514        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3515        // excluded from the LLM classification candidate list, otherwise the
3516        // classifier can pick it for a `medium`/`complex` task it can't serve,
3517        // wasting the classification call. Mirrors the real-world shape where
3518        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3519        crate::engine::cooldown::reset_global_state().await;
3520        let mut config = RouterConfig::default();
3521        config
3522            .model_map
3523            .entry("simple".to_string())
3524            .or_default()
3525            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3526        config
3527            .model_map
3528            .entry("medium".to_string())
3529            .or_default()
3530            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3531        config
3532            .model_map
3533            .entry("complex".to_string())
3534            .or_default()
3535            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3536        // review must be irrelevant to the coding-tier filter
3537        config
3538            .model_map
3539            .entry("review".to_string())
3540            .or_default()
3541            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3542        for tier in ["simple", "medium", "complex", "review"] {
3543            config
3544                .model_map
3545                .get_mut(tier)
3546                .unwrap()
3547                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3548        }
3549
3550        let mut router = Router::new(config);
3551        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3552
3553        record_model_failure("minimax_like", "opus").await;
3554        assert!(is_model_in_cooldown("minimax_like", "opus"));
3555        // No agent-level cooldown, and the simple tier is healthy, so the loose
3556        // "any tier" check still passes — this is exactly the leak #3511 missed.
3557        assert!(!is_agent_in_cooldown("minimax_like"));
3558        assert!(router.agent_has_any_available_model("minimax_like"));
3559        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3560
3561        let uncooled_agents: Vec<String> = router
3562            .available_agents
3563            .iter()
3564            .filter(|a| {
3565                !is_agent_in_cooldown(a)
3566                    && !is_agent_degraded(a)
3567                    && router.agent_has_available_model_for_coding_tiers(a)
3568            })
3569            .cloned()
3570            .collect();
3571
3572        assert_eq!(
3573            uncooled_agents,
3574            vec!["agent_healthy".to_string()],
3575            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3576        );
3577    }
3578}