Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22use crate::backends::ExternalTask;
23use crate::cmd_cache;
24use crate::engine::cooldown;
25use crate::engine::cooldown::{
26    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
27    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
28};
29use crate::engine::runner::agents::opencode;
30use crate::engine::runner::response;
31use crate::store::WaitingOnDependency;
32use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
33pub use config::{RouterConfig, parse_pool_entry};
34use llm::{LlmRouter, TIMEOUT_PREFIX};
35use serde::{Deserialize, Serialize};
36use std::fmt;
37use std::fmt::Debug;
38use std::fmt::Formatter;
39use std::sync::Arc;
40use tokio::task;
41pub use weights::AgentWeights;
42
43fn fmt_degraded(degraded: &[String]) -> String {
44    if degraded.is_empty() {
45        String::new()
46    } else {
47        format!(" (degraded: {})", degraded.join(", "))
48    }
49}
50
51/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
52///
53/// This is a transient error that should not count against `max_route_attempts` since
54/// no agent was actually invoked. The caller should back off and retry later.
55#[derive(Debug, thiserror::Error)]
56pub enum AllAgentsCooledError {
57    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
58    AllCooled {
59        scope: String,
60        remaining_secs: Option<u64>,
61        degraded: Vec<String>,
62    },
63}
64
65impl AllAgentsCooledError {
66    /// Create a new error for the given scope, naming currently degraded agents
67    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
68        Self::AllCooled {
69            scope,
70            remaining_secs,
71            degraded: list_degraded_agents()
72                .into_iter()
73                .map(|(agent, _reason)| agent)
74                .collect(),
75        }
76    }
77
78    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
79
80    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
81    pub fn scope(&self) -> &str {
82        match self {
83            Self::AllCooled { scope, .. } => scope,
84        }
85    }
86
87    /// Remaining cooldown duration in seconds when available.
88    pub fn remaining_secs(&self) -> Option<u64> {
89        match self {
90            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
91        }
92    }
93
94    /// Agents currently marked degraded, shown by `orch cooldown list`
95    pub fn degraded(&self) -> &[String] {
96        match self {
97            Self::AllCooled { degraded, .. } => degraded,
98        }
99    }
100}
101
102/// Typed error returned by [`get_route_result`].
103#[derive(Debug, thiserror::Error)]
104pub enum RouteResultError {
105    /// The task has no agent field — it needs to be re-routed.
106    #[error("no agent configured for task {task_id}")]
107    NoAgent { task_id: String },
108    /// Any other failure (store lookup, SQL, etc.).
109    #[error(transparent)]
110    Other(#[from] anyhow::Error),
111}
112
113/// Result of routing a task to an agent.
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
115pub struct RouteResult {
116    /// The selected agent: "claude", "codex", or "opencode"
117    pub agent: String,
118    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
119    pub model: Option<String>,
120    /// Complexity level: "simple", "medium", or "complex"
121    pub complexity: String,
122    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
123    pub estimate: u8,
124    /// Why this agent was selected
125    pub reason: String,
126    /// Specialized agent profile (skills, tools, constraints)
127    pub profile: AgentProfile,
128    /// Selected skill IDs from the catalog
129    pub selected_skills: Vec<String>,
130    /// Optional warning about routing decision
131    pub warning: Option<String>,
132}
133
134/// Specialized agent profile for a task.
135#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
136pub struct AgentProfile {
137    /// Role name, e.g., "backend specialist"
138    pub role: String,
139    /// Focus skills for this task
140    pub skills: Vec<String>,
141    /// Tools allowed for this task
142    pub tools: Vec<String>,
143    /// Constraints for this task
144    pub constraints: Vec<String>,
145}
146
147impl AgentProfile {
148    /// Build the default "general" profile from router config.
149    pub fn default_from_config(config: &RouterConfig) -> Self {
150        Self {
151            role: "general".to_string(),
152            skills: vec![],
153            tools: config.allowed_tools.clone(),
154            constraints: vec![],
155        }
156    }
157}
158
159impl RouteResult {
160    /// The model for the task's tier, or one from the nearest tier with a free model. The warning
161    /// names the tier used when it differs, so the switch shows in the route activity
162    pub(crate) fn resolve_model(
163        config: &RouterConfig,
164        agent: &str,
165        complexity: &str,
166        task_id: &str,
167    ) -> (Option<String>, Option<String>) {
168        match config.model_for_complexity_or_adjacent(agent, complexity, task_id) {
169            Some((model, tier)) if tier != complexity => {
170                let warning =
171                    format!("no available {complexity} model for {agent}; using {tier} tier");
172                (Some(model), Some(warning))
173            }
174            Some((model, _)) => (Some(model), None),
175            None => (None, None),
176        }
177    }
178
179    /// Build a standard route result from the three task-specific values.
180    ///
181    /// Fills `profile`, `selected_skills`, and `warning` with router-config
182    /// defaults so call sites only need to supply what varies per task.
183    pub fn from_config(
184        agent: String,
185        complexity: String,
186        reason: String,
187        config: &RouterConfig,
188        task_id: &str,
189    ) -> Self {
190        let (model, warning) = Self::resolve_model(config, &agent, &complexity, task_id);
191        Self {
192            agent,
193            model,
194            complexity,
195            estimate: 0,
196            reason,
197            profile: AgentProfile::default_from_config(config),
198            selected_skills: config.default_skills.clone(),
199            warning,
200        }
201    }
202}
203
204/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
205///
206/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
207/// **not** require holding the router's `RwLock`, so async operations can
208/// safely be called without starving write-lock waiters.
209pub struct LlmRouterHandle(Arc<LlmRouter>);
210
211impl Debug for LlmRouterHandle {
212    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
213        f.debug_struct("LlmRouterHandle").finish_non_exhaustive()
214    }
215}
216
217impl LlmRouterHandle {
218    /// Invalidate the skills catalog cache so the next routing call reloads
219    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
220    pub async fn invalidate_skills_catalog(&self) {
221        self.0.invalidate_skills_catalog().await;
222    }
223}
224
225/// The agent router.
226#[expect(
227    clippy::struct_field_names,
228    reason = "the router holds sub-routers whose names say which backend they use"
229)]
230pub struct Router {
231    /// Router configuration
232    pub config: RouterConfig,
233    /// Available agents discovered at runtime
234    pub available_agents: Vec<String>,
235    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
236    pub weights: AgentWeights,
237    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
238    /// without holding the router RwLock across async operations)
239    llm_router: Arc<LlmRouter>,
240    /// Ollama router for local routing (contains reusable reqwest::Client)
241    ollama_router: Option<ollama::OllamaRouter>,
242    /// Round-robin index for task routing
243    pub(crate) rr_index: usize,
244    /// Last agent routed to (for distribution tracking)
245    pub(crate) last_agent: Option<String>,
246    /// Round-robin index for review agent selection
247    pub(crate) review_rr_index: usize,
248    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
249    /// `opencode:free` entries are expanded at construction time.
250    pub(crate) router_pool: Vec<(String, String)>,
251    /// Current round-robin index into router_pool
252    pub(crate) pool_index: usize,
253}
254
255impl Debug for Router {
256    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
257        f.debug_struct("Router")
258            .field("config", &self.config)
259            .field("available_agents", &self.available_agents)
260            .field("weights", &self.weights)
261            .field("ollama_router", &self.ollama_router)
262            .field("rr_index", &self.rr_index)
263            .field("last_agent", &self.last_agent)
264            .field("review_rr_index", &self.review_rr_index)
265            .field("router_pool", &self.router_pool)
266            .field("pool_index", &self.pool_index)
267            .finish_non_exhaustive()
268    }
269}
270
271impl Router {
272    /// Create a new router with the given configuration.
273    pub fn new(config: RouterConfig) -> Self {
274        let available_agents = Self::discover_agents(&config.agents);
275        let mut weights = AgentWeights {
276            base_weights: config.weights.clone(),
277            ..Default::default()
278        };
279        weights.ensure_agents_with_weights(&available_agents, &config.weights);
280        if !config.weights.is_empty() {
281            tracing::info!(
282                weights = ?config.weights,
283                "configured agent routing weights"
284            );
285        }
286        let router_pool = Self::expand_pool(&config);
287        tracing::info!(
288            pool = ?router_pool,
289            fallback = %config.effective_fallback(),
290            "router LLM pool initialized"
291        );
292        // Prime only when this config names opencode, so a default or claude-only
293        // router never runs `opencode models`
294        if config.uses_opencode() {
295            opencode::prime_free_model_cache();
296        }
297        let ollama_router = if config.mode == "local" {
298            Some(ollama::OllamaRouter::new(&config))
299        } else {
300            None
301        };
302        Self {
303            config,
304            available_agents,
305            weights,
306            llm_router: Arc::new(LlmRouter::new()),
307            ollama_router,
308            rr_index: 0,
309            last_agent: None,
310            review_rr_index: 0,
311            router_pool,
312            pool_index: 0,
313        }
314    }
315
316    /// Create a router with default configuration loaded from files.
317    pub fn from_config() -> Self {
318        Self::new(RouterConfig::from_config())
319    }
320
321    /// Create a router with a forced agent list, bypassing CLI discovery.
322    /// Only for use in unit tests.
323    #[cfg(test)]
324    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
325        let mut weights = AgentWeights {
326            base_weights: config.weights.clone(),
327            ..Default::default()
328        };
329        weights.ensure_agents(&available_agents);
330        let router_pool = Self::expand_pool(&config);
331        Self {
332            config,
333            available_agents,
334            weights,
335            llm_router: Arc::new(LlmRouter::new()),
336            ollama_router: None,
337            rr_index: 0,
338            last_agent: None,
339            review_rr_index: 0,
340            router_pool,
341            pool_index: 0,
342        }
343    }
344
345    /// Return a cloned handle to the LLM router subsystem.
346    ///
347    /// Callers that need to invoke async methods on the underlying `LlmRouter`
348    /// should clone this handle *before* releasing any `RwLock` read guard on
349    /// `Router`, then call the async method after the guard is dropped.  This
350    /// prevents holding the read lock across await points (which starves
351    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
352    pub fn llm_router_handle(&self) -> LlmRouterHandle {
353        LlmRouterHandle(Arc::clone(&self.llm_router))
354    }
355
356    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
357        debug_assert!(pool_len > 0, "router pool must not be empty");
358        self.pool_index = (idx + 1) % pool_len;
359    }
360
361    /// Discover available agent CLIs in PATH.
362    /// Checks all agents from the configured list.
363    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
364        let mut agents = Vec::new();
365        for agent in configured_agents {
366            if cmd_cache::command_exists(agent) {
367                agents.push(agent.clone());
368            }
369        }
370        agents
371    }
372
373    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
374    ///
375    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
376    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
377    /// - If `opencode` is not installed or the command fails, the entry is skipped.
378    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
379    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
380        let raw_pool = config.effective_pool();
381        let mut expanded: Vec<(String, String)> = Vec::new();
382
383        for entry in &raw_pool {
384            if entry == "opencode:free" {
385                let free_models = Self::discover_free_opencode_models();
386                if free_models.is_empty() {
387                    tracing::debug!("opencode:free expanded to nothing — skipping");
388                } else {
389                    tracing::debug!(models = ?free_models, "opencode:free expanded");
390                    for model in free_models {
391                        expanded.push(("opencode".to_string(), model));
392                    }
393                }
394            } else {
395                let (agent, model) = parse_pool_entry(entry);
396                expanded.push((agent, model));
397            }
398        }
399
400        if expanded.is_empty() {
401            // All entries were opencode:free and opencode isn't installed — use fallback
402            let (agent, model) = parse_pool_entry(&config.effective_fallback());
403            tracing::debug!(
404                agent = %agent, model = %model,
405                "pool empty after expansion, using fallback as sole pool entry"
406            );
407            expanded.push((agent, model));
408        }
409
410        expanded
411    }
412
413    /// Async version of expand_pool which may query the opencode CLI without
414    /// blocking the Tokio runtime.
415    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
416        let raw_pool = config.effective_pool();
417        let mut expanded: Vec<(String, String)> = Vec::new();
418
419        for entry in &raw_pool {
420            if entry == "opencode:free" {
421                let free_models = Self::discover_free_opencode_models_async().await;
422                if free_models.is_empty() {
423                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
424                } else {
425                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
426                    for model in free_models {
427                        expanded.push(("opencode".to_string(), model));
428                    }
429                }
430            } else {
431                let (agent, model) = parse_pool_entry(entry);
432                expanded.push((agent, model));
433            }
434        }
435
436        if expanded.is_empty() {
437            // All entries were opencode:free and opencode isn't installed — use fallback
438            let (agent, model) = parse_pool_entry(&config.effective_fallback());
439            tracing::debug!(
440                agent = %agent, model = %model,
441                "pool empty after async expansion, using fallback as sole pool entry"
442            );
443            expanded.push((agent, model));
444        }
445
446        expanded
447    }
448
449    /// Async reload that uses async pool expansion to avoid blocking the runtime
450    /// when discovering opencode free models.
451    pub async fn reload_async(&mut self) {
452        let new_config = RouterConfig::from_config();
453        let new_agents = Self::discover_agents(&new_config.agents);
454        let new_pool = Self::expand_pool_async(&new_config).await;
455        tracing::info!(
456            mode = %new_config.mode,
457            agents = ?new_agents,
458            fallback = %new_config.fallback_executor,
459            weighted_rr = new_config.weighted_round_robin,
460            pool = ?new_pool,
461            "router reloaded (async)"
462        );
463        self.config = new_config;
464        self.available_agents.clone_from(&new_agents);
465        // Ensure new agents have weight entries (preserves existing weights)
466        self.weights.ensure_agents(&new_agents);
467        self.router_pool = new_pool;
468        self.pool_index = 0;
469        // Reinitialize Ollama router if mode changed to local
470        self.ollama_router = if self.config.mode == "local" {
471            Some(ollama::OllamaRouter::new(&self.config))
472        } else {
473            None
474        };
475    }
476
477    fn discover_free_opencode_models() -> Vec<String> {
478        opencode::discover_free_opencode_models()
479    }
480
481    async fn discover_free_opencode_models_async() -> Vec<String> {
482        task::spawn_blocking(Self::discover_free_opencode_models)
483            .await
484            .unwrap_or_default()
485    }
486
487    /// Run the pre-emptive health check, refreshing the degraded-agent set.
488    ///
489    /// Queries the `rate_limits` table for recent events and combines with
490    /// cooldown state to mark agents as degraded before routing attempts them.
491    pub async fn refresh_health(&self, store: &Arc<TaskStore>) {
492        // KV is the source of truth for cooldowns. Reconcile the in-memory map
493        // from KV every tick so external mutations (e.g. `orch cooldown clear`
494        // from a separate CLI process) actually take effect on the running
495        // service. Without this, the engine keeps stale entries until they
496        // expire on their own, producing the "all agents cooled" log with
497        // `orch cooldown list` simultaneously reporting "No active cooldowns".
498        sync_from_kv(store).await;
499
500        let agents = self.available_agents.clone();
501        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
502        refresh_degraded_agents(
503            store,
504            &agents,
505            &model_checker,
506            config::health_check_window_hours(),
507            config::degraded_rate_limit_threshold(),
508        )
509        .await;
510    }
511
512    /// Check if an agent is available.
513    pub fn is_agent_available(&self, agent: &str) -> bool {
514        self.available_agents.contains(&agent.to_string())
515    }
516
517    /// True if the agent has at least one uncooled model across any complexity
518    /// tier. Used where the eventual complexity isn't known yet (e.g. before
519    /// LLM classification), so we can't check a single tier.
520    fn agent_has_any_available_model(&self, agent: &str) -> bool {
521        ["simple", "medium", "complex", "review"]
522            .iter()
523            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
524    }
525
526    /// True if the agent has an available (uncooled) model for the `medium`
527    /// and `complex` tiers — the tiers real coding tasks are classified into
528    /// by the LLM classifier. Used to pre-filter the candidate list handed to
529    /// that classifier so an agent whose healthy tiers are only `simple` isn't
530    /// offered: the classifier can't be constrained to `simple`, so offering
531    /// such an agent produces a near-certain wasted classification call whenever
532    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
533    /// outcome of initial classification and is deliberately excluded.
534    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
535        ["medium", "complex"]
536            .iter()
537            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
538    }
539
540    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
541        if !self.is_agent_available(agent) {
542            return false;
543        }
544        if is_agent_in_cooldown(agent) {
545            tracing::debug!(agent, "agent skipped: in cooldown");
546            return false;
547        }
548        if is_agent_degraded(agent) {
549            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
550            return false;
551        }
552        // If weighted routing is used, consider the agent degraded when its
553        // weight has decayed below the threshold. This avoids
554        // proactively routing to agents that recently hit many rate limits.
555        if self.config.weighted_round_robin {
556            let weight = self.weights.get_weight(agent);
557            if weight < config::SKIP_LIMITED_THRESHOLD {
558                tracing::debug!(
559                    agent,
560                    weight,
561                    "agent considered degraded by weight threshold, skipping"
562                );
563                return false;
564            }
565        }
566        if !self
567            .config
568            .has_available_model_in_any_tier(agent, complexity)
569        {
570            tracing::debug!(agent, complexity, "agent skipped: no available model");
571            return false;
572        }
573        true
574    }
575
576    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
577        self.available_agents
578            .iter()
579            .filter(|agent| self.agent_is_routable(agent, complexity))
580            .cloned()
581            .collect()
582    }
583
584    /// Count of healthy (routable) agents for a given complexity.
585    /// Returns the number of agents that are available, not in cooldown, and have an available model.
586    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
587        self.available_agents_for_complexity(complexity).len()
588    }
589
590    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
591    /// Uses "simple" complexity as the baseline since it has the widest model support.
592    pub fn is_degraded(&self, threshold: usize) -> bool {
593        self.healthy_agent_count("simple") < threshold
594    }
595
596    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
597        let mut earliest: Option<i64> = None;
598
599        for agent in &self.available_agents {
600            if let Some(until) = cooldown_until(agent) {
601                earliest = Some(earliest.map_or(until, |current| current.min(until)));
602            }
603
604            if let Some(comp) = complexity
605                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
606            {
607                for model in pool {
608                    let key = format!("{agent}:{model}");
609                    if let Some(until) = cooldown_until(&key) {
610                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
611                    }
612                }
613            }
614        }
615
616        earliest
617    }
618
619    /// Find the earliest cooldown expiration among the router LLM pool entries.
620    ///
621    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
622    /// or `None` if no pool model has a cooldown.
623    fn earliest_pool_cooldown(&self) -> Option<i64> {
624        let mut earliest: Option<i64> = None;
625        for (agent, model) in &self.router_pool {
626            let key = format!("{agent}:{model}");
627            if let Some(until) = cooldown_until(&key) {
628                earliest = Some(earliest.map_or(until, |current| current.min(until)));
629            }
630        }
631        earliest
632    }
633
634    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
635        // If any cooldowns are present, return an error immediately so the
636        // caller (the tick loop) can skip this task and retry on the next tick.
637        // Sleeping inside the router blocks the entire tick loop for the
638        // cooldown duration which starves other tasks. The engine tick already
639        // retries every `tick_interval` (default 10s), so allow the tick loop
640        // to drive retries instead of sleeping here.
641        let now = chrono::Utc::now().timestamp();
642        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
643            // All agents are degraded but none have a timed cooldown entry
644            // (e.g. over-threshold rate limiting without explicit cooldown duration).
645            // Log at warn and return an error so the tick skips this task and retries after backoff.
646            tracing::warn!(
647                scope = complexity.unwrap_or("all agents"),
648                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
649            );
650            // Return an error similar to the cooldown case to trigger backoff in the tick loop
651            return Err(AllAgentsCooledError::new(
652                complexity.unwrap_or("all agents").to_string(),
653                None,
654            )
655            .into());
656        };
657
658        let remaining = earliest.saturating_sub(now);
659        if remaining > 0 {
660            let scope_str = complexity.unwrap_or("all agents").to_string();
661            tracing::warn!(
662                remaining_secs = remaining,
663                scope = %scope_str,
664                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
665            );
666            // Return a domain error indicating all candidates are cooled so the
667            // caller can handle it (tick will skip this task and retry later).
668            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
669        }
670
671        Ok(())
672    }
673
674    /// Get the first available agent.
675    /// Pick next agent via round-robin (for review or other non-task routing).
676    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
677    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
678    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
679        // Use available_agents_for_complexity which already filters using agent_is_routable,
680        // eliminating duplicate cooldown/degraded/model checks.
681        let candidates = self.available_agents_for_complexity(complexity);
682        if candidates.is_empty() {
683            return None;
684        }
685        let idx = self.review_rr_index;
686        let n = candidates.len();
687
688        // Try to find an agent that isn't excluded
689        let agent = (0..n)
690            .map(|offset| &candidates[(idx + offset) % n])
691            .find(|a| !exclude.contains(&a.as_str()))
692            .cloned()
693            // Fallback: use the first available candidate (even if excluded)
694            .or_else(|| candidates.first().cloned())?;
695
696        self.review_rr_index = (idx + 1) % candidates.len().max(1);
697        Some(agent)
698    }
699
700    /// If the selected agent matches `no_code_last_agent` for this task, filter it
701    /// out of the candidates and return the filtered list. Used to prevent the
702    /// same agent from being selected twice in a row after a no-code failure (#2410).
703    async fn skip_no_code_agent(
704        &self,
705        store: &Arc<TaskStore>,
706        repo: &str,
707        task_id: &str,
708        candidates: &[String],
709    ) -> Option<Vec<String>> {
710        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
711            .await
712            .unwrap_or_else(|| {
713                tracing::warn!(
714                    task_id,
715                    "no_code_last_agent DB read returned None — loop detection may be impaired"
716                );
717                String::new()
718            });
719        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
720            return None;
721        }
722        let filtered: Vec<String> = candidates
723            .iter()
724            .filter(|a| a.as_str() != no_code_agent)
725            .cloned()
726            .collect();
727        if filtered.is_empty() {
728            tracing::warn!(
729                task_id,
730                no_code_agent = %no_code_agent,
731                "all available agents are the no-code agent — cannot skip"
732            );
733            return None;
734        }
735        tracing::info!(
736            task_id,
737            skipped = %no_code_agent,
738            remaining = ?filtered,
739            "skipping no-code agent, routing to alternative"
740        );
741        Some(filtered)
742    }
743
744    /// Route a task to the best agent.
745    ///
746    /// Routing logic (in priority order):
747    /// 1. Check for `agent:*` label — use that agent directly
748    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
749    /// 3. If round_robin mode, cycle through agents (stateful)
750    /// 4. Call LLM classifier for intelligent routing
751    /// 5. After max_route_attempts LLM failures, fall back to round-robin
752    pub async fn route(
753        &mut self,
754        task: &ExternalTask,
755        store: &Arc<TaskStore>,
756        repo: &str,
757    ) -> anyhow::Result<RouteResult> {
758        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
759        // before the routing loop, not per-task. Do not re-add it here.
760
761        // Hold the task before any classification, label override, or fallback.
762        // A waiting task must not increment route_attempts.
763        match store.resolve_id_by_external(repo, &task.id.0).await {
764            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
765                Ok(Some(blockers)) => {
766                    return Err(WaitingOnDependency { blockers }.into());
767                }
768                Ok(None) => {}
769                Err(err) => return Err(err),
770            },
771            Ok(None) => {}
772            Err(err) => return Err(err),
773        }
774
775        loop {
776            // 1. Check store agent field first (set by failover, authoritative over labels)
777            // Use a lightweight lookup to avoid deserializing the full Task row when
778            // only the `agent` column is needed. This avoids two full-row loads
779            // per routing pass (resolve_task_id + get) and significantly reduces
780            // CPU work on busy instances.
781            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
782                Ok(Some(agent)) => {
783                    // Router enforces that the agent string matches a configured agent
784                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
785                        Some(agent)
786                    } else {
787                        None
788                    }
789                }
790                Ok(None) => None,
791                Err(e) => {
792                    tracing::warn!(
793                        repo,
794                        task_id = %task.id.0,
795                        err = %e,
796                        "failed to fetch stored agent, falling back to label/LLM routing"
797                    );
798                    None
799                }
800            };
801
802            // 2. Fall back to explicit agent label
803            let resolved_agent = store_agent.or_else(|| {
804                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
805            });
806
807            if let Some(agent) = resolved_agent {
808                let complexity = strategies::extract_complexity_from_labels(&task.labels);
809                // If agent appears routable (not in agent-wide cooldown and has
810                // available models or no model_map configured), check whether
811                // we actually have a concrete model to dispatch. If the
812                // resolved label maps to no model (None), we fall through to
813                // the standard routing logic so the router LLM can pick a
814                // proper agent/model, instead of dispatching with an empty
815                // model which causes some agents (e.g. opencode) to exit
816                // silently.
817                if self.agent_is_routable(&agent, &complexity) {
818                    let (model, tier_warning) =
819                        RouteResult::resolve_model(&self.config, &agent, &complexity, &task.id.0);
820
821                    // Only dispatch label overrides when we have a concrete
822                    // model to send to the executor. Dispatching with an
823                    // empty model led to silent exits for some agents
824                    // (notably opencode). If no concrete model is available
825                    // for the labeled agent, fall through to the standard
826                    // routing logic so the LLM or round-robin can pick an
827                    // appropriate agent/model.
828                    if model.is_some() {
829                        tracing::debug!(
830                            task_id = %task.id.0,
831                            agent = %agent,
832                            complexity = %complexity,
833                            "routed via label"
834                        );
835                        let mut result = RouteResult::from_config(
836                            agent.clone(),
837                            complexity.clone(),
838                            format!("label agent:{agent}"),
839                            &self.config,
840                            &task.id.0,
841                        );
842                        // Label routing uses agent-specific role and the
843                        // resolved model (may differ from from_config's default).
844                        result.profile.role = format!("{agent} specialist");
845                        result.model = model.clone();
846                        result.warning = tier_warning;
847                        self.log_route_activity(store, repo, &task.id.0, &result, None)
848                            .await;
849                        return Ok(result);
850                    }
851                    // No concrete model found for the labeled agent — do
852                    // not honor the label override. Fall through to the
853                    // standard routing logic so another agent or the LLM
854                    // can be chosen with a valid model.
855                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
856                }
857
858                let candidates = self.available_agents_for_complexity(&complexity);
859                if candidates.is_empty() {
860                    self.wait_for_cooldown(Some(&complexity))?;
861                    continue;
862                }
863                // Label target is cooled or lacked model; fall through to standard routing.
864            }
865
866            let complexity = strategies::extract_complexity_from_labels(&task.labels);
867
868            // Check if any agents have available models for this complexity
869            let mut candidates = self.available_agents_for_complexity(&complexity);
870            if candidates.is_empty() {
871                self.wait_for_cooldown(Some(&complexity))?;
872                continue;
873            }
874
875            // Skip the no-code agent if one is recorded for this task (#2410).
876            // If all available agents are filtered out, we fall through to
877            // LLM routing with no constraints (LLM will pick the same agent but
878            // the response_handler will block it immediately via same-agent check).
879            if let Some(filtered) = self
880                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
881                .await
882            {
883                candidates = filtered;
884            }
885
886            // 2. Weighted round-robin — capacity-based selection
887            if self.config.weighted_round_robin {
888                let result = strategies::route_via_weighted_round_robin(
889                    &candidates,
890                    &self.weights,
891                    &self.config,
892                    task,
893                    &mut self.last_agent,
894                )?;
895                self.log_route_activity(store, repo, &task.id.0, &result, None)
896                    .await;
897                return Ok(result);
898            }
899
900            // 3. Round-robin mode — use stateful round-robin
901            if self.config.mode == "round_robin" {
902                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
903                let result = strategies::route_via_round_robin_stateful(
904                    &candidates,
905                    &self.config,
906                    task,
907                    &mut self.rr_index,
908                    &mut self.last_agent,
909                )?;
910                self.log_route_activity(store, repo, &task.id.0, &result, None)
911                    .await;
912                return Ok(result);
913            }
914
915            // 4. Local Ollama routing
916            if self.config.mode == "local" {
917                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
918                let ollama_router = self
919                    .ollama_router
920                    .as_ref()
921                    .expect("ollama_router must be initialized in Router::new when mode=local");
922                match ollama_router.route(task, &self.config).await {
923                    Ok(result) => {
924                        self.log_route_activity(store, repo, &task.id.0, &result, None)
925                            .await;
926                        return Ok(result);
927                    }
928                    Err(e) => {
929                        tracing::warn!(
930                            task_id = %task.id.0,
931                            error = %e,
932                            "Ollama routing failed, falling back to round-robin"
933                        );
934                        let result = strategies::route_via_round_robin_stateful(
935                            &candidates,
936                            &self.config,
937                            task,
938                            &mut self.rr_index,
939                            &mut self.last_agent,
940                        )?;
941                        self.log_route_activity(store, repo, &task.id.0, &result, None)
942                            .await;
943                        return Ok(result);
944                    }
945                }
946            }
947
948            // 5. LLM-based routing with retry tracking
949            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
950
951            if route_attempts >= self.config.max_route_attempts {
952                tracing::warn!(
953                    task_id = %task.id.0,
954                    attempts = route_attempts,
955                    max = self.config.max_route_attempts,
956                    "max LLM route attempts reached, falling back to round-robin"
957                );
958                let candidates = self.available_agents_for_complexity(&complexity);
959                if candidates.is_empty() {
960                    self.wait_for_cooldown(Some(&complexity))?;
961                    continue;
962                }
963                let result = strategies::route_via_round_robin_stateful(
964                    &candidates,
965                    &self.config,
966                    task,
967                    &mut self.rr_index,
968                    &mut self.last_agent,
969                )?;
970                self.log_route_activity(store, repo, &task.id.0, &result, None)
971                    .await;
972                return Ok(result);
973            }
974
975            // Log routing start (before await)
976            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
977
978            match self.route_with_llm(task, repo).await {
979                Ok(result) => {
980                    let candidates = self.available_agents_for_complexity(&result.complexity);
981                    if candidates.is_empty() {
982                        self.wait_for_cooldown(Some(&result.complexity))?;
983                        continue;
984                    }
985
986                    // If the LLM selected the no-code agent, filter it out and
987                    // pick an alternative (#2410). Re-use the existing fallback logic.
988                    let no_code_agent =
989                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
990                             .await
991                             .unwrap_or_else(|| {
992                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
993                                 String::new()
994                             });
995                    if result.agent == no_code_agent {
996                        // Pick the first agent that isn't the no-code agent.
997                        let fallback_agent = candidates
998                            .iter()
999                            .find(|a| a.as_str() != no_code_agent)
1000                            .cloned()
1001                            .unwrap_or_else(|| candidates[0].clone());
1002                        if fallback_agent.as_str() == no_code_agent {
1003                            tracing::warn!(
1004                                task_id = %task.id.0,
1005                                no_code_agent = %no_code_agent,
1006                                complexity = %result.complexity,
1007                                "all available agents are the no-code agent — waiting for another agent"
1008                            );
1009                            self.wait_for_cooldown(Some(&result.complexity))?;
1010                            continue;
1011                        }
1012                        tracing::info!(
1013                            task_id = %task.id.0,
1014                            llm_selected = %result.agent,
1015                            skipped = %no_code_agent,
1016                            fallback = %fallback_agent,
1017                            "LLM selected the no-code agent — routing to alternative"
1018                        );
1019                        let (fallback_model, _) = RouteResult::resolve_model(
1020                            &self.config,
1021                            &fallback_agent,
1022                            &result.complexity,
1023                            &task.id.0,
1024                        );
1025
1026                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1027                        let result = RouteResult {
1028                            agent: fallback_agent.clone(),
1029                            model: fallback_model,
1030                            complexity: result.complexity.clone(),
1031                            estimate: result.estimate,
1032                            reason: format!(
1033                                "LLM selected no-code agent {}; routed to {}",
1034                                result.agent, fallback_agent
1035                            ),
1036                            profile: result.profile.clone(),
1037                            selected_skills: result.selected_skills.clone(),
1038                            warning: Some(format!(
1039                                "LLM selected no-code agent {}; used fallback {}",
1040                                result.agent, fallback_agent
1041                            )),
1042                        };
1043                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1044                            .await;
1045                        return Ok(result);
1046                    }
1047
1048                    let model_cooled = result
1049                        .model
1050                        .as_deref()
1051                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1052                    if !candidates.contains(&result.agent) || model_cooled {
1053                        let fallback_agent = candidates
1054                            .iter()
1055                            .find(|agent| *agent != &result.agent)
1056                            .cloned()
1057                            .unwrap_or_else(|| candidates[0].clone());
1058                        let (fallback_model, _) = RouteResult::resolve_model(
1059                            &self.config,
1060                            &fallback_agent,
1061                            &result.complexity,
1062                            &task.id.0,
1063                        );
1064
1065                        // If the fallback model is None and the fallback agent is the
1066                        // same as the LLM-selected agent, it means all models for the
1067                        // only available agent are cooled. In that case, wait for
1068                        // cooldowns instead of returning a RouteResult with
1069                        // model: None which would be dispatched incorrectly.
1070                        if fallback_model.is_none() && fallback_agent == result.agent {
1071                            tracing::warn!(
1072                                task_id = %task.id.0,
1073                                agent = %result.agent,
1074                                complexity = %result.complexity,
1075                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1076                            );
1077                            self.wait_for_cooldown(Some(&result.complexity))?;
1078                            continue;
1079                        }
1080
1081                        // Distinguish logging and reasons when we actually switch
1082                        // agents vs when we keep the same agent but pick a
1083                        // different (non-cooled) model.
1084                        if fallback_agent == result.agent {
1085                            tracing::warn!(
1086                                task_id = %task.id.0,
1087                                agent = %result.agent,
1088                                complexity = %result.complexity,
1089                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1090                            );
1091                        } else {
1092                            tracing::warn!(
1093                                task_id = %task.id.0,
1094                                agent = %result.agent,
1095                                fallback = %fallback_agent,
1096                                complexity = %result.complexity,
1097                                "LLM selected cooled agent/model; rerouting to available agent"
1098                            );
1099                        }
1100
1101                        // Reset attempts on success
1102                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1103
1104                        let reason = if fallback_agent == result.agent {
1105                            format!(
1106                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1107                            )
1108                        } else {
1109                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1110                        };
1111
1112                        let result = RouteResult {
1113                            agent: fallback_agent.clone(),
1114                            model: fallback_model,
1115                            complexity: result.complexity.clone(),
1116                            estimate: result.estimate,
1117                            reason: reason.clone(),
1118                            profile: result.profile.clone(),
1119                            selected_skills: result.selected_skills.clone(),
1120                            warning: Some(if fallback_agent == result.agent {
1121                                "LLM-selected model was cooled; selected alternative model for same agent"
1122                                    .to_string()
1123                            } else {
1124                                "LLM-selected agent/model was cooled; rerouted to available agent"
1125                                    .to_string()
1126                            }),
1127                        };
1128                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1129                            .await;
1130                        return Ok(result);
1131                    }
1132
1133                    // Reset attempts on success
1134                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1135                    tracing::info!(
1136                        task_id = %task.id.0,
1137                        agent = %result.agent,
1138                        complexity = %result.complexity,
1139                        "routed via LLM"
1140                    );
1141                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1142                        .await;
1143                    return Ok(result);
1144                }
1145                Err(e) => {
1146                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1147                        let scope = err.scope();
1148                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1149                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1150                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1151                        // (no complexity-specific pool lookup, which would fail for "router pool").
1152                        let scope_opt = match scope {
1153                            "all agents" | "router pool" => None,
1154                            _ => Some(scope),
1155                        };
1156                        self.wait_for_cooldown(scope_opt)?;
1157                        continue;
1158                    }
1159
1160                    let new_attempts = route_attempts + 1;
1161                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1162                        .await;
1163                    tracing::warn!(
1164                        task_id = %task.id.0,
1165                        error = %e,
1166                        error_chain = ?e,
1167                        attempt = new_attempts,
1168                        max = self.config.max_route_attempts,
1169                        "LLM routing failed"
1170                    );
1171
1172                    if new_attempts >= self.config.max_route_attempts {
1173                        tracing::info!(
1174                            task_id = %task.id.0,
1175                            "falling back to round-robin after {} failed attempts",
1176                            new_attempts
1177                        );
1178                        let candidates = self.available_agents_for_complexity(&complexity);
1179                        if candidates.is_empty() {
1180                            self.wait_for_cooldown(Some(&complexity))?;
1181                            continue;
1182                        }
1183                        let result = strategies::route_via_round_robin_stateful(
1184                            &candidates,
1185                            &self.config,
1186                            task,
1187                            &mut self.rr_index,
1188                            &mut self.last_agent,
1189                        )?;
1190                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1191                            .await;
1192                        return Ok(result);
1193                    }
1194
1195                    let candidates = self.available_agents_for_complexity(&complexity);
1196                    if candidates.is_empty() {
1197                        self.wait_for_cooldown(Some(&complexity))?;
1198                        continue;
1199                    }
1200                    let result = strategies::route_via_fallback(
1201                        &candidates,
1202                        &self.config,
1203                        task,
1204                        Some(&self.weights),
1205                    )?;
1206                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1207                        .await;
1208                    return Ok(result);
1209                }
1210            }
1211        }
1212    }
1213
1214    /// Get the number of LLM routing attempts for a task from the store.
1215    async fn get_route_attempts(&self, task_id: &str, store: &Arc<TaskStore>, repo: &str) -> u32 {
1216        match store.resolve_task_id(repo, task_id).await {
1217            Ok(Some(store_id)) => store
1218                .get(store_id)
1219                .await
1220                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1221            _ => 0,
1222        }
1223    }
1224
1225    /// Set the number of LLM routing attempts for a task in the store.
1226    async fn set_route_attempts(
1227        &self,
1228        task_id: &str,
1229        attempts: u32,
1230        store: &Arc<TaskStore>,
1231        repo: &str,
1232    ) {
1233        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1234            && let Err(e) = store
1235                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1236                .await
1237        {
1238            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1239        }
1240    }
1241
1242    /// Log a route event to task activity timeline.
1243    async fn log_route_activity(
1244        &self,
1245        store: &Arc<TaskStore>,
1246        repo: &str,
1247        task_id: &str,
1248        result: &RouteResult,
1249        from_agent: Option<&str>,
1250    ) {
1251        let details = serde_json::json!({
1252            "reason": result.reason,
1253            "complexity": result.complexity,
1254            "skills": result.selected_skills,
1255            "role": result.profile.role,
1256            "warning": result.warning,
1257        });
1258        let event_type = if from_agent.is_some() {
1259            "rerouted"
1260        } else {
1261            "routed"
1262        };
1263        store_log_activity(
1264            &Some(Arc::clone(store)),
1265            repo,
1266            task_id,
1267            event_type,
1268            None,
1269            Some("routed"),
1270            Some(&result.agent),
1271            result.model.as_deref(),
1272            Some(&details),
1273        )
1274        .await;
1275    }
1276
1277    /// Route using LLM classification with pool round-robin.
1278    ///
1279    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1280    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1281    /// returns the last error (the caller falls back to round-robin agent selection).
1282    async fn route_with_llm(
1283        &mut self,
1284        task: &ExternalTask,
1285        repo: &str,
1286    ) -> anyhow::Result<RouteResult> {
1287        // Filter out cooled and degraded agents, and agents without an available
1288        // model for the coding tiers (`medium`/`complex`) that real tasks are
1289        // classified into, so the LLM only sees candidates that could actually
1290        // be dispatched for whatever complexity it picks. An agent whose only
1291        // healthy tier is `simple` is excluded too: the classifier can't be
1292        // constrained to `simple`, so leaving such an agent in would waste the
1293        // call whenever it picks `medium`/`complex` (issue #3521).
1294        // Fall back to the full list if all agents are cooled/degraded.
1295        // Cloned to avoid borrow conflict with &mut self in the loop.
1296        let uncooled_agents: Vec<String> = self
1297            .available_agents
1298            .iter()
1299            .filter(|a| {
1300                !is_agent_in_cooldown(a)
1301                    && !is_agent_degraded(a)
1302                    && self.agent_has_available_model_for_coding_tiers(a)
1303            })
1304            .cloned()
1305            .collect();
1306        if uncooled_agents.is_empty() {
1307            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1308        }
1309        let llm_agents = uncooled_agents;
1310
1311        let pool = self.router_pool.clone();
1312        let n = pool.len();
1313        let start = self.pool_index;
1314        let mut last_err: Option<anyhow::Error> = None;
1315        let mut attempted_any = false;
1316
1317        // Try pool entries in round-robin order, skipping cooled ones
1318        for i in 0..n {
1319            let idx = (start + i) % n;
1320            let (agent, model) = &pool[idx];
1321            let model_str = model.as_str();
1322
1323            if response::is_model_in_cooldown(agent, model_str) {
1324                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1325                continue;
1326            }
1327
1328            // At least one pool entry was available to attempt
1329            attempted_any = true;
1330
1331            let model_opt = if model_str.is_empty() {
1332                None
1333            } else {
1334                Some(model_str)
1335            };
1336
1337            match self
1338                .llm_router
1339                .route_with_llm_using(
1340                    task,
1341                    &llm_agents,
1342                    &self.config,
1343                    &mut self.last_agent,
1344                    repo,
1345                    agent,
1346                    model_opt,
1347                )
1348                .await
1349            {
1350                Ok(result) => {
1351                    // Advance index so the next call starts at the next pool entry
1352                    self.advance_pool_index_after_attempt(idx, n);
1353                    tracing::debug!(
1354                        agent,
1355                        model = model_str,
1356                        pool_idx = idx,
1357                        "router LLM pool entry succeeded"
1358                    );
1359                    record_agent_success(agent, model_str).await;
1360                    return Ok(result);
1361                }
1362                Err(e) => {
1363                    let err_msg = e.to_string();
1364                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1365                    if is_timeout {
1366                        // Advance pool index so the next tick starts at the next entry,
1367                        // then return immediately. Continuing to try further pool entries
1368                        // in the same tick would accumulate multiple timeout windows and
1369                        // stall the tick loop for minutes (issue #3187).
1370                        //
1371                        // Also record a standard cooldown, same as the non-timeout branch,
1372                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1373                        // on future ticks instead of burning the full timeout window again
1374                        // every time round-robin reaches it (issue #3422).
1375                        self.advance_pool_index_after_attempt(idx, n);
1376                        tracing::warn!(
1377                            agent,
1378                            model = model_str,
1379                            next_pool_idx = self.pool_index,
1380                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1381                        );
1382                        response::record_model_failure(agent, model_str).await;
1383                        return Err(e);
1384                    }
1385                    tracing::warn!(
1386                        agent,
1387                        model = model_str,
1388                        error = %e,
1389                        "pool entry failed, recording cooldown and trying next"
1390                    );
1391                    if !cooldown::record_credit_exhaustion_from_message(
1392                        agent,
1393                        Some(model_str),
1394                        &err_msg,
1395                    )
1396                    .await
1397                    {
1398                        response::record_model_failure(agent, model_str).await;
1399                    }
1400                    last_err = Some(e);
1401                    self.advance_pool_index_after_attempt(idx, n);
1402                }
1403            }
1404        }
1405
1406        // If every pool entry was pre-cooled (nothing was attempted), do not
1407        // immediately exhaust the pool again — fail fast and let the tick loop
1408        // retry on the next cycle instead of cascading failures.
1409        // However, we should still try the fallback if configured.
1410        if !attempted_any {
1411            let now = chrono::Utc::now().timestamp();
1412            let earliest = self.earliest_pool_cooldown();
1413            if let Some(until) = earliest {
1414                // Pool entries exist but are all cooled - we'll try fallback below
1415                let remaining = until.saturating_sub(now);
1416                tracing::warn!(
1417                    remaining_secs = remaining,
1418                    pool = ?pool,
1419                    "all router LLM pool entries on cooldown — will try fallback"
1420                );
1421                // Continue to fallback logic instead of returning early
1422            } else {
1423                // No pool cooldowns found either — this means the pool was empty
1424                // or every entry was filtered out for some other reason.
1425                return Err(last_err
1426                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1427            }
1428        }
1429
1430        // All pool entries failed or were cooled — try the configured fallback
1431        let fallback = self.config.effective_fallback();
1432        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1433        let fb_model_str = fb_model.as_str();
1434
1435        // Only try fallback if it wasn't already in the pool (avoid double-try)
1436        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1437
1438        if !fallback_already_tried && !response::is_model_in_cooldown(&fb_agent, fb_model_str) {
1439            tracing::info!(
1440                agent = %fb_agent,
1441                model = %fb_model_str,
1442                "all pool entries exhausted — trying fallback router LLM"
1443            );
1444            let fb_model_opt = if fb_model_str.is_empty() {
1445                None
1446            } else {
1447                Some(fb_model_str)
1448            };
1449            match self
1450                .llm_router
1451                .route_with_llm_using(
1452                    task,
1453                    &llm_agents,
1454                    &self.config,
1455                    &mut self.last_agent,
1456                    repo,
1457                    &fb_agent,
1458                    fb_model_opt,
1459                )
1460                .await
1461            {
1462                Ok(result) => {
1463                    record_agent_success(&fb_agent, fb_model_str).await;
1464                    return Ok(result);
1465                }
1466                Err(e) => {
1467                    let err_msg = e.to_string();
1468                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1469                    if is_timeout {
1470                        // Cool down on timeout like pool entries (#3422), else a
1471                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1472                        response::record_model_failure(&fb_agent, fb_model_str).await;
1473                        tracing::warn!(
1474                            agent = %fb_agent,
1475                            model = %fb_model_str,
1476                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1477                        );
1478                        return Err(e);
1479                    }
1480                    tracing::warn!(
1481                        agent = %fb_agent,
1482                        model = %fb_model_str,
1483                        error = %e,
1484                        "fallback router LLM also failed"
1485                    );
1486                    response::record_model_failure(&fb_agent, fb_model_str).await;
1487                    last_err = Some(e);
1488                }
1489            }
1490        }
1491
1492        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1493    }
1494
1495    /// Record a rate limit event for an agent, reducing its routing weight.
1496    ///
1497    /// Called by the engine when an agent returns a 429/rate limit error.
1498    pub fn record_rate_limit(&mut self, agent: &str) {
1499        self.weights.record_rate_limit(agent);
1500    }
1501
1502    /// Record a successful task completion, restoring agent weight.
1503    pub fn record_success(&mut self, agent: &str) {
1504        self.weights.record_success(agent);
1505    }
1506
1507    /// Tick weight recovery for all agents.
1508    ///
1509    /// Called periodically by the engine to gradually restore weights
1510    /// as rate limit windows expire.
1511    pub fn tick_weight_recovery(&mut self) {
1512        self.weights.tick_recovery();
1513    }
1514
1515    /// Store routing result in the task store.
1516    pub async fn store_route_result(
1517        &self,
1518        task_id: &str,
1519        result: &RouteResult,
1520        store: &Arc<TaskStore>,
1521        repo: &str,
1522    ) -> anyhow::Result<()> {
1523        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1524            store
1525                .set_fields(
1526                    store_id,
1527                    &[
1528                        ("agent", serde_json::json!(result.agent)),
1529                        ("complexity", serde_json::json!(result.complexity)),
1530                        ("route_reason", serde_json::json!(result.reason)),
1531                        (
1532                            "model",
1533                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1534                        ),
1535                    ],
1536                )
1537                .await?;
1538        }
1539        Ok(())
1540    }
1541}
1542
1543/// Retrieve routing result from the task store.
1544pub async fn get_route_result(
1545    store: &Arc<TaskStore>,
1546    repo: &str,
1547    task_id: &str,
1548) -> Result<RouteResult, RouteResultError> {
1549    let store_id = store
1550        .resolve_task_id(repo, task_id)
1551        .await
1552        .map_err(RouteResultError::Other)?
1553        .ok_or_else(|| {
1554            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1555        })?;
1556    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1557
1558    let agent = task
1559        .agent
1560        .clone()
1561        .filter(|a| !a.is_empty())
1562        .ok_or_else(|| RouteResultError::NoAgent {
1563            task_id: task_id.to_string(),
1564        })?;
1565
1566    let complexity = if task.complexity.is_empty() {
1567        "medium".to_string()
1568    } else {
1569        task.complexity.clone()
1570    };
1571
1572    let reason = task.route_reason.clone();
1573    let model = task.model.clone().filter(|m| !m.is_empty());
1574
1575    // Validate that the stored model is valid for the resolved agent.
1576    // Agent-specific aliases (e.g. "opus") can leak across agents during
1577    // failover, so we verify against the router's model pools (#1604).
1578    let config = RouterConfig::from_config();
1579    let model = model.filter(|m| {
1580        let complexities = ["simple", "medium", "complex", "review"];
1581        // Collect all model pools configured for this agent across complexity tiers.
1582        let all_pools: Vec<Vec<String>> = complexities
1583            .iter()
1584            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1585            .collect();
1586        // If the agent has no model pools configured, we can't validate — pass through.
1587        if all_pools.is_empty() {
1588            return true;
1589        }
1590        let valid = all_pools.iter().any(|pool| pool.contains(m));
1591        if !valid {
1592            tracing::warn!(
1593                task_id,
1594                agent,
1595                model = %m,
1596                "discarding stale model: not in agent's model pools"
1597            );
1598        }
1599        valid
1600    });
1601
1602    // A stale model is replaced by the agent's model for the task's complexity, an empty
1603    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1604    let original_model = task.model.clone().filter(|m| !m.is_empty());
1605    let mut model = model;
1606    if original_model.is_some() && model.is_none() {
1607        model = config.model_for_complexity(&agent, &complexity, task_id);
1608        let stale_model = original_model.as_deref().unwrap_or("unknown");
1609        let stale_err = format!(
1610            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1611            model.as_deref().unwrap_or("the agent default")
1612        );
1613        if let Err(e) = store
1614            .set_fields(
1615                store_id,
1616                &[
1617                    (
1618                        "model",
1619                        serde_json::json!(model.clone().unwrap_or_default()),
1620                    ),
1621                    ("last_error", serde_json::json!(stale_err)),
1622                ],
1623            )
1624            .await
1625        {
1626            // Log the error so transient DB failures are visible and diagnosable.
1627            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1628        }
1629    }
1630
1631    Ok(RouteResult {
1632        agent,
1633        model,
1634        complexity,
1635        estimate: 0,
1636        reason,
1637        profile: AgentProfile::default(),
1638        selected_skills: vec![],
1639        warning: None,
1640    })
1641}
1642
1643#[cfg(test)]
1644#[expect(
1645    clippy::float_cmp,
1646    reason = "the weights are stored constants, so they compare exactly"
1647)]
1648mod tests {
1649    use super::config::DEFAULT_AGENTS;
1650    use super::llm::LlmRouteResponse;
1651    use super::strategies;
1652    use super::weights::default_missing_weight;
1653    use super::weights::{
1654        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1655    };
1656    use super::*;
1657    use crate::backends::{ExternalId, ExternalTask};
1658    use crate::engine::cooldown::{
1659        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1660        record_model_failure, set_model_cooldown,
1661    };
1662    use crate::store::NewTask;
1663    use crate::store::TaskStore;
1664    use serial_test::serial;
1665    use std::collections::HashMap;
1666    use std::env;
1667    #[cfg(unix)]
1668    use std::ffi::OsString;
1669    use std::fs;
1670    #[cfg(unix)]
1671    use std::fs::Permissions;
1672    #[cfg(unix)]
1673    use std::os::unix::fs::PermissionsExt;
1674    #[cfg(unix)]
1675    use std::process;
1676    #[cfg(unix)]
1677    use std::process::Command;
1678    #[cfg(unix)]
1679    use std::thread;
1680    use std::time::{Duration, Instant};
1681
1682    #[tokio::test]
1683    #[serial_test::serial(orch_home)]
1684    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1685        let home = tempfile::tempdir().unwrap();
1686        fs::write(
1687            home.path().join("config.yml"),
1688            "model_map:\n  medium:\n    claude: sonnet\n",
1689        )
1690        .unwrap();
1691        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1692        unsafe { env::set_var("ORCH_HOME", home.path()) };
1693
1694        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1695        let store_id = store
1696            .create(&NewTask {
1697                external_id: Some("7".to_string()),
1698                repo: "owner/repo".to_string(),
1699                origin: "github".to_string(),
1700                title: "stale model".to_string(),
1701                ..Default::default()
1702            })
1703            .await
1704            .unwrap();
1705        store
1706            .set_fields(
1707                store_id,
1708                &[
1709                    ("agent", serde_json::json!("claude")),
1710                    ("model", serde_json::json!("claude-removed-model")),
1711                    ("complexity", serde_json::json!("medium")),
1712                ],
1713            )
1714            .await
1715            .unwrap();
1716
1717        let route = get_route_result(&store, "owner/repo", "7").await;
1718        let stored = store.get(store_id).await.unwrap().model;
1719        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1720        unsafe { env::remove_var("ORCH_HOME") };
1721
1722        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1723        assert_eq!(stored.as_deref(), Some("sonnet"));
1724    }
1725
1726    // Test-only delegate so tests can call router.parse_llm_response() directly
1727    // without referencing llm_router.
1728    impl Router {
1729        #[expect(
1730            clippy::unused_self,
1731            reason = "the test helper keeps the router method syntax"
1732        )]
1733        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1734            LlmRouter::parse_llm_response("claude", response)
1735        }
1736    }
1737
1738    async fn test_store() -> Arc<TaskStore> {
1739        Arc::new(TaskStore::open_memory().await.unwrap())
1740    }
1741
1742    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1743        ExternalTask {
1744            id: ExternalId(id.to_string()),
1745            title: title.to_string(),
1746            body: "Test body".to_string(),
1747            state: "open".to_string(),
1748            labels,
1749            author: "test".to_string(),
1750            created_at: "2026-01-01T00:00:00Z".to_string(),
1751            updated_at: "2026-01-01T00:00:00Z".to_string(),
1752            url: format!("https://github.com/test/test/issues/{id}"),
1753        }
1754    }
1755
1756    #[serial(cooldown_state)]
1757    #[test]
1758    fn extract_agent_from_labels() {
1759        let mut config = RouterConfig::default();
1760        // Ensure a concrete model exists for the labeled agent so the label
1761        // override is honored by the router (routing requires a model).
1762        config
1763            .model_map
1764            .entry("medium".to_string())
1765            .or_default()
1766            .insert("claude".to_string(), vec!["haiku".to_string()]);
1767
1768        assert_eq!(
1769            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1770            Some("claude".to_string())
1771        );
1772        assert_eq!(
1773            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1774            Some("codex".to_string())
1775        );
1776        assert_eq!(
1777            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1778            Some("opencode".to_string())
1779        );
1780        assert_eq!(
1781            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1782            None
1783        );
1784        // Verify kimi and minimax are recognized from labels
1785        assert_eq!(
1786            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1787            Some("kimi".to_string())
1788        );
1789        assert_eq!(
1790            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1791            Some("minimax".to_string())
1792        );
1793    }
1794
1795    #[serial(cooldown_state)]
1796    #[tokio::test]
1797    async fn all_agents_cooled_error_names_degraded_agents() {
1798        cooldown::reset_global_state().await;
1799        mark_agent_degraded("claude");
1800        mark_agent_degraded("grok");
1801        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1802        let msg = err.to_string();
1803        assert!(
1804            msg.contains("degraded: claude, grok"),
1805            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1806        );
1807        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1808        clear_agent_degraded("claude");
1809        clear_agent_degraded("grok");
1810    }
1811
1812    #[serial(cooldown_state)]
1813    #[test]
1814    fn default_agents_constant() {
1815        assert_eq!(DEFAULT_AGENTS.len(), 7);
1816        assert!(DEFAULT_AGENTS.contains(&"claude"));
1817        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1818        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1819        assert!(DEFAULT_AGENTS.contains(&"glm"));
1820        assert!(DEFAULT_AGENTS.contains(&"grok"));
1821    }
1822
1823    #[serial(cooldown_state)]
1824    #[test]
1825    fn extract_complexity_from_labels() {
1826        assert_eq!(
1827            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1828            "simple"
1829        );
1830        assert_eq!(
1831            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1832            "medium"
1833        );
1834        assert_eq!(
1835            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1836            "complex"
1837        );
1838        assert_eq!(
1839            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1840            "medium"
1841        );
1842    }
1843
1844    #[serial(cooldown_state)]
1845    #[test]
1846    fn route_result_serialization() {
1847        let result = RouteResult {
1848            agent: "claude".to_string(),
1849            model: Some("sonnet".to_string()),
1850            complexity: "medium".to_string(),
1851            estimate: 5,
1852            reason: "test".to_string(),
1853            profile: AgentProfile {
1854                role: "backend".to_string(),
1855                skills: vec!["rust".to_string()],
1856                tools: vec!["git".to_string()],
1857                constraints: vec![],
1858            },
1859            selected_skills: vec!["gh".to_string()],
1860            warning: None,
1861        };
1862
1863        let json = serde_json::to_string(&result).unwrap();
1864        assert!(json.contains("claude"));
1865        assert!(json.contains("sonnet"));
1866
1867        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1868        assert_eq!(deserialized.agent, "claude");
1869        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1870    }
1871
1872    #[serial(cooldown_state)]
1873    #[test]
1874    fn router_config_default() {
1875        let mut config = RouterConfig::default();
1876        // Ensure a concrete model exists for the labeled agent so the label
1877        // override is honored by the router (routing requires a model).
1878        config
1879            .model_map
1880            .entry("medium".to_string())
1881            .or_default()
1882            .insert("claude".to_string(), vec!["haiku".to_string()]);
1883
1884        assert_eq!(config.mode, "round_robin");
1885        assert_eq!(config.router_agent, "claude");
1886        assert_eq!(config.router_model, "haiku");
1887        assert_eq!(config.fallback_executor, "codex");
1888        assert_eq!(config.max_route_attempts, 3);
1889        assert_ne!(config.allowed_tools, [] as [String; 0]);
1890        assert_ne!(config.default_skills, [] as [String; 0]);
1891
1892        // Verify configurable agents list includes all 7 agents
1893        assert_eq!(config.agents.len(), 7);
1894        assert!(config.agents.contains(&"claude".to_string()));
1895        assert!(config.agents.contains(&"codex".to_string()));
1896        assert!(config.agents.contains(&"opencode".to_string()));
1897        assert!(config.agents.contains(&"kimi".to_string()));
1898        assert!(config.agents.contains(&"minimax".to_string()));
1899        assert!(config.agents.contains(&"glm".to_string()));
1900        assert!(config.agents.contains(&"grok".to_string()));
1901    }
1902
1903    #[serial(cooldown_state)]
1904    #[test]
1905    fn model_map_lookup_returns_none_without_config() {
1906        // Default config has no hardcoded models — config.yml is the source of truth.
1907        let config = RouterConfig::default();
1908        assert!(
1909            config
1910                .model_for_complexity("claude", "simple", "")
1911                .is_none()
1912        );
1913        assert!(
1914            config
1915                .model_for_complexity("claude", "medium", "")
1916                .is_none()
1917        );
1918        assert!(
1919            config
1920                .model_for_complexity("claude", "complex", "")
1921                .is_none()
1922        );
1923        assert!(
1924            config
1925                .model_for_complexity("opencode", "review", "")
1926                .is_none()
1927        );
1928    }
1929
1930    #[serial(cooldown_state)]
1931    #[test]
1932    fn model_map_lookup_returns_configured_value() {
1933        // When model_map is populated (from config), it is returned correctly.
1934        let mut config = RouterConfig::default();
1935        config
1936            .model_map
1937            .entry("simple".to_string())
1938            .or_default()
1939            .insert("claude".to_string(), vec!["haiku".to_string()]);
1940        assert_eq!(
1941            config.model_for_complexity("claude", "simple", ""),
1942            Some("haiku".to_string())
1943        );
1944    }
1945
1946    #[serial(cooldown_state)]
1947    #[tokio::test]
1948    async fn model_pool_selection_skips_cooled() {
1949        cooldown::reset_global_state().await;
1950
1951        let mut config = RouterConfig::default();
1952        // Set up a two-model pool for opencode/simple
1953        config
1954            .model_map
1955            .entry("simple".to_string())
1956            .or_default()
1957            .insert(
1958                "opencode".to_string(),
1959                vec!["model-a".to_string(), "model-b".to_string()],
1960            );
1961
1962        // Before any cooldown, both models are candidates
1963        let m = config.model_for_complexity("opencode", "simple", "task-1");
1964        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1965
1966        // Cool model-a
1967        record_model_failure("opencode", "model-a").await;
1968        assert!(is_model_in_cooldown("opencode", "model-a"));
1969
1970        // Now only model-b should be returned
1971        // (try many task_ids to rule out hash coincidence)
1972        for i in 0..20 {
1973            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1974            assert_eq!(
1975                result,
1976                Some("model-b".to_string()),
1977                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1978            );
1979        }
1980
1981        // Cool model-b too — now all models are cooled
1982        record_model_failure("opencode", "model-b").await;
1983        // All cooled → return None so the caller can fall back to a different agent
1984        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1985        assert_eq!(fallback, None);
1986    }
1987
1988    #[serial(cooldown_state)]
1989    #[test]
1990    fn model_pool_single_string_backward_compat() {
1991        // Single-item pools behave identically to the old string format.
1992        // Manually insert a single-item pool and verify it is returned correctly.
1993        let mut config = RouterConfig::default();
1994        config
1995            .model_map
1996            .entry("simple".to_string())
1997            .or_default()
1998            .insert("claude".to_string(), vec!["haiku".to_string()]);
1999        let m = config.model_for_complexity("claude", "simple", "any-task");
2000        assert_eq!(m, Some("haiku".to_string()));
2001    }
2002
2003    #[serial(cooldown_state)]
2004    #[test]
2005    fn parse_llm_response_direct_json() {
2006        let config = RouterConfig::default();
2007        let router = Router::new(config);
2008
2009        let response = r#"{
2010            "executor": "claude",
2011            "complexity": "complex",
2012            "reason": "requires architecture analysis",
2013            "profile": {
2014                "role": "architect",
2015                "skills": ["rust", "design"],
2016                "tools": ["git", "rg"],
2017                "constraints": []
2018            },
2019            "selected_skills": ["gh"]
2020        }"#;
2021
2022        let parsed = router.parse_llm_response(response).unwrap();
2023        assert_eq!(parsed.executor, "claude");
2024        assert_eq!(parsed.complexity, "complex");
2025        assert_eq!(parsed.reason, "requires architecture analysis");
2026        assert_eq!(parsed.profile.role, "architect");
2027    }
2028
2029    #[serial(cooldown_state)]
2030    #[test]
2031    fn parse_llm_response_markdown_fenced() {
2032        let config = RouterConfig::default();
2033        let router = Router::new(config);
2034
2035        let response = r#"Here's my analysis:
2036
2037```json
2038{
2039    "executor": "codex",
2040    "complexity": "medium",
2041    "reason": "coding task",
2042    "profile": {
2043        "role": "developer",
2044        "skills": ["coding"],
2045        "tools": [],
2046        "constraints": []
2047    },
2048    "selected_skills": []
2049}
2050```
2051
2052Hope that helps!"#;
2053
2054        let parsed = router.parse_llm_response(response).unwrap();
2055        assert_eq!(parsed.executor, "codex");
2056        assert_eq!(parsed.complexity, "medium");
2057    }
2058
2059    #[serial(cooldown_state)]
2060    #[test]
2061    fn parse_llm_response_claude_envelope() {
2062        let config = RouterConfig::default();
2063        let router = Router::new(config);
2064
2065        // Claude --output-format json wraps the response in an envelope
2066        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2067
2068        let parsed = router.parse_llm_response(response).unwrap();
2069        assert_eq!(parsed.executor, "claude");
2070        assert_eq!(parsed.complexity, "complex");
2071        assert_eq!(parsed.reason, "requires deep analysis");
2072    }
2073
2074    #[serial(cooldown_state)]
2075    #[test]
2076    fn parse_llm_response_claude_envelope_with_code_block() {
2077        let config = RouterConfig::default();
2078        let router = Router::new(config);
2079
2080        // Claude may return the JSON inside a code block within the envelope
2081        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2082        let envelope = serde_json::json!({
2083            "type": "result",
2084            "subtype": "success",
2085            "is_error": false,
2086            "result": inner,
2087            "usage": {"input_tokens": 50, "output_tokens": 30}
2088        });
2089
2090        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2091        assert_eq!(parsed.executor, "codex");
2092        assert_eq!(parsed.complexity, "simple");
2093    }
2094
2095    #[serial(cooldown_state)]
2096    #[test]
2097    fn parse_llm_response_claude_envelope_error() {
2098        let config = RouterConfig::default();
2099        let router = Router::new(config);
2100
2101        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2102
2103        let err = router.parse_llm_response(response).unwrap_err();
2104        assert!(err.to_string().contains("error"), "got: {err}");
2105    }
2106
2107    #[serial(cooldown_state)]
2108    #[test]
2109    fn parse_llm_response_empty() {
2110        let config = RouterConfig::default();
2111        let router = Router::new(config);
2112
2113        let err = router.parse_llm_response("").unwrap_err();
2114        assert!(err.to_string().contains("empty"), "got: {err}");
2115    }
2116
2117    /// Claude envelope where result is a JSON object, not a string.
2118    /// Some Claude versions may return the result as a nested object.
2119    #[serial(cooldown_state)]
2120    #[test]
2121    fn parse_llm_response_claude_envelope_result_as_object() {
2122        let config = RouterConfig::default();
2123        let router = Router::new(config);
2124
2125        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2126
2127        let parsed = router.parse_llm_response(response);
2128        assert!(
2129            parsed.is_ok(),
2130            "should handle result-as-object: {}",
2131            parsed.unwrap_err()
2132        );
2133        assert_eq!(parsed.unwrap().executor, "claude");
2134    }
2135
2136    /// LLM returns "agent" instead of "executor" — common hallucination.
2137    #[serial(cooldown_state)]
2138    #[test]
2139    fn parse_llm_response_agent_instead_of_executor() {
2140        let config = RouterConfig::default();
2141        let router = Router::new(config);
2142
2143        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2144
2145        let parsed = router.parse_llm_response(response);
2146        assert!(
2147            parsed.is_ok(),
2148            "should accept 'agent' alias: {}",
2149            parsed.unwrap_err()
2150        );
2151        assert_eq!(parsed.unwrap().executor, "claude");
2152    }
2153
2154    /// LLM returns minimal JSON without profile or reason.
2155    #[serial(cooldown_state)]
2156    #[test]
2157    fn parse_llm_response_minimal_json() {
2158        let config = RouterConfig::default();
2159        let router = Router::new(config);
2160
2161        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2162
2163        let parsed = router.parse_llm_response(response);
2164        assert!(
2165            parsed.is_ok(),
2166            "should accept minimal JSON: {}",
2167            parsed.unwrap_err()
2168        );
2169        assert_eq!(parsed.unwrap().executor, "codex");
2170    }
2171
2172    // ── Real fixture tests: route prompt → response → parsed result ──
2173
2174    /// Real Claude envelope with result as escaped JSON string.
2175    #[serial(cooldown_state)]
2176    #[test]
2177    fn fixture_route_response_string() {
2178        let config = RouterConfig::default();
2179        let router = Router::new(config);
2180
2181        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2182        let parsed = router.parse_llm_response(response).unwrap();
2183        assert_eq!(parsed.executor, "codex");
2184        assert_eq!(parsed.complexity, "medium");
2185        assert_ne!(parsed.reason, "");
2186        assert_ne!(parsed.profile.role, "");
2187    }
2188
2189    /// Real Claude envelope with result as JSON object (not string).
2190    /// This was the root cause of all router failures on 2026-02-27.
2191    #[serial(cooldown_state)]
2192    #[test]
2193    fn fixture_route_response_object() {
2194        let config = RouterConfig::default();
2195        let router = Router::new(config);
2196
2197        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2198        let parsed = router.parse_llm_response(response).unwrap();
2199        assert_eq!(parsed.executor, "claude");
2200        assert_eq!(parsed.complexity, "complex");
2201    }
2202
2203    /// Real Claude envelope with result as string containing markdown + JSON.
2204    #[serial(cooldown_state)]
2205    #[test]
2206    fn fixture_route_response_markdown() {
2207        let config = RouterConfig::default();
2208        let router = Router::new(config);
2209
2210        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2211        let parsed = router.parse_llm_response(response).unwrap();
2212        assert_eq!(parsed.executor, "codex");
2213        assert_eq!(parsed.complexity, "medium");
2214    }
2215
2216    /// Claude envelope with escaped JSON string containing newlines and markdown.
2217    #[serial(cooldown_state)]
2218    #[test]
2219    fn parse_llm_response_claude_envelope_with_prose() {
2220        let config = RouterConfig::default();
2221        let router = Router::new(config);
2222
2223        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2224        let envelope = serde_json::json!({
2225            "type": "result",
2226            "subtype": "success",
2227            "is_error": false,
2228            "result": inner,
2229            "usage": {"input_tokens": 100, "output_tokens": 80}
2230        });
2231
2232        let parsed = router.parse_llm_response(&envelope.to_string());
2233        assert!(
2234            parsed.is_ok(),
2235            "should handle prose + code block in envelope: {}",
2236            parsed.unwrap_err()
2237        );
2238        assert_eq!(parsed.unwrap().executor, "codex");
2239    }
2240
2241    #[serial(cooldown_state)]
2242    #[tokio::test]
2243    async fn route_round_robin_basic() {
2244        cooldown::reset_global_state().await;
2245        // Force at least one agent to be available for testing
2246        // In real usage, discover_agents finds installed CLIs
2247        let config = RouterConfig {
2248            mode: "round_robin".to_string(),
2249            ..Default::default()
2250        };
2251
2252        // Create router with mock available agents
2253        let agents = vec!["claude".to_string(), "codex".to_string()];
2254        let mut weights = AgentWeights::default();
2255        weights.ensure_agents(&agents);
2256        let router = Router {
2257            config,
2258            available_agents: agents,
2259            weights,
2260            llm_router: Arc::new(LlmRouter::new()),
2261            ollama_router: None,
2262            rr_index: 0,
2263            last_agent: None,
2264            review_rr_index: 0,
2265            router_pool: vec![],
2266            pool_index: 0,
2267        };
2268
2269        let task = create_test_task("1", "Test task", vec![]);
2270        let result =
2271            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2272                .unwrap();
2273
2274        // Task 1 % 2 agents = agent at index 1 = codex
2275        assert_eq!(result.agent, "codex");
2276        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2277    }
2278
2279    #[serial(cooldown_state)]
2280    #[tokio::test]
2281    async fn route_uses_label_override() {
2282        cooldown::reset_global_state().await;
2283        let mut config = RouterConfig::default();
2284        // Ensure a concrete model exists so the label override can be honored
2285        // in the test environment where no external config.yml is present.
2286        config
2287            .model_map
2288            .entry("medium".to_string())
2289            .or_default()
2290            .insert("claude".to_string(), vec!["haiku".to_string()]);
2291        let agents = vec!["claude".to_string(), "codex".to_string()];
2292        let mut weights = AgentWeights::default();
2293        weights.ensure_agents(&agents);
2294        let mut router = Router {
2295            config,
2296            available_agents: agents,
2297            weights,
2298            llm_router: Arc::new(LlmRouter::new()),
2299            ollama_router: None,
2300            rr_index: 0,
2301            last_agent: None,
2302            review_rr_index: 0,
2303            router_pool: vec![],
2304            pool_index: 0,
2305        };
2306
2307        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2308        let store = test_store().await;
2309
2310        // Should use label override, not LLM
2311        let result = router.route(&task, &store, "test/repo").await.unwrap();
2312        assert_eq!(result.agent, "claude");
2313        assert!(result.reason.contains("label"));
2314    }
2315
2316    #[serial(cooldown_state)]
2317    #[test]
2318    fn check_routing_sanity_warnings() {
2319        // Backend task routed to claude should warn
2320        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2321        let profile = AgentProfile {
2322            role: "general".to_string(),
2323            skills: vec!["api".to_string()],
2324            tools: vec![],
2325            constraints: vec![],
2326        };
2327        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2328        assert!(warning.is_some());
2329        assert!(warning.unwrap().contains("backend"));
2330
2331        // Docs task routed to codex should warn
2332        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2333        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2334        assert!(warning.is_some());
2335        assert!(warning.unwrap().contains("docs"));
2336
2337        // Normal routing should not warn
2338        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2339        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2340        assert!(warning.is_none());
2341    }
2342
2343    #[serial(cooldown_state)]
2344    #[tokio::test]
2345    async fn router_reload_preserves_structure() {
2346        cooldown::reset_global_state().await;
2347        let config = RouterConfig::default();
2348        let agents = vec!["claude".to_string()];
2349        let mut weights = AgentWeights::default();
2350        weights.ensure_agents(&agents);
2351        let mut router = Router {
2352            config,
2353            available_agents: agents,
2354            weights,
2355            llm_router: Arc::new(LlmRouter::new()),
2356            ollama_router: None,
2357            rr_index: 0,
2358            last_agent: None,
2359            review_rr_index: 0,
2360            router_pool: vec![],
2361            pool_index: 0,
2362        };
2363
2364        // Reload — should re-read config and remain valid
2365        router.reload_async().await;
2366
2367        // After reload, mode should be a valid value (llm, local, or round_robin)
2368        assert!(
2369            router.config.mode == "llm"
2370                || router.config.mode == "round_robin"
2371                || router.config.mode == "local",
2372            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2373            router.config.mode
2374        );
2375        // Fallback executor should always be set
2376        assert_ne!(router.config.fallback_executor, "");
2377        // Tools should always be populated
2378        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2379    }
2380
2381    // --- Weighted round-robin tests ---
2382
2383    #[serial(cooldown_state)]
2384    #[test]
2385    fn rate_limit_state_defaults_to_full_weight() {
2386        let state = RateLimitState::default();
2387        assert_eq!(state.weight, DEFAULT_WEIGHT);
2388        assert_eq!(state.consecutive_hits, 0);
2389        assert!(state.last_limited_at.is_none());
2390        assert!(!state.is_limited());
2391    }
2392
2393    #[serial(cooldown_state)]
2394    #[test]
2395    fn rate_limit_state_decays_on_hit() {
2396        let mut state = RateLimitState::default();
2397        state.record_rate_limit();
2398
2399        assert!(state.weight < DEFAULT_WEIGHT);
2400        assert_eq!(state.consecutive_hits, 1);
2401        assert!(state.last_limited_at.is_some());
2402        assert!(state.is_limited());
2403
2404        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2405        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2406        assert!((state.weight - expected).abs() < 1e-10);
2407    }
2408
2409    #[serial(cooldown_state)]
2410    #[test]
2411    fn rate_limit_state_never_drops_below_min() {
2412        let mut state = RateLimitState::default();
2413
2414        // Hit many times
2415        for _ in 0..100 {
2416            state.record_rate_limit();
2417        }
2418
2419        assert!(state.weight >= MIN_WEIGHT);
2420        assert_eq!(state.consecutive_hits, 100);
2421    }
2422
2423    #[serial(cooldown_state)]
2424    #[test]
2425    fn rate_limit_state_recovers_on_success() {
2426        let mut state = RateLimitState::default();
2427
2428        // Decay first
2429        state.record_rate_limit();
2430        state.record_rate_limit();
2431        let after_decay = state.weight;
2432
2433        // Record success
2434        state.record_success();
2435        assert!(state.weight > after_decay);
2436        assert_eq!(state.consecutive_hits, 0);
2437    }
2438
2439    #[serial(cooldown_state)]
2440    #[test]
2441    fn rate_limit_state_success_caps_at_default() {
2442        let mut state = RateLimitState::default();
2443
2444        // Already at full weight, success shouldn't exceed it
2445        state.record_success();
2446        assert_eq!(state.weight, DEFAULT_WEIGHT);
2447    }
2448
2449    #[serial(cooldown_state)]
2450    #[test]
2451    fn agent_weights_ensure_agents() {
2452        let mut weights = AgentWeights::default();
2453        let agents = vec!["claude".to_string(), "codex".to_string()];
2454        weights.ensure_agents(&agents);
2455
2456        assert_eq!(weights.states.len(), 2);
2457        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2458        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2459    }
2460
2461    #[serial(cooldown_state)]
2462    #[test]
2463    fn agent_weights_ensure_agents_preserves_existing() {
2464        let mut weights = AgentWeights::default();
2465        let agents = vec!["claude".to_string(), "codex".to_string()];
2466        weights.ensure_agents(&agents);
2467
2468        // Reduce claude's weight
2469        weights.record_rate_limit("claude");
2470        let claude_weight = weights.get_weight("claude");
2471        assert!(claude_weight < DEFAULT_WEIGHT);
2472
2473        // Ensure agents again — claude's weight should be preserved
2474        let agents2 = vec![
2475            "claude".to_string(),
2476            "codex".to_string(),
2477            "opencode".to_string(),
2478        ];
2479        weights.ensure_agents(&agents2);
2480
2481        assert_eq!(weights.get_weight("claude"), claude_weight);
2482        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2483    }
2484
2485    #[test]
2486    fn default_missing_weight_uses_minimum_of_configured() {
2487        let mut configured = HashMap::new();
2488        configured.insert("claude".to_string(), 0.2);
2489        configured.insert("codex".to_string(), 0.2);
2490        configured.insert("opencode".to_string(), 0.2);
2491        configured.insert("kimi".to_string(), 0.2);
2492
2493        // minimax omitted from router.weights — must not silently outrank
2494        // every explicitly configured agent (issue #3596).
2495        assert_eq!(default_missing_weight(&configured), 0.2);
2496    }
2497
2498    #[test]
2499    fn default_missing_weight_falls_back_when_none_configured() {
2500        let configured = HashMap::new();
2501        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2502    }
2503
2504    #[serial(cooldown_state)]
2505    #[test]
2506    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2507        let mut weights = AgentWeights::default();
2508        let mut configured = HashMap::new();
2509        configured.insert("claude".to_string(), 0.2);
2510        configured.insert("codex".to_string(), 0.2);
2511        configured.insert("opencode".to_string(), 0.2);
2512        configured.insert("kimi".to_string(), 0.2);
2513
2514        let agents = vec![
2515            "claude".to_string(),
2516            "codex".to_string(),
2517            "opencode".to_string(),
2518            "kimi".to_string(),
2519            "minimax".to_string(),
2520        ];
2521        weights.ensure_agents_with_weights(&agents, &configured);
2522
2523        // minimax was omitted from router.weights but must not get the old
2524        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2525        assert_eq!(weights.get_weight("minimax"), 0.2);
2526        assert_eq!(weights.get_weight("claude"), 0.2);
2527    }
2528
2529    #[serial(cooldown_state)]
2530    #[test]
2531    fn agent_weights_record_rate_limit() {
2532        let mut weights = AgentWeights::default();
2533        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2534
2535        weights.record_rate_limit("claude");
2536
2537        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2538        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2539    }
2540
2541    #[serial(cooldown_state)]
2542    #[test]
2543    fn agent_weights_record_success() {
2544        let mut weights = AgentWeights::default();
2545        weights.ensure_agents(&["claude".to_string()]);
2546
2547        weights.record_rate_limit("claude");
2548        let after_limit = weights.get_weight("claude");
2549
2550        weights.record_success("claude");
2551        assert!(weights.get_weight("claude") > after_limit);
2552    }
2553
2554    #[serial(cooldown_state)]
2555    #[test]
2556    fn agent_weights_snapshot() {
2557        let mut weights = AgentWeights::default();
2558        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2559        weights.record_rate_limit("claude");
2560
2561        let snap = weights.snapshot();
2562        assert_eq!(snap.len(), 2);
2563
2564        // Snapshot is sorted alphabetically
2565        assert_eq!(snap[0].0, "claude");
2566        assert!(snap[0].1 < DEFAULT_WEIGHT);
2567        assert_eq!(snap[0].2, 1); // 1 hit
2568
2569        assert_eq!(snap[1].0, "codex");
2570        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2571        assert_eq!(snap[1].2, 0); // 0 hits
2572    }
2573
2574    #[serial(cooldown_state)]
2575    #[test]
2576    fn agent_weights_weighted_select_favors_higher_weight() {
2577        let mut weights = AgentWeights::default();
2578        let agents = vec!["claude".to_string(), "codex".to_string()];
2579        weights.ensure_agents(&agents);
2580
2581        // Heavily penalize claude
2582        for _ in 0..10 {
2583            weights.record_rate_limit("claude");
2584        }
2585
2586        // Run many selections and count
2587        let mut claude_count = 0;
2588        let mut codex_count = 0;
2589        for _ in 0..100 {
2590            match weights.weighted_select(&agents, "task-1") {
2591                Some(ref a) if a == "claude" => claude_count += 1,
2592                Some(ref a) if a == "codex" => codex_count += 1,
2593                _ => {}
2594            }
2595        }
2596
2597        // Codex should get significantly more selections
2598        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2599        assert!(
2600            codex_count > claude_count,
2601            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2602        );
2603    }
2604
2605    #[serial(cooldown_state)]
2606    #[test]
2607    fn agent_weights_weighted_select_empty_returns_none() {
2608        let weights = AgentWeights::default();
2609        assert!(weights.weighted_select(&[], "task-1").is_none());
2610    }
2611
2612    #[serial(cooldown_state)]
2613    #[test]
2614    fn agent_weights_batch_routing_produces_varied_selections() {
2615        // Regression test: when routing a batch of tasks in the same tick,
2616        // different task IDs should not all select the same agent.
2617        let mut weights = AgentWeights::default();
2618        let agents = vec!["claude".to_string(), "codex".to_string()];
2619        weights.ensure_agents(&agents);
2620
2621        // Simulate batch routing: 20 different task IDs called in rapid succession
2622        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2623        let selections: Vec<String> = task_ids
2624            .iter()
2625            .filter_map(|id| weights.weighted_select(&agents, id))
2626            .collect();
2627
2628        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2629        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2630        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2631        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2632        assert!(
2633            claude_count > 0 && codex_count > 0,
2634            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2635        );
2636    }
2637
2638    #[serial(cooldown_state)]
2639    #[test]
2640    fn agent_weights_weighted_select_single_agent() {
2641        let mut weights = AgentWeights::default();
2642        let agents = vec!["claude".to_string()];
2643        weights.ensure_agents(&agents);
2644
2645        let selected = weights.weighted_select(&agents, "task-1");
2646        assert_eq!(selected, Some("claude".to_string()));
2647    }
2648
2649    #[serial(cooldown_state)]
2650    #[test]
2651    fn agent_weights_tick_recovery_restores_weight() {
2652        let mut weights = AgentWeights::default();
2653        weights.ensure_agents(&["claude".to_string()]);
2654
2655        // Set up a rate limit that looks like it happened long ago
2656        let state = weights.states.get_mut("claude").unwrap();
2657        state.weight = 0.5;
2658        state.last_limited_at = Some(
2659            Instant::now()
2660                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2661                .expect("the clock is past the recovery delay"),
2662        );
2663        state.consecutive_hits = 2;
2664
2665        let before = weights.get_weight("claude");
2666        weights.tick_recovery();
2667        let after = weights.get_weight("claude");
2668
2669        assert!(after > before, "weight should increase after recovery tick");
2670    }
2671
2672    #[serial(cooldown_state)]
2673    #[test]
2674    fn agent_weights_tick_recovery_clears_on_full_restore() {
2675        let mut weights = AgentWeights::default();
2676        weights.ensure_agents(&["claude".to_string()]);
2677
2678        // Set weight just below full with an old limit
2679        let state = weights.states.get_mut("claude").unwrap();
2680        state.weight = DEFAULT_WEIGHT - 0.01;
2681        state.last_limited_at = Some(
2682            Instant::now()
2683                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2684                .expect("the clock is past the recovery delay"),
2685        );
2686        state.consecutive_hits = 1;
2687
2688        weights.tick_recovery();
2689
2690        let state = weights.states.get("claude").unwrap();
2691        assert_eq!(state.weight, DEFAULT_WEIGHT);
2692        assert!(state.last_limited_at.is_none());
2693        assert_eq!(state.consecutive_hits, 0);
2694    }
2695
2696    #[serial(cooldown_state)]
2697    #[tokio::test]
2698    async fn route_weighted_round_robin_basic() {
2699        cooldown::reset_global_state().await;
2700        let mut config = RouterConfig {
2701            weighted_round_robin: true,
2702            ..Default::default()
2703        };
2704        // Provide a concrete model for codex so label override can be used.
2705        config
2706            .model_map
2707            .entry("medium".to_string())
2708            .or_default()
2709            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2710
2711        let agents = vec!["claude".to_string(), "codex".to_string()];
2712        let mut weights = AgentWeights::default();
2713        weights.ensure_agents(&agents);
2714        let mut router = Router {
2715            config,
2716            available_agents: agents,
2717            weights,
2718            llm_router: Arc::new(LlmRouter::new()),
2719            ollama_router: None,
2720            rr_index: 0,
2721            last_agent: None,
2722            review_rr_index: 0,
2723            router_pool: vec![],
2724            pool_index: 0,
2725        };
2726
2727        let task = create_test_task("1", "Test task", vec![]);
2728        let store = test_store().await;
2729        let result = router.route(&task, &store, "test/repo").await.unwrap();
2730
2731        // Should use weighted_round_robin
2732        assert!(result.reason.contains("weighted_round_robin"));
2733        assert!(
2734            result.agent == "claude" || result.agent == "codex",
2735            "agent should be claude or codex, got '{}'",
2736            result.agent
2737        );
2738    }
2739
2740    #[serial(cooldown_state)]
2741    #[tokio::test]
2742    async fn route_weighted_round_robin_respects_label_override() {
2743        cooldown::reset_global_state().await;
2744        let mut config = RouterConfig {
2745            weighted_round_robin: true,
2746            ..Default::default()
2747        };
2748        // Provide a concrete model for codex so label override can be used.
2749        config
2750            .model_map
2751            .entry("medium".to_string())
2752            .or_default()
2753            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2754
2755        let agents = vec!["claude".to_string(), "codex".to_string()];
2756        let mut weights = AgentWeights::default();
2757        weights.ensure_agents(&agents);
2758        let mut router = Router {
2759            config,
2760            available_agents: agents,
2761            weights,
2762            llm_router: Arc::new(LlmRouter::new()),
2763            ollama_router: None,
2764            rr_index: 0,
2765            last_agent: None,
2766            review_rr_index: 0,
2767            router_pool: vec![],
2768            pool_index: 0,
2769        };
2770
2771        // Label override should take precedence over weighted routing
2772        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2773        let store = test_store().await;
2774        let result = router.route(&task, &store, "test/repo").await.unwrap();
2775        assert_eq!(result.agent, "codex");
2776        assert!(result.reason.contains("label"));
2777    }
2778
2779    #[serial(cooldown_state)]
2780    #[tokio::test]
2781    async fn route_takes_the_adjacent_tier_model_when_the_own_tier_is_cooled() {
2782        cooldown::reset_global_state().await;
2783        let agent = "tierroute_agent";
2784        let mut config = RouterConfig {
2785            agents: vec![agent.to_string()],
2786            ..Default::default()
2787        };
2788        config.model_map.clear();
2789        for (tier, model) in [("medium", "tr-m"), ("complex", "tr-c")] {
2790            config
2791                .model_map
2792                .entry(tier.to_string())
2793                .or_default()
2794                .insert(agent.to_string(), vec![model.to_string()]);
2795        }
2796        cooldown::record_model_failure(agent, "tr-m").await;
2797        let mut router = Router::new_for_test(config, vec![agent.to_string()]);
2798
2799        let task = create_test_task(
2800            "1",
2801            "Test task",
2802            vec![format!("agent:{agent}"), "complexity:medium".to_string()],
2803        );
2804        let store = test_store().await;
2805        let result = router.route(&task, &store, "test/repo").await.unwrap();
2806        assert_eq!(result.agent, agent);
2807        assert_eq!(result.model.as_deref(), Some("tr-c"));
2808        assert_eq!(result.complexity, "medium");
2809        assert!(result.warning.unwrap().contains("complex tier"));
2810    }
2811
2812    #[serial(cooldown_state)]
2813    #[test]
2814    fn router_config_weighted_round_robin_default_false() {
2815        let config = RouterConfig::default();
2816        assert!(!config.weighted_round_robin);
2817    }
2818
2819    #[serial(cooldown_state)]
2820    #[test]
2821    fn router_record_and_recover_weights() {
2822        let config = RouterConfig {
2823            weighted_round_robin: true,
2824            ..Default::default()
2825        };
2826        let mut router = Router::new(config);
2827        // Override discovered agents for test
2828        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2829        router.weights.ensure_agents(&router.available_agents);
2830
2831        // Record rate limit
2832        router.record_rate_limit("claude");
2833        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2834        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2835
2836        // Record success for claude
2837        router.record_success("claude");
2838        let after_success = router.weights.get_weight("claude");
2839        assert!(after_success > MIN_WEIGHT);
2840    }
2841
2842    #[serial(cooldown_state)]
2843    #[test]
2844    fn review_rr_index_advances() {
2845        let mut config = RouterConfig::default();
2846        // Set up model_map so has_available_model_for_complexity returns true for test agents
2847        for agent in &["test_a", "test_b", "test_c"] {
2848            config
2849                .model_map
2850                .entry("medium".to_string())
2851                .or_default()
2852                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2853        }
2854        let agents = vec![
2855            "test_a".to_string(),
2856            "test_b".to_string(),
2857            "test_c".to_string(),
2858        ];
2859        let mut weights = AgentWeights::default();
2860        weights.ensure_agents(&agents);
2861        let mut router = Router {
2862            config,
2863            available_agents: agents,
2864            weights,
2865            llm_router: Arc::new(LlmRouter::new()),
2866            ollama_router: None,
2867            rr_index: 0,
2868            last_agent: None,
2869            review_rr_index: 0,
2870            router_pool: vec![],
2871            pool_index: 0,
2872        };
2873
2874        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2875        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2876        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2877
2878        // All three agents should appear (order depends on start index)
2879        let mut seen = vec![a1, a2, a3];
2880        seen.sort();
2881        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2882    }
2883
2884    #[serial(cooldown_state)]
2885    #[test]
2886    fn review_rr_fallback_with_cooled_agent() {
2887        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2888        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2889        // candidates.get(idx) to return None even though candidates is non-empty.
2890        let mut config = RouterConfig::default();
2891        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2892        for agent in &["test_a", "test_b", "test_c"] {
2893            config
2894                .model_map
2895                .entry("medium".to_string())
2896                .or_default()
2897                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2898        }
2899        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2900        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2901        let mut weights = AgentWeights::default();
2902        weights.ensure_agents(&agents);
2903        let mut router = Router {
2904            config,
2905            available_agents: agents,
2906            weights,
2907            llm_router: Arc::new(LlmRouter::new()),
2908            ollama_router: None,
2909            rr_index: 0,
2910            last_agent: None,
2911            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2912            review_rr_index: 2,
2913            router_pool: vec![],
2914            pool_index: 0,
2915        };
2916
2917        // All candidates excluded — should fall back to first candidate, not None
2918        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2919        assert!(
2920            result.is_some(),
2921            "next_round_robin_agent must not return None when candidates is non-empty"
2922        );
2923    }
2924
2925    #[serial(cooldown_state)]
2926    #[test]
2927    fn review_rr_index_uses_candidates_len_not_available_len() {
2928        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2929        // not available_agents.len(). When an agent has no model for the complexity,
2930        // candidates.len() < available_agents.len() and the old code produced a drifting
2931        // index that caused non-uniform distribution.
2932        let mut config = RouterConfig::default();
2933        // Only register models for test_a and test_b — test_c has no model, so it is
2934        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2935        for agent in &["test_a", "test_b"] {
2936            config
2937                .model_map
2938                .entry("medium".to_string())
2939                .or_default()
2940                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2941        }
2942        let agents = vec![
2943            "test_a".to_string(),
2944            "test_b".to_string(),
2945            "test_c".to_string(), // available but no model → filtered out of candidates
2946        ];
2947        let mut weights = AgentWeights::default();
2948        weights.ensure_agents(&agents);
2949        let mut router = Router {
2950            config,
2951            available_agents: agents,
2952            weights,
2953            llm_router: Arc::new(LlmRouter::new()),
2954            ollama_router: None,
2955            rr_index: 0,
2956            last_agent: None,
2957            review_rr_index: 0,
2958            router_pool: vec![],
2959            pool_index: 0,
2960        };
2961
2962        // With the bug: index advances modulo 3 (available_agents.len()), so the
2963        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2964        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2965        // looks uniform by accident, but adding a 4th available agent breaks it.
2966        //
2967        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2968        // cycle. Verify this by checking review_rr_index after each call.
2969
2970        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2971        assert_eq!(
2972            router.review_rr_index, 1,
2973            "index should be 1 after first call"
2974        );
2975
2976        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2977        assert_eq!(
2978            router.review_rr_index, 0,
2979            "index should wrap to 0 after second call"
2980        );
2981
2982        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2983        assert_eq!(
2984            router.review_rr_index, 1,
2985            "index should be 1 after third call"
2986        );
2987
2988        // Agents must alternate uniformly
2989        assert_ne!(r1, r2, "consecutive calls should select different agents");
2990        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2991    }
2992
2993    #[serial(cooldown_state)]
2994    #[tokio::test]
2995    async fn last_agent_tracks_routing() {
2996        cooldown::reset_global_state().await;
2997        let mut config = RouterConfig {
2998            mode: "round_robin".to_string(),
2999            ..Default::default()
3000        };
3001        config
3002            .model_map
3003            .entry("medium".to_string())
3004            .or_default()
3005            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
3006        config
3007            .model_map
3008            .entry("medium".to_string())
3009            .or_default()
3010            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
3011        let agents = vec!["test_x".to_string(), "test_y".to_string()];
3012        let mut weights = AgentWeights::default();
3013        weights.ensure_agents(&agents);
3014        let mut router = Router {
3015            config,
3016            available_agents: agents,
3017            weights,
3018            llm_router: Arc::new(LlmRouter::new()),
3019            ollama_router: None,
3020            rr_index: 0,
3021            last_agent: None,
3022            review_rr_index: 0,
3023            router_pool: vec![],
3024            pool_index: 0,
3025        };
3026
3027        assert!(router.last_agent.is_none());
3028
3029        let task = create_test_task("1", "Test routing", vec![]);
3030        let store = test_store().await;
3031        let result = router.route(&task, &store, "test/repo").await.unwrap();
3032
3033        assert!(router.last_agent.is_some());
3034        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
3035    }
3036
3037    // ── pool config ───────────────────────────────────────────────────────────
3038
3039    #[serial(cooldown_state)]
3040    #[test]
3041    fn parse_pool_entry_splits_on_first_colon() {
3042        let (agent, model) = super::parse_pool_entry("claude:haiku");
3043        assert_eq!(agent, "claude");
3044        assert_eq!(model, "haiku");
3045    }
3046
3047    #[serial(cooldown_state)]
3048    #[test]
3049    fn parse_pool_entry_model_with_slash() {
3050        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
3051        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
3052        assert_eq!(agent, "opencode");
3053        assert_eq!(model, "github-copilot/gpt-5-mini");
3054    }
3055
3056    #[serial(cooldown_state)]
3057    #[test]
3058    fn parse_pool_entry_no_colon() {
3059        let (agent, model) = super::parse_pool_entry("claude");
3060        assert_eq!(agent, "claude");
3061        assert_eq!(model, "");
3062    }
3063
3064    #[serial(cooldown_state)]
3065    #[test]
3066    fn effective_pool_uses_configured_pool() {
3067        let config = RouterConfig {
3068            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3069            ..RouterConfig::default()
3070        };
3071        let pool = config.effective_pool();
3072        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3073    }
3074
3075    #[serial(cooldown_state)]
3076    #[test]
3077    fn effective_pool_falls_back_to_router_agent_model() {
3078        let config = RouterConfig::default();
3079        // Default has no pool → derives from router_agent:router_model
3080        let pool = config.effective_pool();
3081        assert_eq!(pool.len(), 1);
3082        assert!(pool[0].starts_with(&config.router_agent));
3083        assert!(pool[0].contains(&config.router_model));
3084    }
3085
3086    #[serial(cooldown_state)]
3087    #[test]
3088    fn effective_fallback_uses_configured_fallback() {
3089        let config = RouterConfig {
3090            fallback: "claude:haiku".to_string(),
3091            ..RouterConfig::default()
3092        };
3093        assert_eq!(config.effective_fallback(), "claude:haiku");
3094    }
3095
3096    #[serial(cooldown_state)]
3097    #[test]
3098    fn effective_fallback_derives_from_router_agent_model() {
3099        let config = RouterConfig::default();
3100        let fallback = config.effective_fallback();
3101        assert!(fallback.starts_with(&config.router_agent));
3102        assert!(fallback.contains(&config.router_model));
3103    }
3104
3105    // ── pool expansion ────────────────────────────────────────────────────────
3106
3107    #[serial(cooldown_state)]
3108    #[test]
3109    fn expand_pool_basic_entries() {
3110        let config = RouterConfig {
3111            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3112            ..RouterConfig::default()
3113        };
3114        let expanded = Router::expand_pool(&config);
3115        assert_eq!(expanded.len(), 2);
3116        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3117        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3118    }
3119
3120    #[serial(cooldown_state)]
3121    #[test]
3122    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3123        let config = RouterConfig {
3124            pool: vec![],
3125            router_agent: "claude".to_string(),
3126            router_model: "haiku".to_string(),
3127            ..RouterConfig::default()
3128        };
3129        let expanded = Router::expand_pool(&config);
3130        assert_eq!(expanded.len(), 1);
3131        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3132    }
3133
3134    #[serial(cooldown_state)]
3135    #[test]
3136    fn expand_pool_opencode_free_skipped_when_not_installed() {
3137        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3138        // to nothing and the pool falls back to effective_fallback
3139        let config = RouterConfig {
3140            pool: vec!["opencode:free".to_string()],
3141            fallback: "claude:haiku".to_string(),
3142            ..RouterConfig::default()
3143        };
3144        let expanded = Router::expand_pool(&config);
3145        assert_eq!(expanded.len(), 1);
3146        assert_eq!(expanded[0].0, "claude");
3147        assert_eq!(expanded[0].1, "haiku");
3148    }
3149
3150    #[serial(cooldown_state)]
3151    #[test]
3152    fn expand_pool_single_entry_from_default() {
3153        let config = RouterConfig::default();
3154        let expanded = Router::expand_pool(&config);
3155        assert!(
3156            !expanded.is_empty(),
3157            "pool must never be empty after expansion"
3158        );
3159    }
3160
3161    // ── pool round-robin index ────────────────────────────────────────────────
3162
3163    #[serial(cooldown_state)]
3164    #[test]
3165    fn pool_index_initializes_to_zero() {
3166        let config = RouterConfig::default();
3167        let router = Router::new(config);
3168        assert_eq!(router.pool_index, 0);
3169    }
3170
3171    #[serial(cooldown_state)]
3172    #[test]
3173    fn router_pool_non_empty_after_construction() {
3174        let config = RouterConfig::default();
3175        let router = Router::new(config);
3176        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3177    }
3178
3179    /**
3180     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3181     * `opencode serve --service` child behind, so a suite run leaked one server per test
3182     * process. The stub on PATH records any exec, so this fails if discovery comes back
3183     **/
3184    #[cfg(unix)]
3185    #[serial(cooldown_state)]
3186    #[test]
3187    fn building_a_router_never_runs_opencode() {
3188        struct PathRestore(Option<OsString>);
3189        impl Drop for PathRestore {
3190            fn drop(&mut self) {
3191                match &self.0 {
3192                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3193                    Some(p) => unsafe { env::set_var("PATH", p) },
3194                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3195                    None => unsafe { env::remove_var("PATH") },
3196                }
3197                cmd_cache::reset_for_test();
3198            }
3199        }
3200
3201        let dir = tempfile::tempdir().unwrap();
3202        let marker = dir.path().join("spawned");
3203        let bin_dir = dir.path().join("bin");
3204        fs::create_dir(&bin_dir).unwrap();
3205        let stub = bin_dir.join("opencode");
3206        fs::write(
3207            &stub,
3208            format!(
3209                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3210                marker.display()
3211            ),
3212        )
3213        .unwrap();
3214        fs::set_permissions(&stub, Permissions::from_mode(0o755)).unwrap();
3215
3216        let prev_path = env::var_os("PATH");
3217        let mut new_path = bin_dir.as_os_str().to_os_string();
3218        new_path.push(":");
3219        if let Some(ref p) = prev_path {
3220            new_path.push(p);
3221        }
3222        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3223        unsafe { env::set_var("PATH", &new_path) };
3224        cmd_cache::reset_for_test();
3225        let _restore = PathRestore(prev_path);
3226
3227        // A config that names opencode everywhere discovery reads from
3228        let mut config = RouterConfig {
3229            pool: vec!["opencode:free".to_string()],
3230            fallback: "opencode:free".to_string(),
3231            ..RouterConfig::default()
3232        };
3233        config
3234            .model_map
3235            .entry("simple".to_string())
3236            .or_default()
3237            .insert(
3238                "opencode".to_string(),
3239                vec!["canary-model-free".to_string()],
3240            );
3241
3242        let _router = Router::new(config.clone());
3243        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3244        let _pool = Router::expand_pool(&config);
3245        // Discovery refreshes on a background thread, give it room to exec
3246        thread::sleep(Duration::from_millis(300));
3247
3248        assert!(
3249            !marker.exists(),
3250            "building a router ran opencode: {}",
3251            fs::read_to_string(&marker).unwrap_or_default().trim()
3252        );
3253        assert!(
3254            opencode_children().is_empty(),
3255            "building a router left opencode processes: {:?}",
3256            opencode_children()
3257        );
3258    }
3259
3260    /// Commands of this process's direct children that mention opencode
3261    #[cfg(unix)]
3262    fn opencode_children() -> Vec<String> {
3263        let me = process::id().to_string();
3264        let output = Command::new("ps")
3265            .args(["-axo", "ppid=,command="])
3266            .output()
3267            .expect("ps");
3268        String::from_utf8_lossy(&output.stdout)
3269            .lines()
3270            .filter_map(|line| {
3271                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3272                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3273            })
3274            .collect()
3275    }
3276
3277    #[serial(cooldown_state)]
3278    #[tokio::test]
3279    async fn reload_resets_pool_index() {
3280        cooldown::reset_global_state().await;
3281        let config = RouterConfig::default();
3282        let mut router = Router::new(config);
3283        router.pool_index = 5;
3284        router.reload_async().await;
3285        assert_eq!(router.pool_index, 0);
3286    }
3287
3288    #[serial(cooldown_state)]
3289    #[test]
3290    fn pool_index_advances_after_failed_attempt() {
3291        let mut router = Router::new(RouterConfig::default());
3292        router.router_pool = vec![
3293            ("claude".to_string(), "haiku".to_string()),
3294            ("codex".to_string(), "gpt-5.2".to_string()),
3295            (
3296                "opencode".to_string(),
3297                "github-copilot/gpt-5-mini".to_string(),
3298            ),
3299        ];
3300        router.pool_index = 0;
3301
3302        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3303
3304        assert_eq!(router.pool_index, 1);
3305    }
3306
3307    #[serial(cooldown_state)]
3308    #[test]
3309    fn pool_index_wraps_after_last_attempt() {
3310        let mut router = Router::new(RouterConfig::default());
3311        router.router_pool = vec![
3312            ("claude".to_string(), "haiku".to_string()),
3313            ("codex".to_string(), "gpt-5.2".to_string()),
3314            (
3315                "opencode".to_string(),
3316                "github-copilot/gpt-5-mini".to_string(),
3317            ),
3318        ];
3319        router.pool_index = 2;
3320
3321        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3322
3323        assert_eq!(router.pool_index, 0);
3324    }
3325
3326    #[serial(cooldown_state)]
3327    #[test]
3328    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3329        let router = Router::new(RouterConfig::default());
3330        assert!(router.earliest_pool_cooldown().is_none());
3331    }
3332
3333    #[serial(cooldown_state)]
3334    #[tokio::test]
3335    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3336        cooldown::reset_global_state().await;
3337        // Use explicit pool with predictable model names to avoid opencode discovery.
3338        let config = RouterConfig {
3339            pool: vec![
3340                "claude:haiku-4-5-20251001".to_string(),
3341                "opencode:haiku-4-5-20251001".to_string(),
3342            ],
3343            ..Default::default()
3344        };
3345        let router = Router::new(config);
3346        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3347        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3348
3349        let earliest = router.earliest_pool_cooldown();
3350        assert!(earliest.is_some());
3351        let now = chrono::Utc::now().timestamp();
3352        let remaining = earliest.unwrap() - now;
3353        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3354        assert!((295..=300).contains(&remaining));
3355    }
3356
3357    #[serial(cooldown_state)]
3358    #[tokio::test]
3359    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3360        cooldown::reset_global_state().await;
3361        // Use explicit pool so we can predict what entries exist.
3362        let config = RouterConfig {
3363            pool: vec![
3364                "claude:pool-model-a".to_string(),
3365                "opencode:pool-model-b".to_string(),
3366            ],
3367            ..Default::default()
3368        };
3369        let router = Router::new(config);
3370        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3371        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3372        // Set cooldown on a pool entry (opencode:pool-model-b)
3373        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3374
3375        let earliest = router.earliest_pool_cooldown();
3376        assert!(earliest.is_some());
3377        let now = chrono::Utc::now().timestamp();
3378        let remaining = earliest.unwrap() - now;
3379        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3380        assert!((295..=300).contains(&remaining));
3381    }
3382
3383    // ---- Pre-emptive health check: degraded agent exclusion ----
3384
3385    #[serial(cooldown_state)]
3386    #[test]
3387    fn degraded_agent_excluded_from_routing() {
3388        let mut config = RouterConfig::default();
3389        config
3390            .model_map
3391            .entry("medium".to_string())
3392            .or_default()
3393            .insert(
3394                "test_degraded_routing".to_string(),
3395                vec!["model-a".to_string()],
3396            );
3397
3398        let mut router = Router::new(config);
3399        router.available_agents = vec!["test_degraded_routing".to_string()];
3400
3401        // Agent is routable before degradation
3402        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3403
3404        // Mark as degraded
3405        mark_agent_degraded("test_degraded_routing");
3406
3407        // Now excluded from routing
3408        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3409
3410        // available_agents_for_complexity returns empty
3411        assert!(
3412            router.available_agents_for_complexity("medium").is_empty(),
3413            "degraded agent should be excluded from available agents"
3414        );
3415
3416        // healthy_agent_count reflects exclusion
3417        assert_eq!(router.healthy_agent_count("medium"), 0);
3418
3419        // Cleanup
3420        clear_agent_degraded("test_degraded_routing");
3421    }
3422
3423    #[serial(cooldown_state)]
3424    #[test]
3425    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3426        let mut config = RouterConfig::default();
3427        for agent in &["agent_healthy", "agent_degraded"] {
3428            config
3429                .model_map
3430                .entry("medium".to_string())
3431                .or_default()
3432                .insert(agent.to_string(), vec!["model-x".to_string()]);
3433        }
3434
3435        let mut router = Router::new(config);
3436        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3437
3438        mark_agent_degraded("agent_degraded");
3439
3440        let candidates = router.available_agents_for_complexity("medium");
3441        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3442        assert_eq!(router.healthy_agent_count("medium"), 1);
3443
3444        // Cleanup
3445        clear_agent_degraded("agent_degraded");
3446    }
3447
3448    #[serial(cooldown_state)]
3449    #[tokio::test]
3450    async fn refresh_health_marks_degraded_from_rate_limits() {
3451        cooldown::reset_global_state().await;
3452        let store = test_store().await;
3453        let agent = "test_refresh_health_agent";
3454
3455        let mut config = RouterConfig::default();
3456        config
3457            .model_map
3458            .entry("medium".to_string())
3459            .or_default()
3460            .insert(agent.to_string(), vec!["model-z".to_string()]);
3461
3462        let mut router = Router::new(config);
3463        router.available_agents = vec![agent.to_string()];
3464
3465        // Insert rate limit events exceeding default threshold (3)
3466        for _ in 0..4 {
3467            store.record_rate_limit(agent, "rate", None).await.unwrap();
3468        }
3469
3470        router.refresh_health(&store).await;
3471
3472        assert!(
3473            is_agent_degraded(agent),
3474            "agent should be degraded after exceeding rate limit threshold"
3475        );
3476        assert!(
3477            !router.agent_is_routable(agent, "medium"),
3478            "degraded agent should not be routable"
3479        );
3480
3481        // Cleanup
3482        clear_agent_degraded(agent);
3483    }
3484
3485    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3486
3487    #[serial(cooldown_state)]
3488    #[tokio::test]
3489    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3490        cooldown::reset_global_state().await;
3491        let mut config = RouterConfig::default();
3492        // Agent's only model, shared across every complexity tier it resolves to.
3493        for tier in ["simple", "medium", "complex", "review"] {
3494            config
3495                .model_map
3496                .entry(tier.to_string())
3497                .or_default()
3498                .insert(
3499                    "test_all_cooled".to_string(),
3500                    vec!["only-model".to_string()],
3501                );
3502        }
3503
3504        let mut router = Router::new(config);
3505        router.available_agents = vec!["test_all_cooled".to_string()];
3506
3507        assert!(router.agent_has_any_available_model("test_all_cooled"));
3508
3509        record_model_failure("test_all_cooled", "only-model").await;
3510        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3511
3512        assert!(
3513            !router.agent_has_any_available_model("test_all_cooled"),
3514            "agent with its only model cooled across every tier should have no available model"
3515        );
3516    }
3517
3518    #[serial(cooldown_state)]
3519    #[tokio::test]
3520    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3521        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3522        // agent that has no agent-level cooldown but whose only resolvable model (across
3523        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3524        cooldown::reset_global_state().await;
3525        let mut config = RouterConfig::default();
3526        for tier in ["simple", "medium", "complex", "review"] {
3527            config
3528                .model_map
3529                .entry(tier.to_string())
3530                .or_default()
3531                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3532            config
3533                .model_map
3534                .get_mut(tier)
3535                .unwrap()
3536                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3537        }
3538
3539        let mut router = Router::new(config);
3540        router.available_agents = vec![
3541            "agent_cooled_model".to_string(),
3542            "agent_healthy".to_string(),
3543        ];
3544
3545        record_model_failure("agent_cooled_model", "opus").await;
3546        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3547        // No agent-level cooldown — the old agent-only filter would still offer it.
3548        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3549
3550        let uncooled_agents: Vec<String> = router
3551            .available_agents
3552            .iter()
3553            .filter(|a| {
3554                !is_agent_in_cooldown(a)
3555                    && !is_agent_degraded(a)
3556                    && router.agent_has_available_model_for_coding_tiers(a)
3557            })
3558            .cloned()
3559            .collect();
3560
3561        assert_eq!(
3562            uncooled_agents,
3563            vec!["agent_healthy".to_string()],
3564            "agent whose only model is cooled must be excluded from the LLM candidate list"
3565        );
3566    }
3567
3568    #[serial(cooldown_state)]
3569    #[tokio::test]
3570    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3571        // Regression test for #3521: an agent whose only healthy tier is `simple`
3572        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3573        // excluded from the LLM classification candidate list, otherwise the
3574        // classifier can pick it for a `medium`/`complex` task it can't serve,
3575        // wasting the classification call. Mirrors the real-world shape where
3576        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3577        cooldown::reset_global_state().await;
3578        let mut config = RouterConfig::default();
3579        config
3580            .model_map
3581            .entry("simple".to_string())
3582            .or_default()
3583            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3584        config
3585            .model_map
3586            .entry("medium".to_string())
3587            .or_default()
3588            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3589        config
3590            .model_map
3591            .entry("complex".to_string())
3592            .or_default()
3593            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3594        // review must be irrelevant to the coding-tier filter
3595        config
3596            .model_map
3597            .entry("review".to_string())
3598            .or_default()
3599            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3600        for tier in ["simple", "medium", "complex", "review"] {
3601            config
3602                .model_map
3603                .get_mut(tier)
3604                .unwrap()
3605                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3606        }
3607
3608        let mut router = Router::new(config);
3609        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3610
3611        record_model_failure("minimax_like", "opus").await;
3612        assert!(is_model_in_cooldown("minimax_like", "opus"));
3613        // No agent-level cooldown, and the simple tier is healthy, so the loose
3614        // "any tier" check still passes — this is exactly the leak #3511 missed.
3615        assert!(!is_agent_in_cooldown("minimax_like"));
3616        assert!(router.agent_has_any_available_model("minimax_like"));
3617        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3618
3619        let uncooled_agents: Vec<String> = router
3620            .available_agents
3621            .iter()
3622            .filter(|a| {
3623                !is_agent_in_cooldown(a)
3624                    && !is_agent_degraded(a)
3625                    && router.agent_has_available_model_for_coding_tiers(a)
3626            })
3627            .cloned()
3628            .collect();
3629
3630        assert_eq!(
3631            uncooled_agents,
3632            vec!["agent_healthy".to_string()],
3633            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3634        );
3635    }
3636}