Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22use crate::backends::ExternalTask;
23use crate::cmd_cache;
24use crate::engine::cooldown;
25use crate::engine::cooldown::{
26    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
27    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
28};
29use crate::engine::runner::agents::opencode;
30use crate::engine::runner::response;
31use crate::store::WaitingOnDependency;
32use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
33pub use config::{RouterConfig, parse_pool_entry};
34use llm::{LlmRouter, TIMEOUT_PREFIX};
35use serde::{Deserialize, Serialize};
36use std::fmt;
37use std::fmt::Debug;
38use std::fmt::Formatter;
39use std::sync::Arc;
40use tokio::task;
41pub use weights::AgentWeights;
42
43fn fmt_degraded(degraded: &[String]) -> String {
44    if degraded.is_empty() {
45        String::new()
46    } else {
47        format!(" (degraded: {})", degraded.join(", "))
48    }
49}
50
51/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
52///
53/// This is a transient error that should not count against `max_route_attempts` since
54/// no agent was actually invoked. The caller should back off and retry later.
55#[derive(Debug, thiserror::Error)]
56pub enum AllAgentsCooledError {
57    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
58    AllCooled {
59        scope: String,
60        remaining_secs: Option<u64>,
61        degraded: Vec<String>,
62    },
63}
64
65impl AllAgentsCooledError {
66    /// Create a new error for the given scope, naming currently degraded agents
67    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
68        Self::AllCooled {
69            scope,
70            remaining_secs,
71            degraded: list_degraded_agents()
72                .into_iter()
73                .map(|(agent, _reason)| agent)
74                .collect(),
75        }
76    }
77
78    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
79
80    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
81    pub fn scope(&self) -> &str {
82        match self {
83            Self::AllCooled { scope, .. } => scope,
84        }
85    }
86
87    /// Remaining cooldown duration in seconds when available.
88    pub fn remaining_secs(&self) -> Option<u64> {
89        match self {
90            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
91        }
92    }
93
94    /// Agents currently marked degraded, shown by `orch cooldown list`
95    pub fn degraded(&self) -> &[String] {
96        match self {
97            Self::AllCooled { degraded, .. } => degraded,
98        }
99    }
100}
101
102/// Typed error returned by [`get_route_result`].
103#[derive(Debug, thiserror::Error)]
104pub enum RouteResultError {
105    /// The task has no agent field — it needs to be re-routed.
106    #[error("no agent configured for task {task_id}")]
107    NoAgent { task_id: String },
108    /// Any other failure (store lookup, SQL, etc.).
109    #[error(transparent)]
110    Other(#[from] anyhow::Error),
111}
112
113/// Result of routing a task to an agent.
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
115pub struct RouteResult {
116    /// The selected agent: "claude", "codex", or "opencode"
117    pub agent: String,
118    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
119    pub model: Option<String>,
120    /// Complexity level: "simple", "medium", or "complex"
121    pub complexity: String,
122    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
123    pub estimate: u8,
124    /// Why this agent was selected
125    pub reason: String,
126    /// Specialized agent profile (skills, tools, constraints)
127    pub profile: AgentProfile,
128    /// Selected skill IDs from the catalog
129    pub selected_skills: Vec<String>,
130    /// Optional warning about routing decision
131    pub warning: Option<String>,
132}
133
134/// Specialized agent profile for a task.
135#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
136pub struct AgentProfile {
137    /// Role name, e.g., "backend specialist"
138    pub role: String,
139    /// Focus skills for this task
140    pub skills: Vec<String>,
141    /// Tools allowed for this task
142    pub tools: Vec<String>,
143    /// Constraints for this task
144    pub constraints: Vec<String>,
145}
146
147impl AgentProfile {
148    /// Build the default "general" profile from router config.
149    pub fn default_from_config(config: &RouterConfig) -> Self {
150        Self {
151            role: "general".to_string(),
152            skills: vec![],
153            tools: config.allowed_tools.clone(),
154            constraints: vec![],
155        }
156    }
157}
158
159impl RouteResult {
160    /// The model for the task's tier, or one from the nearest tier with a free model. The warning
161    /// names the tier used when it differs, so the switch shows in the route activity
162    pub(crate) fn resolve_model(
163        config: &RouterConfig,
164        agent: &str,
165        complexity: &str,
166        task_id: &str,
167    ) -> (Option<String>, Option<String>) {
168        match config.model_for_complexity_or_adjacent(agent, complexity, task_id) {
169            Some((model, tier)) if tier != complexity => {
170                let warning =
171                    format!("no available {complexity} model for {agent}; using {tier} tier");
172                (Some(model), Some(warning))
173            }
174            Some((model, _)) => (Some(model), None),
175            None => (None, None),
176        }
177    }
178
179    /// Build a standard route result from the three task-specific values.
180    ///
181    /// Fills `profile`, `selected_skills`, and `warning` with router-config
182    /// defaults so call sites only need to supply what varies per task.
183    pub fn from_config(
184        agent: String,
185        complexity: String,
186        reason: String,
187        config: &RouterConfig,
188        task_id: &str,
189    ) -> Self {
190        let (model, warning) = Self::resolve_model(config, &agent, &complexity, task_id);
191        Self {
192            agent,
193            model,
194            complexity,
195            estimate: 0,
196            reason,
197            profile: AgentProfile::default_from_config(config),
198            selected_skills: config.default_skills.clone(),
199            warning,
200        }
201    }
202}
203
204/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
205///
206/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
207/// **not** require holding the router's `RwLock`, so async operations can
208/// safely be called without starving write-lock waiters.
209pub struct LlmRouterHandle(Arc<LlmRouter>);
210
211impl Debug for LlmRouterHandle {
212    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
213        f.debug_struct("LlmRouterHandle").finish_non_exhaustive()
214    }
215}
216
217impl LlmRouterHandle {
218    /// Invalidate the skills catalog cache so the next routing call reloads
219    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
220    pub async fn invalidate_skills_catalog(&self) {
221        self.0.invalidate_skills_catalog().await;
222    }
223}
224
225/// The agent router.
226#[expect(
227    clippy::struct_field_names,
228    reason = "the router holds sub-routers whose names say which backend they use"
229)]
230pub struct Router {
231    /// Router configuration
232    pub config: RouterConfig,
233    /// Available agents discovered at runtime
234    pub available_agents: Vec<String>,
235    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
236    pub weights: AgentWeights,
237    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
238    /// without holding the router RwLock across async operations)
239    llm_router: Arc<LlmRouter>,
240    /// Ollama router for local routing (contains reusable reqwest::Client)
241    ollama_router: Option<ollama::OllamaRouter>,
242    /// Round-robin index for task routing
243    pub(crate) rr_index: usize,
244    /// Last agent routed to (for distribution tracking)
245    pub(crate) last_agent: Option<String>,
246    /// Round-robin index for review agent selection
247    pub(crate) review_rr_index: usize,
248    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
249    /// `opencode:free` entries are expanded at construction time.
250    pub(crate) router_pool: Vec<(String, String)>,
251    /// Current round-robin index into router_pool
252    pub(crate) pool_index: usize,
253}
254
255impl Debug for Router {
256    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
257        f.debug_struct("Router")
258            .field("config", &self.config)
259            .field("available_agents", &self.available_agents)
260            .field("weights", &self.weights)
261            .field("ollama_router", &self.ollama_router)
262            .field("rr_index", &self.rr_index)
263            .field("last_agent", &self.last_agent)
264            .field("review_rr_index", &self.review_rr_index)
265            .field("router_pool", &self.router_pool)
266            .field("pool_index", &self.pool_index)
267            .finish_non_exhaustive()
268    }
269}
270
271impl Router {
272    /// Create a new router with the given configuration.
273    pub fn new(config: RouterConfig) -> Self {
274        let available_agents = Self::discover_agents(&config.agents);
275        let mut weights = AgentWeights {
276            base_weights: config.weights.clone(),
277            ..Default::default()
278        };
279        weights.ensure_agents_with_weights(&available_agents, &config.weights);
280        if !config.weights.is_empty() {
281            tracing::info!(
282                weights = ?config.weights,
283                "configured agent routing weights"
284            );
285        }
286        let router_pool = Self::expand_pool(&config);
287        tracing::info!(
288            pool = ?router_pool,
289            fallback = %config.effective_fallback(),
290            "router LLM pool initialized"
291        );
292        // Prime only when this config names opencode, so a default or claude-only
293        // router never runs `opencode models`
294        if config.uses_opencode() {
295            opencode::prime_free_model_cache();
296        }
297        let ollama_router = if config.mode == "local" {
298            Some(ollama::OllamaRouter::new(&config))
299        } else {
300            None
301        };
302        Self {
303            config,
304            available_agents,
305            weights,
306            llm_router: Arc::new(LlmRouter::new()),
307            ollama_router,
308            rr_index: 0,
309            last_agent: None,
310            review_rr_index: 0,
311            router_pool,
312            pool_index: 0,
313        }
314    }
315
316    /// Create a router with default configuration loaded from files.
317    pub fn from_config() -> Self {
318        Self::new(RouterConfig::from_config())
319    }
320
321    /// Create a router with a forced agent list, bypassing CLI discovery.
322    /// Only for use in unit tests.
323    #[cfg(test)]
324    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
325        let mut weights = AgentWeights {
326            base_weights: config.weights.clone(),
327            ..Default::default()
328        };
329        weights.ensure_agents(&available_agents);
330        let router_pool = Self::expand_pool(&config);
331        Self {
332            config,
333            available_agents,
334            weights,
335            llm_router: Arc::new(LlmRouter::new()),
336            ollama_router: None,
337            rr_index: 0,
338            last_agent: None,
339            review_rr_index: 0,
340            router_pool,
341            pool_index: 0,
342        }
343    }
344
345    /// Return a cloned handle to the LLM router subsystem.
346    ///
347    /// Callers that need to invoke async methods on the underlying `LlmRouter`
348    /// should clone this handle *before* releasing any `RwLock` read guard on
349    /// `Router`, then call the async method after the guard is dropped.  This
350    /// prevents holding the read lock across await points (which starves
351    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
352    pub fn llm_router_handle(&self) -> LlmRouterHandle {
353        LlmRouterHandle(Arc::clone(&self.llm_router))
354    }
355
356    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
357        debug_assert!(pool_len > 0, "router pool must not be empty");
358        self.pool_index = (idx + 1) % pool_len;
359    }
360
361    /// Discover available agent CLIs in PATH.
362    /// Checks all agents from the configured list.
363    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
364        let mut agents = Vec::new();
365        for agent in configured_agents {
366            if cmd_cache::command_exists(agent) {
367                agents.push(agent.clone());
368            }
369        }
370        agents
371    }
372
373    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
374    ///
375    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
376    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
377    /// - If `opencode` is not installed or the command fails, the entry is skipped.
378    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
379    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
380        let raw_pool = config.effective_pool();
381        let mut expanded: Vec<(String, String)> = Vec::new();
382
383        for entry in &raw_pool {
384            if entry == "opencode:free" {
385                let free_models = Self::discover_free_opencode_models();
386                if free_models.is_empty() {
387                    tracing::debug!("opencode:free expanded to nothing — skipping");
388                } else {
389                    tracing::debug!(models = ?free_models, "opencode:free expanded");
390                    for model in free_models {
391                        expanded.push(("opencode".to_string(), model));
392                    }
393                }
394            } else {
395                let (agent, model) = parse_pool_entry(entry);
396                expanded.push((agent, model));
397            }
398        }
399
400        if expanded.is_empty() {
401            // All entries were opencode:free and opencode isn't installed — use fallback
402            let (agent, model) = parse_pool_entry(&config.effective_fallback());
403            tracing::debug!(
404                agent = %agent, model = %model,
405                "pool empty after expansion, using fallback as sole pool entry"
406            );
407            expanded.push((agent, model));
408        }
409
410        expanded
411    }
412
413    /// Async version of expand_pool which may query the opencode CLI without
414    /// blocking the Tokio runtime.
415    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
416        let raw_pool = config.effective_pool();
417        let mut expanded: Vec<(String, String)> = Vec::new();
418
419        for entry in &raw_pool {
420            if entry == "opencode:free" {
421                let free_models = Self::discover_free_opencode_models_async().await;
422                if free_models.is_empty() {
423                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
424                } else {
425                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
426                    for model in free_models {
427                        expanded.push(("opencode".to_string(), model));
428                    }
429                }
430            } else {
431                let (agent, model) = parse_pool_entry(entry);
432                expanded.push((agent, model));
433            }
434        }
435
436        if expanded.is_empty() {
437            // All entries were opencode:free and opencode isn't installed — use fallback
438            let (agent, model) = parse_pool_entry(&config.effective_fallback());
439            tracing::debug!(
440                agent = %agent, model = %model,
441                "pool empty after async expansion, using fallback as sole pool entry"
442            );
443            expanded.push((agent, model));
444        }
445
446        expanded
447    }
448
449    /// Async reload that uses async pool expansion to avoid blocking the runtime
450    /// when discovering opencode free models.
451    pub async fn reload_async(&mut self) {
452        let new_config = RouterConfig::from_config();
453        let new_agents = Self::discover_agents(&new_config.agents);
454        let new_pool = Self::expand_pool_async(&new_config).await;
455        tracing::info!(
456            mode = %new_config.mode,
457            agents = ?new_agents,
458            fallback = %new_config.fallback_executor,
459            weighted_rr = new_config.weighted_round_robin,
460            pool = ?new_pool,
461            "router reloaded (async)"
462        );
463        self.config = new_config;
464        self.available_agents.clone_from(&new_agents);
465        // Ensure new agents have weight entries (preserves existing weights)
466        self.weights.ensure_agents(&new_agents);
467        self.router_pool = new_pool;
468        self.pool_index = 0;
469        // Reinitialize Ollama router if mode changed to local
470        self.ollama_router = if self.config.mode == "local" {
471            Some(ollama::OllamaRouter::new(&self.config))
472        } else {
473            None
474        };
475    }
476
477    fn discover_free_opencode_models() -> Vec<String> {
478        opencode::discover_free_opencode_models()
479    }
480
481    async fn discover_free_opencode_models_async() -> Vec<String> {
482        task::spawn_blocking(Self::discover_free_opencode_models)
483            .await
484            .unwrap_or_default()
485    }
486
487    /// Run the pre-emptive health check, refreshing the degraded-agent set.
488    ///
489    /// Queries the `rate_limits` table for recent events and combines with
490    /// cooldown state to mark agents as degraded before routing attempts them.
491    pub async fn refresh_health(&self, store: &Arc<TaskStore>) {
492        // KV is the source of truth for cooldowns. Reconcile the in-memory map
493        // from KV every tick so external mutations (e.g. `orch cooldown clear`
494        // from a separate CLI process) actually take effect on the running
495        // service. Without this, the engine keeps stale entries until they
496        // expire on their own, producing the "all agents cooled" log with
497        // `orch cooldown list` simultaneously reporting "No active cooldowns".
498        sync_from_kv(store).await;
499
500        let agents = self.available_agents.clone();
501        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
502        refresh_degraded_agents(
503            store,
504            &agents,
505            &model_checker,
506            config::health_check_window_hours(),
507            config::degraded_rate_limit_threshold(),
508        )
509        .await;
510    }
511
512    /// Check if an agent is available.
513    pub fn is_agent_available(&self, agent: &str) -> bool {
514        self.available_agents.contains(&agent.to_string())
515    }
516
517    /// True if the agent has at least one uncooled model across any complexity
518    /// tier. Used where the eventual complexity isn't known yet (e.g. before
519    /// LLM classification), so we can't check a single tier.
520    fn agent_has_any_available_model(&self, agent: &str) -> bool {
521        ["simple", "medium", "complex", "review"]
522            .iter()
523            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
524    }
525
526    /// True if the agent has an available (uncooled) model for the `medium`
527    /// and `complex` tiers — the tiers real coding tasks are classified into
528    /// by the LLM classifier. Used to pre-filter the candidate list handed to
529    /// that classifier so an agent whose healthy tiers are only `simple` isn't
530    /// offered: the classifier can't be constrained to `simple`, so offering
531    /// such an agent produces a near-certain wasted classification call whenever
532    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
533    /// outcome of initial classification and is deliberately excluded.
534    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
535        ["medium", "complex"]
536            .iter()
537            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
538    }
539
540    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
541        if !self.is_agent_available(agent) {
542            return false;
543        }
544        if is_agent_in_cooldown(agent) {
545            tracing::debug!(agent, "agent skipped: in cooldown");
546            return false;
547        }
548        if is_agent_degraded(agent) {
549            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
550            return false;
551        }
552        // If weighted routing is used, consider the agent degraded when its
553        // weight has decayed below the threshold. This avoids
554        // proactively routing to agents that recently hit many rate limits.
555        if self.config.weighted_round_robin {
556            let weight = self.weights.get_weight(agent);
557            if weight < config::SKIP_LIMITED_THRESHOLD {
558                tracing::debug!(
559                    agent,
560                    weight,
561                    "agent considered degraded by weight threshold, skipping"
562                );
563                return false;
564            }
565        }
566        if !self
567            .config
568            .has_available_model_in_any_tier(agent, complexity)
569        {
570            tracing::debug!(agent, complexity, "agent skipped: no available model");
571            return false;
572        }
573        true
574    }
575
576    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
577        self.available_agents
578            .iter()
579            .filter(|agent| self.agent_is_routable(agent, complexity))
580            .cloned()
581            .collect()
582    }
583
584    /// Count of healthy (routable) agents for a given complexity.
585    /// Returns the number of agents that are available, not in cooldown, and have an available model.
586    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
587        self.available_agents_for_complexity(complexity).len()
588    }
589
590    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
591    /// Uses "simple" complexity as the baseline since it has the widest model support.
592    pub fn is_degraded(&self, threshold: usize) -> bool {
593        self.healthy_agent_count("simple") < threshold
594    }
595
596    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
597        let mut earliest: Option<i64> = None;
598
599        for agent in &self.available_agents {
600            if let Some(until) = cooldown_until(agent) {
601                earliest = Some(earliest.map_or(until, |current| current.min(until)));
602            }
603
604            if let Some(comp) = complexity
605                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
606            {
607                for model in pool {
608                    let key = format!("{agent}:{model}");
609                    if let Some(until) = cooldown_until(&key) {
610                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
611                    }
612                }
613            }
614        }
615
616        earliest
617    }
618
619    /// Find the earliest cooldown expiration among the router LLM pool entries.
620    ///
621    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
622    /// or `None` if no pool model has a cooldown.
623    fn earliest_pool_cooldown(&self) -> Option<i64> {
624        let mut earliest: Option<i64> = None;
625        for (agent, model) in &self.router_pool {
626            let key = format!("{agent}:{model}");
627            if let Some(until) = cooldown_until(&key) {
628                earliest = Some(earliest.map_or(until, |current| current.min(until)));
629            }
630        }
631        earliest
632    }
633
634    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
635        // If any cooldowns are present, return an error immediately so the
636        // caller (the tick loop) can skip this task and retry on the next tick.
637        // Sleeping inside the router blocks the entire tick loop for the
638        // cooldown duration which starves other tasks. The engine tick already
639        // retries every `tick_interval` (default 10s), so allow the tick loop
640        // to drive retries instead of sleeping here.
641        let now = chrono::Utc::now().timestamp();
642        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
643            // All agents are degraded but none have a timed cooldown entry
644            // (e.g. over-threshold rate limiting without explicit cooldown duration).
645            // Log at warn and return an error so the tick skips this task and retries after backoff.
646            tracing::warn!(
647                scope = complexity.unwrap_or("all agents"),
648                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
649            );
650            // Return an error similar to the cooldown case to trigger backoff in the tick loop
651            return Err(AllAgentsCooledError::new(
652                complexity.unwrap_or("all agents").to_string(),
653                None,
654            )
655            .into());
656        };
657
658        let remaining = earliest.saturating_sub(now);
659        if remaining > 0 {
660            let scope_str = complexity.unwrap_or("all agents").to_string();
661            tracing::warn!(
662                remaining_secs = remaining,
663                scope = %scope_str,
664                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
665            );
666            // Return a domain error indicating all candidates are cooled so the
667            // caller can handle it (tick will skip this task and retry later).
668            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
669        }
670
671        Ok(())
672    }
673
674    /// Get the first available agent.
675    /// Pick next agent via round-robin (for review or other non-task routing).
676    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
677    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
678    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
679        // Use available_agents_for_complexity which already filters using agent_is_routable,
680        // eliminating duplicate cooldown/degraded/model checks.
681        let candidates = self.available_agents_for_complexity(complexity);
682        if candidates.is_empty() {
683            return None;
684        }
685        let idx = self.review_rr_index;
686        let n = candidates.len();
687
688        // Try to find an agent that isn't excluded
689        let agent = (0..n)
690            .map(|offset| &candidates[(idx + offset) % n])
691            .find(|a| !exclude.contains(&a.as_str()))
692            .cloned()
693            // Fallback: use the first available candidate (even if excluded)
694            .or_else(|| candidates.first().cloned())?;
695
696        self.review_rr_index = (idx + 1) % candidates.len().max(1);
697        Some(agent)
698    }
699
700    /// If the selected agent matches `no_code_last_agent` for this task, filter it
701    /// out of the candidates and return the filtered list. Used to prevent the
702    /// same agent from being selected twice in a row after a no-code failure (#2410).
703    async fn skip_no_code_agent(
704        &self,
705        store: &Arc<TaskStore>,
706        repo: &str,
707        task_id: &str,
708        candidates: &[String],
709    ) -> Option<Vec<String>> {
710        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
711            .await
712            .unwrap_or_else(|| {
713                tracing::warn!(
714                    task_id,
715                    "no_code_last_agent DB read returned None — loop detection may be impaired"
716                );
717                String::new()
718            });
719        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
720            return None;
721        }
722        let filtered: Vec<String> = candidates
723            .iter()
724            .filter(|a| a.as_str() != no_code_agent)
725            .cloned()
726            .collect();
727        if filtered.is_empty() {
728            tracing::warn!(
729                task_id,
730                no_code_agent = %no_code_agent,
731                "all available agents are the no-code agent — cannot skip"
732            );
733            return None;
734        }
735        tracing::info!(
736            task_id,
737            skipped = %no_code_agent,
738            remaining = ?filtered,
739            "skipping no-code agent, routing to alternative"
740        );
741        Some(filtered)
742    }
743
744    /// Route a task to the best agent.
745    ///
746    /// Routing logic (in priority order):
747    /// 1. Check for `agent:*` label — use that agent directly
748    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
749    /// 3. If round_robin mode, cycle through agents (stateful)
750    /// 4. Call LLM classifier for intelligent routing
751    /// 5. After max_route_attempts LLM failures, fall back to round-robin
752    pub async fn route(
753        &mut self,
754        task: &ExternalTask,
755        store: &Arc<TaskStore>,
756        repo: &str,
757    ) -> anyhow::Result<RouteResult> {
758        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
759        // before the routing loop, not per-task. Do not re-add it here.
760
761        // Hold the task before any classification, label override, or fallback.
762        // A waiting task must not increment route_attempts.
763        match store.resolve_id_by_external(repo, &task.id.0).await {
764            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
765                Ok(Some(blockers)) => {
766                    return Err(WaitingOnDependency { blockers }.into());
767                }
768                Ok(None) => {}
769                Err(err) => return Err(err),
770            },
771            Ok(None) => {}
772            Err(err) => return Err(err),
773        }
774
775        loop {
776            // 1. Check store agent field first (set by failover, authoritative over labels)
777            // Use a lightweight lookup to avoid deserializing the full Task row when
778            // only the `agent` column is needed. This avoids two full-row loads
779            // per routing pass (resolve_task_id + get) and significantly reduces
780            // CPU work on busy instances.
781            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
782                Ok(Some(agent)) => {
783                    // Router enforces that the agent string matches a configured agent
784                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
785                        Some(agent)
786                    } else {
787                        None
788                    }
789                }
790                Ok(None) => None,
791                Err(e) => {
792                    tracing::warn!(
793                        repo,
794                        task_id = %task.id.0,
795                        err = %e,
796                        "failed to fetch stored agent, falling back to label/LLM routing"
797                    );
798                    None
799                }
800            };
801
802            // 2. Fall back to explicit agent label
803            let resolved_agent = store_agent.or_else(|| {
804                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
805            });
806
807            if let Some(agent) = resolved_agent {
808                let complexity = strategies::extract_complexity_from_labels(&task.labels);
809                // If agent appears routable (not in agent-wide cooldown and has
810                // available models or no model_map configured), check whether
811                // we actually have a concrete model to dispatch. If the
812                // resolved label maps to no model (None), we fall through to
813                // the standard routing logic so the router LLM can pick a
814                // proper agent/model, instead of dispatching with an empty
815                // model which causes some agents (e.g. opencode) to exit
816                // silently.
817                if self.agent_is_routable(&agent, &complexity) {
818                    let (model, tier_warning) =
819                        RouteResult::resolve_model(&self.config, &agent, &complexity, &task.id.0);
820
821                    // Only dispatch label overrides when we have a concrete
822                    // model to send to the executor. Dispatching with an
823                    // empty model led to silent exits for some agents
824                    // (notably opencode). If no concrete model is available
825                    // for the labeled agent, fall through to the standard
826                    // routing logic so the LLM or round-robin can pick an
827                    // appropriate agent/model.
828                    if model.is_some() {
829                        tracing::debug!(
830                            task_id = %task.id.0,
831                            agent = %agent,
832                            complexity = %complexity,
833                            "routed via label"
834                        );
835                        let mut result = RouteResult::from_config(
836                            agent.clone(),
837                            complexity.clone(),
838                            format!("label agent:{agent}"),
839                            &self.config,
840                            &task.id.0,
841                        );
842                        // Label routing uses agent-specific role and the
843                        // resolved model (may differ from from_config's default).
844                        result.profile.role = format!("{agent} specialist");
845                        result.model = model.clone();
846                        result.warning = tier_warning;
847                        self.log_route_activity(store, repo, &task.id.0, &result, None)
848                            .await;
849                        return Ok(result);
850                    }
851                    // No concrete model found for the labeled agent — do
852                    // not honor the label override. Fall through to the
853                    // standard routing logic so another agent or the LLM
854                    // can be chosen with a valid model.
855                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
856                }
857
858                let candidates = self.available_agents_for_complexity(&complexity);
859                if candidates.is_empty() {
860                    self.wait_for_cooldown(Some(&complexity))?;
861                    continue;
862                }
863                // Label target is cooled or lacked model; fall through to standard routing.
864            }
865
866            let complexity = strategies::extract_complexity_from_labels(&task.labels);
867
868            // Check if any agents have available models for this complexity
869            let mut candidates = self.available_agents_for_complexity(&complexity);
870            if candidates.is_empty() {
871                self.wait_for_cooldown(Some(&complexity))?;
872                continue;
873            }
874
875            // Skip the no-code agent if one is recorded for this task (#2410).
876            // If all available agents are filtered out, we fall through to
877            // LLM routing with no constraints (LLM will pick the same agent but
878            // the response_handler will block it immediately via same-agent check).
879            if let Some(filtered) = self
880                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
881                .await
882            {
883                candidates = filtered;
884            }
885
886            // 2. Weighted round-robin — capacity-based selection
887            if self.config.weighted_round_robin {
888                let result = strategies::route_via_weighted_round_robin(
889                    &candidates,
890                    &self.weights,
891                    &self.config,
892                    task,
893                    &mut self.last_agent,
894                )?;
895                self.log_route_activity(store, repo, &task.id.0, &result, None)
896                    .await;
897                return Ok(result);
898            }
899
900            // 3. Round-robin mode — use stateful round-robin
901            if self.config.mode == "round_robin" {
902                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
903                let result = strategies::route_via_round_robin_stateful(
904                    &candidates,
905                    &self.config,
906                    task,
907                    &mut self.rr_index,
908                    &mut self.last_agent,
909                )?;
910                self.log_route_activity(store, repo, &task.id.0, &result, None)
911                    .await;
912                return Ok(result);
913            }
914
915            // 4. Local Ollama routing
916            if self.config.mode == "local" {
917                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
918                let ollama_router = self
919                    .ollama_router
920                    .as_ref()
921                    .expect("ollama_router must be initialized in Router::new when mode=local");
922                match ollama_router.route(task, &self.config).await {
923                    Ok(result) => {
924                        self.log_route_activity(store, repo, &task.id.0, &result, None)
925                            .await;
926                        return Ok(result);
927                    }
928                    Err(e) => {
929                        tracing::warn!(
930                            task_id = %task.id.0,
931                            error = %e,
932                            "Ollama routing failed, falling back to round-robin"
933                        );
934                        let result = strategies::route_via_round_robin_stateful(
935                            &candidates,
936                            &self.config,
937                            task,
938                            &mut self.rr_index,
939                            &mut self.last_agent,
940                        )?;
941                        self.log_route_activity(store, repo, &task.id.0, &result, None)
942                            .await;
943                        return Ok(result);
944                    }
945                }
946            }
947
948            // 5. LLM-based routing with retry tracking
949            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
950
951            if route_attempts >= self.config.max_route_attempts {
952                tracing::warn!(
953                    task_id = %task.id.0,
954                    attempts = route_attempts,
955                    max = self.config.max_route_attempts,
956                    "max LLM route attempts reached, falling back to round-robin"
957                );
958                let candidates = self.available_agents_for_complexity(&complexity);
959                if candidates.is_empty() {
960                    self.wait_for_cooldown(Some(&complexity))?;
961                    continue;
962                }
963                let result = strategies::route_via_round_robin_stateful(
964                    &candidates,
965                    &self.config,
966                    task,
967                    &mut self.rr_index,
968                    &mut self.last_agent,
969                )?;
970                self.log_route_activity(store, repo, &task.id.0, &result, None)
971                    .await;
972                return Ok(result);
973            }
974
975            // Log routing start (before await)
976            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
977
978            match self.route_with_llm(task, repo).await {
979                Ok(result) => {
980                    let candidates = self.available_agents_for_complexity(&result.complexity);
981                    if candidates.is_empty() {
982                        self.wait_for_cooldown(Some(&result.complexity))?;
983                        continue;
984                    }
985
986                    // If the LLM selected the no-code agent, filter it out and
987                    // pick an alternative (#2410). Re-use the existing fallback logic.
988                    let no_code_agent =
989                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
990                             .await
991                             .unwrap_or_else(|| {
992                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
993                                 String::new()
994                             });
995                    if result.agent == no_code_agent {
996                        // Pick the first agent that isn't the no-code agent.
997                        let fallback_agent = candidates
998                            .iter()
999                            .find(|a| a.as_str() != no_code_agent)
1000                            .cloned()
1001                            .unwrap_or_else(|| candidates[0].clone());
1002                        if fallback_agent.as_str() == no_code_agent {
1003                            tracing::warn!(
1004                                task_id = %task.id.0,
1005                                no_code_agent = %no_code_agent,
1006                                complexity = %result.complexity,
1007                                "all available agents are the no-code agent — waiting for another agent"
1008                            );
1009                            self.wait_for_cooldown(Some(&result.complexity))?;
1010                            continue;
1011                        }
1012                        tracing::info!(
1013                            task_id = %task.id.0,
1014                            llm_selected = %result.agent,
1015                            skipped = %no_code_agent,
1016                            fallback = %fallback_agent,
1017                            "LLM selected the no-code agent — routing to alternative"
1018                        );
1019                        let (fallback_model, _) = RouteResult::resolve_model(
1020                            &self.config,
1021                            &fallback_agent,
1022                            &result.complexity,
1023                            &task.id.0,
1024                        );
1025
1026                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1027                        let result = RouteResult {
1028                            agent: fallback_agent.clone(),
1029                            model: fallback_model,
1030                            complexity: result.complexity.clone(),
1031                            estimate: result.estimate,
1032                            reason: format!(
1033                                "LLM selected no-code agent {}; routed to {}",
1034                                result.agent, fallback_agent
1035                            ),
1036                            profile: result.profile.clone(),
1037                            selected_skills: result.selected_skills.clone(),
1038                            warning: Some(format!(
1039                                "LLM selected no-code agent {}; used fallback {}",
1040                                result.agent, fallback_agent
1041                            )),
1042                        };
1043                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1044                            .await;
1045                        return Ok(result);
1046                    }
1047
1048                    let model_cooled = result
1049                        .model
1050                        .as_deref()
1051                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1052                    if !candidates.contains(&result.agent) || model_cooled {
1053                        let fallback_agent = candidates
1054                            .iter()
1055                            .find(|agent| *agent != &result.agent)
1056                            .cloned()
1057                            .unwrap_or_else(|| candidates[0].clone());
1058                        let (fallback_model, _) = RouteResult::resolve_model(
1059                            &self.config,
1060                            &fallback_agent,
1061                            &result.complexity,
1062                            &task.id.0,
1063                        );
1064
1065                        // If the fallback model is None and the fallback agent is the
1066                        // same as the LLM-selected agent, it means all models for the
1067                        // only available agent are cooled. In that case, wait for
1068                        // cooldowns instead of returning a RouteResult with
1069                        // model: None which would be dispatched incorrectly.
1070                        if fallback_model.is_none() && fallback_agent == result.agent {
1071                            tracing::warn!(
1072                                task_id = %task.id.0,
1073                                agent = %result.agent,
1074                                complexity = %result.complexity,
1075                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1076                            );
1077                            self.wait_for_cooldown(Some(&result.complexity))?;
1078                            continue;
1079                        }
1080
1081                        // Distinguish logging and reasons when we actually switch
1082                        // agents vs when we keep the same agent but pick a
1083                        // different (non-cooled) model.
1084                        if fallback_agent == result.agent {
1085                            tracing::warn!(
1086                                task_id = %task.id.0,
1087                                agent = %result.agent,
1088                                complexity = %result.complexity,
1089                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1090                            );
1091                        } else {
1092                            tracing::warn!(
1093                                task_id = %task.id.0,
1094                                agent = %result.agent,
1095                                fallback = %fallback_agent,
1096                                complexity = %result.complexity,
1097                                "LLM selected cooled agent/model; rerouting to available agent"
1098                            );
1099                        }
1100
1101                        // Reset attempts on success
1102                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1103
1104                        let reason = if fallback_agent == result.agent {
1105                            format!(
1106                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1107                            )
1108                        } else {
1109                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1110                        };
1111
1112                        let result = RouteResult {
1113                            agent: fallback_agent.clone(),
1114                            model: fallback_model,
1115                            complexity: result.complexity.clone(),
1116                            estimate: result.estimate,
1117                            reason: reason.clone(),
1118                            profile: result.profile.clone(),
1119                            selected_skills: result.selected_skills.clone(),
1120                            warning: Some(if fallback_agent == result.agent {
1121                                "LLM-selected model was cooled; selected alternative model for same agent"
1122                                    .to_string()
1123                            } else {
1124                                "LLM-selected agent/model was cooled; rerouted to available agent"
1125                                    .to_string()
1126                            }),
1127                        };
1128                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1129                            .await;
1130                        return Ok(result);
1131                    }
1132
1133                    // Reset attempts on success
1134                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1135                    tracing::info!(
1136                        task_id = %task.id.0,
1137                        agent = %result.agent,
1138                        complexity = %result.complexity,
1139                        "routed via LLM"
1140                    );
1141                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1142                        .await;
1143                    return Ok(result);
1144                }
1145                Err(e) => {
1146                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1147                        let scope = err.scope();
1148                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1149                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1150                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1151                        // (no complexity-specific pool lookup, which would fail for "router pool").
1152                        let scope_opt = match scope {
1153                            "all agents" | "router pool" => None,
1154                            _ => Some(scope),
1155                        };
1156                        self.wait_for_cooldown(scope_opt)?;
1157                        continue;
1158                    }
1159
1160                    let new_attempts = route_attempts + 1;
1161                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1162                        .await;
1163                    tracing::warn!(
1164                        task_id = %task.id.0,
1165                        error = %e,
1166                        error_chain = ?e,
1167                        attempt = new_attempts,
1168                        max = self.config.max_route_attempts,
1169                        "LLM routing failed"
1170                    );
1171
1172                    if new_attempts >= self.config.max_route_attempts {
1173                        tracing::info!(
1174                            task_id = %task.id.0,
1175                            "falling back to round-robin after {} failed attempts",
1176                            new_attempts
1177                        );
1178                        let candidates = self.available_agents_for_complexity(&complexity);
1179                        if candidates.is_empty() {
1180                            self.wait_for_cooldown(Some(&complexity))?;
1181                            continue;
1182                        }
1183                        let result = strategies::route_via_round_robin_stateful(
1184                            &candidates,
1185                            &self.config,
1186                            task,
1187                            &mut self.rr_index,
1188                            &mut self.last_agent,
1189                        )?;
1190                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1191                            .await;
1192                        return Ok(result);
1193                    }
1194
1195                    let candidates = self.available_agents_for_complexity(&complexity);
1196                    if candidates.is_empty() {
1197                        self.wait_for_cooldown(Some(&complexity))?;
1198                        continue;
1199                    }
1200                    let result = strategies::route_via_fallback(
1201                        &candidates,
1202                        &self.config,
1203                        task,
1204                        Some(&self.weights),
1205                    )?;
1206                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1207                        .await;
1208                    return Ok(result);
1209                }
1210            }
1211        }
1212    }
1213
1214    /// Get the number of LLM routing attempts for a task from the store.
1215    async fn get_route_attempts(&self, task_id: &str, store: &Arc<TaskStore>, repo: &str) -> u32 {
1216        match store.resolve_task_id(repo, task_id).await {
1217            Ok(Some(store_id)) => store
1218                .get(store_id)
1219                .await
1220                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1221            _ => 0,
1222        }
1223    }
1224
1225    /// Set the number of LLM routing attempts for a task in the store.
1226    async fn set_route_attempts(
1227        &self,
1228        task_id: &str,
1229        attempts: u32,
1230        store: &Arc<TaskStore>,
1231        repo: &str,
1232    ) {
1233        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1234            && let Err(e) = store
1235                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1236                .await
1237        {
1238            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1239        }
1240    }
1241
1242    /// Log a route event to task activity timeline.
1243    async fn log_route_activity(
1244        &self,
1245        store: &Arc<TaskStore>,
1246        repo: &str,
1247        task_id: &str,
1248        result: &RouteResult,
1249        from_agent: Option<&str>,
1250    ) {
1251        let details = serde_json::json!({
1252            "reason": result.reason,
1253            "complexity": result.complexity,
1254            "skills": result.selected_skills,
1255            "role": result.profile.role,
1256            "warning": result.warning,
1257        });
1258        let event_type = if from_agent.is_some() {
1259            "rerouted"
1260        } else {
1261            "routed"
1262        };
1263        store_log_activity(
1264            &Some(Arc::clone(store)),
1265            repo,
1266            task_id,
1267            event_type,
1268            None,
1269            Some("routed"),
1270            Some(&result.agent),
1271            result.model.as_deref(),
1272            Some(&details),
1273        )
1274        .await;
1275    }
1276
1277    /// Route using LLM classification with pool round-robin.
1278    ///
1279    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1280    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1281    /// returns the last error (the caller falls back to round-robin agent selection).
1282    async fn route_with_llm(
1283        &mut self,
1284        task: &ExternalTask,
1285        repo: &str,
1286    ) -> anyhow::Result<RouteResult> {
1287        // Filter out cooled and degraded agents, and agents without an available
1288        // model for the coding tiers (`medium`/`complex`) that real tasks are
1289        // classified into, so the LLM only sees candidates that could actually
1290        // be dispatched for whatever complexity it picks. An agent whose only
1291        // healthy tier is `simple` is excluded too: the classifier can't be
1292        // constrained to `simple`, so leaving such an agent in would waste the
1293        // call whenever it picks `medium`/`complex` (issue #3521).
1294        // Fall back to the full list if all agents are cooled/degraded.
1295        // Cloned to avoid borrow conflict with &mut self in the loop.
1296        let uncooled_agents: Vec<String> = self
1297            .available_agents
1298            .iter()
1299            .filter(|a| {
1300                !is_agent_in_cooldown(a)
1301                    && !is_agent_degraded(a)
1302                    && self.agent_has_available_model_for_coding_tiers(a)
1303            })
1304            .cloned()
1305            .collect();
1306        if uncooled_agents.is_empty() {
1307            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1308        }
1309        let llm_agents = uncooled_agents;
1310
1311        let pool = self.router_pool.clone();
1312        let n = pool.len();
1313        let start = self.pool_index;
1314        let mut last_err: Option<anyhow::Error> = None;
1315        let mut attempted_any = false;
1316
1317        // Try pool entries in round-robin order, skipping cooled ones
1318        for i in 0..n {
1319            let idx = (start + i) % n;
1320            let (agent, model) = &pool[idx];
1321            let model_str = model.as_str();
1322
1323            if response::is_model_in_cooldown(agent, model_str) {
1324                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1325                continue;
1326            }
1327
1328            // At least one pool entry was available to attempt
1329            attempted_any = true;
1330
1331            let model_opt = if model_str.is_empty() {
1332                None
1333            } else {
1334                Some(model_str)
1335            };
1336
1337            match self
1338                .llm_router
1339                .route_with_llm_using(
1340                    task,
1341                    &llm_agents,
1342                    &self.config,
1343                    &mut self.last_agent,
1344                    repo,
1345                    agent,
1346                    model_opt,
1347                )
1348                .await
1349            {
1350                Ok(result) => {
1351                    // Advance index so the next call starts at the next pool entry
1352                    self.advance_pool_index_after_attempt(idx, n);
1353                    tracing::debug!(
1354                        agent,
1355                        model = model_str,
1356                        pool_idx = idx,
1357                        "router LLM pool entry succeeded"
1358                    );
1359                    record_agent_success(agent, model_str).await;
1360                    return Ok(result);
1361                }
1362                Err(e) => {
1363                    let err_msg = e.to_string();
1364                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1365                    if is_timeout {
1366                        // Advance pool index so the next tick starts at the next entry,
1367                        // then return immediately. Continuing to try further pool entries
1368                        // in the same tick would accumulate multiple timeout windows and
1369                        // stall the tick loop for minutes (issue #3187).
1370                        //
1371                        // Also record a standard cooldown, same as the non-timeout branch,
1372                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1373                        // on future ticks instead of burning the full timeout window again
1374                        // every time round-robin reaches it (issue #3422).
1375                        self.advance_pool_index_after_attempt(idx, n);
1376                        tracing::warn!(
1377                            agent,
1378                            model = model_str,
1379                            next_pool_idx = self.pool_index,
1380                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1381                        );
1382                        response::record_model_failure(agent, model_str).await;
1383                        return Err(e);
1384                    }
1385                    tracing::warn!(
1386                        agent,
1387                        model = model_str,
1388                        error = %e,
1389                        "pool entry failed, recording cooldown and trying next"
1390                    );
1391                    if !cooldown::record_credit_exhaustion_from_message(
1392                        agent,
1393                        Some(model_str),
1394                        &err_msg,
1395                    )
1396                    .await
1397                    {
1398                        response::record_model_failure(agent, model_str).await;
1399                    }
1400                    last_err = Some(e);
1401                    self.advance_pool_index_after_attempt(idx, n);
1402                }
1403            }
1404        }
1405
1406        // If every pool entry was pre-cooled (nothing was attempted), do not
1407        // immediately exhaust the pool again — fail fast and let the tick loop
1408        // retry on the next cycle instead of cascading failures.
1409        // However, we should still try the fallback if configured.
1410        if !attempted_any {
1411            let now = chrono::Utc::now().timestamp();
1412            let earliest = self.earliest_pool_cooldown();
1413            if let Some(until) = earliest {
1414                // Pool entries exist but are all cooled - we'll try fallback below
1415                let remaining = until.saturating_sub(now);
1416                tracing::warn!(
1417                    remaining_secs = remaining,
1418                    pool = ?pool,
1419                    "all router LLM pool entries on cooldown — will try fallback"
1420                );
1421                // Continue to fallback logic instead of returning early
1422            } else {
1423                // No pool cooldowns found either — this means the pool was empty
1424                // or every entry was filtered out for some other reason.
1425                return Err(last_err
1426                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1427            }
1428        }
1429
1430        // All pool entries failed or were cooled — try the configured fallback
1431        let fallback = self.config.effective_fallback();
1432        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1433        let fb_model_str = fb_model.as_str();
1434
1435        // Only try fallback if it wasn't already in the pool (avoid double-try)
1436        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1437
1438        if !fallback_already_tried && !response::is_model_in_cooldown(&fb_agent, fb_model_str) {
1439            tracing::info!(
1440                agent = %fb_agent,
1441                model = %fb_model_str,
1442                "all pool entries exhausted — trying fallback router LLM"
1443            );
1444            let fb_model_opt = if fb_model_str.is_empty() {
1445                None
1446            } else {
1447                Some(fb_model_str)
1448            };
1449            match self
1450                .llm_router
1451                .route_with_llm_using(
1452                    task,
1453                    &llm_agents,
1454                    &self.config,
1455                    &mut self.last_agent,
1456                    repo,
1457                    &fb_agent,
1458                    fb_model_opt,
1459                )
1460                .await
1461            {
1462                Ok(result) => {
1463                    record_agent_success(&fb_agent, fb_model_str).await;
1464                    return Ok(result);
1465                }
1466                Err(e) => {
1467                    let err_msg = e.to_string();
1468                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1469                    if is_timeout {
1470                        // Cool down on timeout like pool entries (#3422), else a
1471                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1472                        response::record_model_failure(&fb_agent, fb_model_str).await;
1473                        tracing::warn!(
1474                            agent = %fb_agent,
1475                            model = %fb_model_str,
1476                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1477                        );
1478                        return Err(e);
1479                    }
1480                    tracing::warn!(
1481                        agent = %fb_agent,
1482                        model = %fb_model_str,
1483                        error = %e,
1484                        "fallback router LLM also failed"
1485                    );
1486                    response::record_model_failure(&fb_agent, fb_model_str).await;
1487                    last_err = Some(e);
1488                }
1489            }
1490        }
1491
1492        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1493    }
1494
1495    /// Record a rate limit event for an agent, reducing its routing weight.
1496    ///
1497    /// Called by the engine when an agent returns a 429/rate limit error.
1498    pub fn record_rate_limit(&mut self, agent: &str) {
1499        self.weights.record_rate_limit(agent);
1500    }
1501
1502    /// Record a successful task completion, restoring agent weight.
1503    pub fn record_success(&mut self, agent: &str) {
1504        self.weights.record_success(agent);
1505    }
1506
1507    /// Tick weight recovery for all agents.
1508    ///
1509    /// Called periodically by the engine to gradually restore weights
1510    /// as rate limit windows expire.
1511    pub fn tick_weight_recovery(&mut self) {
1512        self.weights.tick_recovery();
1513    }
1514
1515    /// Store routing result in the task store.
1516    pub async fn store_route_result(
1517        &self,
1518        task_id: &str,
1519        result: &RouteResult,
1520        store: &Arc<TaskStore>,
1521        repo: &str,
1522    ) -> anyhow::Result<()> {
1523        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1524            store
1525                .set_fields(
1526                    store_id,
1527                    &[
1528                        ("agent", serde_json::json!(result.agent)),
1529                        ("complexity", serde_json::json!(result.complexity)),
1530                        ("route_reason", serde_json::json!(result.reason)),
1531                        (
1532                            "model",
1533                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1534                        ),
1535                    ],
1536                )
1537                .await?;
1538        }
1539        Ok(())
1540    }
1541}
1542
1543/// Retrieve routing result from the task store.
1544pub async fn get_route_result(
1545    store: &Arc<TaskStore>,
1546    repo: &str,
1547    task_id: &str,
1548) -> Result<RouteResult, RouteResultError> {
1549    let store_id = store
1550        .resolve_task_id(repo, task_id)
1551        .await
1552        .map_err(RouteResultError::Other)?
1553        .ok_or_else(|| {
1554            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1555        })?;
1556    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1557
1558    let agent = task
1559        .agent
1560        .clone()
1561        .filter(|a| !a.is_empty())
1562        .ok_or_else(|| RouteResultError::NoAgent {
1563            task_id: task_id.to_string(),
1564        })?;
1565
1566    let complexity = if task.complexity.is_empty() {
1567        "medium".to_string()
1568    } else {
1569        task.complexity.clone()
1570    };
1571
1572    let reason = task.route_reason.clone();
1573    let model = task.model.clone().filter(|m| !m.is_empty());
1574
1575    // Validate that the stored model is valid for the resolved agent.
1576    // Agent-specific aliases (e.g. "opus") can leak across agents during
1577    // failover, so we verify against the router's model pools (#1604).
1578    let config = RouterConfig::from_config();
1579    let model = model.filter(|m| {
1580        let complexities = ["simple", "medium", "complex", "review"];
1581        // Collect all model pools configured for this agent across complexity tiers.
1582        let all_pools: Vec<Vec<String>> = complexities
1583            .iter()
1584            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1585            .collect();
1586        // If the agent has no model pools configured, we can't validate — pass through.
1587        if all_pools.is_empty() {
1588            return true;
1589        }
1590        let valid = all_pools.iter().any(|pool| pool.contains(m));
1591        if !valid {
1592            tracing::warn!(
1593                task_id,
1594                agent,
1595                model = %m,
1596                "discarding stale model: not in agent's model pools"
1597            );
1598        }
1599        valid
1600    });
1601
1602    // A stale model is replaced by the agent's model for the task's complexity, an empty
1603    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1604    let original_model = task.model.clone().filter(|m| !m.is_empty());
1605    let mut model = model;
1606    if original_model.is_some() && model.is_none() {
1607        model = config.model_for_complexity(&agent, &complexity, task_id);
1608        let stale_model = original_model.as_deref().unwrap_or("unknown");
1609        let stale_err = format!(
1610            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1611            model.as_deref().unwrap_or("the agent default")
1612        );
1613        if let Err(e) = store
1614            .set_fields(
1615                store_id,
1616                &[
1617                    (
1618                        "model",
1619                        serde_json::json!(model.clone().unwrap_or_default()),
1620                    ),
1621                    ("last_error", serde_json::json!(stale_err)),
1622                ],
1623            )
1624            .await
1625        {
1626            // Log the error so transient DB failures are visible and diagnosable.
1627            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1628        }
1629    }
1630
1631    Ok(RouteResult {
1632        agent,
1633        model,
1634        complexity,
1635        estimate: 0,
1636        reason,
1637        profile: AgentProfile::default(),
1638        selected_skills: vec![],
1639        warning: None,
1640    })
1641}
1642
1643#[cfg(test)]
1644#[expect(
1645    clippy::float_cmp,
1646    reason = "the weights are stored constants, so they compare exactly"
1647)]
1648mod tests {
1649    use super::config::DEFAULT_AGENTS;
1650    use super::llm::LlmRouteResponse;
1651    use super::strategies;
1652    use super::weights::default_missing_weight;
1653    use super::weights::{
1654        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1655    };
1656    use super::*;
1657    use crate::backends::{ExternalId, ExternalTask};
1658    use crate::engine::cooldown::{
1659        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1660        record_model_failure, set_model_cooldown,
1661    };
1662    use crate::store::NewTask;
1663    use crate::store::TaskStore;
1664    use serial_test::serial;
1665    use std::collections::HashMap;
1666    use std::env;
1667    #[cfg(unix)]
1668    use std::ffi::OsString;
1669    use std::fs;
1670    #[cfg(unix)]
1671    use std::fs::Permissions;
1672    #[cfg(unix)]
1673    use std::os::unix::fs::PermissionsExt;
1674    #[cfg(unix)]
1675    use std::process;
1676    #[cfg(unix)]
1677    use std::process::Command;
1678    #[cfg(unix)]
1679    use std::thread;
1680    use std::time::{Duration, Instant};
1681
1682    #[tokio::test]
1683    #[serial_test::serial(orch_home)]
1684    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1685        let home = tempfile::tempdir().unwrap();
1686        fs::write(
1687            home.path().join("config.yml"),
1688            "model_map:\n  medium:\n    claude: sonnet\n",
1689        )
1690        .unwrap();
1691        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1692        unsafe { env::set_var("ORCH_HOME", home.path()) };
1693
1694        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1695        let store_id = store
1696            .create(&NewTask {
1697                external_id: Some("7".to_string()),
1698                repo: "owner/repo".to_string(),
1699                origin: "github".to_string(),
1700                title: "stale model".to_string(),
1701                ..Default::default()
1702            })
1703            .await
1704            .unwrap();
1705        store
1706            .set_fields(
1707                store_id,
1708                &[
1709                    ("agent", serde_json::json!("claude")),
1710                    ("model", serde_json::json!("claude-removed-model")),
1711                    ("complexity", serde_json::json!("medium")),
1712                ],
1713            )
1714            .await
1715            .unwrap();
1716
1717        let route = get_route_result(&store, "owner/repo", "7").await;
1718        let stored = store.get(store_id).await.unwrap().model;
1719        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1720        unsafe { env::remove_var("ORCH_HOME") };
1721
1722        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1723        assert_eq!(stored.as_deref(), Some("sonnet"));
1724    }
1725
1726    // Test-only delegate so tests can call router.parse_llm_response() directly
1727    // without referencing llm_router.
1728    impl Router {
1729        #[expect(
1730            clippy::unused_self,
1731            reason = "the test helper keeps the router method syntax"
1732        )]
1733        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1734            LlmRouter::parse_llm_response("claude", response)
1735        }
1736    }
1737
1738    async fn test_store() -> Arc<TaskStore> {
1739        Arc::new(TaskStore::open_memory().await.unwrap())
1740    }
1741
1742    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1743        ExternalTask {
1744            id: ExternalId(id.to_string()),
1745            title: title.to_string(),
1746            body: "Test body".to_string(),
1747            state: "open".to_string(),
1748            labels,
1749            author: "test".to_string(),
1750            created_at: "2026-01-01T00:00:00Z".to_string(),
1751            updated_at: "2026-01-01T00:00:00Z".to_string(),
1752            url: format!("https://github.com/test/test/issues/{id}"),
1753        }
1754    }
1755
1756    #[serial(cooldown_state)]
1757    #[test]
1758    fn extract_agent_from_labels() {
1759        let mut config = RouterConfig::default();
1760        // Ensure a concrete model exists for the labeled agent so the label
1761        // override is honored by the router (routing requires a model).
1762        config
1763            .model_map
1764            .entry("medium".to_string())
1765            .or_default()
1766            .insert("claude".to_string(), vec!["haiku".to_string()]);
1767
1768        assert_eq!(
1769            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1770            Some("claude".to_string())
1771        );
1772        assert_eq!(
1773            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1774            Some("codex".to_string())
1775        );
1776        assert_eq!(
1777            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1778            Some("opencode".to_string())
1779        );
1780        assert_eq!(
1781            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1782            None
1783        );
1784        // Verify kimi and minimax are recognized from labels
1785        assert_eq!(
1786            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1787            Some("kimi".to_string())
1788        );
1789        assert_eq!(
1790            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1791            Some("minimax".to_string())
1792        );
1793    }
1794
1795    #[serial(cooldown_state)]
1796    #[tokio::test]
1797    async fn all_agents_cooled_error_names_degraded_agents() {
1798        cooldown::reset_global_state().await;
1799        mark_agent_degraded("claude");
1800        mark_agent_degraded("grok");
1801        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1802        let msg = err.to_string();
1803        assert!(
1804            msg.contains("degraded: claude, grok"),
1805            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1806        );
1807        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1808        clear_agent_degraded("claude");
1809        clear_agent_degraded("grok");
1810    }
1811
1812    #[serial(cooldown_state)]
1813    #[test]
1814    fn default_agents_constant() {
1815        assert_eq!(DEFAULT_AGENTS.len(), 7);
1816        assert!(DEFAULT_AGENTS.contains(&"claude"));
1817        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1818        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1819        assert!(DEFAULT_AGENTS.contains(&"glm"));
1820        assert!(DEFAULT_AGENTS.contains(&"grok"));
1821    }
1822
1823    #[serial(cooldown_state)]
1824    #[test]
1825    fn extract_complexity_from_labels() {
1826        assert_eq!(
1827            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1828            "simple"
1829        );
1830        assert_eq!(
1831            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1832            "medium"
1833        );
1834        assert_eq!(
1835            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1836            "complex"
1837        );
1838        assert_eq!(
1839            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1840            "medium"
1841        );
1842    }
1843
1844    #[serial(cooldown_state)]
1845    #[test]
1846    fn route_result_serialization() {
1847        let result = RouteResult {
1848            agent: "claude".to_string(),
1849            model: Some("sonnet".to_string()),
1850            complexity: "medium".to_string(),
1851            estimate: 5,
1852            reason: "test".to_string(),
1853            profile: AgentProfile {
1854                role: "backend".to_string(),
1855                skills: vec!["rust".to_string()],
1856                tools: vec!["git".to_string()],
1857                constraints: vec![],
1858            },
1859            selected_skills: vec!["gh".to_string()],
1860            warning: None,
1861        };
1862
1863        let json = serde_json::to_string(&result).unwrap();
1864        assert!(json.contains("claude"));
1865        assert!(json.contains("sonnet"));
1866
1867        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1868        assert_eq!(deserialized.agent, "claude");
1869        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1870    }
1871
1872    #[serial(cooldown_state)]
1873    #[test]
1874    fn router_config_default() {
1875        let mut config = RouterConfig::default();
1876        // Ensure a concrete model exists for the labeled agent so the label
1877        // override is honored by the router (routing requires a model).
1878        config
1879            .model_map
1880            .entry("medium".to_string())
1881            .or_default()
1882            .insert("claude".to_string(), vec!["haiku".to_string()]);
1883
1884        assert_eq!(config.mode, "round_robin");
1885        assert_eq!(config.router_agent, "claude");
1886        assert_eq!(
1887            config.router_model, "",
1888            "no router.model means the router agent's CLI default"
1889        );
1890        assert_eq!(config.fallback_executor, "codex");
1891        assert_eq!(config.max_route_attempts, 3);
1892        assert_ne!(config.allowed_tools, [] as [String; 0]);
1893        assert_ne!(config.default_skills, [] as [String; 0]);
1894
1895        // Verify configurable agents list includes all 7 agents
1896        assert_eq!(config.agents.len(), 7);
1897        assert!(config.agents.contains(&"claude".to_string()));
1898        assert!(config.agents.contains(&"codex".to_string()));
1899        assert!(config.agents.contains(&"opencode".to_string()));
1900        assert!(config.agents.contains(&"kimi".to_string()));
1901        assert!(config.agents.contains(&"minimax".to_string()));
1902        assert!(config.agents.contains(&"glm".to_string()));
1903        assert!(config.agents.contains(&"grok".to_string()));
1904    }
1905
1906    #[serial(cooldown_state)]
1907    #[test]
1908    fn model_map_lookup_returns_none_without_config() {
1909        // Default config has no hardcoded models — config.yml is the source of truth.
1910        let config = RouterConfig::default();
1911        assert!(
1912            config
1913                .model_for_complexity("claude", "simple", "")
1914                .is_none()
1915        );
1916        assert!(
1917            config
1918                .model_for_complexity("claude", "medium", "")
1919                .is_none()
1920        );
1921        assert!(
1922            config
1923                .model_for_complexity("claude", "complex", "")
1924                .is_none()
1925        );
1926        assert!(
1927            config
1928                .model_for_complexity("opencode", "review", "")
1929                .is_none()
1930        );
1931    }
1932
1933    #[serial(cooldown_state)]
1934    #[test]
1935    fn model_map_lookup_returns_configured_value() {
1936        // When model_map is populated (from config), it is returned correctly.
1937        let mut config = RouterConfig::default();
1938        config
1939            .model_map
1940            .entry("simple".to_string())
1941            .or_default()
1942            .insert("claude".to_string(), vec!["haiku".to_string()]);
1943        assert_eq!(
1944            config.model_for_complexity("claude", "simple", ""),
1945            Some("haiku".to_string())
1946        );
1947    }
1948
1949    #[serial(cooldown_state)]
1950    #[tokio::test]
1951    async fn model_pool_selection_skips_cooled() {
1952        cooldown::reset_global_state().await;
1953
1954        let mut config = RouterConfig::default();
1955        // Set up a two-model pool for opencode/simple
1956        config
1957            .model_map
1958            .entry("simple".to_string())
1959            .or_default()
1960            .insert(
1961                "opencode".to_string(),
1962                vec!["model-a".to_string(), "model-b".to_string()],
1963            );
1964
1965        // Before any cooldown, both models are candidates
1966        let m = config.model_for_complexity("opencode", "simple", "task-1");
1967        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1968
1969        // Cool model-a
1970        record_model_failure("opencode", "model-a").await;
1971        assert!(is_model_in_cooldown("opencode", "model-a"));
1972
1973        // Now only model-b should be returned
1974        // (try many task_ids to rule out hash coincidence)
1975        for i in 0..20 {
1976            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1977            assert_eq!(
1978                result,
1979                Some("model-b".to_string()),
1980                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1981            );
1982        }
1983
1984        // Cool model-b too — now all models are cooled
1985        record_model_failure("opencode", "model-b").await;
1986        // All cooled → return None so the caller can fall back to a different agent
1987        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1988        assert_eq!(fallback, None);
1989    }
1990
1991    #[serial(cooldown_state)]
1992    #[test]
1993    fn model_pool_single_string_backward_compat() {
1994        // Single-item pools behave identically to the old string format.
1995        // Manually insert a single-item pool and verify it is returned correctly.
1996        let mut config = RouterConfig::default();
1997        config
1998            .model_map
1999            .entry("simple".to_string())
2000            .or_default()
2001            .insert("claude".to_string(), vec!["haiku".to_string()]);
2002        let m = config.model_for_complexity("claude", "simple", "any-task");
2003        assert_eq!(m, Some("haiku".to_string()));
2004    }
2005
2006    #[serial(cooldown_state)]
2007    #[test]
2008    fn parse_llm_response_direct_json() {
2009        let config = RouterConfig::default();
2010        let router = Router::new(config);
2011
2012        let response = r#"{
2013            "executor": "claude",
2014            "complexity": "complex",
2015            "reason": "requires architecture analysis",
2016            "profile": {
2017                "role": "architect",
2018                "skills": ["rust", "design"],
2019                "tools": ["git", "rg"],
2020                "constraints": []
2021            },
2022            "selected_skills": ["gh"]
2023        }"#;
2024
2025        let parsed = router.parse_llm_response(response).unwrap();
2026        assert_eq!(parsed.executor, "claude");
2027        assert_eq!(parsed.complexity, "complex");
2028        assert_eq!(parsed.reason, "requires architecture analysis");
2029        assert_eq!(parsed.profile.role, "architect");
2030    }
2031
2032    #[serial(cooldown_state)]
2033    #[test]
2034    fn parse_llm_response_markdown_fenced() {
2035        let config = RouterConfig::default();
2036        let router = Router::new(config);
2037
2038        let response = r#"Here's my analysis:
2039
2040```json
2041{
2042    "executor": "codex",
2043    "complexity": "medium",
2044    "reason": "coding task",
2045    "profile": {
2046        "role": "developer",
2047        "skills": ["coding"],
2048        "tools": [],
2049        "constraints": []
2050    },
2051    "selected_skills": []
2052}
2053```
2054
2055Hope that helps!"#;
2056
2057        let parsed = router.parse_llm_response(response).unwrap();
2058        assert_eq!(parsed.executor, "codex");
2059        assert_eq!(parsed.complexity, "medium");
2060    }
2061
2062    #[serial(cooldown_state)]
2063    #[test]
2064    fn parse_llm_response_claude_envelope() {
2065        let config = RouterConfig::default();
2066        let router = Router::new(config);
2067
2068        // Claude --output-format json wraps the response in an envelope
2069        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2070
2071        let parsed = router.parse_llm_response(response).unwrap();
2072        assert_eq!(parsed.executor, "claude");
2073        assert_eq!(parsed.complexity, "complex");
2074        assert_eq!(parsed.reason, "requires deep analysis");
2075    }
2076
2077    #[serial(cooldown_state)]
2078    #[test]
2079    fn parse_llm_response_claude_envelope_with_code_block() {
2080        let config = RouterConfig::default();
2081        let router = Router::new(config);
2082
2083        // Claude may return the JSON inside a code block within the envelope
2084        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2085        let envelope = serde_json::json!({
2086            "type": "result",
2087            "subtype": "success",
2088            "is_error": false,
2089            "result": inner,
2090            "usage": {"input_tokens": 50, "output_tokens": 30}
2091        });
2092
2093        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2094        assert_eq!(parsed.executor, "codex");
2095        assert_eq!(parsed.complexity, "simple");
2096    }
2097
2098    #[serial(cooldown_state)]
2099    #[test]
2100    fn parse_llm_response_claude_envelope_error() {
2101        let config = RouterConfig::default();
2102        let router = Router::new(config);
2103
2104        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2105
2106        let err = router.parse_llm_response(response).unwrap_err();
2107        assert!(err.to_string().contains("error"), "got: {err}");
2108    }
2109
2110    #[serial(cooldown_state)]
2111    #[test]
2112    fn parse_llm_response_empty() {
2113        let config = RouterConfig::default();
2114        let router = Router::new(config);
2115
2116        let err = router.parse_llm_response("").unwrap_err();
2117        assert!(err.to_string().contains("empty"), "got: {err}");
2118    }
2119
2120    /// Claude envelope where result is a JSON object, not a string.
2121    /// Some Claude versions may return the result as a nested object.
2122    #[serial(cooldown_state)]
2123    #[test]
2124    fn parse_llm_response_claude_envelope_result_as_object() {
2125        let config = RouterConfig::default();
2126        let router = Router::new(config);
2127
2128        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2129
2130        let parsed = router.parse_llm_response(response);
2131        assert!(
2132            parsed.is_ok(),
2133            "should handle result-as-object: {}",
2134            parsed.unwrap_err()
2135        );
2136        assert_eq!(parsed.unwrap().executor, "claude");
2137    }
2138
2139    /// LLM returns "agent" instead of "executor" — common hallucination.
2140    #[serial(cooldown_state)]
2141    #[test]
2142    fn parse_llm_response_agent_instead_of_executor() {
2143        let config = RouterConfig::default();
2144        let router = Router::new(config);
2145
2146        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2147
2148        let parsed = router.parse_llm_response(response);
2149        assert!(
2150            parsed.is_ok(),
2151            "should accept 'agent' alias: {}",
2152            parsed.unwrap_err()
2153        );
2154        assert_eq!(parsed.unwrap().executor, "claude");
2155    }
2156
2157    /// LLM returns minimal JSON without profile or reason.
2158    #[serial(cooldown_state)]
2159    #[test]
2160    fn parse_llm_response_minimal_json() {
2161        let config = RouterConfig::default();
2162        let router = Router::new(config);
2163
2164        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2165
2166        let parsed = router.parse_llm_response(response);
2167        assert!(
2168            parsed.is_ok(),
2169            "should accept minimal JSON: {}",
2170            parsed.unwrap_err()
2171        );
2172        assert_eq!(parsed.unwrap().executor, "codex");
2173    }
2174
2175    // ── Real fixture tests: route prompt → response → parsed result ──
2176
2177    /// Real Claude envelope with result as escaped JSON string.
2178    #[serial(cooldown_state)]
2179    #[test]
2180    fn fixture_route_response_string() {
2181        let config = RouterConfig::default();
2182        let router = Router::new(config);
2183
2184        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2185        let parsed = router.parse_llm_response(response).unwrap();
2186        assert_eq!(parsed.executor, "codex");
2187        assert_eq!(parsed.complexity, "medium");
2188        assert_ne!(parsed.reason, "");
2189        assert_ne!(parsed.profile.role, "");
2190    }
2191
2192    /// Real Claude envelope with result as JSON object (not string).
2193    /// This was the root cause of all router failures on 2026-02-27.
2194    #[serial(cooldown_state)]
2195    #[test]
2196    fn fixture_route_response_object() {
2197        let config = RouterConfig::default();
2198        let router = Router::new(config);
2199
2200        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2201        let parsed = router.parse_llm_response(response).unwrap();
2202        assert_eq!(parsed.executor, "claude");
2203        assert_eq!(parsed.complexity, "complex");
2204    }
2205
2206    /// Real Claude envelope with result as string containing markdown + JSON.
2207    #[serial(cooldown_state)]
2208    #[test]
2209    fn fixture_route_response_markdown() {
2210        let config = RouterConfig::default();
2211        let router = Router::new(config);
2212
2213        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2214        let parsed = router.parse_llm_response(response).unwrap();
2215        assert_eq!(parsed.executor, "codex");
2216        assert_eq!(parsed.complexity, "medium");
2217    }
2218
2219    /// Claude envelope with escaped JSON string containing newlines and markdown.
2220    #[serial(cooldown_state)]
2221    #[test]
2222    fn parse_llm_response_claude_envelope_with_prose() {
2223        let config = RouterConfig::default();
2224        let router = Router::new(config);
2225
2226        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2227        let envelope = serde_json::json!({
2228            "type": "result",
2229            "subtype": "success",
2230            "is_error": false,
2231            "result": inner,
2232            "usage": {"input_tokens": 100, "output_tokens": 80}
2233        });
2234
2235        let parsed = router.parse_llm_response(&envelope.to_string());
2236        assert!(
2237            parsed.is_ok(),
2238            "should handle prose + code block in envelope: {}",
2239            parsed.unwrap_err()
2240        );
2241        assert_eq!(parsed.unwrap().executor, "codex");
2242    }
2243
2244    #[serial(cooldown_state)]
2245    #[tokio::test]
2246    async fn route_round_robin_basic() {
2247        cooldown::reset_global_state().await;
2248        // Force at least one agent to be available for testing
2249        // In real usage, discover_agents finds installed CLIs
2250        let config = RouterConfig {
2251            mode: "round_robin".to_string(),
2252            ..Default::default()
2253        };
2254
2255        // Create router with mock available agents
2256        let agents = vec!["claude".to_string(), "codex".to_string()];
2257        let mut weights = AgentWeights::default();
2258        weights.ensure_agents(&agents);
2259        let router = Router {
2260            config,
2261            available_agents: agents,
2262            weights,
2263            llm_router: Arc::new(LlmRouter::new()),
2264            ollama_router: None,
2265            rr_index: 0,
2266            last_agent: None,
2267            review_rr_index: 0,
2268            router_pool: vec![],
2269            pool_index: 0,
2270        };
2271
2272        let task = create_test_task("1", "Test task", vec![]);
2273        let result =
2274            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2275                .unwrap();
2276
2277        // Task 1 % 2 agents = agent at index 1 = codex
2278        assert_eq!(result.agent, "codex");
2279        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2280    }
2281
2282    #[serial(cooldown_state)]
2283    #[tokio::test]
2284    async fn route_uses_label_override() {
2285        cooldown::reset_global_state().await;
2286        let mut config = RouterConfig::default();
2287        // Ensure a concrete model exists so the label override can be honored
2288        // in the test environment where no external config.yml is present.
2289        config
2290            .model_map
2291            .entry("medium".to_string())
2292            .or_default()
2293            .insert("claude".to_string(), vec!["haiku".to_string()]);
2294        let agents = vec!["claude".to_string(), "codex".to_string()];
2295        let mut weights = AgentWeights::default();
2296        weights.ensure_agents(&agents);
2297        let mut router = Router {
2298            config,
2299            available_agents: agents,
2300            weights,
2301            llm_router: Arc::new(LlmRouter::new()),
2302            ollama_router: None,
2303            rr_index: 0,
2304            last_agent: None,
2305            review_rr_index: 0,
2306            router_pool: vec![],
2307            pool_index: 0,
2308        };
2309
2310        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2311        let store = test_store().await;
2312
2313        // Should use label override, not LLM
2314        let result = router.route(&task, &store, "test/repo").await.unwrap();
2315        assert_eq!(result.agent, "claude");
2316        assert!(result.reason.contains("label"));
2317    }
2318
2319    #[serial(cooldown_state)]
2320    #[test]
2321    fn check_routing_sanity_warnings() {
2322        // Backend task routed to claude should warn
2323        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2324        let profile = AgentProfile {
2325            role: "general".to_string(),
2326            skills: vec!["api".to_string()],
2327            tools: vec![],
2328            constraints: vec![],
2329        };
2330        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2331        assert!(warning.is_some());
2332        assert!(warning.unwrap().contains("backend"));
2333
2334        // Docs task routed to codex should warn
2335        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2336        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2337        assert!(warning.is_some());
2338        assert!(warning.unwrap().contains("docs"));
2339
2340        // Normal routing should not warn
2341        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2342        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2343        assert!(warning.is_none());
2344    }
2345
2346    #[serial(cooldown_state)]
2347    #[tokio::test]
2348    async fn router_reload_preserves_structure() {
2349        cooldown::reset_global_state().await;
2350        let config = RouterConfig::default();
2351        let agents = vec!["claude".to_string()];
2352        let mut weights = AgentWeights::default();
2353        weights.ensure_agents(&agents);
2354        let mut router = Router {
2355            config,
2356            available_agents: agents,
2357            weights,
2358            llm_router: Arc::new(LlmRouter::new()),
2359            ollama_router: None,
2360            rr_index: 0,
2361            last_agent: None,
2362            review_rr_index: 0,
2363            router_pool: vec![],
2364            pool_index: 0,
2365        };
2366
2367        // Reload — should re-read config and remain valid
2368        router.reload_async().await;
2369
2370        // After reload, mode should be a valid value (llm, local, or round_robin)
2371        assert!(
2372            router.config.mode == "llm"
2373                || router.config.mode == "round_robin"
2374                || router.config.mode == "local",
2375            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2376            router.config.mode
2377        );
2378        // Fallback executor should always be set
2379        assert_ne!(router.config.fallback_executor, "");
2380        // Tools should always be populated
2381        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2382    }
2383
2384    // --- Weighted round-robin tests ---
2385
2386    #[serial(cooldown_state)]
2387    #[test]
2388    fn rate_limit_state_defaults_to_full_weight() {
2389        let state = RateLimitState::default();
2390        assert_eq!(state.weight, DEFAULT_WEIGHT);
2391        assert_eq!(state.consecutive_hits, 0);
2392        assert!(state.last_limited_at.is_none());
2393        assert!(!state.is_limited());
2394    }
2395
2396    #[serial(cooldown_state)]
2397    #[test]
2398    fn rate_limit_state_decays_on_hit() {
2399        let mut state = RateLimitState::default();
2400        state.record_rate_limit();
2401
2402        assert!(state.weight < DEFAULT_WEIGHT);
2403        assert_eq!(state.consecutive_hits, 1);
2404        assert!(state.last_limited_at.is_some());
2405        assert!(state.is_limited());
2406
2407        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2408        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2409        assert!((state.weight - expected).abs() < 1e-10);
2410    }
2411
2412    #[serial(cooldown_state)]
2413    #[test]
2414    fn rate_limit_state_never_drops_below_min() {
2415        let mut state = RateLimitState::default();
2416
2417        // Hit many times
2418        for _ in 0..100 {
2419            state.record_rate_limit();
2420        }
2421
2422        assert!(state.weight >= MIN_WEIGHT);
2423        assert_eq!(state.consecutive_hits, 100);
2424    }
2425
2426    #[serial(cooldown_state)]
2427    #[test]
2428    fn rate_limit_state_recovers_on_success() {
2429        let mut state = RateLimitState::default();
2430
2431        // Decay first
2432        state.record_rate_limit();
2433        state.record_rate_limit();
2434        let after_decay = state.weight;
2435
2436        // Record success
2437        state.record_success();
2438        assert!(state.weight > after_decay);
2439        assert_eq!(state.consecutive_hits, 0);
2440    }
2441
2442    #[serial(cooldown_state)]
2443    #[test]
2444    fn rate_limit_state_success_caps_at_default() {
2445        let mut state = RateLimitState::default();
2446
2447        // Already at full weight, success shouldn't exceed it
2448        state.record_success();
2449        assert_eq!(state.weight, DEFAULT_WEIGHT);
2450    }
2451
2452    #[serial(cooldown_state)]
2453    #[test]
2454    fn agent_weights_ensure_agents() {
2455        let mut weights = AgentWeights::default();
2456        let agents = vec!["claude".to_string(), "codex".to_string()];
2457        weights.ensure_agents(&agents);
2458
2459        assert_eq!(weights.states.len(), 2);
2460        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2461        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2462    }
2463
2464    #[serial(cooldown_state)]
2465    #[test]
2466    fn agent_weights_ensure_agents_preserves_existing() {
2467        let mut weights = AgentWeights::default();
2468        let agents = vec!["claude".to_string(), "codex".to_string()];
2469        weights.ensure_agents(&agents);
2470
2471        // Reduce claude's weight
2472        weights.record_rate_limit("claude");
2473        let claude_weight = weights.get_weight("claude");
2474        assert!(claude_weight < DEFAULT_WEIGHT);
2475
2476        // Ensure agents again — claude's weight should be preserved
2477        let agents2 = vec![
2478            "claude".to_string(),
2479            "codex".to_string(),
2480            "opencode".to_string(),
2481        ];
2482        weights.ensure_agents(&agents2);
2483
2484        assert_eq!(weights.get_weight("claude"), claude_weight);
2485        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2486    }
2487
2488    #[test]
2489    fn default_missing_weight_uses_minimum_of_configured() {
2490        let mut configured = HashMap::new();
2491        configured.insert("claude".to_string(), 0.2);
2492        configured.insert("codex".to_string(), 0.2);
2493        configured.insert("opencode".to_string(), 0.2);
2494        configured.insert("kimi".to_string(), 0.2);
2495
2496        // minimax omitted from router.weights — must not silently outrank
2497        // every explicitly configured agent (issue #3596).
2498        assert_eq!(default_missing_weight(&configured), 0.2);
2499    }
2500
2501    #[test]
2502    fn default_missing_weight_falls_back_when_none_configured() {
2503        let configured = HashMap::new();
2504        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2505    }
2506
2507    #[serial(cooldown_state)]
2508    #[test]
2509    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2510        let mut weights = AgentWeights::default();
2511        let mut configured = HashMap::new();
2512        configured.insert("claude".to_string(), 0.2);
2513        configured.insert("codex".to_string(), 0.2);
2514        configured.insert("opencode".to_string(), 0.2);
2515        configured.insert("kimi".to_string(), 0.2);
2516
2517        let agents = vec![
2518            "claude".to_string(),
2519            "codex".to_string(),
2520            "opencode".to_string(),
2521            "kimi".to_string(),
2522            "minimax".to_string(),
2523        ];
2524        weights.ensure_agents_with_weights(&agents, &configured);
2525
2526        // minimax was omitted from router.weights but must not get the old
2527        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2528        assert_eq!(weights.get_weight("minimax"), 0.2);
2529        assert_eq!(weights.get_weight("claude"), 0.2);
2530    }
2531
2532    #[serial(cooldown_state)]
2533    #[test]
2534    fn agent_weights_record_rate_limit() {
2535        let mut weights = AgentWeights::default();
2536        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2537
2538        weights.record_rate_limit("claude");
2539
2540        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2541        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2542    }
2543
2544    #[serial(cooldown_state)]
2545    #[test]
2546    fn agent_weights_record_success() {
2547        let mut weights = AgentWeights::default();
2548        weights.ensure_agents(&["claude".to_string()]);
2549
2550        weights.record_rate_limit("claude");
2551        let after_limit = weights.get_weight("claude");
2552
2553        weights.record_success("claude");
2554        assert!(weights.get_weight("claude") > after_limit);
2555    }
2556
2557    #[serial(cooldown_state)]
2558    #[test]
2559    fn agent_weights_snapshot() {
2560        let mut weights = AgentWeights::default();
2561        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2562        weights.record_rate_limit("claude");
2563
2564        let snap = weights.snapshot();
2565        assert_eq!(snap.len(), 2);
2566
2567        // Snapshot is sorted alphabetically
2568        assert_eq!(snap[0].0, "claude");
2569        assert!(snap[0].1 < DEFAULT_WEIGHT);
2570        assert_eq!(snap[0].2, 1); // 1 hit
2571
2572        assert_eq!(snap[1].0, "codex");
2573        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2574        assert_eq!(snap[1].2, 0); // 0 hits
2575    }
2576
2577    #[serial(cooldown_state)]
2578    #[test]
2579    fn agent_weights_weighted_select_favors_higher_weight() {
2580        let mut weights = AgentWeights::default();
2581        let agents = vec!["claude".to_string(), "codex".to_string()];
2582        weights.ensure_agents(&agents);
2583
2584        // Heavily penalize claude
2585        for _ in 0..10 {
2586            weights.record_rate_limit("claude");
2587        }
2588
2589        // Run many selections and count
2590        let mut claude_count = 0;
2591        let mut codex_count = 0;
2592        for _ in 0..100 {
2593            match weights.weighted_select(&agents, "task-1") {
2594                Some(ref a) if a == "claude" => claude_count += 1,
2595                Some(ref a) if a == "codex" => codex_count += 1,
2596                _ => {}
2597            }
2598        }
2599
2600        // Codex should get significantly more selections
2601        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2602        assert!(
2603            codex_count > claude_count,
2604            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2605        );
2606    }
2607
2608    #[serial(cooldown_state)]
2609    #[test]
2610    fn agent_weights_weighted_select_empty_returns_none() {
2611        let weights = AgentWeights::default();
2612        assert!(weights.weighted_select(&[], "task-1").is_none());
2613    }
2614
2615    #[serial(cooldown_state)]
2616    #[test]
2617    fn agent_weights_batch_routing_produces_varied_selections() {
2618        // Regression test: when routing a batch of tasks in the same tick,
2619        // different task IDs should not all select the same agent.
2620        let mut weights = AgentWeights::default();
2621        let agents = vec!["claude".to_string(), "codex".to_string()];
2622        weights.ensure_agents(&agents);
2623
2624        // Simulate batch routing: 20 different task IDs called in rapid succession
2625        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2626        let selections: Vec<String> = task_ids
2627            .iter()
2628            .filter_map(|id| weights.weighted_select(&agents, id))
2629            .collect();
2630
2631        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2632        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2633        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2634        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2635        assert!(
2636            claude_count > 0 && codex_count > 0,
2637            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2638        );
2639    }
2640
2641    #[serial(cooldown_state)]
2642    #[test]
2643    fn agent_weights_weighted_select_single_agent() {
2644        let mut weights = AgentWeights::default();
2645        let agents = vec!["claude".to_string()];
2646        weights.ensure_agents(&agents);
2647
2648        let selected = weights.weighted_select(&agents, "task-1");
2649        assert_eq!(selected, Some("claude".to_string()));
2650    }
2651
2652    #[serial(cooldown_state)]
2653    #[test]
2654    fn agent_weights_tick_recovery_restores_weight() {
2655        let mut weights = AgentWeights::default();
2656        weights.ensure_agents(&["claude".to_string()]);
2657
2658        // Set up a rate limit that looks like it happened long ago
2659        let state = weights.states.get_mut("claude").unwrap();
2660        state.weight = 0.5;
2661        state.last_limited_at = Some(
2662            Instant::now()
2663                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2664                .expect("the clock is past the recovery delay"),
2665        );
2666        state.consecutive_hits = 2;
2667
2668        let before = weights.get_weight("claude");
2669        weights.tick_recovery();
2670        let after = weights.get_weight("claude");
2671
2672        assert!(after > before, "weight should increase after recovery tick");
2673    }
2674
2675    #[serial(cooldown_state)]
2676    #[test]
2677    fn agent_weights_tick_recovery_clears_on_full_restore() {
2678        let mut weights = AgentWeights::default();
2679        weights.ensure_agents(&["claude".to_string()]);
2680
2681        // Set weight just below full with an old limit
2682        let state = weights.states.get_mut("claude").unwrap();
2683        state.weight = DEFAULT_WEIGHT - 0.01;
2684        state.last_limited_at = Some(
2685            Instant::now()
2686                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2687                .expect("the clock is past the recovery delay"),
2688        );
2689        state.consecutive_hits = 1;
2690
2691        weights.tick_recovery();
2692
2693        let state = weights.states.get("claude").unwrap();
2694        assert_eq!(state.weight, DEFAULT_WEIGHT);
2695        assert!(state.last_limited_at.is_none());
2696        assert_eq!(state.consecutive_hits, 0);
2697    }
2698
2699    #[serial(cooldown_state)]
2700    #[tokio::test]
2701    async fn route_weighted_round_robin_basic() {
2702        cooldown::reset_global_state().await;
2703        let mut config = RouterConfig {
2704            weighted_round_robin: true,
2705            ..Default::default()
2706        };
2707        // Provide a concrete model for codex so label override can be used.
2708        config
2709            .model_map
2710            .entry("medium".to_string())
2711            .or_default()
2712            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2713
2714        let agents = vec!["claude".to_string(), "codex".to_string()];
2715        let mut weights = AgentWeights::default();
2716        weights.ensure_agents(&agents);
2717        let mut router = Router {
2718            config,
2719            available_agents: agents,
2720            weights,
2721            llm_router: Arc::new(LlmRouter::new()),
2722            ollama_router: None,
2723            rr_index: 0,
2724            last_agent: None,
2725            review_rr_index: 0,
2726            router_pool: vec![],
2727            pool_index: 0,
2728        };
2729
2730        let task = create_test_task("1", "Test task", vec![]);
2731        let store = test_store().await;
2732        let result = router.route(&task, &store, "test/repo").await.unwrap();
2733
2734        // Should use weighted_round_robin
2735        assert!(result.reason.contains("weighted_round_robin"));
2736        assert!(
2737            result.agent == "claude" || result.agent == "codex",
2738            "agent should be claude or codex, got '{}'",
2739            result.agent
2740        );
2741    }
2742
2743    #[serial(cooldown_state)]
2744    #[tokio::test]
2745    async fn route_weighted_round_robin_respects_label_override() {
2746        cooldown::reset_global_state().await;
2747        let mut config = RouterConfig {
2748            weighted_round_robin: true,
2749            ..Default::default()
2750        };
2751        // Provide a concrete model for codex so label override can be used.
2752        config
2753            .model_map
2754            .entry("medium".to_string())
2755            .or_default()
2756            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2757
2758        let agents = vec!["claude".to_string(), "codex".to_string()];
2759        let mut weights = AgentWeights::default();
2760        weights.ensure_agents(&agents);
2761        let mut router = Router {
2762            config,
2763            available_agents: agents,
2764            weights,
2765            llm_router: Arc::new(LlmRouter::new()),
2766            ollama_router: None,
2767            rr_index: 0,
2768            last_agent: None,
2769            review_rr_index: 0,
2770            router_pool: vec![],
2771            pool_index: 0,
2772        };
2773
2774        // Label override should take precedence over weighted routing
2775        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2776        let store = test_store().await;
2777        let result = router.route(&task, &store, "test/repo").await.unwrap();
2778        assert_eq!(result.agent, "codex");
2779        assert!(result.reason.contains("label"));
2780    }
2781
2782    #[serial(cooldown_state)]
2783    #[tokio::test]
2784    async fn route_takes_the_adjacent_tier_model_when_the_own_tier_is_cooled() {
2785        cooldown::reset_global_state().await;
2786        let agent = "tierroute_agent";
2787        let mut config = RouterConfig {
2788            agents: vec![agent.to_string()],
2789            ..Default::default()
2790        };
2791        config.model_map.clear();
2792        for (tier, model) in [("medium", "tr-m"), ("complex", "tr-c")] {
2793            config
2794                .model_map
2795                .entry(tier.to_string())
2796                .or_default()
2797                .insert(agent.to_string(), vec![model.to_string()]);
2798        }
2799        cooldown::record_model_failure(agent, "tr-m").await;
2800        let mut router = Router::new_for_test(config, vec![agent.to_string()]);
2801
2802        let task = create_test_task(
2803            "1",
2804            "Test task",
2805            vec![format!("agent:{agent}"), "complexity:medium".to_string()],
2806        );
2807        let store = test_store().await;
2808        let result = router.route(&task, &store, "test/repo").await.unwrap();
2809        assert_eq!(result.agent, agent);
2810        assert_eq!(result.model.as_deref(), Some("tr-c"));
2811        assert_eq!(result.complexity, "medium");
2812        assert!(result.warning.unwrap().contains("complex tier"));
2813    }
2814
2815    #[serial(cooldown_state)]
2816    #[test]
2817    fn router_config_weighted_round_robin_default_false() {
2818        let config = RouterConfig::default();
2819        assert!(!config.weighted_round_robin);
2820    }
2821
2822    #[serial(cooldown_state)]
2823    #[test]
2824    fn router_record_and_recover_weights() {
2825        let config = RouterConfig {
2826            weighted_round_robin: true,
2827            ..Default::default()
2828        };
2829        let mut router = Router::new(config);
2830        // Override discovered agents for test
2831        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2832        router.weights.ensure_agents(&router.available_agents);
2833
2834        // Record rate limit
2835        router.record_rate_limit("claude");
2836        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2837        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2838
2839        // Record success for claude
2840        router.record_success("claude");
2841        let after_success = router.weights.get_weight("claude");
2842        assert!(after_success > MIN_WEIGHT);
2843    }
2844
2845    #[serial(cooldown_state)]
2846    #[test]
2847    fn review_rr_index_advances() {
2848        let mut config = RouterConfig::default();
2849        // Set up model_map so has_available_model_for_complexity returns true for test agents
2850        for agent in &["test_a", "test_b", "test_c"] {
2851            config
2852                .model_map
2853                .entry("medium".to_string())
2854                .or_default()
2855                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2856        }
2857        let agents = vec![
2858            "test_a".to_string(),
2859            "test_b".to_string(),
2860            "test_c".to_string(),
2861        ];
2862        let mut weights = AgentWeights::default();
2863        weights.ensure_agents(&agents);
2864        let mut router = Router {
2865            config,
2866            available_agents: agents,
2867            weights,
2868            llm_router: Arc::new(LlmRouter::new()),
2869            ollama_router: None,
2870            rr_index: 0,
2871            last_agent: None,
2872            review_rr_index: 0,
2873            router_pool: vec![],
2874            pool_index: 0,
2875        };
2876
2877        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2878        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2879        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2880
2881        // All three agents should appear (order depends on start index)
2882        let mut seen = vec![a1, a2, a3];
2883        seen.sort();
2884        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2885    }
2886
2887    #[serial(cooldown_state)]
2888    #[test]
2889    fn review_rr_fallback_with_cooled_agent() {
2890        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2891        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2892        // candidates.get(idx) to return None even though candidates is non-empty.
2893        let mut config = RouterConfig::default();
2894        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2895        for agent in &["test_a", "test_b", "test_c"] {
2896            config
2897                .model_map
2898                .entry("medium".to_string())
2899                .or_default()
2900                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2901        }
2902        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2903        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2904        let mut weights = AgentWeights::default();
2905        weights.ensure_agents(&agents);
2906        let mut router = Router {
2907            config,
2908            available_agents: agents,
2909            weights,
2910            llm_router: Arc::new(LlmRouter::new()),
2911            ollama_router: None,
2912            rr_index: 0,
2913            last_agent: None,
2914            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2915            review_rr_index: 2,
2916            router_pool: vec![],
2917            pool_index: 0,
2918        };
2919
2920        // All candidates excluded — should fall back to first candidate, not None
2921        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2922        assert!(
2923            result.is_some(),
2924            "next_round_robin_agent must not return None when candidates is non-empty"
2925        );
2926    }
2927
2928    #[serial(cooldown_state)]
2929    #[test]
2930    fn review_rr_index_uses_candidates_len_not_available_len() {
2931        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2932        // not available_agents.len(). When an agent has no model for the complexity,
2933        // candidates.len() < available_agents.len() and the old code produced a drifting
2934        // index that caused non-uniform distribution.
2935        let mut config = RouterConfig::default();
2936        // Only register models for test_a and test_b — test_c has no model, so it is
2937        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2938        for agent in &["test_a", "test_b"] {
2939            config
2940                .model_map
2941                .entry("medium".to_string())
2942                .or_default()
2943                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2944        }
2945        let agents = vec![
2946            "test_a".to_string(),
2947            "test_b".to_string(),
2948            "test_c".to_string(), // available but no model → filtered out of candidates
2949        ];
2950        let mut weights = AgentWeights::default();
2951        weights.ensure_agents(&agents);
2952        let mut router = Router {
2953            config,
2954            available_agents: agents,
2955            weights,
2956            llm_router: Arc::new(LlmRouter::new()),
2957            ollama_router: None,
2958            rr_index: 0,
2959            last_agent: None,
2960            review_rr_index: 0,
2961            router_pool: vec![],
2962            pool_index: 0,
2963        };
2964
2965        // With the bug: index advances modulo 3 (available_agents.len()), so the
2966        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2967        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2968        // looks uniform by accident, but adding a 4th available agent breaks it.
2969        //
2970        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2971        // cycle. Verify this by checking review_rr_index after each call.
2972
2973        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2974        assert_eq!(
2975            router.review_rr_index, 1,
2976            "index should be 1 after first call"
2977        );
2978
2979        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2980        assert_eq!(
2981            router.review_rr_index, 0,
2982            "index should wrap to 0 after second call"
2983        );
2984
2985        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2986        assert_eq!(
2987            router.review_rr_index, 1,
2988            "index should be 1 after third call"
2989        );
2990
2991        // Agents must alternate uniformly
2992        assert_ne!(r1, r2, "consecutive calls should select different agents");
2993        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2994    }
2995
2996    #[serial(cooldown_state)]
2997    #[tokio::test]
2998    async fn last_agent_tracks_routing() {
2999        cooldown::reset_global_state().await;
3000        let mut config = RouterConfig {
3001            mode: "round_robin".to_string(),
3002            ..Default::default()
3003        };
3004        config
3005            .model_map
3006            .entry("medium".to_string())
3007            .or_default()
3008            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
3009        config
3010            .model_map
3011            .entry("medium".to_string())
3012            .or_default()
3013            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
3014        let agents = vec!["test_x".to_string(), "test_y".to_string()];
3015        let mut weights = AgentWeights::default();
3016        weights.ensure_agents(&agents);
3017        let mut router = Router {
3018            config,
3019            available_agents: agents,
3020            weights,
3021            llm_router: Arc::new(LlmRouter::new()),
3022            ollama_router: None,
3023            rr_index: 0,
3024            last_agent: None,
3025            review_rr_index: 0,
3026            router_pool: vec![],
3027            pool_index: 0,
3028        };
3029
3030        assert!(router.last_agent.is_none());
3031
3032        let task = create_test_task("1", "Test routing", vec![]);
3033        let store = test_store().await;
3034        let result = router.route(&task, &store, "test/repo").await.unwrap();
3035
3036        assert!(router.last_agent.is_some());
3037        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
3038    }
3039
3040    // ── pool config ───────────────────────────────────────────────────────────
3041
3042    #[serial(cooldown_state)]
3043    #[test]
3044    fn parse_pool_entry_splits_on_first_colon() {
3045        let (agent, model) = super::parse_pool_entry("claude:haiku");
3046        assert_eq!(agent, "claude");
3047        assert_eq!(model, "haiku");
3048    }
3049
3050    #[serial(cooldown_state)]
3051    #[test]
3052    fn parse_pool_entry_model_with_slash() {
3053        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
3054        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
3055        assert_eq!(agent, "opencode");
3056        assert_eq!(model, "github-copilot/gpt-5-mini");
3057    }
3058
3059    #[serial(cooldown_state)]
3060    #[test]
3061    fn parse_pool_entry_no_colon() {
3062        let (agent, model) = super::parse_pool_entry("claude");
3063        assert_eq!(agent, "claude");
3064        assert_eq!(model, "");
3065    }
3066
3067    #[serial(cooldown_state)]
3068    #[test]
3069    fn effective_pool_uses_configured_pool() {
3070        let config = RouterConfig {
3071            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3072            ..RouterConfig::default()
3073        };
3074        let pool = config.effective_pool();
3075        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3076    }
3077
3078    #[serial(cooldown_state)]
3079    #[test]
3080    fn effective_pool_falls_back_to_router_agent_model() {
3081        let config = RouterConfig::default();
3082        // Default has no pool → derives from router_agent:router_model
3083        let pool = config.effective_pool();
3084        assert_eq!(pool.len(), 1);
3085        assert!(pool[0].starts_with(&config.router_agent));
3086        assert!(pool[0].contains(&config.router_model));
3087    }
3088
3089    #[serial(cooldown_state)]
3090    #[test]
3091    fn effective_fallback_uses_configured_fallback() {
3092        let config = RouterConfig {
3093            fallback: "claude:haiku".to_string(),
3094            ..RouterConfig::default()
3095        };
3096        assert_eq!(config.effective_fallback(), "claude:haiku");
3097    }
3098
3099    #[serial(cooldown_state)]
3100    #[test]
3101    fn effective_fallback_derives_from_router_agent_model() {
3102        let config = RouterConfig::default();
3103        let fallback = config.effective_fallback();
3104        assert!(fallback.starts_with(&config.router_agent));
3105        assert!(fallback.contains(&config.router_model));
3106    }
3107
3108    // ── pool expansion ────────────────────────────────────────────────────────
3109
3110    #[serial(cooldown_state)]
3111    #[test]
3112    fn expand_pool_basic_entries() {
3113        let config = RouterConfig {
3114            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3115            ..RouterConfig::default()
3116        };
3117        let expanded = Router::expand_pool(&config);
3118        assert_eq!(expanded.len(), 2);
3119        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3120        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3121    }
3122
3123    #[serial(cooldown_state)]
3124    #[test]
3125    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3126        let config = RouterConfig {
3127            pool: vec![],
3128            router_agent: "claude".to_string(),
3129            router_model: "haiku".to_string(),
3130            ..RouterConfig::default()
3131        };
3132        let expanded = Router::expand_pool(&config);
3133        assert_eq!(expanded.len(), 1);
3134        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3135    }
3136
3137    #[serial(cooldown_state)]
3138    #[test]
3139    fn expand_pool_opencode_free_skipped_when_not_installed() {
3140        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3141        // to nothing and the pool falls back to effective_fallback
3142        let config = RouterConfig {
3143            pool: vec!["opencode:free".to_string()],
3144            fallback: "claude:haiku".to_string(),
3145            ..RouterConfig::default()
3146        };
3147        let expanded = Router::expand_pool(&config);
3148        assert_eq!(expanded.len(), 1);
3149        assert_eq!(expanded[0].0, "claude");
3150        assert_eq!(expanded[0].1, "haiku");
3151    }
3152
3153    #[serial(cooldown_state)]
3154    #[test]
3155    fn expand_pool_single_entry_from_default() {
3156        let config = RouterConfig::default();
3157        let expanded = Router::expand_pool(&config);
3158        assert!(
3159            !expanded.is_empty(),
3160            "pool must never be empty after expansion"
3161        );
3162    }
3163
3164    // ── pool round-robin index ────────────────────────────────────────────────
3165
3166    #[serial(cooldown_state)]
3167    #[test]
3168    fn pool_index_initializes_to_zero() {
3169        let config = RouterConfig::default();
3170        let router = Router::new(config);
3171        assert_eq!(router.pool_index, 0);
3172    }
3173
3174    #[serial(cooldown_state)]
3175    #[test]
3176    fn router_pool_non_empty_after_construction() {
3177        let config = RouterConfig::default();
3178        let router = Router::new(config);
3179        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3180    }
3181
3182    /**
3183     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3184     * `opencode serve --service` child behind, so a suite run leaked one server per test
3185     * process. The stub on PATH records any exec, so this fails if discovery comes back
3186     **/
3187    #[cfg(unix)]
3188    #[serial(cooldown_state)]
3189    #[test]
3190    fn building_a_router_never_runs_opencode() {
3191        struct PathRestore(Option<OsString>);
3192        impl Drop for PathRestore {
3193            fn drop(&mut self) {
3194                match &self.0 {
3195                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3196                    Some(p) => unsafe { env::set_var("PATH", p) },
3197                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3198                    None => unsafe { env::remove_var("PATH") },
3199                }
3200                cmd_cache::reset_for_test();
3201            }
3202        }
3203
3204        let dir = tempfile::tempdir().unwrap();
3205        let marker = dir.path().join("spawned");
3206        let bin_dir = dir.path().join("bin");
3207        fs::create_dir(&bin_dir).unwrap();
3208        let stub = bin_dir.join("opencode");
3209        fs::write(
3210            &stub,
3211            format!(
3212                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3213                marker.display()
3214            ),
3215        )
3216        .unwrap();
3217        fs::set_permissions(&stub, Permissions::from_mode(0o755)).unwrap();
3218
3219        let prev_path = env::var_os("PATH");
3220        let mut new_path = bin_dir.as_os_str().to_os_string();
3221        new_path.push(":");
3222        if let Some(ref p) = prev_path {
3223            new_path.push(p);
3224        }
3225        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3226        unsafe { env::set_var("PATH", &new_path) };
3227        cmd_cache::reset_for_test();
3228        let _restore = PathRestore(prev_path);
3229
3230        // A config that names opencode everywhere discovery reads from
3231        let mut config = RouterConfig {
3232            pool: vec!["opencode:free".to_string()],
3233            fallback: "opencode:free".to_string(),
3234            ..RouterConfig::default()
3235        };
3236        config
3237            .model_map
3238            .entry("simple".to_string())
3239            .or_default()
3240            .insert(
3241                "opencode".to_string(),
3242                vec!["canary-model-free".to_string()],
3243            );
3244
3245        let _router = Router::new(config.clone());
3246        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3247        let _pool = Router::expand_pool(&config);
3248        // Discovery refreshes on a background thread, give it room to exec
3249        thread::sleep(Duration::from_millis(300));
3250
3251        assert!(
3252            !marker.exists(),
3253            "building a router ran opencode: {}",
3254            fs::read_to_string(&marker).unwrap_or_default().trim()
3255        );
3256        assert!(
3257            opencode_children().is_empty(),
3258            "building a router left opencode processes: {:?}",
3259            opencode_children()
3260        );
3261    }
3262
3263    /// Commands of this process's direct children that mention opencode
3264    #[cfg(unix)]
3265    fn opencode_children() -> Vec<String> {
3266        let me = process::id().to_string();
3267        let output = Command::new("ps")
3268            .args(["-axo", "ppid=,command="])
3269            .output()
3270            .expect("ps");
3271        String::from_utf8_lossy(&output.stdout)
3272            .lines()
3273            .filter_map(|line| {
3274                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3275                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3276            })
3277            .collect()
3278    }
3279
3280    #[serial(cooldown_state)]
3281    #[tokio::test]
3282    async fn reload_resets_pool_index() {
3283        cooldown::reset_global_state().await;
3284        let config = RouterConfig::default();
3285        let mut router = Router::new(config);
3286        router.pool_index = 5;
3287        router.reload_async().await;
3288        assert_eq!(router.pool_index, 0);
3289    }
3290
3291    #[serial(cooldown_state)]
3292    #[test]
3293    fn pool_index_advances_after_failed_attempt() {
3294        let mut router = Router::new(RouterConfig::default());
3295        router.router_pool = vec![
3296            ("claude".to_string(), "haiku".to_string()),
3297            ("codex".to_string(), "gpt-5.2".to_string()),
3298            (
3299                "opencode".to_string(),
3300                "github-copilot/gpt-5-mini".to_string(),
3301            ),
3302        ];
3303        router.pool_index = 0;
3304
3305        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3306
3307        assert_eq!(router.pool_index, 1);
3308    }
3309
3310    #[serial(cooldown_state)]
3311    #[test]
3312    fn pool_index_wraps_after_last_attempt() {
3313        let mut router = Router::new(RouterConfig::default());
3314        router.router_pool = vec![
3315            ("claude".to_string(), "haiku".to_string()),
3316            ("codex".to_string(), "gpt-5.2".to_string()),
3317            (
3318                "opencode".to_string(),
3319                "github-copilot/gpt-5-mini".to_string(),
3320            ),
3321        ];
3322        router.pool_index = 2;
3323
3324        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3325
3326        assert_eq!(router.pool_index, 0);
3327    }
3328
3329    #[serial(cooldown_state)]
3330    #[test]
3331    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3332        let router = Router::new(RouterConfig::default());
3333        assert!(router.earliest_pool_cooldown().is_none());
3334    }
3335
3336    #[serial(cooldown_state)]
3337    #[tokio::test]
3338    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3339        cooldown::reset_global_state().await;
3340        // Use explicit pool with predictable model names to avoid opencode discovery.
3341        let config = RouterConfig {
3342            pool: vec![
3343                "claude:haiku-4-5-20251001".to_string(),
3344                "opencode:haiku-4-5-20251001".to_string(),
3345            ],
3346            ..Default::default()
3347        };
3348        let router = Router::new(config);
3349        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3350        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3351
3352        let earliest = router.earliest_pool_cooldown();
3353        assert!(earliest.is_some());
3354        let now = chrono::Utc::now().timestamp();
3355        let remaining = earliest.unwrap() - now;
3356        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3357        assert!((295..=300).contains(&remaining));
3358    }
3359
3360    #[serial(cooldown_state)]
3361    #[tokio::test]
3362    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3363        cooldown::reset_global_state().await;
3364        // Use explicit pool so we can predict what entries exist.
3365        let config = RouterConfig {
3366            pool: vec![
3367                "claude:pool-model-a".to_string(),
3368                "opencode:pool-model-b".to_string(),
3369            ],
3370            ..Default::default()
3371        };
3372        let router = Router::new(config);
3373        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3374        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3375        // Set cooldown on a pool entry (opencode:pool-model-b)
3376        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3377
3378        let earliest = router.earliest_pool_cooldown();
3379        assert!(earliest.is_some());
3380        let now = chrono::Utc::now().timestamp();
3381        let remaining = earliest.unwrap() - now;
3382        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3383        assert!((295..=300).contains(&remaining));
3384    }
3385
3386    // ---- Pre-emptive health check: degraded agent exclusion ----
3387
3388    #[serial(cooldown_state)]
3389    #[test]
3390    fn degraded_agent_excluded_from_routing() {
3391        let mut config = RouterConfig::default();
3392        config
3393            .model_map
3394            .entry("medium".to_string())
3395            .or_default()
3396            .insert(
3397                "test_degraded_routing".to_string(),
3398                vec!["model-a".to_string()],
3399            );
3400
3401        let mut router = Router::new(config);
3402        router.available_agents = vec!["test_degraded_routing".to_string()];
3403
3404        // Agent is routable before degradation
3405        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3406
3407        // Mark as degraded
3408        mark_agent_degraded("test_degraded_routing");
3409
3410        // Now excluded from routing
3411        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3412
3413        // available_agents_for_complexity returns empty
3414        assert!(
3415            router.available_agents_for_complexity("medium").is_empty(),
3416            "degraded agent should be excluded from available agents"
3417        );
3418
3419        // healthy_agent_count reflects exclusion
3420        assert_eq!(router.healthy_agent_count("medium"), 0);
3421
3422        // Cleanup
3423        clear_agent_degraded("test_degraded_routing");
3424    }
3425
3426    #[serial(cooldown_state)]
3427    #[test]
3428    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3429        let mut config = RouterConfig::default();
3430        for agent in &["agent_healthy", "agent_degraded"] {
3431            config
3432                .model_map
3433                .entry("medium".to_string())
3434                .or_default()
3435                .insert(agent.to_string(), vec!["model-x".to_string()]);
3436        }
3437
3438        let mut router = Router::new(config);
3439        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3440
3441        mark_agent_degraded("agent_degraded");
3442
3443        let candidates = router.available_agents_for_complexity("medium");
3444        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3445        assert_eq!(router.healthy_agent_count("medium"), 1);
3446
3447        // Cleanup
3448        clear_agent_degraded("agent_degraded");
3449    }
3450
3451    #[serial(cooldown_state)]
3452    #[tokio::test]
3453    async fn refresh_health_marks_degraded_from_rate_limits() {
3454        cooldown::reset_global_state().await;
3455        let store = test_store().await;
3456        let agent = "test_refresh_health_agent";
3457
3458        let mut config = RouterConfig::default();
3459        config
3460            .model_map
3461            .entry("medium".to_string())
3462            .or_default()
3463            .insert(agent.to_string(), vec!["model-z".to_string()]);
3464
3465        let mut router = Router::new(config);
3466        router.available_agents = vec![agent.to_string()];
3467
3468        // Insert rate limit events exceeding default threshold (3)
3469        for _ in 0..4 {
3470            store.record_rate_limit(agent, "rate", None).await.unwrap();
3471        }
3472
3473        router.refresh_health(&store).await;
3474
3475        assert!(
3476            is_agent_degraded(agent),
3477            "agent should be degraded after exceeding rate limit threshold"
3478        );
3479        assert!(
3480            !router.agent_is_routable(agent, "medium"),
3481            "degraded agent should not be routable"
3482        );
3483
3484        // Cleanup
3485        clear_agent_degraded(agent);
3486    }
3487
3488    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3489
3490    #[serial(cooldown_state)]
3491    #[tokio::test]
3492    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3493        cooldown::reset_global_state().await;
3494        let mut config = RouterConfig::default();
3495        // Agent's only model, shared across every complexity tier it resolves to.
3496        for tier in ["simple", "medium", "complex", "review"] {
3497            config
3498                .model_map
3499                .entry(tier.to_string())
3500                .or_default()
3501                .insert(
3502                    "test_all_cooled".to_string(),
3503                    vec!["only-model".to_string()],
3504                );
3505        }
3506
3507        let mut router = Router::new(config);
3508        router.available_agents = vec!["test_all_cooled".to_string()];
3509
3510        assert!(router.agent_has_any_available_model("test_all_cooled"));
3511
3512        record_model_failure("test_all_cooled", "only-model").await;
3513        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3514
3515        assert!(
3516            !router.agent_has_any_available_model("test_all_cooled"),
3517            "agent with its only model cooled across every tier should have no available model"
3518        );
3519    }
3520
3521    #[serial(cooldown_state)]
3522    #[tokio::test]
3523    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3524        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3525        // agent that has no agent-level cooldown but whose only resolvable model (across
3526        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3527        cooldown::reset_global_state().await;
3528        let mut config = RouterConfig::default();
3529        for tier in ["simple", "medium", "complex", "review"] {
3530            config
3531                .model_map
3532                .entry(tier.to_string())
3533                .or_default()
3534                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3535            config
3536                .model_map
3537                .get_mut(tier)
3538                .unwrap()
3539                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3540        }
3541
3542        let mut router = Router::new(config);
3543        router.available_agents = vec![
3544            "agent_cooled_model".to_string(),
3545            "agent_healthy".to_string(),
3546        ];
3547
3548        record_model_failure("agent_cooled_model", "opus").await;
3549        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3550        // No agent-level cooldown — the old agent-only filter would still offer it.
3551        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3552
3553        let uncooled_agents: Vec<String> = router
3554            .available_agents
3555            .iter()
3556            .filter(|a| {
3557                !is_agent_in_cooldown(a)
3558                    && !is_agent_degraded(a)
3559                    && router.agent_has_available_model_for_coding_tiers(a)
3560            })
3561            .cloned()
3562            .collect();
3563
3564        assert_eq!(
3565            uncooled_agents,
3566            vec!["agent_healthy".to_string()],
3567            "agent whose only model is cooled must be excluded from the LLM candidate list"
3568        );
3569    }
3570
3571    #[serial(cooldown_state)]
3572    #[tokio::test]
3573    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3574        // Regression test for #3521: an agent whose only healthy tier is `simple`
3575        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3576        // excluded from the LLM classification candidate list, otherwise the
3577        // classifier can pick it for a `medium`/`complex` task it can't serve,
3578        // wasting the classification call. Mirrors the real-world shape where
3579        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3580        cooldown::reset_global_state().await;
3581        let mut config = RouterConfig::default();
3582        config
3583            .model_map
3584            .entry("simple".to_string())
3585            .or_default()
3586            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3587        config
3588            .model_map
3589            .entry("medium".to_string())
3590            .or_default()
3591            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3592        config
3593            .model_map
3594            .entry("complex".to_string())
3595            .or_default()
3596            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3597        // review must be irrelevant to the coding-tier filter
3598        config
3599            .model_map
3600            .entry("review".to_string())
3601            .or_default()
3602            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3603        for tier in ["simple", "medium", "complex", "review"] {
3604            config
3605                .model_map
3606                .get_mut(tier)
3607                .unwrap()
3608                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3609        }
3610
3611        let mut router = Router::new(config);
3612        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3613
3614        record_model_failure("minimax_like", "opus").await;
3615        assert!(is_model_in_cooldown("minimax_like", "opus"));
3616        // No agent-level cooldown, and the simple tier is healthy, so the loose
3617        // "any tier" check still passes — this is exactly the leak #3511 missed.
3618        assert!(!is_agent_in_cooldown("minimax_like"));
3619        assert!(router.agent_has_any_available_model("minimax_like"));
3620        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3621
3622        let uncooled_agents: Vec<String> = router
3623            .available_agents
3624            .iter()
3625            .filter(|a| {
3626                !is_agent_in_cooldown(a)
3627                    && !is_agent_degraded(a)
3628                    && router.agent_has_available_model_for_coding_tiers(a)
3629            })
3630            .cloned()
3631            .collect();
3632
3633        assert_eq!(
3634            uncooled_agents,
3635            vec!["agent_healthy".to_string()],
3636            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3637        );
3638    }
3639}