Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22pub use config::{RouterConfig, parse_pool_entry};
23pub use weights::AgentWeights;
24
25fn fmt_degraded(degraded: &[String]) -> String {
26    if degraded.is_empty() {
27        String::new()
28    } else {
29        format!(" (degraded: {})", degraded.join(", "))
30    }
31}
32
33/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
34///
35/// This is a transient error that should not count against `max_route_attempts` since
36/// no agent was actually invoked. The caller should back off and retry later.
37#[derive(Debug, thiserror::Error)]
38pub enum AllAgentsCooledError {
39    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
40    AllCooled {
41        scope: String,
42        remaining_secs: Option<u64>,
43        degraded: Vec<String>,
44    },
45}
46
47impl AllAgentsCooledError {
48    /// Create a new error for the given scope, naming currently degraded agents
49    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
50        Self::AllCooled {
51            scope,
52            remaining_secs,
53            degraded: list_degraded_agents()
54                .into_iter()
55                .map(|(agent, _reason)| agent)
56                .collect(),
57        }
58    }
59
60    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
61
62    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
63    pub fn scope(&self) -> &str {
64        match self {
65            Self::AllCooled { scope, .. } => scope,
66        }
67    }
68
69    /// Remaining cooldown duration in seconds when available.
70    pub fn remaining_secs(&self) -> Option<u64> {
71        match self {
72            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
73        }
74    }
75
76    /// Agents currently marked degraded, shown by `orch cooldown list`
77    pub fn degraded(&self) -> &[String] {
78        match self {
79            Self::AllCooled { degraded, .. } => degraded,
80        }
81    }
82}
83
84/// Typed error returned by [`get_route_result`].
85#[derive(Debug, thiserror::Error)]
86pub enum RouteResultError {
87    /// The task has no agent field — it needs to be re-routed.
88    #[error("no agent configured for task {task_id}")]
89    NoAgent { task_id: String },
90    /// Any other failure (store lookup, SQL, etc.).
91    #[error(transparent)]
92    Other(#[from] anyhow::Error),
93}
94
95use crate::backends::ExternalTask;
96use crate::engine::cooldown::{
97    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
98    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
99};
100use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
101use serde::{Deserialize, Serialize};
102use std::sync::Arc;
103
104use llm::{LlmRouter, TIMEOUT_PREFIX};
105
106/// Result of routing a task to an agent.
107#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
108pub struct RouteResult {
109    /// The selected agent: "claude", "codex", or "opencode"
110    pub agent: String,
111    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
112    pub model: Option<String>,
113    /// Complexity level: "simple", "medium", or "complex"
114    pub complexity: String,
115    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
116    pub estimate: u8,
117    /// Why this agent was selected
118    pub reason: String,
119    /// Specialized agent profile (skills, tools, constraints)
120    pub profile: AgentProfile,
121    /// Selected skill IDs from the catalog
122    pub selected_skills: Vec<String>,
123    /// Optional warning about routing decision
124    pub warning: Option<String>,
125}
126
127/// Specialized agent profile for a task.
128#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
129pub struct AgentProfile {
130    /// Role name, e.g., "backend specialist"
131    pub role: String,
132    /// Focus skills for this task
133    pub skills: Vec<String>,
134    /// Tools allowed for this task
135    pub tools: Vec<String>,
136    /// Constraints for this task
137    pub constraints: Vec<String>,
138}
139
140impl AgentProfile {
141    /// Build the default "general" profile from router config.
142    pub fn default_from_config(config: &RouterConfig) -> Self {
143        Self {
144            role: "general".to_string(),
145            skills: vec![],
146            tools: config.allowed_tools.clone(),
147            constraints: vec![],
148        }
149    }
150}
151
152impl RouteResult {
153    /// Build a standard route result from the three task-specific values.
154    ///
155    /// Fills `profile`, `selected_skills`, and `warning` with router-config
156    /// defaults so call sites only need to supply what varies per task.
157    pub fn from_config(
158        agent: String,
159        complexity: String,
160        reason: String,
161        config: &RouterConfig,
162        task_id: &str,
163    ) -> Self {
164        let model = config.model_for_complexity(&agent, &complexity, task_id);
165        Self {
166            agent,
167            model,
168            complexity,
169            estimate: 0,
170            reason,
171            profile: AgentProfile::default_from_config(config),
172            selected_skills: config.default_skills.clone(),
173            warning: None,
174        }
175    }
176}
177
178/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
179///
180/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
181/// **not** require holding the router's `RwLock`, so async operations can
182/// safely be called without starving write-lock waiters.
183pub struct LlmRouterHandle(std::sync::Arc<LlmRouter>);
184
185impl std::fmt::Debug for LlmRouterHandle {
186    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
187        f.debug_struct("LlmRouterHandle").finish_non_exhaustive()
188    }
189}
190
191impl LlmRouterHandle {
192    /// Invalidate the skills catalog cache so the next routing call reloads
193    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
194    pub async fn invalidate_skills_catalog(&self) {
195        self.0.invalidate_skills_catalog().await;
196    }
197}
198
199/// The agent router.
200#[expect(
201    clippy::struct_field_names,
202    reason = "the router holds sub-routers whose names say which backend they use"
203)]
204pub struct Router {
205    /// Router configuration
206    pub config: RouterConfig,
207    /// Available agents discovered at runtime
208    pub available_agents: Vec<String>,
209    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
210    pub weights: AgentWeights,
211    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
212    /// without holding the router RwLock across async operations)
213    llm_router: std::sync::Arc<LlmRouter>,
214    /// Ollama router for local routing (contains reusable reqwest::Client)
215    ollama_router: Option<ollama::OllamaRouter>,
216    /// Round-robin index for task routing
217    pub(crate) rr_index: usize,
218    /// Last agent routed to (for distribution tracking)
219    pub(crate) last_agent: Option<String>,
220    /// Round-robin index for review agent selection
221    pub(crate) review_rr_index: usize,
222    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
223    /// `opencode:free` entries are expanded at construction time.
224    pub(crate) router_pool: Vec<(String, String)>,
225    /// Current round-robin index into router_pool
226    pub(crate) pool_index: usize,
227}
228
229impl std::fmt::Debug for Router {
230    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
231        f.debug_struct("Router")
232            .field("config", &self.config)
233            .field("available_agents", &self.available_agents)
234            .field("weights", &self.weights)
235            .field("ollama_router", &self.ollama_router)
236            .field("rr_index", &self.rr_index)
237            .field("last_agent", &self.last_agent)
238            .field("review_rr_index", &self.review_rr_index)
239            .field("router_pool", &self.router_pool)
240            .field("pool_index", &self.pool_index)
241            .finish_non_exhaustive()
242    }
243}
244
245impl Router {
246    /// Create a new router with the given configuration.
247    pub fn new(config: RouterConfig) -> Self {
248        let available_agents = Self::discover_agents(&config.agents);
249        let mut weights = AgentWeights {
250            base_weights: config.weights.clone(),
251            ..Default::default()
252        };
253        weights.ensure_agents_with_weights(&available_agents, &config.weights);
254        if !config.weights.is_empty() {
255            tracing::info!(
256                weights = ?config.weights,
257                "configured agent routing weights"
258            );
259        }
260        let router_pool = Self::expand_pool(&config);
261        tracing::info!(
262            pool = ?router_pool,
263            fallback = %config.effective_fallback(),
264            "router LLM pool initialized"
265        );
266        // Prime only when this config names opencode, so a default or claude-only
267        // router never runs `opencode models`
268        if config.uses_opencode() {
269            crate::engine::runner::agents::opencode::prime_free_model_cache();
270        }
271        let ollama_router = if config.mode == "local" {
272            Some(ollama::OllamaRouter::new(&config))
273        } else {
274            None
275        };
276        Self {
277            config,
278            available_agents,
279            weights,
280            llm_router: std::sync::Arc::new(LlmRouter::new()),
281            ollama_router,
282            rr_index: 0,
283            last_agent: None,
284            review_rr_index: 0,
285            router_pool,
286            pool_index: 0,
287        }
288    }
289
290    /// Create a router with default configuration loaded from files.
291    pub fn from_config() -> Self {
292        Self::new(RouterConfig::from_config())
293    }
294
295    /// Create a router with a forced agent list, bypassing CLI discovery.
296    /// Only for use in unit tests.
297    #[cfg(test)]
298    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
299        let mut weights = AgentWeights {
300            base_weights: config.weights.clone(),
301            ..Default::default()
302        };
303        weights.ensure_agents(&available_agents);
304        let router_pool = Self::expand_pool(&config);
305        Self {
306            config,
307            available_agents,
308            weights,
309            llm_router: std::sync::Arc::new(LlmRouter::new()),
310            ollama_router: None,
311            rr_index: 0,
312            last_agent: None,
313            review_rr_index: 0,
314            router_pool,
315            pool_index: 0,
316        }
317    }
318
319    /// Return a cloned handle to the LLM router subsystem.
320    ///
321    /// Callers that need to invoke async methods on the underlying `LlmRouter`
322    /// should clone this handle *before* releasing any `RwLock` read guard on
323    /// `Router`, then call the async method after the guard is dropped.  This
324    /// prevents holding the read lock across await points (which starves
325    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
326    pub fn llm_router_handle(&self) -> LlmRouterHandle {
327        LlmRouterHandle(std::sync::Arc::clone(&self.llm_router))
328    }
329
330    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
331        debug_assert!(pool_len > 0, "router pool must not be empty");
332        self.pool_index = (idx + 1) % pool_len;
333    }
334
335    /// Discover available agent CLIs in PATH.
336    /// Checks all agents from the configured list.
337    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
338        let mut agents = Vec::new();
339        for agent in configured_agents {
340            if crate::cmd_cache::command_exists(agent) {
341                agents.push(agent.clone());
342            }
343        }
344        agents
345    }
346
347    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
348    ///
349    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
350    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
351    /// - If `opencode` is not installed or the command fails, the entry is skipped.
352    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
353    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
354        let raw_pool = config.effective_pool();
355        let mut expanded: Vec<(String, String)> = Vec::new();
356
357        for entry in &raw_pool {
358            if entry == "opencode:free" {
359                let free_models = Self::discover_free_opencode_models();
360                if free_models.is_empty() {
361                    tracing::debug!("opencode:free expanded to nothing — skipping");
362                } else {
363                    tracing::debug!(models = ?free_models, "opencode:free expanded");
364                    for model in free_models {
365                        expanded.push(("opencode".to_string(), model));
366                    }
367                }
368            } else {
369                let (agent, model) = parse_pool_entry(entry);
370                expanded.push((agent, model));
371            }
372        }
373
374        if expanded.is_empty() {
375            // All entries were opencode:free and opencode isn't installed — use fallback
376            let (agent, model) = parse_pool_entry(&config.effective_fallback());
377            tracing::debug!(
378                agent = %agent, model = %model,
379                "pool empty after expansion, using fallback as sole pool entry"
380            );
381            expanded.push((agent, model));
382        }
383
384        expanded
385    }
386
387    /// Async version of expand_pool which may query the opencode CLI without
388    /// blocking the Tokio runtime.
389    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
390        let raw_pool = config.effective_pool();
391        let mut expanded: Vec<(String, String)> = Vec::new();
392
393        for entry in &raw_pool {
394            if entry == "opencode:free" {
395                let free_models = Self::discover_free_opencode_models_async().await;
396                if free_models.is_empty() {
397                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
398                } else {
399                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
400                    for model in free_models {
401                        expanded.push(("opencode".to_string(), model));
402                    }
403                }
404            } else {
405                let (agent, model) = parse_pool_entry(entry);
406                expanded.push((agent, model));
407            }
408        }
409
410        if expanded.is_empty() {
411            // All entries were opencode:free and opencode isn't installed — use fallback
412            let (agent, model) = parse_pool_entry(&config.effective_fallback());
413            tracing::debug!(
414                agent = %agent, model = %model,
415                "pool empty after async expansion, using fallback as sole pool entry"
416            );
417            expanded.push((agent, model));
418        }
419
420        expanded
421    }
422
423    /// Async reload that uses async pool expansion to avoid blocking the runtime
424    /// when discovering opencode free models.
425    pub async fn reload_async(&mut self) {
426        let new_config = RouterConfig::from_config();
427        let new_agents = Self::discover_agents(&new_config.agents);
428        let new_pool = Self::expand_pool_async(&new_config).await;
429        tracing::info!(
430            mode = %new_config.mode,
431            agents = ?new_agents,
432            fallback = %new_config.fallback_executor,
433            weighted_rr = new_config.weighted_round_robin,
434            pool = ?new_pool,
435            "router reloaded (async)"
436        );
437        self.config = new_config;
438        self.available_agents.clone_from(&new_agents);
439        // Ensure new agents have weight entries (preserves existing weights)
440        self.weights.ensure_agents(&new_agents);
441        self.router_pool = new_pool;
442        self.pool_index = 0;
443        // Reinitialize Ollama router if mode changed to local
444        self.ollama_router = if self.config.mode == "local" {
445            Some(ollama::OllamaRouter::new(&self.config))
446        } else {
447            None
448        };
449    }
450
451    fn discover_free_opencode_models() -> Vec<String> {
452        crate::engine::runner::agents::opencode::discover_free_opencode_models()
453    }
454
455    async fn discover_free_opencode_models_async() -> Vec<String> {
456        tokio::task::spawn_blocking(Self::discover_free_opencode_models)
457            .await
458            .unwrap_or_default()
459    }
460
461    /// Run the pre-emptive health check, refreshing the degraded-agent set.
462    ///
463    /// Queries the `rate_limits` table for recent events and combines with
464    /// cooldown state to mark agents as degraded before routing attempts them.
465    pub async fn refresh_health(&self, store: &std::sync::Arc<TaskStore>) {
466        // KV is the source of truth for cooldowns. Reconcile the in-memory map
467        // from KV every tick so external mutations (e.g. `orch cooldown clear`
468        // from a separate CLI process) actually take effect on the running
469        // service. Without this, the engine keeps stale entries until they
470        // expire on their own, producing the "all agents cooled" log with
471        // `orch cooldown list` simultaneously reporting "No active cooldowns".
472        sync_from_kv(store).await;
473
474        let agents = self.available_agents.clone();
475        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
476        refresh_degraded_agents(
477            store,
478            &agents,
479            &model_checker,
480            config::health_check_window_hours(),
481            config::degraded_rate_limit_threshold(),
482        )
483        .await;
484    }
485
486    /// Check if an agent is available.
487    pub fn is_agent_available(&self, agent: &str) -> bool {
488        self.available_agents.contains(&agent.to_string())
489    }
490
491    /// True if the agent has at least one uncooled model across any complexity
492    /// tier. Used where the eventual complexity isn't known yet (e.g. before
493    /// LLM classification), so we can't check a single tier.
494    fn agent_has_any_available_model(&self, agent: &str) -> bool {
495        ["simple", "medium", "complex", "review"]
496            .iter()
497            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
498    }
499
500    /// True if the agent has an available (uncooled) model for the `medium`
501    /// and `complex` tiers — the tiers real coding tasks are classified into
502    /// by the LLM classifier. Used to pre-filter the candidate list handed to
503    /// that classifier so an agent whose healthy tiers are only `simple` isn't
504    /// offered: the classifier can't be constrained to `simple`, so offering
505    /// such an agent produces a near-certain wasted classification call whenever
506    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
507    /// outcome of initial classification and is deliberately excluded.
508    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
509        ["medium", "complex"]
510            .iter()
511            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
512    }
513
514    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
515        if !self.is_agent_available(agent) {
516            return false;
517        }
518        if is_agent_in_cooldown(agent) {
519            tracing::debug!(agent, "agent skipped: in cooldown");
520            return false;
521        }
522        if is_agent_degraded(agent) {
523            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
524            return false;
525        }
526        // If weighted routing is used, consider the agent degraded when its
527        // weight has decayed below the threshold. This avoids
528        // proactively routing to agents that recently hit many rate limits.
529        if self.config.weighted_round_robin {
530            let weight = self.weights.get_weight(agent);
531            if weight < config::SKIP_LIMITED_THRESHOLD {
532                tracing::debug!(
533                    agent,
534                    weight,
535                    "agent considered degraded by weight threshold, skipping"
536                );
537                return false;
538            }
539        }
540        if !self
541            .config
542            .has_available_model_for_complexity(agent, complexity)
543        {
544            tracing::debug!(agent, complexity, "agent skipped: no available model");
545            return false;
546        }
547        true
548    }
549
550    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
551        self.available_agents
552            .iter()
553            .filter(|agent| self.agent_is_routable(agent, complexity))
554            .cloned()
555            .collect()
556    }
557
558    /// Count of healthy (routable) agents for a given complexity.
559    /// Returns the number of agents that are available, not in cooldown, and have an available model.
560    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
561        self.available_agents_for_complexity(complexity).len()
562    }
563
564    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
565    /// Uses "simple" complexity as the baseline since it has the widest model support.
566    pub fn is_degraded(&self, threshold: usize) -> bool {
567        self.healthy_agent_count("simple") < threshold
568    }
569
570    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
571        let mut earliest: Option<i64> = None;
572
573        for agent in &self.available_agents {
574            if let Some(until) = cooldown_until(agent) {
575                earliest = Some(earliest.map_or(until, |current| current.min(until)));
576            }
577
578            if let Some(comp) = complexity
579                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
580            {
581                for model in pool {
582                    let key = format!("{agent}:{model}");
583                    if let Some(until) = cooldown_until(&key) {
584                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
585                    }
586                }
587            }
588        }
589
590        earliest
591    }
592
593    /// Find the earliest cooldown expiration among the router LLM pool entries.
594    ///
595    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
596    /// or `None` if no pool model has a cooldown.
597    fn earliest_pool_cooldown(&self) -> Option<i64> {
598        let mut earliest: Option<i64> = None;
599        for (agent, model) in &self.router_pool {
600            let key = format!("{agent}:{model}");
601            if let Some(until) = cooldown_until(&key) {
602                earliest = Some(earliest.map_or(until, |current| current.min(until)));
603            }
604        }
605        earliest
606    }
607
608    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
609        // If any cooldowns are present, return an error immediately so the
610        // caller (the tick loop) can skip this task and retry on the next tick.
611        // Sleeping inside the router blocks the entire tick loop for the
612        // cooldown duration which starves other tasks. The engine tick already
613        // retries every `tick_interval` (default 10s), so allow the tick loop
614        // to drive retries instead of sleeping here.
615        let now = chrono::Utc::now().timestamp();
616        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
617            // All agents are degraded but none have a timed cooldown entry
618            // (e.g. over-threshold rate limiting without explicit cooldown duration).
619            // Log at warn and return an error so the tick skips this task and retries after backoff.
620            tracing::warn!(
621                scope = complexity.unwrap_or("all agents"),
622                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
623            );
624            // Return an error similar to the cooldown case to trigger backoff in the tick loop
625            return Err(AllAgentsCooledError::new(
626                complexity.unwrap_or("all agents").to_string(),
627                None,
628            )
629            .into());
630        };
631
632        let remaining = earliest.saturating_sub(now);
633        if remaining > 0 {
634            let scope_str = complexity.unwrap_or("all agents").to_string();
635            tracing::warn!(
636                remaining_secs = remaining,
637                scope = %scope_str,
638                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
639            );
640            // Return a domain error indicating all candidates are cooled so the
641            // caller can handle it (tick will skip this task and retry later).
642            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
643        }
644
645        Ok(())
646    }
647
648    /// Get the first available agent.
649    /// Pick next agent via round-robin (for review or other non-task routing).
650    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
651    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
652    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
653        // Use available_agents_for_complexity which already filters using agent_is_routable,
654        // eliminating duplicate cooldown/degraded/model checks.
655        let candidates = self.available_agents_for_complexity(complexity);
656        if candidates.is_empty() {
657            return None;
658        }
659        let idx = self.review_rr_index;
660        let n = candidates.len();
661
662        // Try to find an agent that isn't excluded
663        let agent = (0..n)
664            .map(|offset| &candidates[(idx + offset) % n])
665            .find(|a| !exclude.contains(&a.as_str()))
666            .cloned()
667            // Fallback: use the first available candidate (even if excluded)
668            .or_else(|| candidates.first().cloned())?;
669
670        self.review_rr_index = (idx + 1) % candidates.len().max(1);
671        Some(agent)
672    }
673
674    /// If the selected agent matches `no_code_last_agent` for this task, filter it
675    /// out of the candidates and return the filtered list. Used to prevent the
676    /// same agent from being selected twice in a row after a no-code failure (#2410).
677    async fn skip_no_code_agent(
678        &self,
679        store: &std::sync::Arc<TaskStore>,
680        repo: &str,
681        task_id: &str,
682        candidates: &[String],
683    ) -> Option<Vec<String>> {
684        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
685            .await
686            .unwrap_or_else(|| {
687                tracing::warn!(
688                    task_id,
689                    "no_code_last_agent DB read returned None — loop detection may be impaired"
690                );
691                String::new()
692            });
693        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
694            return None;
695        }
696        let filtered: Vec<String> = candidates
697            .iter()
698            .filter(|a| a.as_str() != no_code_agent)
699            .cloned()
700            .collect();
701        if filtered.is_empty() {
702            tracing::warn!(
703                task_id,
704                no_code_agent = %no_code_agent,
705                "all available agents are the no-code agent — cannot skip"
706            );
707            return None;
708        }
709        tracing::info!(
710            task_id,
711            skipped = %no_code_agent,
712            remaining = ?filtered,
713            "skipping no-code agent, routing to alternative"
714        );
715        Some(filtered)
716    }
717
718    /// Route a task to the best agent.
719    ///
720    /// Routing logic (in priority order):
721    /// 1. Check for `agent:*` label — use that agent directly
722    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
723    /// 3. If round_robin mode, cycle through agents (stateful)
724    /// 4. Call LLM classifier for intelligent routing
725    /// 5. After max_route_attempts LLM failures, fall back to round-robin
726    pub async fn route(
727        &mut self,
728        task: &ExternalTask,
729        store: &std::sync::Arc<TaskStore>,
730        repo: &str,
731    ) -> anyhow::Result<RouteResult> {
732        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
733        // before the routing loop, not per-task. Do not re-add it here.
734
735        // Hold the task before any classification, label override, or fallback.
736        // A waiting task must not increment route_attempts.
737        match store.resolve_id_by_external(repo, &task.id.0).await {
738            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
739                Ok(Some(blockers)) => {
740                    return Err(crate::store::WaitingOnDependency { blockers }.into());
741                }
742                Ok(None) => {}
743                Err(err) => return Err(err),
744            },
745            Ok(None) => {}
746            Err(err) => return Err(err),
747        }
748
749        loop {
750            // 1. Check store agent field first (set by failover, authoritative over labels)
751            // Use a lightweight lookup to avoid deserializing the full Task row when
752            // only the `agent` column is needed. This avoids two full-row loads
753            // per routing pass (resolve_task_id + get) and significantly reduces
754            // CPU work on busy instances.
755            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
756                Ok(Some(agent)) => {
757                    // Router enforces that the agent string matches a configured agent
758                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
759                        Some(agent)
760                    } else {
761                        None
762                    }
763                }
764                Ok(None) => None,
765                Err(e) => {
766                    tracing::warn!(
767                        repo,
768                        task_id = %task.id.0,
769                        err = %e,
770                        "failed to fetch stored agent, falling back to label/LLM routing"
771                    );
772                    None
773                }
774            };
775
776            // 2. Fall back to explicit agent label
777            let resolved_agent = store_agent.or_else(|| {
778                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
779            });
780
781            if let Some(agent) = resolved_agent {
782                let complexity = strategies::extract_complexity_from_labels(&task.labels);
783                // If agent appears routable (not in agent-wide cooldown and has
784                // available models or no model_map configured), check whether
785                // we actually have a concrete model to dispatch. If the
786                // resolved label maps to no model (None), we fall through to
787                // the standard routing logic so the router LLM can pick a
788                // proper agent/model, instead of dispatching with an empty
789                // model which causes some agents (e.g. opencode) to exit
790                // silently.
791                if self.agent_is_routable(&agent, &complexity) {
792                    let model = self
793                        .config
794                        .model_for_complexity(&agent, &complexity, &task.id.0);
795
796                    // Only dispatch label overrides when we have a concrete
797                    // model to send to the executor. Dispatching with an
798                    // empty model led to silent exits for some agents
799                    // (notably opencode). If no concrete model is available
800                    // for the labeled agent, fall through to the standard
801                    // routing logic so the LLM or round-robin can pick an
802                    // appropriate agent/model.
803                    if model.is_some() {
804                        tracing::debug!(
805                            task_id = %task.id.0,
806                            agent = %agent,
807                            complexity = %complexity,
808                            "routed via label"
809                        );
810                        let mut result = RouteResult::from_config(
811                            agent.clone(),
812                            complexity.clone(),
813                            format!("label agent:{agent}"),
814                            &self.config,
815                            &task.id.0,
816                        );
817                        // Label routing uses agent-specific role and the
818                        // resolved model (may differ from from_config's default).
819                        result.profile.role = format!("{agent} specialist");
820                        result.model = model.clone();
821                        self.log_route_activity(store, repo, &task.id.0, &result, None)
822                            .await;
823                        return Ok(result);
824                    }
825                    // No concrete model found for the labeled agent — do
826                    // not honor the label override. Fall through to the
827                    // standard routing logic so another agent or the LLM
828                    // can be chosen with a valid model.
829                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
830                }
831
832                let candidates = self.available_agents_for_complexity(&complexity);
833                if candidates.is_empty() {
834                    self.wait_for_cooldown(Some(&complexity))?;
835                    continue;
836                }
837                // Label target is cooled or lacked model; fall through to standard routing.
838            }
839
840            let complexity = strategies::extract_complexity_from_labels(&task.labels);
841
842            // Check if any agents have available models for this complexity
843            let mut candidates = self.available_agents_for_complexity(&complexity);
844            if candidates.is_empty() {
845                self.wait_for_cooldown(Some(&complexity))?;
846                continue;
847            }
848
849            // Skip the no-code agent if one is recorded for this task (#2410).
850            // If all available agents are filtered out, we fall through to
851            // LLM routing with no constraints (LLM will pick the same agent but
852            // the response_handler will block it immediately via same-agent check).
853            if let Some(filtered) = self
854                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
855                .await
856            {
857                candidates = filtered;
858            }
859
860            // 2. Weighted round-robin — capacity-based selection
861            if self.config.weighted_round_robin {
862                let result = strategies::route_via_weighted_round_robin(
863                    &candidates,
864                    &self.weights,
865                    &self.config,
866                    task,
867                    &mut self.last_agent,
868                )?;
869                self.log_route_activity(store, repo, &task.id.0, &result, None)
870                    .await;
871                return Ok(result);
872            }
873
874            // 3. Round-robin mode — use stateful round-robin
875            if self.config.mode == "round_robin" {
876                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
877                let result = strategies::route_via_round_robin_stateful(
878                    &candidates,
879                    &self.config,
880                    task,
881                    &mut self.rr_index,
882                    &mut self.last_agent,
883                )?;
884                self.log_route_activity(store, repo, &task.id.0, &result, None)
885                    .await;
886                return Ok(result);
887            }
888
889            // 4. Local Ollama routing
890            if self.config.mode == "local" {
891                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
892                let ollama_router = self
893                    .ollama_router
894                    .as_ref()
895                    .expect("ollama_router must be initialized in Router::new when mode=local");
896                match ollama_router.route(task, &self.config).await {
897                    Ok(result) => {
898                        self.log_route_activity(store, repo, &task.id.0, &result, None)
899                            .await;
900                        return Ok(result);
901                    }
902                    Err(e) => {
903                        tracing::warn!(
904                            task_id = %task.id.0,
905                            error = %e,
906                            "Ollama routing failed, falling back to round-robin"
907                        );
908                        let result = strategies::route_via_round_robin_stateful(
909                            &candidates,
910                            &self.config,
911                            task,
912                            &mut self.rr_index,
913                            &mut self.last_agent,
914                        )?;
915                        self.log_route_activity(store, repo, &task.id.0, &result, None)
916                            .await;
917                        return Ok(result);
918                    }
919                }
920            }
921
922            // 5. LLM-based routing with retry tracking
923            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
924
925            if route_attempts >= self.config.max_route_attempts {
926                tracing::warn!(
927                    task_id = %task.id.0,
928                    attempts = route_attempts,
929                    max = self.config.max_route_attempts,
930                    "max LLM route attempts reached, falling back to round-robin"
931                );
932                let candidates = self.available_agents_for_complexity(&complexity);
933                if candidates.is_empty() {
934                    self.wait_for_cooldown(Some(&complexity))?;
935                    continue;
936                }
937                let result = strategies::route_via_round_robin_stateful(
938                    &candidates,
939                    &self.config,
940                    task,
941                    &mut self.rr_index,
942                    &mut self.last_agent,
943                )?;
944                self.log_route_activity(store, repo, &task.id.0, &result, None)
945                    .await;
946                return Ok(result);
947            }
948
949            // Log routing start (before await)
950            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
951
952            match self.route_with_llm(task, repo).await {
953                Ok(result) => {
954                    let candidates = self.available_agents_for_complexity(&result.complexity);
955                    if candidates.is_empty() {
956                        self.wait_for_cooldown(Some(&result.complexity))?;
957                        continue;
958                    }
959
960                    // If the LLM selected the no-code agent, filter it out and
961                    // pick an alternative (#2410). Re-use the existing fallback logic.
962                    let no_code_agent =
963                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
964                             .await
965                             .unwrap_or_else(|| {
966                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
967                                 String::new()
968                             });
969                    if result.agent == no_code_agent {
970                        // Pick the first agent that isn't the no-code agent.
971                        let fallback_agent = candidates
972                            .iter()
973                            .find(|a| a.as_str() != no_code_agent)
974                            .cloned()
975                            .unwrap_or_else(|| candidates[0].clone());
976                        if fallback_agent.as_str() == no_code_agent {
977                            tracing::warn!(
978                                task_id = %task.id.0,
979                                no_code_agent = %no_code_agent,
980                                complexity = %result.complexity,
981                                "all available agents are the no-code agent — waiting for another agent"
982                            );
983                            self.wait_for_cooldown(Some(&result.complexity))?;
984                            continue;
985                        }
986                        tracing::info!(
987                            task_id = %task.id.0,
988                            llm_selected = %result.agent,
989                            skipped = %no_code_agent,
990                            fallback = %fallback_agent,
991                            "LLM selected the no-code agent — routing to alternative"
992                        );
993                        let fallback_model = self.config.model_for_complexity(
994                            &fallback_agent,
995                            &result.complexity,
996                            &task.id.0,
997                        );
998
999                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1000                        let result = RouteResult {
1001                            agent: fallback_agent.clone(),
1002                            model: fallback_model,
1003                            complexity: result.complexity.clone(),
1004                            estimate: result.estimate,
1005                            reason: format!(
1006                                "LLM selected no-code agent {}; routed to {}",
1007                                result.agent, fallback_agent
1008                            ),
1009                            profile: result.profile.clone(),
1010                            selected_skills: result.selected_skills.clone(),
1011                            warning: Some(format!(
1012                                "LLM selected no-code agent {}; used fallback {}",
1013                                result.agent, fallback_agent
1014                            )),
1015                        };
1016                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1017                            .await;
1018                        return Ok(result);
1019                    }
1020
1021                    let model_cooled = result
1022                        .model
1023                        .as_deref()
1024                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1025                    if !candidates.contains(&result.agent) || model_cooled {
1026                        let fallback_agent = candidates
1027                            .iter()
1028                            .find(|agent| *agent != &result.agent)
1029                            .cloned()
1030                            .unwrap_or_else(|| candidates[0].clone());
1031                        let fallback_model = self.config.model_for_complexity(
1032                            &fallback_agent,
1033                            &result.complexity,
1034                            &task.id.0,
1035                        );
1036
1037                        // If the fallback model is None and the fallback agent is the
1038                        // same as the LLM-selected agent, it means all models for the
1039                        // only available agent are cooled. In that case, wait for
1040                        // cooldowns instead of returning a RouteResult with
1041                        // model: None which would be dispatched incorrectly.
1042                        if fallback_model.is_none() && fallback_agent == result.agent {
1043                            tracing::warn!(
1044                                task_id = %task.id.0,
1045                                agent = %result.agent,
1046                                complexity = %result.complexity,
1047                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1048                            );
1049                            self.wait_for_cooldown(Some(&result.complexity))?;
1050                            continue;
1051                        }
1052
1053                        // Distinguish logging and reasons when we actually switch
1054                        // agents vs when we keep the same agent but pick a
1055                        // different (non-cooled) model.
1056                        if fallback_agent == result.agent {
1057                            tracing::warn!(
1058                                task_id = %task.id.0,
1059                                agent = %result.agent,
1060                                complexity = %result.complexity,
1061                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1062                            );
1063                        } else {
1064                            tracing::warn!(
1065                                task_id = %task.id.0,
1066                                agent = %result.agent,
1067                                fallback = %fallback_agent,
1068                                complexity = %result.complexity,
1069                                "LLM selected cooled agent/model; rerouting to available agent"
1070                            );
1071                        }
1072
1073                        // Reset attempts on success
1074                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1075
1076                        let reason = if fallback_agent == result.agent {
1077                            format!(
1078                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1079                            )
1080                        } else {
1081                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1082                        };
1083
1084                        let result = RouteResult {
1085                            agent: fallback_agent.clone(),
1086                            model: fallback_model,
1087                            complexity: result.complexity.clone(),
1088                            estimate: result.estimate,
1089                            reason: reason.clone(),
1090                            profile: result.profile.clone(),
1091                            selected_skills: result.selected_skills.clone(),
1092                            warning: Some(if fallback_agent == result.agent {
1093                                "LLM-selected model was cooled; selected alternative model for same agent"
1094                                    .to_string()
1095                            } else {
1096                                "LLM-selected agent/model was cooled; rerouted to available agent"
1097                                    .to_string()
1098                            }),
1099                        };
1100                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1101                            .await;
1102                        return Ok(result);
1103                    }
1104
1105                    // Reset attempts on success
1106                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1107                    tracing::info!(
1108                        task_id = %task.id.0,
1109                        agent = %result.agent,
1110                        complexity = %result.complexity,
1111                        "routed via LLM"
1112                    );
1113                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1114                        .await;
1115                    return Ok(result);
1116                }
1117                Err(e) => {
1118                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1119                        let scope = err.scope();
1120                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1121                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1122                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1123                        // (no complexity-specific pool lookup, which would fail for "router pool").
1124                        let scope_opt = match scope {
1125                            "all agents" | "router pool" => None,
1126                            _ => Some(scope),
1127                        };
1128                        self.wait_for_cooldown(scope_opt)?;
1129                        continue;
1130                    }
1131
1132                    let new_attempts = route_attempts + 1;
1133                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1134                        .await;
1135                    tracing::warn!(
1136                        task_id = %task.id.0,
1137                        error = %e,
1138                        error_chain = ?e,
1139                        attempt = new_attempts,
1140                        max = self.config.max_route_attempts,
1141                        "LLM routing failed"
1142                    );
1143
1144                    if new_attempts >= self.config.max_route_attempts {
1145                        tracing::info!(
1146                            task_id = %task.id.0,
1147                            "falling back to round-robin after {} failed attempts",
1148                            new_attempts
1149                        );
1150                        let candidates = self.available_agents_for_complexity(&complexity);
1151                        if candidates.is_empty() {
1152                            self.wait_for_cooldown(Some(&complexity))?;
1153                            continue;
1154                        }
1155                        let result = strategies::route_via_round_robin_stateful(
1156                            &candidates,
1157                            &self.config,
1158                            task,
1159                            &mut self.rr_index,
1160                            &mut self.last_agent,
1161                        )?;
1162                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1163                            .await;
1164                        return Ok(result);
1165                    }
1166
1167                    let candidates = self.available_agents_for_complexity(&complexity);
1168                    if candidates.is_empty() {
1169                        self.wait_for_cooldown(Some(&complexity))?;
1170                        continue;
1171                    }
1172                    let result = strategies::route_via_fallback(
1173                        &candidates,
1174                        &self.config,
1175                        task,
1176                        Some(&self.weights),
1177                    )?;
1178                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1179                        .await;
1180                    return Ok(result);
1181                }
1182            }
1183        }
1184    }
1185
1186    /// Get the number of LLM routing attempts for a task from the store.
1187    async fn get_route_attempts(
1188        &self,
1189        task_id: &str,
1190        store: &std::sync::Arc<TaskStore>,
1191        repo: &str,
1192    ) -> u32 {
1193        match store.resolve_task_id(repo, task_id).await {
1194            Ok(Some(store_id)) => store
1195                .get(store_id)
1196                .await
1197                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1198            _ => 0,
1199        }
1200    }
1201
1202    /// Set the number of LLM routing attempts for a task in the store.
1203    async fn set_route_attempts(
1204        &self,
1205        task_id: &str,
1206        attempts: u32,
1207        store: &std::sync::Arc<TaskStore>,
1208        repo: &str,
1209    ) {
1210        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1211            && let Err(e) = store
1212                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1213                .await
1214        {
1215            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1216        }
1217    }
1218
1219    /// Log a route event to task activity timeline.
1220    async fn log_route_activity(
1221        &self,
1222        store: &std::sync::Arc<TaskStore>,
1223        repo: &str,
1224        task_id: &str,
1225        result: &RouteResult,
1226        from_agent: Option<&str>,
1227    ) {
1228        let details = serde_json::json!({
1229            "reason": result.reason,
1230            "complexity": result.complexity,
1231            "skills": result.selected_skills,
1232            "role": result.profile.role,
1233            "warning": result.warning,
1234        });
1235        let event_type = if from_agent.is_some() {
1236            "rerouted"
1237        } else {
1238            "routed"
1239        };
1240        store_log_activity(
1241            &Some(Arc::clone(store)),
1242            repo,
1243            task_id,
1244            event_type,
1245            None,
1246            Some("routed"),
1247            Some(&result.agent),
1248            result.model.as_deref(),
1249            Some(&details),
1250        )
1251        .await;
1252    }
1253
1254    /// Route using LLM classification with pool round-robin.
1255    ///
1256    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1257    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1258    /// returns the last error (the caller falls back to round-robin agent selection).
1259    async fn route_with_llm(
1260        &mut self,
1261        task: &ExternalTask,
1262        repo: &str,
1263    ) -> anyhow::Result<RouteResult> {
1264        // Filter out cooled and degraded agents, and agents without an available
1265        // model for the coding tiers (`medium`/`complex`) that real tasks are
1266        // classified into, so the LLM only sees candidates that could actually
1267        // be dispatched for whatever complexity it picks. An agent whose only
1268        // healthy tier is `simple` is excluded too: the classifier can't be
1269        // constrained to `simple`, so leaving such an agent in would waste the
1270        // call whenever it picks `medium`/`complex` (issue #3521).
1271        // Fall back to the full list if all agents are cooled/degraded.
1272        // Cloned to avoid borrow conflict with &mut self in the loop.
1273        let uncooled_agents: Vec<String> = self
1274            .available_agents
1275            .iter()
1276            .filter(|a| {
1277                !is_agent_in_cooldown(a)
1278                    && !is_agent_degraded(a)
1279                    && self.agent_has_available_model_for_coding_tiers(a)
1280            })
1281            .cloned()
1282            .collect();
1283        if uncooled_agents.is_empty() {
1284            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1285        }
1286        let llm_agents = uncooled_agents;
1287
1288        let pool = self.router_pool.clone();
1289        let n = pool.len();
1290        let start = self.pool_index;
1291        let mut last_err: Option<anyhow::Error> = None;
1292        let mut attempted_any = false;
1293
1294        // Try pool entries in round-robin order, skipping cooled ones
1295        for i in 0..n {
1296            let idx = (start + i) % n;
1297            let (agent, model) = &pool[idx];
1298            let model_str = model.as_str();
1299
1300            if crate::engine::runner::response::is_model_in_cooldown(agent, model_str) {
1301                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1302                continue;
1303            }
1304
1305            // At least one pool entry was available to attempt
1306            attempted_any = true;
1307
1308            let model_opt = if model_str.is_empty() {
1309                None
1310            } else {
1311                Some(model_str)
1312            };
1313
1314            match self
1315                .llm_router
1316                .route_with_llm_using(
1317                    task,
1318                    &llm_agents,
1319                    &self.config,
1320                    &mut self.last_agent,
1321                    repo,
1322                    agent,
1323                    model_opt,
1324                )
1325                .await
1326            {
1327                Ok(result) => {
1328                    // Advance index so the next call starts at the next pool entry
1329                    self.advance_pool_index_after_attempt(idx, n);
1330                    tracing::debug!(
1331                        agent,
1332                        model = model_str,
1333                        pool_idx = idx,
1334                        "router LLM pool entry succeeded"
1335                    );
1336                    record_agent_success(agent, model_str).await;
1337                    return Ok(result);
1338                }
1339                Err(e) => {
1340                    let err_msg = e.to_string();
1341                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1342                    if is_timeout {
1343                        // Advance pool index so the next tick starts at the next entry,
1344                        // then return immediately. Continuing to try further pool entries
1345                        // in the same tick would accumulate multiple timeout windows and
1346                        // stall the tick loop for minutes (issue #3187).
1347                        //
1348                        // Also record a standard cooldown, same as the non-timeout branch,
1349                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1350                        // on future ticks instead of burning the full timeout window again
1351                        // every time round-robin reaches it (issue #3422).
1352                        self.advance_pool_index_after_attempt(idx, n);
1353                        tracing::warn!(
1354                            agent,
1355                            model = model_str,
1356                            next_pool_idx = self.pool_index,
1357                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1358                        );
1359                        crate::engine::runner::response::record_model_failure(agent, model_str)
1360                            .await;
1361                        return Err(e);
1362                    }
1363                    tracing::warn!(
1364                        agent,
1365                        model = model_str,
1366                        error = %e,
1367                        "pool entry failed, recording cooldown and trying next"
1368                    );
1369                    crate::engine::runner::response::record_model_failure(agent, model_str).await;
1370                    last_err = Some(e);
1371                    self.advance_pool_index_after_attempt(idx, n);
1372                }
1373            }
1374        }
1375
1376        // If every pool entry was pre-cooled (nothing was attempted), do not
1377        // immediately exhaust the pool again — fail fast and let the tick loop
1378        // retry on the next cycle instead of cascading failures.
1379        // However, we should still try the fallback if configured.
1380        if !attempted_any {
1381            let now = chrono::Utc::now().timestamp();
1382            let earliest = self.earliest_pool_cooldown();
1383            if let Some(until) = earliest {
1384                // Pool entries exist but are all cooled - we'll try fallback below
1385                let remaining = until.saturating_sub(now);
1386                tracing::warn!(
1387                    remaining_secs = remaining,
1388                    pool = ?pool,
1389                    "all router LLM pool entries on cooldown — will try fallback"
1390                );
1391                // Continue to fallback logic instead of returning early
1392            } else {
1393                // No pool cooldowns found either — this means the pool was empty
1394                // or every entry was filtered out for some other reason.
1395                return Err(last_err
1396                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1397            }
1398        }
1399
1400        // All pool entries failed or were cooled — try the configured fallback
1401        let fallback = self.config.effective_fallback();
1402        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1403        let fb_model_str = fb_model.as_str();
1404
1405        // Only try fallback if it wasn't already in the pool (avoid double-try)
1406        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1407
1408        if !fallback_already_tried
1409            && !crate::engine::runner::response::is_model_in_cooldown(&fb_agent, fb_model_str)
1410        {
1411            tracing::info!(
1412                agent = %fb_agent,
1413                model = %fb_model_str,
1414                "all pool entries exhausted — trying fallback router LLM"
1415            );
1416            let fb_model_opt = if fb_model_str.is_empty() {
1417                None
1418            } else {
1419                Some(fb_model_str)
1420            };
1421            match self
1422                .llm_router
1423                .route_with_llm_using(
1424                    task,
1425                    &llm_agents,
1426                    &self.config,
1427                    &mut self.last_agent,
1428                    repo,
1429                    &fb_agent,
1430                    fb_model_opt,
1431                )
1432                .await
1433            {
1434                Ok(result) => {
1435                    record_agent_success(&fb_agent, fb_model_str).await;
1436                    return Ok(result);
1437                }
1438                Err(e) => {
1439                    let err_msg = e.to_string();
1440                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1441                    if is_timeout {
1442                        // Cool down on timeout like pool entries (#3422), else a
1443                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1444                        crate::engine::runner::response::record_model_failure(
1445                            &fb_agent,
1446                            fb_model_str,
1447                        )
1448                        .await;
1449                        tracing::warn!(
1450                            agent = %fb_agent,
1451                            model = %fb_model_str,
1452                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1453                        );
1454                        return Err(e);
1455                    }
1456                    tracing::warn!(
1457                        agent = %fb_agent,
1458                        model = %fb_model_str,
1459                        error = %e,
1460                        "fallback router LLM also failed"
1461                    );
1462                    crate::engine::runner::response::record_model_failure(&fb_agent, fb_model_str)
1463                        .await;
1464                    last_err = Some(e);
1465                }
1466            }
1467        }
1468
1469        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1470    }
1471
1472    /// Record a rate limit event for an agent, reducing its routing weight.
1473    ///
1474    /// Called by the engine when an agent returns a 429/rate limit error.
1475    pub fn record_rate_limit(&mut self, agent: &str) {
1476        self.weights.record_rate_limit(agent);
1477    }
1478
1479    /// Record a successful task completion, restoring agent weight.
1480    pub fn record_success(&mut self, agent: &str) {
1481        self.weights.record_success(agent);
1482    }
1483
1484    /// Tick weight recovery for all agents.
1485    ///
1486    /// Called periodically by the engine to gradually restore weights
1487    /// as rate limit windows expire.
1488    pub fn tick_weight_recovery(&mut self) {
1489        self.weights.tick_recovery();
1490    }
1491
1492    /// Store routing result in the task store.
1493    pub async fn store_route_result(
1494        &self,
1495        task_id: &str,
1496        result: &RouteResult,
1497        store: &std::sync::Arc<TaskStore>,
1498        repo: &str,
1499    ) -> anyhow::Result<()> {
1500        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1501            store
1502                .set_fields(
1503                    store_id,
1504                    &[
1505                        ("agent", serde_json::json!(result.agent)),
1506                        ("complexity", serde_json::json!(result.complexity)),
1507                        ("route_reason", serde_json::json!(result.reason)),
1508                        (
1509                            "model",
1510                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1511                        ),
1512                    ],
1513                )
1514                .await?;
1515        }
1516        Ok(())
1517    }
1518}
1519
1520/// Retrieve routing result from the task store.
1521pub async fn get_route_result(
1522    store: &std::sync::Arc<TaskStore>,
1523    repo: &str,
1524    task_id: &str,
1525) -> Result<RouteResult, RouteResultError> {
1526    let store_id = store
1527        .resolve_task_id(repo, task_id)
1528        .await
1529        .map_err(RouteResultError::Other)?
1530        .ok_or_else(|| {
1531            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1532        })?;
1533    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1534
1535    let agent = task
1536        .agent
1537        .clone()
1538        .filter(|a| !a.is_empty())
1539        .ok_or_else(|| RouteResultError::NoAgent {
1540            task_id: task_id.to_string(),
1541        })?;
1542
1543    let complexity = if task.complexity.is_empty() {
1544        "medium".to_string()
1545    } else {
1546        task.complexity.clone()
1547    };
1548
1549    let reason = task.route_reason.clone();
1550    let model = task.model.clone().filter(|m| !m.is_empty());
1551
1552    // Validate that the stored model is valid for the resolved agent.
1553    // Agent-specific aliases (e.g. "opus") can leak across agents during
1554    // failover, so we verify against the router's model pools (#1604).
1555    let config = RouterConfig::from_config();
1556    let model = model.filter(|m| {
1557        let complexities = ["simple", "medium", "complex", "review"];
1558        // Collect all model pools configured for this agent across complexity tiers.
1559        let all_pools: Vec<Vec<String>> = complexities
1560            .iter()
1561            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1562            .collect();
1563        // If the agent has no model pools configured, we can't validate — pass through.
1564        if all_pools.is_empty() {
1565            return true;
1566        }
1567        let valid = all_pools.iter().any(|pool| pool.contains(m));
1568        if !valid {
1569            tracing::warn!(
1570                task_id,
1571                agent,
1572                model = %m,
1573                "discarding stale model: not in agent's model pools"
1574            );
1575        }
1576        valid
1577    });
1578
1579    // A stale model is replaced by the agent's model for the task's complexity, an empty
1580    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1581    let original_model = task.model.clone().filter(|m| !m.is_empty());
1582    let mut model = model;
1583    if original_model.is_some() && model.is_none() {
1584        model = config.model_for_complexity(&agent, &complexity, task_id);
1585        let stale_model = original_model.as_deref().unwrap_or("unknown");
1586        let stale_err = format!(
1587            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1588            model.as_deref().unwrap_or("the agent default")
1589        );
1590        if let Err(e) = store
1591            .set_fields(
1592                store_id,
1593                &[
1594                    (
1595                        "model",
1596                        serde_json::json!(model.clone().unwrap_or_default()),
1597                    ),
1598                    ("last_error", serde_json::json!(stale_err)),
1599                ],
1600            )
1601            .await
1602        {
1603            // Log the error so transient DB failures are visible and diagnosable.
1604            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1605        }
1606    }
1607
1608    Ok(RouteResult {
1609        agent,
1610        model,
1611        complexity,
1612        estimate: 0,
1613        reason,
1614        profile: AgentProfile::default(),
1615        selected_skills: vec![],
1616        warning: None,
1617    })
1618}
1619
1620#[cfg(test)]
1621#[expect(
1622    clippy::float_cmp,
1623    reason = "the weights are stored constants, so they compare exactly"
1624)]
1625mod tests {
1626    use super::config::DEFAULT_AGENTS;
1627    use super::llm::LlmRouteResponse;
1628    use super::strategies;
1629    use super::weights::{
1630        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1631    };
1632    use super::*;
1633    use crate::backends::{ExternalId, ExternalTask};
1634    use crate::engine::cooldown::{
1635        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1636        record_model_failure, set_model_cooldown,
1637    };
1638    use crate::store::TaskStore;
1639    use serial_test::serial;
1640    use std::time::{Duration, Instant};
1641
1642    #[tokio::test]
1643    #[serial_test::serial(orch_home)]
1644    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1645        let home = tempfile::tempdir().unwrap();
1646        std::fs::write(
1647            home.path().join("config.yml"),
1648            "model_map:\n  medium:\n    claude: sonnet\n",
1649        )
1650        .unwrap();
1651        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1652        unsafe { std::env::set_var("ORCH_HOME", home.path()) };
1653
1654        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1655        let store_id = store
1656            .create(&crate::store::NewTask {
1657                external_id: Some("7".to_string()),
1658                repo: "owner/repo".to_string(),
1659                origin: "github".to_string(),
1660                title: "stale model".to_string(),
1661                ..Default::default()
1662            })
1663            .await
1664            .unwrap();
1665        store
1666            .set_fields(
1667                store_id,
1668                &[
1669                    ("agent", serde_json::json!("claude")),
1670                    ("model", serde_json::json!("claude-removed-model")),
1671                    ("complexity", serde_json::json!("medium")),
1672                ],
1673            )
1674            .await
1675            .unwrap();
1676
1677        let route = get_route_result(&store, "owner/repo", "7").await;
1678        let stored = store.get(store_id).await.unwrap().model;
1679        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1680        unsafe { std::env::remove_var("ORCH_HOME") };
1681
1682        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1683        assert_eq!(stored.as_deref(), Some("sonnet"));
1684    }
1685
1686    // Test-only delegate so tests can call router.parse_llm_response() directly
1687    // without referencing llm_router.
1688    impl Router {
1689        #[expect(
1690            clippy::unused_self,
1691            reason = "the test helper keeps the router method syntax"
1692        )]
1693        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1694            LlmRouter::parse_llm_response("claude", response)
1695        }
1696    }
1697
1698    async fn test_store() -> std::sync::Arc<TaskStore> {
1699        std::sync::Arc::new(TaskStore::open_memory().await.unwrap())
1700    }
1701
1702    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1703        ExternalTask {
1704            id: ExternalId(id.to_string()),
1705            title: title.to_string(),
1706            body: "Test body".to_string(),
1707            state: "open".to_string(),
1708            labels,
1709            author: "test".to_string(),
1710            created_at: "2026-01-01T00:00:00Z".to_string(),
1711            updated_at: "2026-01-01T00:00:00Z".to_string(),
1712            url: format!("https://github.com/test/test/issues/{id}"),
1713        }
1714    }
1715
1716    #[serial(cooldown_state)]
1717    #[test]
1718    fn extract_agent_from_labels() {
1719        let mut config = RouterConfig::default();
1720        // Ensure a concrete model exists for the labeled agent so the label
1721        // override is honored by the router (routing requires a model).
1722        config
1723            .model_map
1724            .entry("medium".to_string())
1725            .or_default()
1726            .insert("claude".to_string(), vec!["haiku".to_string()]);
1727
1728        assert_eq!(
1729            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1730            Some("claude".to_string())
1731        );
1732        assert_eq!(
1733            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1734            Some("codex".to_string())
1735        );
1736        assert_eq!(
1737            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1738            Some("opencode".to_string())
1739        );
1740        assert_eq!(
1741            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1742            None
1743        );
1744        // Verify kimi and minimax are recognized from labels
1745        assert_eq!(
1746            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1747            Some("kimi".to_string())
1748        );
1749        assert_eq!(
1750            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1751            Some("minimax".to_string())
1752        );
1753    }
1754
1755    #[serial(cooldown_state)]
1756    #[tokio::test]
1757    async fn all_agents_cooled_error_names_degraded_agents() {
1758        crate::engine::cooldown::reset_global_state().await;
1759        mark_agent_degraded("claude");
1760        mark_agent_degraded("grok");
1761        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1762        let msg = err.to_string();
1763        assert!(
1764            msg.contains("degraded: claude, grok"),
1765            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1766        );
1767        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1768        clear_agent_degraded("claude");
1769        clear_agent_degraded("grok");
1770    }
1771
1772    #[serial(cooldown_state)]
1773    #[test]
1774    fn default_agents_constant() {
1775        assert_eq!(DEFAULT_AGENTS.len(), 7);
1776        assert!(DEFAULT_AGENTS.contains(&"claude"));
1777        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1778        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1779        assert!(DEFAULT_AGENTS.contains(&"glm"));
1780        assert!(DEFAULT_AGENTS.contains(&"grok"));
1781    }
1782
1783    #[serial(cooldown_state)]
1784    #[test]
1785    fn extract_complexity_from_labels() {
1786        assert_eq!(
1787            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1788            "simple"
1789        );
1790        assert_eq!(
1791            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1792            "medium"
1793        );
1794        assert_eq!(
1795            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1796            "complex"
1797        );
1798        assert_eq!(
1799            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1800            "medium"
1801        );
1802    }
1803
1804    #[serial(cooldown_state)]
1805    #[test]
1806    fn route_result_serialization() {
1807        let result = RouteResult {
1808            agent: "claude".to_string(),
1809            model: Some("sonnet".to_string()),
1810            complexity: "medium".to_string(),
1811            estimate: 5,
1812            reason: "test".to_string(),
1813            profile: AgentProfile {
1814                role: "backend".to_string(),
1815                skills: vec!["rust".to_string()],
1816                tools: vec!["git".to_string()],
1817                constraints: vec![],
1818            },
1819            selected_skills: vec!["gh".to_string()],
1820            warning: None,
1821        };
1822
1823        let json = serde_json::to_string(&result).unwrap();
1824        assert!(json.contains("claude"));
1825        assert!(json.contains("sonnet"));
1826
1827        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1828        assert_eq!(deserialized.agent, "claude");
1829        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1830    }
1831
1832    #[serial(cooldown_state)]
1833    #[test]
1834    fn router_config_default() {
1835        let mut config = RouterConfig::default();
1836        // Ensure a concrete model exists for the labeled agent so the label
1837        // override is honored by the router (routing requires a model).
1838        config
1839            .model_map
1840            .entry("medium".to_string())
1841            .or_default()
1842            .insert("claude".to_string(), vec!["haiku".to_string()]);
1843
1844        assert_eq!(config.mode, "round_robin");
1845        assert_eq!(config.router_agent, "claude");
1846        assert_eq!(config.router_model, "haiku");
1847        assert_eq!(config.fallback_executor, "codex");
1848        assert_eq!(config.max_route_attempts, 3);
1849        assert_ne!(config.allowed_tools, [] as [String; 0]);
1850        assert_ne!(config.default_skills, [] as [String; 0]);
1851
1852        // Verify configurable agents list includes all 7 agents
1853        assert_eq!(config.agents.len(), 7);
1854        assert!(config.agents.contains(&"claude".to_string()));
1855        assert!(config.agents.contains(&"codex".to_string()));
1856        assert!(config.agents.contains(&"opencode".to_string()));
1857        assert!(config.agents.contains(&"kimi".to_string()));
1858        assert!(config.agents.contains(&"minimax".to_string()));
1859        assert!(config.agents.contains(&"glm".to_string()));
1860        assert!(config.agents.contains(&"grok".to_string()));
1861    }
1862
1863    #[serial(cooldown_state)]
1864    #[test]
1865    fn model_map_lookup_returns_none_without_config() {
1866        // Default config has no hardcoded models — config.yml is the source of truth.
1867        let config = RouterConfig::default();
1868        assert!(
1869            config
1870                .model_for_complexity("claude", "simple", "")
1871                .is_none()
1872        );
1873        assert!(
1874            config
1875                .model_for_complexity("claude", "medium", "")
1876                .is_none()
1877        );
1878        assert!(
1879            config
1880                .model_for_complexity("claude", "complex", "")
1881                .is_none()
1882        );
1883        assert!(
1884            config
1885                .model_for_complexity("opencode", "review", "")
1886                .is_none()
1887        );
1888    }
1889
1890    #[serial(cooldown_state)]
1891    #[test]
1892    fn model_map_lookup_returns_configured_value() {
1893        // When model_map is populated (from config), it is returned correctly.
1894        let mut config = RouterConfig::default();
1895        config
1896            .model_map
1897            .entry("simple".to_string())
1898            .or_default()
1899            .insert("claude".to_string(), vec!["haiku".to_string()]);
1900        assert_eq!(
1901            config.model_for_complexity("claude", "simple", ""),
1902            Some("haiku".to_string())
1903        );
1904    }
1905
1906    #[serial(cooldown_state)]
1907    #[tokio::test]
1908    async fn model_pool_selection_skips_cooled() {
1909        crate::engine::cooldown::reset_global_state().await;
1910
1911        let mut config = RouterConfig::default();
1912        // Set up a two-model pool for opencode/simple
1913        config
1914            .model_map
1915            .entry("simple".to_string())
1916            .or_default()
1917            .insert(
1918                "opencode".to_string(),
1919                vec!["model-a".to_string(), "model-b".to_string()],
1920            );
1921
1922        // Before any cooldown, both models are candidates
1923        let m = config.model_for_complexity("opencode", "simple", "task-1");
1924        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1925
1926        // Cool model-a
1927        record_model_failure("opencode", "model-a").await;
1928        assert!(is_model_in_cooldown("opencode", "model-a"));
1929
1930        // Now only model-b should be returned
1931        // (try many task_ids to rule out hash coincidence)
1932        for i in 0..20 {
1933            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1934            assert_eq!(
1935                result,
1936                Some("model-b".to_string()),
1937                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1938            );
1939        }
1940
1941        // Cool model-b too — now all models are cooled
1942        record_model_failure("opencode", "model-b").await;
1943        // All cooled → return None so the caller can fall back to a different agent
1944        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1945        assert_eq!(fallback, None);
1946    }
1947
1948    #[serial(cooldown_state)]
1949    #[test]
1950    fn model_pool_single_string_backward_compat() {
1951        // Single-item pools behave identically to the old string format.
1952        // Manually insert a single-item pool and verify it is returned correctly.
1953        let mut config = RouterConfig::default();
1954        config
1955            .model_map
1956            .entry("simple".to_string())
1957            .or_default()
1958            .insert("claude".to_string(), vec!["haiku".to_string()]);
1959        let m = config.model_for_complexity("claude", "simple", "any-task");
1960        assert_eq!(m, Some("haiku".to_string()));
1961    }
1962
1963    #[serial(cooldown_state)]
1964    #[test]
1965    fn parse_llm_response_direct_json() {
1966        let config = RouterConfig::default();
1967        let router = Router::new(config);
1968
1969        let response = r#"{
1970            "executor": "claude",
1971            "complexity": "complex",
1972            "reason": "requires architecture analysis",
1973            "profile": {
1974                "role": "architect",
1975                "skills": ["rust", "design"],
1976                "tools": ["git", "rg"],
1977                "constraints": []
1978            },
1979            "selected_skills": ["gh"]
1980        }"#;
1981
1982        let parsed = router.parse_llm_response(response).unwrap();
1983        assert_eq!(parsed.executor, "claude");
1984        assert_eq!(parsed.complexity, "complex");
1985        assert_eq!(parsed.reason, "requires architecture analysis");
1986        assert_eq!(parsed.profile.role, "architect");
1987    }
1988
1989    #[serial(cooldown_state)]
1990    #[test]
1991    fn parse_llm_response_markdown_fenced() {
1992        let config = RouterConfig::default();
1993        let router = Router::new(config);
1994
1995        let response = r#"Here's my analysis:
1996
1997```json
1998{
1999    "executor": "codex",
2000    "complexity": "medium",
2001    "reason": "coding task",
2002    "profile": {
2003        "role": "developer",
2004        "skills": ["coding"],
2005        "tools": [],
2006        "constraints": []
2007    },
2008    "selected_skills": []
2009}
2010```
2011
2012Hope that helps!"#;
2013
2014        let parsed = router.parse_llm_response(response).unwrap();
2015        assert_eq!(parsed.executor, "codex");
2016        assert_eq!(parsed.complexity, "medium");
2017    }
2018
2019    #[serial(cooldown_state)]
2020    #[test]
2021    fn parse_llm_response_claude_envelope() {
2022        let config = RouterConfig::default();
2023        let router = Router::new(config);
2024
2025        // Claude --output-format json wraps the response in an envelope
2026        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2027
2028        let parsed = router.parse_llm_response(response).unwrap();
2029        assert_eq!(parsed.executor, "claude");
2030        assert_eq!(parsed.complexity, "complex");
2031        assert_eq!(parsed.reason, "requires deep analysis");
2032    }
2033
2034    #[serial(cooldown_state)]
2035    #[test]
2036    fn parse_llm_response_claude_envelope_with_code_block() {
2037        let config = RouterConfig::default();
2038        let router = Router::new(config);
2039
2040        // Claude may return the JSON inside a code block within the envelope
2041        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2042        let envelope = serde_json::json!({
2043            "type": "result",
2044            "subtype": "success",
2045            "is_error": false,
2046            "result": inner,
2047            "usage": {"input_tokens": 50, "output_tokens": 30}
2048        });
2049
2050        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2051        assert_eq!(parsed.executor, "codex");
2052        assert_eq!(parsed.complexity, "simple");
2053    }
2054
2055    #[serial(cooldown_state)]
2056    #[test]
2057    fn parse_llm_response_claude_envelope_error() {
2058        let config = RouterConfig::default();
2059        let router = Router::new(config);
2060
2061        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2062
2063        let err = router.parse_llm_response(response).unwrap_err();
2064        assert!(err.to_string().contains("error"), "got: {err}");
2065    }
2066
2067    #[serial(cooldown_state)]
2068    #[test]
2069    fn parse_llm_response_empty() {
2070        let config = RouterConfig::default();
2071        let router = Router::new(config);
2072
2073        let err = router.parse_llm_response("").unwrap_err();
2074        assert!(err.to_string().contains("empty"), "got: {err}");
2075    }
2076
2077    /// Claude envelope where result is a JSON object, not a string.
2078    /// Some Claude versions may return the result as a nested object.
2079    #[serial(cooldown_state)]
2080    #[test]
2081    fn parse_llm_response_claude_envelope_result_as_object() {
2082        let config = RouterConfig::default();
2083        let router = Router::new(config);
2084
2085        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2086
2087        let parsed = router.parse_llm_response(response);
2088        assert!(
2089            parsed.is_ok(),
2090            "should handle result-as-object: {}",
2091            parsed.unwrap_err()
2092        );
2093        assert_eq!(parsed.unwrap().executor, "claude");
2094    }
2095
2096    /// LLM returns "agent" instead of "executor" — common hallucination.
2097    #[serial(cooldown_state)]
2098    #[test]
2099    fn parse_llm_response_agent_instead_of_executor() {
2100        let config = RouterConfig::default();
2101        let router = Router::new(config);
2102
2103        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2104
2105        let parsed = router.parse_llm_response(response);
2106        assert!(
2107            parsed.is_ok(),
2108            "should accept 'agent' alias: {}",
2109            parsed.unwrap_err()
2110        );
2111        assert_eq!(parsed.unwrap().executor, "claude");
2112    }
2113
2114    /// LLM returns minimal JSON without profile or reason.
2115    #[serial(cooldown_state)]
2116    #[test]
2117    fn parse_llm_response_minimal_json() {
2118        let config = RouterConfig::default();
2119        let router = Router::new(config);
2120
2121        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2122
2123        let parsed = router.parse_llm_response(response);
2124        assert!(
2125            parsed.is_ok(),
2126            "should accept minimal JSON: {}",
2127            parsed.unwrap_err()
2128        );
2129        assert_eq!(parsed.unwrap().executor, "codex");
2130    }
2131
2132    // ── Real fixture tests: route prompt → response → parsed result ──
2133
2134    /// Real Claude envelope with result as escaped JSON string.
2135    #[serial(cooldown_state)]
2136    #[test]
2137    fn fixture_route_response_string() {
2138        let config = RouterConfig::default();
2139        let router = Router::new(config);
2140
2141        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2142        let parsed = router.parse_llm_response(response).unwrap();
2143        assert_eq!(parsed.executor, "codex");
2144        assert_eq!(parsed.complexity, "medium");
2145        assert_ne!(parsed.reason, "");
2146        assert_ne!(parsed.profile.role, "");
2147    }
2148
2149    /// Real Claude envelope with result as JSON object (not string).
2150    /// This was the root cause of all router failures on 2026-02-27.
2151    #[serial(cooldown_state)]
2152    #[test]
2153    fn fixture_route_response_object() {
2154        let config = RouterConfig::default();
2155        let router = Router::new(config);
2156
2157        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2158        let parsed = router.parse_llm_response(response).unwrap();
2159        assert_eq!(parsed.executor, "claude");
2160        assert_eq!(parsed.complexity, "complex");
2161    }
2162
2163    /// Real Claude envelope with result as string containing markdown + JSON.
2164    #[serial(cooldown_state)]
2165    #[test]
2166    fn fixture_route_response_markdown() {
2167        let config = RouterConfig::default();
2168        let router = Router::new(config);
2169
2170        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2171        let parsed = router.parse_llm_response(response).unwrap();
2172        assert_eq!(parsed.executor, "codex");
2173        assert_eq!(parsed.complexity, "medium");
2174    }
2175
2176    /// Claude envelope with escaped JSON string containing newlines and markdown.
2177    #[serial(cooldown_state)]
2178    #[test]
2179    fn parse_llm_response_claude_envelope_with_prose() {
2180        let config = RouterConfig::default();
2181        let router = Router::new(config);
2182
2183        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2184        let envelope = serde_json::json!({
2185            "type": "result",
2186            "subtype": "success",
2187            "is_error": false,
2188            "result": inner,
2189            "usage": {"input_tokens": 100, "output_tokens": 80}
2190        });
2191
2192        let parsed = router.parse_llm_response(&envelope.to_string());
2193        assert!(
2194            parsed.is_ok(),
2195            "should handle prose + code block in envelope: {}",
2196            parsed.unwrap_err()
2197        );
2198        assert_eq!(parsed.unwrap().executor, "codex");
2199    }
2200
2201    #[serial(cooldown_state)]
2202    #[tokio::test]
2203    async fn route_round_robin_basic() {
2204        crate::engine::cooldown::reset_global_state().await;
2205        // Force at least one agent to be available for testing
2206        // In real usage, discover_agents finds installed CLIs
2207        let config = RouterConfig {
2208            mode: "round_robin".to_string(),
2209            ..Default::default()
2210        };
2211
2212        // Create router with mock available agents
2213        let agents = vec!["claude".to_string(), "codex".to_string()];
2214        let mut weights = AgentWeights::default();
2215        weights.ensure_agents(&agents);
2216        let router = Router {
2217            config,
2218            available_agents: agents,
2219            weights,
2220            llm_router: std::sync::Arc::new(LlmRouter::new()),
2221            ollama_router: None,
2222            rr_index: 0,
2223            last_agent: None,
2224            review_rr_index: 0,
2225            router_pool: vec![],
2226            pool_index: 0,
2227        };
2228
2229        let task = create_test_task("1", "Test task", vec![]);
2230        let result =
2231            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2232                .unwrap();
2233
2234        // Task 1 % 2 agents = agent at index 1 = codex
2235        assert_eq!(result.agent, "codex");
2236        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2237    }
2238
2239    #[serial(cooldown_state)]
2240    #[tokio::test]
2241    async fn route_uses_label_override() {
2242        crate::engine::cooldown::reset_global_state().await;
2243        let mut config = RouterConfig::default();
2244        // Ensure a concrete model exists so the label override can be honored
2245        // in the test environment where no external config.yml is present.
2246        config
2247            .model_map
2248            .entry("medium".to_string())
2249            .or_default()
2250            .insert("claude".to_string(), vec!["haiku".to_string()]);
2251        let agents = vec!["claude".to_string(), "codex".to_string()];
2252        let mut weights = AgentWeights::default();
2253        weights.ensure_agents(&agents);
2254        let mut router = Router {
2255            config,
2256            available_agents: agents,
2257            weights,
2258            llm_router: std::sync::Arc::new(LlmRouter::new()),
2259            ollama_router: None,
2260            rr_index: 0,
2261            last_agent: None,
2262            review_rr_index: 0,
2263            router_pool: vec![],
2264            pool_index: 0,
2265        };
2266
2267        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2268        let store = test_store().await;
2269
2270        // Should use label override, not LLM
2271        let result = router.route(&task, &store, "test/repo").await.unwrap();
2272        assert_eq!(result.agent, "claude");
2273        assert!(result.reason.contains("label"));
2274    }
2275
2276    #[serial(cooldown_state)]
2277    #[test]
2278    fn check_routing_sanity_warnings() {
2279        // Backend task routed to claude should warn
2280        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2281        let profile = AgentProfile {
2282            role: "general".to_string(),
2283            skills: vec!["api".to_string()],
2284            tools: vec![],
2285            constraints: vec![],
2286        };
2287        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2288        assert!(warning.is_some());
2289        assert!(warning.unwrap().contains("backend"));
2290
2291        // Docs task routed to codex should warn
2292        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2293        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2294        assert!(warning.is_some());
2295        assert!(warning.unwrap().contains("docs"));
2296
2297        // Normal routing should not warn
2298        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2299        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2300        assert!(warning.is_none());
2301    }
2302
2303    #[serial(cooldown_state)]
2304    #[tokio::test]
2305    async fn router_reload_preserves_structure() {
2306        crate::engine::cooldown::reset_global_state().await;
2307        let config = RouterConfig::default();
2308        let agents = vec!["claude".to_string()];
2309        let mut weights = AgentWeights::default();
2310        weights.ensure_agents(&agents);
2311        let mut router = Router {
2312            config,
2313            available_agents: agents,
2314            weights,
2315            llm_router: std::sync::Arc::new(LlmRouter::new()),
2316            ollama_router: None,
2317            rr_index: 0,
2318            last_agent: None,
2319            review_rr_index: 0,
2320            router_pool: vec![],
2321            pool_index: 0,
2322        };
2323
2324        // Reload — should re-read config and remain valid
2325        router.reload_async().await;
2326
2327        // After reload, mode should be a valid value (llm, local, or round_robin)
2328        assert!(
2329            router.config.mode == "llm"
2330                || router.config.mode == "round_robin"
2331                || router.config.mode == "local",
2332            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2333            router.config.mode
2334        );
2335        // Fallback executor should always be set
2336        assert_ne!(router.config.fallback_executor, "");
2337        // Tools should always be populated
2338        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2339    }
2340
2341    // --- Weighted round-robin tests ---
2342
2343    #[serial(cooldown_state)]
2344    #[test]
2345    fn rate_limit_state_defaults_to_full_weight() {
2346        let state = RateLimitState::default();
2347        assert_eq!(state.weight, DEFAULT_WEIGHT);
2348        assert_eq!(state.consecutive_hits, 0);
2349        assert!(state.last_limited_at.is_none());
2350        assert!(!state.is_limited());
2351    }
2352
2353    #[serial(cooldown_state)]
2354    #[test]
2355    fn rate_limit_state_decays_on_hit() {
2356        let mut state = RateLimitState::default();
2357        state.record_rate_limit();
2358
2359        assert!(state.weight < DEFAULT_WEIGHT);
2360        assert_eq!(state.consecutive_hits, 1);
2361        assert!(state.last_limited_at.is_some());
2362        assert!(state.is_limited());
2363
2364        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2365        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2366        assert!((state.weight - expected).abs() < 1e-10);
2367    }
2368
2369    #[serial(cooldown_state)]
2370    #[test]
2371    fn rate_limit_state_never_drops_below_min() {
2372        let mut state = RateLimitState::default();
2373
2374        // Hit many times
2375        for _ in 0..100 {
2376            state.record_rate_limit();
2377        }
2378
2379        assert!(state.weight >= MIN_WEIGHT);
2380        assert_eq!(state.consecutive_hits, 100);
2381    }
2382
2383    #[serial(cooldown_state)]
2384    #[test]
2385    fn rate_limit_state_recovers_on_success() {
2386        let mut state = RateLimitState::default();
2387
2388        // Decay first
2389        state.record_rate_limit();
2390        state.record_rate_limit();
2391        let after_decay = state.weight;
2392
2393        // Record success
2394        state.record_success();
2395        assert!(state.weight > after_decay);
2396        assert_eq!(state.consecutive_hits, 0);
2397    }
2398
2399    #[serial(cooldown_state)]
2400    #[test]
2401    fn rate_limit_state_success_caps_at_default() {
2402        let mut state = RateLimitState::default();
2403
2404        // Already at full weight, success shouldn't exceed it
2405        state.record_success();
2406        assert_eq!(state.weight, DEFAULT_WEIGHT);
2407    }
2408
2409    #[serial(cooldown_state)]
2410    #[test]
2411    fn agent_weights_ensure_agents() {
2412        let mut weights = AgentWeights::default();
2413        let agents = vec!["claude".to_string(), "codex".to_string()];
2414        weights.ensure_agents(&agents);
2415
2416        assert_eq!(weights.states.len(), 2);
2417        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2418        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2419    }
2420
2421    #[serial(cooldown_state)]
2422    #[test]
2423    fn agent_weights_ensure_agents_preserves_existing() {
2424        let mut weights = AgentWeights::default();
2425        let agents = vec!["claude".to_string(), "codex".to_string()];
2426        weights.ensure_agents(&agents);
2427
2428        // Reduce claude's weight
2429        weights.record_rate_limit("claude");
2430        let claude_weight = weights.get_weight("claude");
2431        assert!(claude_weight < DEFAULT_WEIGHT);
2432
2433        // Ensure agents again — claude's weight should be preserved
2434        let agents2 = vec![
2435            "claude".to_string(),
2436            "codex".to_string(),
2437            "opencode".to_string(),
2438        ];
2439        weights.ensure_agents(&agents2);
2440
2441        assert_eq!(weights.get_weight("claude"), claude_weight);
2442        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2443    }
2444
2445    #[test]
2446    fn default_missing_weight_uses_minimum_of_configured() {
2447        use super::weights::default_missing_weight;
2448
2449        let mut configured = std::collections::HashMap::new();
2450        configured.insert("claude".to_string(), 0.2);
2451        configured.insert("codex".to_string(), 0.2);
2452        configured.insert("opencode".to_string(), 0.2);
2453        configured.insert("kimi".to_string(), 0.2);
2454
2455        // minimax omitted from router.weights — must not silently outrank
2456        // every explicitly configured agent (issue #3596).
2457        assert_eq!(default_missing_weight(&configured), 0.2);
2458    }
2459
2460    #[test]
2461    fn default_missing_weight_falls_back_when_none_configured() {
2462        use super::weights::default_missing_weight;
2463
2464        let configured = std::collections::HashMap::new();
2465        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2466    }
2467
2468    #[serial(cooldown_state)]
2469    #[test]
2470    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2471        let mut weights = AgentWeights::default();
2472        let mut configured = std::collections::HashMap::new();
2473        configured.insert("claude".to_string(), 0.2);
2474        configured.insert("codex".to_string(), 0.2);
2475        configured.insert("opencode".to_string(), 0.2);
2476        configured.insert("kimi".to_string(), 0.2);
2477
2478        let agents = vec![
2479            "claude".to_string(),
2480            "codex".to_string(),
2481            "opencode".to_string(),
2482            "kimi".to_string(),
2483            "minimax".to_string(),
2484        ];
2485        weights.ensure_agents_with_weights(&agents, &configured);
2486
2487        // minimax was omitted from router.weights but must not get the old
2488        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2489        assert_eq!(weights.get_weight("minimax"), 0.2);
2490        assert_eq!(weights.get_weight("claude"), 0.2);
2491    }
2492
2493    #[serial(cooldown_state)]
2494    #[test]
2495    fn agent_weights_record_rate_limit() {
2496        let mut weights = AgentWeights::default();
2497        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2498
2499        weights.record_rate_limit("claude");
2500
2501        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2502        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2503    }
2504
2505    #[serial(cooldown_state)]
2506    #[test]
2507    fn agent_weights_record_success() {
2508        let mut weights = AgentWeights::default();
2509        weights.ensure_agents(&["claude".to_string()]);
2510
2511        weights.record_rate_limit("claude");
2512        let after_limit = weights.get_weight("claude");
2513
2514        weights.record_success("claude");
2515        assert!(weights.get_weight("claude") > after_limit);
2516    }
2517
2518    #[serial(cooldown_state)]
2519    #[test]
2520    fn agent_weights_snapshot() {
2521        let mut weights = AgentWeights::default();
2522        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2523        weights.record_rate_limit("claude");
2524
2525        let snap = weights.snapshot();
2526        assert_eq!(snap.len(), 2);
2527
2528        // Snapshot is sorted alphabetically
2529        assert_eq!(snap[0].0, "claude");
2530        assert!(snap[0].1 < DEFAULT_WEIGHT);
2531        assert_eq!(snap[0].2, 1); // 1 hit
2532
2533        assert_eq!(snap[1].0, "codex");
2534        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2535        assert_eq!(snap[1].2, 0); // 0 hits
2536    }
2537
2538    #[serial(cooldown_state)]
2539    #[test]
2540    fn agent_weights_weighted_select_favors_higher_weight() {
2541        let mut weights = AgentWeights::default();
2542        let agents = vec!["claude".to_string(), "codex".to_string()];
2543        weights.ensure_agents(&agents);
2544
2545        // Heavily penalize claude
2546        for _ in 0..10 {
2547            weights.record_rate_limit("claude");
2548        }
2549
2550        // Run many selections and count
2551        let mut claude_count = 0;
2552        let mut codex_count = 0;
2553        for _ in 0..100 {
2554            match weights.weighted_select(&agents, "task-1") {
2555                Some(ref a) if a == "claude" => claude_count += 1,
2556                Some(ref a) if a == "codex" => codex_count += 1,
2557                _ => {}
2558            }
2559        }
2560
2561        // Codex should get significantly more selections
2562        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2563        assert!(
2564            codex_count > claude_count,
2565            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2566        );
2567    }
2568
2569    #[serial(cooldown_state)]
2570    #[test]
2571    fn agent_weights_weighted_select_empty_returns_none() {
2572        let weights = AgentWeights::default();
2573        assert!(weights.weighted_select(&[], "task-1").is_none());
2574    }
2575
2576    #[serial(cooldown_state)]
2577    #[test]
2578    fn agent_weights_batch_routing_produces_varied_selections() {
2579        // Regression test: when routing a batch of tasks in the same tick,
2580        // different task IDs should not all select the same agent.
2581        let mut weights = AgentWeights::default();
2582        let agents = vec!["claude".to_string(), "codex".to_string()];
2583        weights.ensure_agents(&agents);
2584
2585        // Simulate batch routing: 20 different task IDs called in rapid succession
2586        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2587        let selections: Vec<String> = task_ids
2588            .iter()
2589            .filter_map(|id| weights.weighted_select(&agents, id))
2590            .collect();
2591
2592        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2593        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2594        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2595        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2596        assert!(
2597            claude_count > 0 && codex_count > 0,
2598            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2599        );
2600    }
2601
2602    #[serial(cooldown_state)]
2603    #[test]
2604    fn agent_weights_weighted_select_single_agent() {
2605        let mut weights = AgentWeights::default();
2606        let agents = vec!["claude".to_string()];
2607        weights.ensure_agents(&agents);
2608
2609        let selected = weights.weighted_select(&agents, "task-1");
2610        assert_eq!(selected, Some("claude".to_string()));
2611    }
2612
2613    #[serial(cooldown_state)]
2614    #[test]
2615    fn agent_weights_tick_recovery_restores_weight() {
2616        let mut weights = AgentWeights::default();
2617        weights.ensure_agents(&["claude".to_string()]);
2618
2619        // Set up a rate limit that looks like it happened long ago
2620        let state = weights.states.get_mut("claude").unwrap();
2621        state.weight = 0.5;
2622        state.last_limited_at = Some(
2623            Instant::now()
2624                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2625                .expect("the clock is past the recovery delay"),
2626        );
2627        state.consecutive_hits = 2;
2628
2629        let before = weights.get_weight("claude");
2630        weights.tick_recovery();
2631        let after = weights.get_weight("claude");
2632
2633        assert!(after > before, "weight should increase after recovery tick");
2634    }
2635
2636    #[serial(cooldown_state)]
2637    #[test]
2638    fn agent_weights_tick_recovery_clears_on_full_restore() {
2639        let mut weights = AgentWeights::default();
2640        weights.ensure_agents(&["claude".to_string()]);
2641
2642        // Set weight just below full with an old limit
2643        let state = weights.states.get_mut("claude").unwrap();
2644        state.weight = DEFAULT_WEIGHT - 0.01;
2645        state.last_limited_at = Some(
2646            Instant::now()
2647                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2648                .expect("the clock is past the recovery delay"),
2649        );
2650        state.consecutive_hits = 1;
2651
2652        weights.tick_recovery();
2653
2654        let state = weights.states.get("claude").unwrap();
2655        assert_eq!(state.weight, DEFAULT_WEIGHT);
2656        assert!(state.last_limited_at.is_none());
2657        assert_eq!(state.consecutive_hits, 0);
2658    }
2659
2660    #[serial(cooldown_state)]
2661    #[tokio::test]
2662    async fn route_weighted_round_robin_basic() {
2663        crate::engine::cooldown::reset_global_state().await;
2664        let mut config = RouterConfig {
2665            weighted_round_robin: true,
2666            ..Default::default()
2667        };
2668        // Provide a concrete model for codex so label override can be used.
2669        config
2670            .model_map
2671            .entry("medium".to_string())
2672            .or_default()
2673            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2674
2675        let agents = vec!["claude".to_string(), "codex".to_string()];
2676        let mut weights = AgentWeights::default();
2677        weights.ensure_agents(&agents);
2678        let mut router = Router {
2679            config,
2680            available_agents: agents,
2681            weights,
2682            llm_router: std::sync::Arc::new(LlmRouter::new()),
2683            ollama_router: None,
2684            rr_index: 0,
2685            last_agent: None,
2686            review_rr_index: 0,
2687            router_pool: vec![],
2688            pool_index: 0,
2689        };
2690
2691        let task = create_test_task("1", "Test task", vec![]);
2692        let store = test_store().await;
2693        let result = router.route(&task, &store, "test/repo").await.unwrap();
2694
2695        // Should use weighted_round_robin
2696        assert!(result.reason.contains("weighted_round_robin"));
2697        assert!(
2698            result.agent == "claude" || result.agent == "codex",
2699            "agent should be claude or codex, got '{}'",
2700            result.agent
2701        );
2702    }
2703
2704    #[serial(cooldown_state)]
2705    #[tokio::test]
2706    async fn route_weighted_round_robin_respects_label_override() {
2707        crate::engine::cooldown::reset_global_state().await;
2708        let mut config = RouterConfig {
2709            weighted_round_robin: true,
2710            ..Default::default()
2711        };
2712        // Provide a concrete model for codex so label override can be used.
2713        config
2714            .model_map
2715            .entry("medium".to_string())
2716            .or_default()
2717            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2718
2719        let agents = vec!["claude".to_string(), "codex".to_string()];
2720        let mut weights = AgentWeights::default();
2721        weights.ensure_agents(&agents);
2722        let mut router = Router {
2723            config,
2724            available_agents: agents,
2725            weights,
2726            llm_router: std::sync::Arc::new(LlmRouter::new()),
2727            ollama_router: None,
2728            rr_index: 0,
2729            last_agent: None,
2730            review_rr_index: 0,
2731            router_pool: vec![],
2732            pool_index: 0,
2733        };
2734
2735        // Label override should take precedence over weighted routing
2736        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2737        let store = test_store().await;
2738        let result = router.route(&task, &store, "test/repo").await.unwrap();
2739        assert_eq!(result.agent, "codex");
2740        assert!(result.reason.contains("label"));
2741    }
2742
2743    #[serial(cooldown_state)]
2744    #[test]
2745    fn router_config_weighted_round_robin_default_false() {
2746        let config = RouterConfig::default();
2747        assert!(!config.weighted_round_robin);
2748    }
2749
2750    #[serial(cooldown_state)]
2751    #[test]
2752    fn router_record_and_recover_weights() {
2753        let config = RouterConfig {
2754            weighted_round_robin: true,
2755            ..Default::default()
2756        };
2757        let mut router = Router::new(config);
2758        // Override discovered agents for test
2759        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2760        router.weights.ensure_agents(&router.available_agents);
2761
2762        // Record rate limit
2763        router.record_rate_limit("claude");
2764        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2765        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2766
2767        // Record success for claude
2768        router.record_success("claude");
2769        let after_success = router.weights.get_weight("claude");
2770        assert!(after_success > MIN_WEIGHT);
2771    }
2772
2773    #[serial(cooldown_state)]
2774    #[test]
2775    fn review_rr_index_advances() {
2776        let mut config = RouterConfig::default();
2777        // Set up model_map so has_available_model_for_complexity returns true for test agents
2778        for agent in &["test_a", "test_b", "test_c"] {
2779            config
2780                .model_map
2781                .entry("medium".to_string())
2782                .or_default()
2783                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2784        }
2785        let agents = vec![
2786            "test_a".to_string(),
2787            "test_b".to_string(),
2788            "test_c".to_string(),
2789        ];
2790        let mut weights = AgentWeights::default();
2791        weights.ensure_agents(&agents);
2792        let mut router = Router {
2793            config,
2794            available_agents: agents,
2795            weights,
2796            llm_router: std::sync::Arc::new(LlmRouter::new()),
2797            ollama_router: None,
2798            rr_index: 0,
2799            last_agent: None,
2800            review_rr_index: 0,
2801            router_pool: vec![],
2802            pool_index: 0,
2803        };
2804
2805        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2806        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2807        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2808
2809        // All three agents should appear (order depends on start index)
2810        let mut seen = vec![a1, a2, a3];
2811        seen.sort();
2812        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2813    }
2814
2815    #[serial(cooldown_state)]
2816    #[test]
2817    fn review_rr_fallback_with_cooled_agent() {
2818        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2819        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2820        // candidates.get(idx) to return None even though candidates is non-empty.
2821        let mut config = RouterConfig::default();
2822        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2823        for agent in &["test_a", "test_b", "test_c"] {
2824            config
2825                .model_map
2826                .entry("medium".to_string())
2827                .or_default()
2828                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2829        }
2830        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2831        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2832        let mut weights = AgentWeights::default();
2833        weights.ensure_agents(&agents);
2834        let mut router = Router {
2835            config,
2836            available_agents: agents,
2837            weights,
2838            llm_router: std::sync::Arc::new(LlmRouter::new()),
2839            ollama_router: None,
2840            rr_index: 0,
2841            last_agent: None,
2842            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2843            review_rr_index: 2,
2844            router_pool: vec![],
2845            pool_index: 0,
2846        };
2847
2848        // All candidates excluded — should fall back to first candidate, not None
2849        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2850        assert!(
2851            result.is_some(),
2852            "next_round_robin_agent must not return None when candidates is non-empty"
2853        );
2854    }
2855
2856    #[serial(cooldown_state)]
2857    #[test]
2858    fn review_rr_index_uses_candidates_len_not_available_len() {
2859        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2860        // not available_agents.len(). When an agent has no model for the complexity,
2861        // candidates.len() < available_agents.len() and the old code produced a drifting
2862        // index that caused non-uniform distribution.
2863        let mut config = RouterConfig::default();
2864        // Only register models for test_a and test_b — test_c has no model, so it is
2865        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2866        for agent in &["test_a", "test_b"] {
2867            config
2868                .model_map
2869                .entry("medium".to_string())
2870                .or_default()
2871                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2872        }
2873        let agents = vec![
2874            "test_a".to_string(),
2875            "test_b".to_string(),
2876            "test_c".to_string(), // available but no model → filtered out of candidates
2877        ];
2878        let mut weights = AgentWeights::default();
2879        weights.ensure_agents(&agents);
2880        let mut router = Router {
2881            config,
2882            available_agents: agents,
2883            weights,
2884            llm_router: std::sync::Arc::new(LlmRouter::new()),
2885            ollama_router: None,
2886            rr_index: 0,
2887            last_agent: None,
2888            review_rr_index: 0,
2889            router_pool: vec![],
2890            pool_index: 0,
2891        };
2892
2893        // With the bug: index advances modulo 3 (available_agents.len()), so the
2894        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2895        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2896        // looks uniform by accident, but adding a 4th available agent breaks it.
2897        //
2898        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2899        // cycle. Verify this by checking review_rr_index after each call.
2900
2901        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2902        assert_eq!(
2903            router.review_rr_index, 1,
2904            "index should be 1 after first call"
2905        );
2906
2907        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2908        assert_eq!(
2909            router.review_rr_index, 0,
2910            "index should wrap to 0 after second call"
2911        );
2912
2913        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2914        assert_eq!(
2915            router.review_rr_index, 1,
2916            "index should be 1 after third call"
2917        );
2918
2919        // Agents must alternate uniformly
2920        assert_ne!(r1, r2, "consecutive calls should select different agents");
2921        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2922    }
2923
2924    #[serial(cooldown_state)]
2925    #[tokio::test]
2926    async fn last_agent_tracks_routing() {
2927        crate::engine::cooldown::reset_global_state().await;
2928        let mut config = RouterConfig {
2929            mode: "round_robin".to_string(),
2930            ..Default::default()
2931        };
2932        config
2933            .model_map
2934            .entry("medium".to_string())
2935            .or_default()
2936            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
2937        config
2938            .model_map
2939            .entry("medium".to_string())
2940            .or_default()
2941            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
2942        let agents = vec!["test_x".to_string(), "test_y".to_string()];
2943        let mut weights = AgentWeights::default();
2944        weights.ensure_agents(&agents);
2945        let mut router = Router {
2946            config,
2947            available_agents: agents,
2948            weights,
2949            llm_router: std::sync::Arc::new(LlmRouter::new()),
2950            ollama_router: None,
2951            rr_index: 0,
2952            last_agent: None,
2953            review_rr_index: 0,
2954            router_pool: vec![],
2955            pool_index: 0,
2956        };
2957
2958        assert!(router.last_agent.is_none());
2959
2960        let task = create_test_task("1", "Test routing", vec![]);
2961        let store = test_store().await;
2962        let result = router.route(&task, &store, "test/repo").await.unwrap();
2963
2964        assert!(router.last_agent.is_some());
2965        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
2966    }
2967
2968    // ── pool config ───────────────────────────────────────────────────────────
2969
2970    #[serial(cooldown_state)]
2971    #[test]
2972    fn parse_pool_entry_splits_on_first_colon() {
2973        let (agent, model) = super::parse_pool_entry("claude:haiku");
2974        assert_eq!(agent, "claude");
2975        assert_eq!(model, "haiku");
2976    }
2977
2978    #[serial(cooldown_state)]
2979    #[test]
2980    fn parse_pool_entry_model_with_slash() {
2981        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
2982        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
2983        assert_eq!(agent, "opencode");
2984        assert_eq!(model, "github-copilot/gpt-5-mini");
2985    }
2986
2987    #[serial(cooldown_state)]
2988    #[test]
2989    fn parse_pool_entry_no_colon() {
2990        let (agent, model) = super::parse_pool_entry("claude");
2991        assert_eq!(agent, "claude");
2992        assert_eq!(model, "");
2993    }
2994
2995    #[serial(cooldown_state)]
2996    #[test]
2997    fn effective_pool_uses_configured_pool() {
2998        let config = RouterConfig {
2999            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3000            ..RouterConfig::default()
3001        };
3002        let pool = config.effective_pool();
3003        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3004    }
3005
3006    #[serial(cooldown_state)]
3007    #[test]
3008    fn effective_pool_falls_back_to_router_agent_model() {
3009        let config = RouterConfig::default();
3010        // Default has no pool → derives from router_agent:router_model
3011        let pool = config.effective_pool();
3012        assert_eq!(pool.len(), 1);
3013        assert!(pool[0].starts_with(&config.router_agent));
3014        assert!(pool[0].contains(&config.router_model));
3015    }
3016
3017    #[serial(cooldown_state)]
3018    #[test]
3019    fn effective_fallback_uses_configured_fallback() {
3020        let config = RouterConfig {
3021            fallback: "claude:haiku".to_string(),
3022            ..RouterConfig::default()
3023        };
3024        assert_eq!(config.effective_fallback(), "claude:haiku");
3025    }
3026
3027    #[serial(cooldown_state)]
3028    #[test]
3029    fn effective_fallback_derives_from_router_agent_model() {
3030        let config = RouterConfig::default();
3031        let fallback = config.effective_fallback();
3032        assert!(fallback.starts_with(&config.router_agent));
3033        assert!(fallback.contains(&config.router_model));
3034    }
3035
3036    // ── pool expansion ────────────────────────────────────────────────────────
3037
3038    #[serial(cooldown_state)]
3039    #[test]
3040    fn expand_pool_basic_entries() {
3041        let config = RouterConfig {
3042            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3043            ..RouterConfig::default()
3044        };
3045        let expanded = Router::expand_pool(&config);
3046        assert_eq!(expanded.len(), 2);
3047        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3048        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3049    }
3050
3051    #[serial(cooldown_state)]
3052    #[test]
3053    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3054        let config = RouterConfig {
3055            pool: vec![],
3056            router_agent: "claude".to_string(),
3057            router_model: "haiku".to_string(),
3058            ..RouterConfig::default()
3059        };
3060        let expanded = Router::expand_pool(&config);
3061        assert_eq!(expanded.len(), 1);
3062        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3063    }
3064
3065    #[serial(cooldown_state)]
3066    #[test]
3067    fn expand_pool_opencode_free_skipped_when_not_installed() {
3068        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3069        // to nothing and the pool falls back to effective_fallback
3070        let config = RouterConfig {
3071            pool: vec!["opencode:free".to_string()],
3072            fallback: "claude:haiku".to_string(),
3073            ..RouterConfig::default()
3074        };
3075        let expanded = Router::expand_pool(&config);
3076        assert_eq!(expanded.len(), 1);
3077        assert_eq!(expanded[0].0, "claude");
3078        assert_eq!(expanded[0].1, "haiku");
3079    }
3080
3081    #[serial(cooldown_state)]
3082    #[test]
3083    fn expand_pool_single_entry_from_default() {
3084        let config = RouterConfig::default();
3085        let expanded = Router::expand_pool(&config);
3086        assert!(
3087            !expanded.is_empty(),
3088            "pool must never be empty after expansion"
3089        );
3090    }
3091
3092    // ── pool round-robin index ────────────────────────────────────────────────
3093
3094    #[serial(cooldown_state)]
3095    #[test]
3096    fn pool_index_initializes_to_zero() {
3097        let config = RouterConfig::default();
3098        let router = Router::new(config);
3099        assert_eq!(router.pool_index, 0);
3100    }
3101
3102    #[serial(cooldown_state)]
3103    #[test]
3104    fn router_pool_non_empty_after_construction() {
3105        let config = RouterConfig::default();
3106        let router = Router::new(config);
3107        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3108    }
3109
3110    /**
3111     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3112     * `opencode serve --service` child behind, so a suite run leaked one server per test
3113     * process. The stub on PATH records any exec, so this fails if discovery comes back
3114     **/
3115    #[cfg(unix)]
3116    #[serial(cooldown_state)]
3117    #[test]
3118    fn building_a_router_never_runs_opencode() {
3119        use std::os::unix::fs::PermissionsExt;
3120
3121        struct PathRestore(Option<std::ffi::OsString>);
3122        impl Drop for PathRestore {
3123            fn drop(&mut self) {
3124                match &self.0 {
3125                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3126                    Some(p) => unsafe { std::env::set_var("PATH", p) },
3127                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3128                    None => unsafe { std::env::remove_var("PATH") },
3129                }
3130                crate::cmd_cache::reset_for_test();
3131            }
3132        }
3133
3134        let dir = tempfile::tempdir().unwrap();
3135        let marker = dir.path().join("spawned");
3136        let bin_dir = dir.path().join("bin");
3137        std::fs::create_dir(&bin_dir).unwrap();
3138        let stub = bin_dir.join("opencode");
3139        std::fs::write(
3140            &stub,
3141            format!(
3142                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3143                marker.display()
3144            ),
3145        )
3146        .unwrap();
3147        std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap();
3148
3149        let prev_path = std::env::var_os("PATH");
3150        let mut new_path = bin_dir.as_os_str().to_os_string();
3151        new_path.push(":");
3152        if let Some(ref p) = prev_path {
3153            new_path.push(p);
3154        }
3155        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3156        unsafe { std::env::set_var("PATH", &new_path) };
3157        crate::cmd_cache::reset_for_test();
3158        let _restore = PathRestore(prev_path);
3159
3160        // A config that names opencode everywhere discovery reads from
3161        let mut config = RouterConfig {
3162            pool: vec!["opencode:free".to_string()],
3163            fallback: "opencode:free".to_string(),
3164            ..RouterConfig::default()
3165        };
3166        config
3167            .model_map
3168            .entry("simple".to_string())
3169            .or_default()
3170            .insert(
3171                "opencode".to_string(),
3172                vec!["canary-model-free".to_string()],
3173            );
3174
3175        let _router = Router::new(config.clone());
3176        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3177        let _pool = Router::expand_pool(&config);
3178        // Discovery refreshes on a background thread, give it room to exec
3179        std::thread::sleep(Duration::from_millis(300));
3180
3181        assert!(
3182            !marker.exists(),
3183            "building a router ran opencode: {}",
3184            std::fs::read_to_string(&marker).unwrap_or_default().trim()
3185        );
3186        assert!(
3187            opencode_children().is_empty(),
3188            "building a router left opencode processes: {:?}",
3189            opencode_children()
3190        );
3191    }
3192
3193    /// Commands of this process's direct children that mention opencode
3194    #[cfg(unix)]
3195    fn opencode_children() -> Vec<String> {
3196        let me = std::process::id().to_string();
3197        let output = std::process::Command::new("ps")
3198            .args(["-axo", "ppid=,command="])
3199            .output()
3200            .expect("ps");
3201        String::from_utf8_lossy(&output.stdout)
3202            .lines()
3203            .filter_map(|line| {
3204                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3205                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3206            })
3207            .collect()
3208    }
3209
3210    #[serial(cooldown_state)]
3211    #[tokio::test]
3212    async fn reload_resets_pool_index() {
3213        crate::engine::cooldown::reset_global_state().await;
3214        let config = RouterConfig::default();
3215        let mut router = Router::new(config);
3216        router.pool_index = 5;
3217        router.reload_async().await;
3218        assert_eq!(router.pool_index, 0);
3219    }
3220
3221    #[serial(cooldown_state)]
3222    #[test]
3223    fn pool_index_advances_after_failed_attempt() {
3224        let mut router = Router::new(RouterConfig::default());
3225        router.router_pool = vec![
3226            ("claude".to_string(), "haiku".to_string()),
3227            ("codex".to_string(), "gpt-5.2".to_string()),
3228            (
3229                "opencode".to_string(),
3230                "github-copilot/gpt-5-mini".to_string(),
3231            ),
3232        ];
3233        router.pool_index = 0;
3234
3235        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3236
3237        assert_eq!(router.pool_index, 1);
3238    }
3239
3240    #[serial(cooldown_state)]
3241    #[test]
3242    fn pool_index_wraps_after_last_attempt() {
3243        let mut router = Router::new(RouterConfig::default());
3244        router.router_pool = vec![
3245            ("claude".to_string(), "haiku".to_string()),
3246            ("codex".to_string(), "gpt-5.2".to_string()),
3247            (
3248                "opencode".to_string(),
3249                "github-copilot/gpt-5-mini".to_string(),
3250            ),
3251        ];
3252        router.pool_index = 2;
3253
3254        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3255
3256        assert_eq!(router.pool_index, 0);
3257    }
3258
3259    #[serial(cooldown_state)]
3260    #[test]
3261    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3262        let router = Router::new(RouterConfig::default());
3263        assert!(router.earliest_pool_cooldown().is_none());
3264    }
3265
3266    #[serial(cooldown_state)]
3267    #[tokio::test]
3268    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3269        crate::engine::cooldown::reset_global_state().await;
3270        // Use explicit pool with predictable model names to avoid opencode discovery.
3271        let config = RouterConfig {
3272            pool: vec![
3273                "claude:haiku-4-5-20251001".to_string(),
3274                "opencode:haiku-4-5-20251001".to_string(),
3275            ],
3276            ..Default::default()
3277        };
3278        let router = Router::new(config);
3279        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3280        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3281
3282        let earliest = router.earliest_pool_cooldown();
3283        assert!(earliest.is_some());
3284        let now = chrono::Utc::now().timestamp();
3285        let remaining = earliest.unwrap() - now;
3286        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3287        assert!((295..=300).contains(&remaining));
3288    }
3289
3290    #[serial(cooldown_state)]
3291    #[tokio::test]
3292    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3293        crate::engine::cooldown::reset_global_state().await;
3294        // Use explicit pool so we can predict what entries exist.
3295        let config = RouterConfig {
3296            pool: vec![
3297                "claude:pool-model-a".to_string(),
3298                "opencode:pool-model-b".to_string(),
3299            ],
3300            ..Default::default()
3301        };
3302        let router = Router::new(config);
3303        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3304        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3305        // Set cooldown on a pool entry (opencode:pool-model-b)
3306        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3307
3308        let earliest = router.earliest_pool_cooldown();
3309        assert!(earliest.is_some());
3310        let now = chrono::Utc::now().timestamp();
3311        let remaining = earliest.unwrap() - now;
3312        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3313        assert!((295..=300).contains(&remaining));
3314    }
3315
3316    // ---- Pre-emptive health check: degraded agent exclusion ----
3317
3318    #[serial(cooldown_state)]
3319    #[test]
3320    fn degraded_agent_excluded_from_routing() {
3321        let mut config = RouterConfig::default();
3322        config
3323            .model_map
3324            .entry("medium".to_string())
3325            .or_default()
3326            .insert(
3327                "test_degraded_routing".to_string(),
3328                vec!["model-a".to_string()],
3329            );
3330
3331        let mut router = Router::new(config);
3332        router.available_agents = vec!["test_degraded_routing".to_string()];
3333
3334        // Agent is routable before degradation
3335        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3336
3337        // Mark as degraded
3338        mark_agent_degraded("test_degraded_routing");
3339
3340        // Now excluded from routing
3341        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3342
3343        // available_agents_for_complexity returns empty
3344        assert!(
3345            router.available_agents_for_complexity("medium").is_empty(),
3346            "degraded agent should be excluded from available agents"
3347        );
3348
3349        // healthy_agent_count reflects exclusion
3350        assert_eq!(router.healthy_agent_count("medium"), 0);
3351
3352        // Cleanup
3353        clear_agent_degraded("test_degraded_routing");
3354    }
3355
3356    #[serial(cooldown_state)]
3357    #[test]
3358    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3359        let mut config = RouterConfig::default();
3360        for agent in &["agent_healthy", "agent_degraded"] {
3361            config
3362                .model_map
3363                .entry("medium".to_string())
3364                .or_default()
3365                .insert(agent.to_string(), vec!["model-x".to_string()]);
3366        }
3367
3368        let mut router = Router::new(config);
3369        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3370
3371        mark_agent_degraded("agent_degraded");
3372
3373        let candidates = router.available_agents_for_complexity("medium");
3374        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3375        assert_eq!(router.healthy_agent_count("medium"), 1);
3376
3377        // Cleanup
3378        clear_agent_degraded("agent_degraded");
3379    }
3380
3381    #[serial(cooldown_state)]
3382    #[tokio::test]
3383    async fn refresh_health_marks_degraded_from_rate_limits() {
3384        crate::engine::cooldown::reset_global_state().await;
3385        let store = test_store().await;
3386        let agent = "test_refresh_health_agent";
3387
3388        let mut config = RouterConfig::default();
3389        config
3390            .model_map
3391            .entry("medium".to_string())
3392            .or_default()
3393            .insert(agent.to_string(), vec!["model-z".to_string()]);
3394
3395        let mut router = Router::new(config);
3396        router.available_agents = vec![agent.to_string()];
3397
3398        // Insert rate limit events exceeding default threshold (3)
3399        for _ in 0..4 {
3400            store.record_rate_limit(agent, "rate", None).await.unwrap();
3401        }
3402
3403        router.refresh_health(&store).await;
3404
3405        assert!(
3406            is_agent_degraded(agent),
3407            "agent should be degraded after exceeding rate limit threshold"
3408        );
3409        assert!(
3410            !router.agent_is_routable(agent, "medium"),
3411            "degraded agent should not be routable"
3412        );
3413
3414        // Cleanup
3415        clear_agent_degraded(agent);
3416    }
3417
3418    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3419
3420    #[serial(cooldown_state)]
3421    #[tokio::test]
3422    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3423        crate::engine::cooldown::reset_global_state().await;
3424        let mut config = RouterConfig::default();
3425        // Agent's only model, shared across every complexity tier it resolves to.
3426        for tier in ["simple", "medium", "complex", "review"] {
3427            config
3428                .model_map
3429                .entry(tier.to_string())
3430                .or_default()
3431                .insert(
3432                    "test_all_cooled".to_string(),
3433                    vec!["only-model".to_string()],
3434                );
3435        }
3436
3437        let mut router = Router::new(config);
3438        router.available_agents = vec!["test_all_cooled".to_string()];
3439
3440        assert!(router.agent_has_any_available_model("test_all_cooled"));
3441
3442        record_model_failure("test_all_cooled", "only-model").await;
3443        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3444
3445        assert!(
3446            !router.agent_has_any_available_model("test_all_cooled"),
3447            "agent with its only model cooled across every tier should have no available model"
3448        );
3449    }
3450
3451    #[serial(cooldown_state)]
3452    #[tokio::test]
3453    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3454        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3455        // agent that has no agent-level cooldown but whose only resolvable model (across
3456        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3457        crate::engine::cooldown::reset_global_state().await;
3458        let mut config = RouterConfig::default();
3459        for tier in ["simple", "medium", "complex", "review"] {
3460            config
3461                .model_map
3462                .entry(tier.to_string())
3463                .or_default()
3464                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3465            config
3466                .model_map
3467                .get_mut(tier)
3468                .unwrap()
3469                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3470        }
3471
3472        let mut router = Router::new(config);
3473        router.available_agents = vec![
3474            "agent_cooled_model".to_string(),
3475            "agent_healthy".to_string(),
3476        ];
3477
3478        record_model_failure("agent_cooled_model", "opus").await;
3479        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3480        // No agent-level cooldown — the old agent-only filter would still offer it.
3481        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3482
3483        let uncooled_agents: Vec<String> = router
3484            .available_agents
3485            .iter()
3486            .filter(|a| {
3487                !is_agent_in_cooldown(a)
3488                    && !is_agent_degraded(a)
3489                    && router.agent_has_available_model_for_coding_tiers(a)
3490            })
3491            .cloned()
3492            .collect();
3493
3494        assert_eq!(
3495            uncooled_agents,
3496            vec!["agent_healthy".to_string()],
3497            "agent whose only model is cooled must be excluded from the LLM candidate list"
3498        );
3499    }
3500
3501    #[serial(cooldown_state)]
3502    #[tokio::test]
3503    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3504        // Regression test for #3521: an agent whose only healthy tier is `simple`
3505        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3506        // excluded from the LLM classification candidate list, otherwise the
3507        // classifier can pick it for a `medium`/`complex` task it can't serve,
3508        // wasting the classification call. Mirrors the real-world shape where
3509        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3510        crate::engine::cooldown::reset_global_state().await;
3511        let mut config = RouterConfig::default();
3512        config
3513            .model_map
3514            .entry("simple".to_string())
3515            .or_default()
3516            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3517        config
3518            .model_map
3519            .entry("medium".to_string())
3520            .or_default()
3521            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3522        config
3523            .model_map
3524            .entry("complex".to_string())
3525            .or_default()
3526            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3527        // review must be irrelevant to the coding-tier filter
3528        config
3529            .model_map
3530            .entry("review".to_string())
3531            .or_default()
3532            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3533        for tier in ["simple", "medium", "complex", "review"] {
3534            config
3535                .model_map
3536                .get_mut(tier)
3537                .unwrap()
3538                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3539        }
3540
3541        let mut router = Router::new(config);
3542        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3543
3544        record_model_failure("minimax_like", "opus").await;
3545        assert!(is_model_in_cooldown("minimax_like", "opus"));
3546        // No agent-level cooldown, and the simple tier is healthy, so the loose
3547        // "any tier" check still passes — this is exactly the leak #3511 missed.
3548        assert!(!is_agent_in_cooldown("minimax_like"));
3549        assert!(router.agent_has_any_available_model("minimax_like"));
3550        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3551
3552        let uncooled_agents: Vec<String> = router
3553            .available_agents
3554            .iter()
3555            .filter(|a| {
3556                !is_agent_in_cooldown(a)
3557                    && !is_agent_degraded(a)
3558                    && router.agent_has_available_model_for_coding_tiers(a)
3559            })
3560            .cloned()
3561            .collect();
3562
3563        assert_eq!(
3564            uncooled_agents,
3565            vec!["agent_healthy".to_string()],
3566            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3567        );
3568    }
3569}