Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22pub use config::{parse_pool_entry, RouterConfig};
23pub use weights::AgentWeights;
24
25fn fmt_degraded(degraded: &[String]) -> String {
26    if degraded.is_empty() {
27        String::new()
28    } else {
29        format!(" (degraded: {})", degraded.join(", "))
30    }
31}
32
33/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
34///
35/// This is a transient error that should not count against `max_route_attempts` since
36/// no agent was actually invoked. The caller should back off and retry later.
37#[derive(Debug, thiserror::Error)]
38pub enum AllAgentsCooledError {
39    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
40    AllCooled {
41        scope: String,
42        remaining_secs: Option<u64>,
43        degraded: Vec<String>,
44    },
45}
46
47impl AllAgentsCooledError {
48    /// Create a new error for the given scope, naming currently degraded agents
49    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
50        Self::AllCooled {
51            scope,
52            remaining_secs,
53            degraded: list_degraded_agents()
54                .into_iter()
55                .map(|(agent, _reason)| agent)
56                .collect(),
57        }
58    }
59
60    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
61
62    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
63    pub fn scope(&self) -> &str {
64        match self {
65            Self::AllCooled { scope, .. } => scope,
66        }
67    }
68
69    /// Remaining cooldown duration in seconds when available.
70    pub fn remaining_secs(&self) -> Option<u64> {
71        match self {
72            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
73        }
74    }
75
76    /// Agents currently marked degraded, shown by `orch cooldown list`
77    pub fn degraded(&self) -> &[String] {
78        match self {
79            Self::AllCooled { degraded, .. } => degraded,
80        }
81    }
82}
83
84/// Typed error returned by [`get_route_result`].
85#[derive(Debug, thiserror::Error)]
86pub enum RouteResultError {
87    /// The task has no agent field — it needs to be re-routed.
88    #[error("no agent configured for task {task_id}")]
89    NoAgent { task_id: String },
90    /// Any other failure (store lookup, SQL, etc.).
91    #[error(transparent)]
92    Other(#[from] anyhow::Error),
93}
94
95use crate::backends::ExternalTask;
96use crate::engine::cooldown::{
97    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
98    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
99};
100use crate::store::{get_task_field_direct, store_log_activity, TaskStore};
101use serde::{Deserialize, Serialize};
102use std::sync::Arc;
103
104use llm::{LlmRouter, TIMEOUT_PREFIX};
105
106/// Result of routing a task to an agent.
107#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
108pub struct RouteResult {
109    /// The selected agent: "claude", "codex", or "opencode"
110    pub agent: String,
111    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
112    pub model: Option<String>,
113    /// Complexity level: "simple", "medium", or "complex"
114    pub complexity: String,
115    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
116    pub estimate: u8,
117    /// Why this agent was selected
118    pub reason: String,
119    /// Specialized agent profile (skills, tools, constraints)
120    pub profile: AgentProfile,
121    /// Selected skill IDs from the catalog
122    pub selected_skills: Vec<String>,
123    /// Optional warning about routing decision
124    pub warning: Option<String>,
125}
126
127/// Specialized agent profile for a task.
128#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
129pub struct AgentProfile {
130    /// Role name, e.g., "backend specialist"
131    pub role: String,
132    /// Focus skills for this task
133    pub skills: Vec<String>,
134    /// Tools allowed for this task
135    pub tools: Vec<String>,
136    /// Constraints for this task
137    pub constraints: Vec<String>,
138}
139
140impl AgentProfile {
141    /// Build the default "general" profile from router config.
142    pub fn default_from_config(config: &RouterConfig) -> Self {
143        Self {
144            role: "general".to_string(),
145            skills: vec![],
146            tools: config.allowed_tools.clone(),
147            constraints: vec![],
148        }
149    }
150}
151
152impl RouteResult {
153    /// Build a standard route result from the three task-specific values.
154    ///
155    /// Fills `profile`, `selected_skills`, and `warning` with router-config
156    /// defaults so call sites only need to supply what varies per task.
157    pub fn from_config(
158        agent: String,
159        complexity: String,
160        reason: String,
161        config: &RouterConfig,
162        task_id: &str,
163    ) -> Self {
164        let model = config.model_for_complexity(&agent, &complexity, task_id);
165        Self {
166            agent,
167            model,
168            complexity,
169            estimate: 0,
170            reason,
171            profile: AgentProfile::default_from_config(config),
172            selected_skills: config.default_skills.clone(),
173            warning: None,
174        }
175    }
176}
177
178/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
179///
180/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
181/// **not** require holding the router's `RwLock`, so async operations can
182/// safely be called without starving write-lock waiters.
183pub struct LlmRouterHandle(std::sync::Arc<LlmRouter>);
184
185impl LlmRouterHandle {
186    /// Invalidate the skills catalog cache so the next routing call reloads
187    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
188    pub async fn invalidate_skills_catalog(&self) {
189        self.0.invalidate_skills_catalog().await;
190    }
191}
192
193/// The agent router.
194pub struct Router {
195    /// Router configuration
196    pub config: RouterConfig,
197    /// Available agents discovered at runtime
198    pub available_agents: Vec<String>,
199    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
200    pub weights: AgentWeights,
201    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
202    /// without holding the router RwLock across async operations)
203    llm_router: std::sync::Arc<LlmRouter>,
204    /// Ollama router for local routing (contains reusable reqwest::Client)
205    ollama_router: Option<ollama::OllamaRouter>,
206    /// Round-robin index for task routing
207    pub(crate) rr_index: usize,
208    /// Last agent routed to (for distribution tracking)
209    pub(crate) last_agent: Option<String>,
210    /// Round-robin index for review agent selection
211    pub(crate) review_rr_index: usize,
212    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
213    /// `opencode:free` entries are expanded at construction time.
214    pub(crate) router_pool: Vec<(String, String)>,
215    /// Current round-robin index into router_pool
216    pub(crate) pool_index: usize,
217}
218
219impl Router {
220    /// Create a new router with the given configuration.
221    pub fn new(config: RouterConfig) -> Self {
222        let available_agents = Self::discover_agents(&config.agents);
223        let mut weights = AgentWeights {
224            base_weights: config.weights.clone(),
225            ..Default::default()
226        };
227        weights.ensure_agents_with_weights(&available_agents, &config.weights);
228        if !config.weights.is_empty() {
229            tracing::info!(
230                weights = ?config.weights,
231                "configured agent routing weights"
232            );
233        }
234        let router_pool = Self::expand_pool(&config);
235        tracing::info!(
236            pool = ?router_pool,
237            fallback = %config.effective_fallback(),
238            "router LLM pool initialized"
239        );
240        // Prime only when this config names opencode, so a default or claude-only
241        // router never runs `opencode models`
242        if config.uses_opencode() {
243            crate::engine::runner::agents::opencode::prime_free_model_cache();
244        }
245        let ollama_router = if config.mode == "local" {
246            Some(ollama::OllamaRouter::new(&config))
247        } else {
248            None
249        };
250        Self {
251            config,
252            available_agents,
253            weights,
254            llm_router: std::sync::Arc::new(LlmRouter::new()),
255            ollama_router,
256            rr_index: 0,
257            last_agent: None,
258            review_rr_index: 0,
259            router_pool,
260            pool_index: 0,
261        }
262    }
263
264    /// Create a router with default configuration loaded from files.
265    pub fn from_config() -> Self {
266        Self::new(RouterConfig::from_config())
267    }
268
269    /// Create a router with a forced agent list, bypassing CLI discovery.
270    /// Only for use in unit tests.
271    #[cfg(test)]
272    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
273        let mut weights = AgentWeights {
274            base_weights: config.weights.clone(),
275            ..Default::default()
276        };
277        weights.ensure_agents(&available_agents);
278        let router_pool = Self::expand_pool(&config);
279        Self {
280            config,
281            available_agents,
282            weights,
283            llm_router: std::sync::Arc::new(LlmRouter::new()),
284            ollama_router: None,
285            rr_index: 0,
286            last_agent: None,
287            review_rr_index: 0,
288            router_pool,
289            pool_index: 0,
290        }
291    }
292
293    /// Return a cloned handle to the LLM router subsystem.
294    ///
295    /// Callers that need to invoke async methods on the underlying `LlmRouter`
296    /// should clone this handle *before* releasing any `RwLock` read guard on
297    /// `Router`, then call the async method after the guard is dropped.  This
298    /// prevents holding the read lock across await points (which starves
299    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
300    pub fn llm_router_handle(&self) -> LlmRouterHandle {
301        LlmRouterHandle(self.llm_router.clone())
302    }
303
304    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
305        debug_assert!(pool_len > 0, "router pool must not be empty");
306        self.pool_index = (idx + 1) % pool_len;
307    }
308
309    /// Discover available agent CLIs in PATH.
310    /// Checks all agents from the configured list.
311    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
312        let mut agents = Vec::new();
313        for agent in configured_agents {
314            if crate::cmd_cache::command_exists(agent) {
315                agents.push(agent.to_string());
316            }
317        }
318        agents
319    }
320
321    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
322    ///
323    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
324    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
325    /// - If `opencode` is not installed or the command fails, the entry is skipped.
326    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
327    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
328        let raw_pool = config.effective_pool();
329        let mut expanded: Vec<(String, String)> = Vec::new();
330
331        for entry in &raw_pool {
332            if entry == "opencode:free" {
333                let free_models = Self::discover_free_opencode_models();
334                if free_models.is_empty() {
335                    tracing::debug!("opencode:free expanded to nothing — skipping");
336                } else {
337                    tracing::debug!(models = ?free_models, "opencode:free expanded");
338                    for model in free_models {
339                        expanded.push(("opencode".to_string(), model));
340                    }
341                }
342            } else {
343                let (agent, model) = parse_pool_entry(entry);
344                expanded.push((agent, model));
345            }
346        }
347
348        if expanded.is_empty() {
349            // All entries were opencode:free and opencode isn't installed — use fallback
350            let (agent, model) = parse_pool_entry(&config.effective_fallback());
351            tracing::debug!(
352                agent = %agent, model = %model,
353                "pool empty after expansion, using fallback as sole pool entry"
354            );
355            expanded.push((agent, model));
356        }
357
358        expanded
359    }
360
361    /// Async version of expand_pool which may query the opencode CLI without
362    /// blocking the Tokio runtime.
363    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
364        let raw_pool = config.effective_pool();
365        let mut expanded: Vec<(String, String)> = Vec::new();
366
367        for entry in &raw_pool {
368            if entry == "opencode:free" {
369                let free_models = Self::discover_free_opencode_models_async().await;
370                if free_models.is_empty() {
371                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
372                } else {
373                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
374                    for model in free_models {
375                        expanded.push(("opencode".to_string(), model));
376                    }
377                }
378            } else {
379                let (agent, model) = parse_pool_entry(entry);
380                expanded.push((agent, model));
381            }
382        }
383
384        if expanded.is_empty() {
385            // All entries were opencode:free and opencode isn't installed — use fallback
386            let (agent, model) = parse_pool_entry(&config.effective_fallback());
387            tracing::debug!(
388                agent = %agent, model = %model,
389                "pool empty after async expansion, using fallback as sole pool entry"
390            );
391            expanded.push((agent, model));
392        }
393
394        expanded
395    }
396
397    /// Async reload that uses async pool expansion to avoid blocking the runtime
398    /// when discovering opencode free models.
399    pub async fn reload_async(&mut self) {
400        let new_config = RouterConfig::from_config();
401        let new_agents = Self::discover_agents(&new_config.agents);
402        let new_pool = Self::expand_pool_async(&new_config).await;
403        tracing::info!(
404            mode = %new_config.mode,
405            agents = ?new_agents,
406            fallback = %new_config.fallback_executor,
407            weighted_rr = new_config.weighted_round_robin,
408            pool = ?new_pool,
409            "router reloaded (async)"
410        );
411        self.config = new_config;
412        self.available_agents = new_agents.clone();
413        // Ensure new agents have weight entries (preserves existing weights)
414        self.weights.ensure_agents(&new_agents);
415        self.router_pool = new_pool;
416        self.pool_index = 0;
417        // Reinitialize Ollama router if mode changed to local
418        self.ollama_router = if self.config.mode == "local" {
419            Some(ollama::OllamaRouter::new(&self.config))
420        } else {
421            None
422        };
423    }
424
425    fn discover_free_opencode_models() -> Vec<String> {
426        crate::engine::runner::agents::opencode::discover_free_opencode_models()
427    }
428
429    async fn discover_free_opencode_models_async() -> Vec<String> {
430        tokio::task::spawn_blocking(Self::discover_free_opencode_models)
431            .await
432            .unwrap_or_default()
433    }
434
435    /// Run the pre-emptive health check, refreshing the degraded-agent set.
436    ///
437    /// Queries the `rate_limits` table for recent events and combines with
438    /// cooldown state to mark agents as degraded before routing attempts them.
439    pub async fn refresh_health(&self, store: &std::sync::Arc<TaskStore>) {
440        // KV is the source of truth for cooldowns. Reconcile the in-memory map
441        // from KV every tick so external mutations (e.g. `orch cooldown clear`
442        // from a separate CLI process) actually take effect on the running
443        // service. Without this, the engine keeps stale entries until they
444        // expire on their own, producing the "all agents cooled" log with
445        // `orch cooldown list` simultaneously reporting "No active cooldowns".
446        sync_from_kv(store).await;
447
448        let agents = self.available_agents.clone();
449        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
450        refresh_degraded_agents(
451            store,
452            &agents,
453            &model_checker,
454            config::health_check_window_hours(),
455            config::degraded_rate_limit_threshold(),
456        )
457        .await;
458    }
459
460    /// Check if an agent is available.
461    pub fn is_agent_available(&self, agent: &str) -> bool {
462        self.available_agents.contains(&agent.to_string())
463    }
464
465    /// True if the agent has at least one uncooled model across any complexity
466    /// tier. Used where the eventual complexity isn't known yet (e.g. before
467    /// LLM classification), so we can't check a single tier.
468    fn agent_has_any_available_model(&self, agent: &str) -> bool {
469        ["simple", "medium", "complex", "review"]
470            .iter()
471            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
472    }
473
474    /// True if the agent has an available (uncooled) model for the `medium`
475    /// and `complex` tiers — the tiers real coding tasks are classified into
476    /// by the LLM classifier. Used to pre-filter the candidate list handed to
477    /// that classifier so an agent whose healthy tiers are only `simple` isn't
478    /// offered: the classifier can't be constrained to `simple`, so offering
479    /// such an agent produces a near-certain wasted classification call whenever
480    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
481    /// outcome of initial classification and is deliberately excluded.
482    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
483        ["medium", "complex"]
484            .iter()
485            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
486    }
487
488    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
489        if !self.is_agent_available(agent) {
490            return false;
491        }
492        if is_agent_in_cooldown(agent) {
493            tracing::debug!(agent, "agent skipped: in cooldown");
494            return false;
495        }
496        if is_agent_degraded(agent) {
497            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
498            return false;
499        }
500        // If weighted routing is used, consider the agent degraded when its
501        // weight has decayed below the configured threshold. This avoids
502        // proactively routing to agents that recently hit many rate limits.
503        if self.config.weighted_round_robin {
504            let weight = self.weights.get_weight(agent);
505            if weight < self.config.skip_limited_threshold {
506                tracing::debug!(
507                    agent,
508                    weight,
509                    "agent considered degraded by weight threshold, skipping"
510                );
511                return false;
512            }
513        }
514        if !self
515            .config
516            .has_available_model_for_complexity(agent, complexity)
517        {
518            tracing::debug!(agent, complexity, "agent skipped: no available model");
519            return false;
520        }
521        true
522    }
523
524    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
525        self.available_agents
526            .iter()
527            .filter(|agent| self.agent_is_routable(agent, complexity))
528            .cloned()
529            .collect()
530    }
531
532    /// Count of healthy (routable) agents for a given complexity.
533    /// Returns the number of agents that are available, not in cooldown, and have an available model.
534    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
535        self.available_agents_for_complexity(complexity).len()
536    }
537
538    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
539    /// Uses "simple" complexity as the baseline since it has the widest model support.
540    pub fn is_degraded(&self, threshold: usize) -> bool {
541        self.healthy_agent_count("simple") < threshold
542    }
543
544    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
545        let mut earliest: Option<i64> = None;
546
547        for agent in &self.available_agents {
548            if let Some(until) = cooldown_until(agent) {
549                earliest = Some(earliest.map_or(until, |current| current.min(until)));
550            }
551
552            if let Some(comp) = complexity {
553                if let Some(pool) = self.config.model_pool_for_complexity(agent, comp) {
554                    for model in pool {
555                        let key = format!("{agent}:{model}");
556                        if let Some(until) = cooldown_until(&key) {
557                            earliest = Some(earliest.map_or(until, |current| current.min(until)));
558                        }
559                    }
560                }
561            }
562        }
563
564        earliest
565    }
566
567    /// Find the earliest cooldown expiration among the router LLM pool entries.
568    ///
569    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
570    /// or `None` if no pool model has a cooldown.
571    fn earliest_pool_cooldown(&self) -> Option<i64> {
572        let mut earliest: Option<i64> = None;
573        for (agent, model) in &self.router_pool {
574            let key = format!("{agent}:{model}");
575            if let Some(until) = cooldown_until(&key) {
576                earliest = Some(earliest.map_or(until, |current| current.min(until)));
577            }
578        }
579        earliest
580    }
581
582    async fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
583        // If any cooldowns are present, return an error immediately so the
584        // caller (the tick loop) can skip this task and retry on the next tick.
585        // Sleeping inside the router blocks the entire tick loop for the
586        // cooldown duration which starves other tasks. The engine tick already
587        // retries every `tick_interval` (default 10s), so allow the tick loop
588        // to drive retries instead of sleeping here.
589        let now = chrono::Utc::now().timestamp();
590        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
591            // All agents are degraded but none have a timed cooldown entry
592            // (e.g. over-threshold rate limiting without explicit cooldown duration).
593            // Log at warn and return an error so the tick skips this task and retries after backoff.
594            tracing::warn!(
595        scope = complexity.unwrap_or("all agents"),
596        "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
597    );
598            // Return an error similar to the cooldown case to trigger backoff in the tick loop
599            return Err(AllAgentsCooledError::new(
600                complexity.unwrap_or("all agents").to_string(),
601                None,
602            )
603            .into());
604        };
605
606        let remaining = earliest.saturating_sub(now);
607        if remaining > 0 {
608            let scope_str = complexity.unwrap_or("all agents").to_string();
609            tracing::warn!(
610                remaining_secs = remaining,
611                scope = %scope_str,
612                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
613            );
614            // Return a domain error indicating all candidates are cooled so the
615            // caller can handle it (tick will skip this task and retry later).
616            return Err(AllAgentsCooledError::new(scope_str, Some(remaining as u64)).into());
617        }
618
619        Ok(())
620    }
621
622    /// Get the first available agent.
623    /// Pick next agent via round-robin (for review or other non-task routing).
624    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
625    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
626    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
627        // Use available_agents_for_complexity which already filters using agent_is_routable,
628        // eliminating duplicate cooldown/degraded/model checks.
629        let candidates = self.available_agents_for_complexity(complexity);
630        if candidates.is_empty() {
631            return None;
632        }
633        let idx = self.review_rr_index;
634        let n = candidates.len();
635
636        // Try to find an agent that isn't excluded
637        let agent = (0..n)
638            .map(|offset| &candidates[(idx + offset) % n])
639            .find(|a| !exclude.contains(&a.as_str()))
640            .cloned()
641            // Fallback: use the first available candidate (even if excluded)
642            .or_else(|| candidates.first().cloned())?;
643
644        self.review_rr_index = (idx + 1) % candidates.len().max(1);
645        Some(agent)
646    }
647
648    /// If the selected agent matches `no_code_last_agent` for this task, filter it
649    /// out of the candidates and return the filtered list. Used to prevent the
650    /// same agent from being selected twice in a row after a no-code failure (#2410).
651    async fn skip_no_code_agent(
652        &self,
653        store: &std::sync::Arc<TaskStore>,
654        repo: &str,
655        task_id: &str,
656        candidates: &[String],
657    ) -> Option<Vec<String>> {
658        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
659            .await
660            .unwrap_or_else(|| {
661                tracing::warn!(
662                    task_id,
663                    "no_code_last_agent DB read returned None — loop detection may be impaired"
664                );
665                String::new()
666            });
667        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
668            return None;
669        }
670        let filtered: Vec<String> = candidates
671            .iter()
672            .filter(|a| a.as_str() != no_code_agent)
673            .cloned()
674            .collect();
675        if filtered.is_empty() {
676            tracing::warn!(
677                task_id,
678                no_code_agent = %no_code_agent,
679                "all available agents are the no-code agent — cannot skip"
680            );
681            return None;
682        }
683        tracing::info!(
684            task_id,
685            skipped = %no_code_agent,
686            remaining = ?filtered,
687            "skipping no-code agent, routing to alternative"
688        );
689        Some(filtered)
690    }
691
692    /// Route a task to the best agent.
693    ///
694    /// Routing logic (in priority order):
695    /// 1. Check for `agent:*` label — use that agent directly
696    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
697    /// 3. If round_robin mode, cycle through agents (stateful)
698    /// 4. Call LLM classifier for intelligent routing
699    /// 5. After max_route_attempts LLM failures, fall back to round-robin
700    pub async fn route(
701        &mut self,
702        task: &ExternalTask,
703        store: &std::sync::Arc<TaskStore>,
704        repo: &str,
705    ) -> anyhow::Result<RouteResult> {
706        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
707        // before the routing loop, not per-task. Do not re-add it here.
708
709        // Hold the task before any classification, label override, or fallback.
710        // A waiting task must not increment route_attempts.
711        match store.resolve_id_by_external(repo, &task.id.0).await {
712            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
713                Ok(Some(blockers)) => {
714                    return Err(crate::store::WaitingOnDependency { blockers }.into());
715                }
716                Ok(None) => {}
717                Err(err) => return Err(err),
718            },
719            Ok(None) => {}
720            Err(err) => return Err(err),
721        }
722
723        loop {
724            // 1. Check store agent field first (set by failover, authoritative over labels)
725            // Use a lightweight lookup to avoid deserializing the full Task row when
726            // only the `agent` column is needed. This avoids two full-row loads
727            // per routing pass (resolve_task_id + get) and significantly reduces
728            // CPU work on busy instances.
729            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
730                Ok(Some(agent)) => {
731                    // Router enforces that the agent string matches a configured agent
732                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
733                        Some(agent)
734                    } else {
735                        None
736                    }
737                }
738                Ok(None) => None,
739                Err(e) => {
740                    tracing::warn!(
741                        repo,
742                        task_id = %task.id.0,
743                        err = %e,
744                        "failed to fetch stored agent, falling back to label/LLM routing"
745                    );
746                    None
747                }
748            };
749
750            // 2. Fall back to explicit agent label
751            let resolved_agent = store_agent.or_else(|| {
752                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
753            });
754
755            if let Some(agent) = resolved_agent {
756                let complexity = strategies::extract_complexity_from_labels(&task.labels);
757                // If agent appears routable (not in agent-wide cooldown and has
758                // available models or no model_map configured), check whether
759                // we actually have a concrete model to dispatch. If the
760                // resolved label maps to no model (None), we fall through to
761                // the standard routing logic so the router LLM can pick a
762                // proper agent/model, instead of dispatching with an empty
763                // model which causes some agents (e.g. opencode) to exit
764                // silently.
765                if self.agent_is_routable(&agent, &complexity) {
766                    let model = self
767                        .config
768                        .model_for_complexity(&agent, &complexity, &task.id.0);
769
770                    // Only dispatch label overrides when we have a concrete
771                    // model to send to the executor. Dispatching with an
772                    // empty model led to silent exits for some agents
773                    // (notably opencode). If no concrete model is available
774                    // for the labeled agent, fall through to the standard
775                    // routing logic so the LLM or round-robin can pick an
776                    // appropriate agent/model.
777                    if model.is_some() {
778                        tracing::debug!(
779                            task_id = %task.id.0,
780                            agent = %agent,
781                            complexity = %complexity,
782                            "routed via label"
783                        );
784                        let mut result = RouteResult::from_config(
785                            agent.clone(),
786                            complexity.clone(),
787                            format!("label agent:{agent}"),
788                            &self.config,
789                            &task.id.0,
790                        );
791                        // Label routing uses agent-specific role and the
792                        // resolved model (may differ from from_config's default).
793                        result.profile.role = format!("{} specialist", agent);
794                        result.model = model.clone();
795                        self.log_route_activity(store, repo, &task.id.0, &result, None)
796                            .await;
797                        return Ok(result);
798                    } else {
799                        // No concrete model found for the labeled agent — do
800                        // not honor the label override. Fall through to the
801                        // standard routing logic so another agent or the LLM
802                        // can be chosen with a valid model.
803                        tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
804                    }
805                }
806
807                let candidates = self.available_agents_for_complexity(&complexity);
808                if candidates.is_empty() {
809                    self.wait_for_cooldown(Some(&complexity)).await?;
810                    continue;
811                }
812                // Label target is cooled or lacked model; fall through to standard routing.
813            }
814
815            let complexity = strategies::extract_complexity_from_labels(&task.labels);
816
817            // Check if any agents have available models for this complexity
818            let mut candidates = self.available_agents_for_complexity(&complexity);
819            if candidates.is_empty() {
820                self.wait_for_cooldown(Some(&complexity)).await?;
821                continue;
822            }
823
824            // Skip the no-code agent if one is recorded for this task (#2410).
825            // If all available agents are filtered out, we fall through to
826            // LLM routing with no constraints (LLM will pick the same agent but
827            // the response_handler will block it immediately via same-agent check).
828            if let Some(filtered) = self
829                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
830                .await
831            {
832                candidates = filtered;
833            }
834
835            // 2. Weighted round-robin — capacity-based selection
836            if self.config.weighted_round_robin {
837                let result = strategies::route_via_weighted_round_robin(
838                    &candidates,
839                    &self.weights,
840                    &self.config,
841                    task,
842                    &mut self.last_agent,
843                )?;
844                self.log_route_activity(store, repo, &task.id.0, &result, None)
845                    .await;
846                return Ok(result);
847            }
848
849            // 3. Round-robin mode — use stateful round-robin
850            if self.config.mode == "round_robin" {
851                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
852                let result = strategies::route_via_round_robin_stateful(
853                    &candidates,
854                    &self.config,
855                    task,
856                    &mut self.rr_index,
857                    &mut self.last_agent,
858                )?;
859                self.log_route_activity(store, repo, &task.id.0, &result, None)
860                    .await;
861                return Ok(result);
862            }
863
864            // 4. Local Ollama routing
865            if self.config.mode == "local" {
866                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
867                let ollama_router = self
868                    .ollama_router
869                    .as_ref()
870                    .expect("ollama_router must be initialized in Router::new when mode=local");
871                match ollama_router.route(task, &self.config).await {
872                    Ok(result) => {
873                        self.log_route_activity(store, repo, &task.id.0, &result, None)
874                            .await;
875                        return Ok(result);
876                    }
877                    Err(e) => {
878                        tracing::warn!(
879                            task_id = %task.id.0,
880                            error = %e,
881                            "Ollama routing failed, falling back to round-robin"
882                        );
883                        let result = strategies::route_via_round_robin_stateful(
884                            &candidates,
885                            &self.config,
886                            task,
887                            &mut self.rr_index,
888                            &mut self.last_agent,
889                        )?;
890                        self.log_route_activity(store, repo, &task.id.0, &result, None)
891                            .await;
892                        return Ok(result);
893                    }
894                }
895            }
896
897            // 5. LLM-based routing with retry tracking
898            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
899
900            if route_attempts >= self.config.max_route_attempts {
901                tracing::warn!(
902                    task_id = %task.id.0,
903                    attempts = route_attempts,
904                    max = self.config.max_route_attempts,
905                    "max LLM route attempts reached, falling back to round-robin"
906                );
907                let candidates = self.available_agents_for_complexity(&complexity);
908                if candidates.is_empty() {
909                    self.wait_for_cooldown(Some(&complexity)).await?;
910                    continue;
911                }
912                let result = strategies::route_via_round_robin_stateful(
913                    &candidates,
914                    &self.config,
915                    task,
916                    &mut self.rr_index,
917                    &mut self.last_agent,
918                )?;
919                self.log_route_activity(store, repo, &task.id.0, &result, None)
920                    .await;
921                return Ok(result);
922            }
923
924            // Log routing start (before await)
925            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
926
927            match self.route_with_llm(task, repo).await {
928                Ok(result) => {
929                    let candidates = self.available_agents_for_complexity(&result.complexity);
930                    if candidates.is_empty() {
931                        self.wait_for_cooldown(Some(&result.complexity)).await?;
932                        continue;
933                    }
934
935                    // If the LLM selected the no-code agent, filter it out and
936                    // pick an alternative (#2410). Re-use the existing fallback logic.
937                    let no_code_agent =
938                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
939                             .await
940                             .unwrap_or_else(|| {
941                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
942                                 String::new()
943                             });
944                    if result.agent == no_code_agent {
945                        // Pick the first agent that isn't the no-code agent.
946                        let fallback_agent = candidates
947                            .iter()
948                            .find(|a| a.as_str() != no_code_agent)
949                            .cloned()
950                            .unwrap_or_else(|| candidates[0].clone());
951                        if fallback_agent.as_str() == no_code_agent {
952                            tracing::warn!(
953                                task_id = %task.id.0,
954                                no_code_agent = %no_code_agent,
955                                complexity = %result.complexity,
956                                "all available agents are the no-code agent — waiting for another agent"
957                            );
958                            self.wait_for_cooldown(Some(&result.complexity)).await?;
959                            continue;
960                        }
961                        tracing::info!(
962                            task_id = %task.id.0,
963                            llm_selected = %result.agent,
964                            skipped = %no_code_agent,
965                            fallback = %fallback_agent,
966                            "LLM selected the no-code agent — routing to alternative"
967                        );
968                        let fallback_model = self.config.model_for_complexity(
969                            &fallback_agent,
970                            &result.complexity,
971                            &task.id.0,
972                        );
973
974                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
975                        let result = RouteResult {
976                            agent: fallback_agent.clone(),
977                            model: fallback_model,
978                            complexity: result.complexity.clone(),
979                            estimate: result.estimate,
980                            reason: format!(
981                                "LLM selected no-code agent {}; routed to {}",
982                                result.agent, fallback_agent
983                            ),
984                            profile: result.profile.clone(),
985                            selected_skills: result.selected_skills.clone(),
986                            warning: Some(format!(
987                                "LLM selected no-code agent {}; used fallback {}",
988                                result.agent, fallback_agent
989                            )),
990                        };
991                        self.log_route_activity(store, repo, &task.id.0, &result, None)
992                            .await;
993                        return Ok(result);
994                    }
995
996                    let model_cooled = result
997                        .model
998                        .as_deref()
999                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1000                    if !candidates.contains(&result.agent) || model_cooled {
1001                        let fallback_agent = candidates
1002                            .iter()
1003                            .find(|agent| *agent != &result.agent)
1004                            .cloned()
1005                            .unwrap_or_else(|| candidates[0].clone());
1006                        let fallback_model = self.config.model_for_complexity(
1007                            &fallback_agent,
1008                            &result.complexity,
1009                            &task.id.0,
1010                        );
1011
1012                        // If the fallback model is None and the fallback agent is the
1013                        // same as the LLM-selected agent, it means all models for the
1014                        // only available agent are cooled. In that case, wait for
1015                        // cooldowns instead of returning a RouteResult with
1016                        // model: None which would be dispatched incorrectly.
1017                        if fallback_model.is_none() && fallback_agent == result.agent {
1018                            tracing::warn!(
1019                                task_id = %task.id.0,
1020                                agent = %result.agent,
1021                                complexity = %result.complexity,
1022                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1023                            );
1024                            self.wait_for_cooldown(Some(&result.complexity)).await?;
1025                            continue;
1026                        }
1027
1028                        // Distinguish logging and reasons when we actually switch
1029                        // agents vs when we keep the same agent but pick a
1030                        // different (non-cooled) model.
1031                        if fallback_agent == result.agent {
1032                            tracing::warn!(
1033                                task_id = %task.id.0,
1034                                agent = %result.agent,
1035                                complexity = %result.complexity,
1036                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1037                            );
1038                        } else {
1039                            tracing::warn!(
1040                                task_id = %task.id.0,
1041                                agent = %result.agent,
1042                                fallback = %fallback_agent,
1043                                complexity = %result.complexity,
1044                                "LLM selected cooled agent/model; rerouting to available agent"
1045                            );
1046                        }
1047
1048                        // Reset attempts on success
1049                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1050
1051                        let reason = if fallback_agent == result.agent {
1052                            format!(
1053                                "LLM selected cooled model; using same agent {} with different model",
1054                                fallback_agent
1055                            )
1056                        } else {
1057                            format!(
1058                                "LLM selected cooled agent/model; rerouted to {}",
1059                                fallback_agent
1060                            )
1061                        };
1062
1063                        let result = RouteResult {
1064                            agent: fallback_agent.clone(),
1065                            model: fallback_model,
1066                            complexity: result.complexity.clone(),
1067                            estimate: result.estimate,
1068                            reason: reason.clone(),
1069                            profile: result.profile.clone(),
1070                            selected_skills: result.selected_skills.clone(),
1071                            warning: Some(if fallback_agent == result.agent {
1072                                "LLM-selected model was cooled; selected alternative model for same agent"
1073                                    .to_string()
1074                            } else {
1075                                "LLM-selected agent/model was cooled; rerouted to available agent"
1076                                    .to_string()
1077                            }),
1078                        };
1079                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1080                            .await;
1081                        return Ok(result);
1082                    }
1083
1084                    // Reset attempts on success
1085                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1086                    tracing::info!(
1087                        task_id = %task.id.0,
1088                        agent = %result.agent,
1089                        complexity = %result.complexity,
1090                        "routed via LLM"
1091                    );
1092                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1093                        .await;
1094                    return Ok(result);
1095                }
1096                Err(e) => {
1097                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1098                        let scope = err.scope();
1099                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1100                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1101                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1102                        // (no complexity-specific pool lookup, which would fail for "router pool").
1103                        let scope_opt = match scope {
1104                            "all agents" | "router pool" => None,
1105                            _ => Some(scope),
1106                        };
1107                        self.wait_for_cooldown(scope_opt).await?;
1108                        continue;
1109                    }
1110
1111                    let new_attempts = route_attempts + 1;
1112                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1113                        .await;
1114                    tracing::warn!(
1115                        task_id = %task.id.0,
1116                        error = %e,
1117                        error_chain = ?e,
1118                        attempt = new_attempts,
1119                        max = self.config.max_route_attempts,
1120                        "LLM routing failed"
1121                    );
1122
1123                    if new_attempts >= self.config.max_route_attempts {
1124                        tracing::info!(
1125                            task_id = %task.id.0,
1126                            "falling back to round-robin after {} failed attempts",
1127                            new_attempts
1128                        );
1129                        let candidates = self.available_agents_for_complexity(&complexity);
1130                        if candidates.is_empty() {
1131                            self.wait_for_cooldown(Some(&complexity)).await?;
1132                            continue;
1133                        }
1134                        let result = strategies::route_via_round_robin_stateful(
1135                            &candidates,
1136                            &self.config,
1137                            task,
1138                            &mut self.rr_index,
1139                            &mut self.last_agent,
1140                        )?;
1141                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1142                            .await;
1143                        return Ok(result);
1144                    }
1145
1146                    let candidates = self.available_agents_for_complexity(&complexity);
1147                    if candidates.is_empty() {
1148                        self.wait_for_cooldown(Some(&complexity)).await?;
1149                        continue;
1150                    }
1151                    let result = strategies::route_via_fallback(
1152                        &candidates,
1153                        &self.config,
1154                        task,
1155                        Some(&self.weights),
1156                    )?;
1157                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1158                        .await;
1159                    return Ok(result);
1160                }
1161            }
1162        }
1163    }
1164
1165    /// Get the number of LLM routing attempts for a task from the store.
1166    async fn get_route_attempts(
1167        &self,
1168        task_id: &str,
1169        store: &std::sync::Arc<TaskStore>,
1170        repo: &str,
1171    ) -> u32 {
1172        match store.resolve_task_id(repo, task_id).await {
1173            Ok(Some(store_id)) => store
1174                .get(store_id)
1175                .await
1176                .map(|t| t.route_attempts as u32)
1177                .unwrap_or(0),
1178            _ => 0,
1179        }
1180    }
1181
1182    /// Set the number of LLM routing attempts for a task in the store.
1183    async fn set_route_attempts(
1184        &self,
1185        task_id: &str,
1186        attempts: u32,
1187        store: &std::sync::Arc<TaskStore>,
1188        repo: &str,
1189    ) {
1190        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1191            if let Err(e) = store
1192                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1193                .await
1194            {
1195                tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1196            }
1197        }
1198    }
1199
1200    /// Log a route event to task activity timeline.
1201    async fn log_route_activity(
1202        &self,
1203        store: &std::sync::Arc<TaskStore>,
1204        repo: &str,
1205        task_id: &str,
1206        result: &RouteResult,
1207        from_agent: Option<&str>,
1208    ) {
1209        let details = serde_json::json!({
1210            "reason": result.reason,
1211            "complexity": result.complexity,
1212            "skills": result.selected_skills,
1213            "role": result.profile.role,
1214            "warning": result.warning,
1215        });
1216        let event_type = if from_agent.is_some() {
1217            "rerouted"
1218        } else {
1219            "routed"
1220        };
1221        store_log_activity(
1222            &Some(Arc::clone(store)),
1223            repo,
1224            task_id,
1225            event_type,
1226            None,
1227            Some("routed"),
1228            Some(&result.agent),
1229            result.model.as_deref(),
1230            Some(&details),
1231        )
1232        .await;
1233    }
1234
1235    /// Route using LLM classification with pool round-robin.
1236    ///
1237    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1238    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1239    /// returns the last error (the caller falls back to round-robin agent selection).
1240    async fn route_with_llm(
1241        &mut self,
1242        task: &ExternalTask,
1243        repo: &str,
1244    ) -> anyhow::Result<RouteResult> {
1245        // Filter out cooled and degraded agents, and agents without an available
1246        // model for the coding tiers (`medium`/`complex`) that real tasks are
1247        // classified into, so the LLM only sees candidates that could actually
1248        // be dispatched for whatever complexity it picks. An agent whose only
1249        // healthy tier is `simple` is excluded too: the classifier can't be
1250        // constrained to `simple`, so leaving such an agent in would waste the
1251        // call whenever it picks `medium`/`complex` (issue #3521).
1252        // Fall back to the full list if all agents are cooled/degraded.
1253        // Cloned to avoid borrow conflict with &mut self in the loop.
1254        let uncooled_agents: Vec<String> = self
1255            .available_agents
1256            .iter()
1257            .filter(|a| {
1258                !is_agent_in_cooldown(a)
1259                    && !is_agent_degraded(a)
1260                    && self.agent_has_available_model_for_coding_tiers(a)
1261            })
1262            .cloned()
1263            .collect();
1264        if uncooled_agents.is_empty() {
1265            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1266        }
1267        let llm_agents = uncooled_agents;
1268
1269        let pool = self.router_pool.clone();
1270        let n = pool.len();
1271        let start = self.pool_index;
1272        let mut last_err: Option<anyhow::Error> = None;
1273        let mut attempted_any = false;
1274
1275        // Try pool entries in round-robin order, skipping cooled ones
1276        for i in 0..n {
1277            let idx = (start + i) % n;
1278            let (agent, model) = &pool[idx];
1279            let model_str = model.as_str();
1280
1281            if crate::engine::runner::response::is_model_in_cooldown(agent, model_str) {
1282                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1283                continue;
1284            }
1285
1286            // At least one pool entry was available to attempt
1287            attempted_any = true;
1288
1289            let model_opt = if model_str.is_empty() {
1290                None
1291            } else {
1292                Some(model_str)
1293            };
1294
1295            match self
1296                .llm_router
1297                .route_with_llm_using(
1298                    task,
1299                    &llm_agents,
1300                    &self.config,
1301                    &mut self.last_agent,
1302                    repo,
1303                    agent,
1304                    model_opt,
1305                )
1306                .await
1307            {
1308                Ok(result) => {
1309                    // Advance index so the next call starts at the next pool entry
1310                    self.advance_pool_index_after_attempt(idx, n);
1311                    tracing::debug!(
1312                        agent,
1313                        model = model_str,
1314                        pool_idx = idx,
1315                        "router LLM pool entry succeeded"
1316                    );
1317                    record_agent_success(agent, model_str).await;
1318                    return Ok(result);
1319                }
1320                Err(e) => {
1321                    let err_msg = e.to_string();
1322                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1323                    if is_timeout {
1324                        // Advance pool index so the next tick starts at the next entry,
1325                        // then return immediately. Continuing to try further pool entries
1326                        // in the same tick would accumulate multiple timeout windows and
1327                        // stall the tick loop for minutes (issue #3187).
1328                        //
1329                        // Also record a standard cooldown, same as the non-timeout branch,
1330                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1331                        // on future ticks instead of burning the full timeout window again
1332                        // every time round-robin reaches it (issue #3422).
1333                        self.advance_pool_index_after_attempt(idx, n);
1334                        tracing::warn!(
1335                            agent,
1336                            model = model_str,
1337                            next_pool_idx = self.pool_index,
1338                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1339                        );
1340                        crate::engine::runner::response::record_model_failure(agent, model_str)
1341                            .await;
1342                        return Err(e);
1343                    } else {
1344                        tracing::warn!(
1345                            agent,
1346                            model = model_str,
1347                            error = %e,
1348                            "pool entry failed, recording cooldown and trying next"
1349                        );
1350                        crate::engine::runner::response::record_model_failure(agent, model_str)
1351                            .await;
1352                    }
1353                    last_err = Some(e);
1354                    self.advance_pool_index_after_attempt(idx, n);
1355                }
1356            }
1357        }
1358
1359        // If every pool entry was pre-cooled (nothing was attempted), do not
1360        // immediately exhaust the pool again — fail fast and let the tick loop
1361        // retry on the next cycle instead of cascading failures.
1362        // However, we should still try the fallback if configured.
1363        if !attempted_any {
1364            let now = chrono::Utc::now().timestamp();
1365            let earliest = self.earliest_pool_cooldown();
1366            if let Some(until) = earliest {
1367                // Pool entries exist but are all cooled - we'll try fallback below
1368                let remaining = until.saturating_sub(now);
1369                tracing::warn!(
1370                    remaining_secs = remaining,
1371                    pool = ?pool,
1372                    "all router LLM pool entries on cooldown — will try fallback"
1373                );
1374                // Continue to fallback logic instead of returning early
1375            } else {
1376                // No pool cooldowns found either — this means the pool was empty
1377                // or every entry was filtered out for some other reason.
1378                return Err(last_err
1379                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1380            }
1381        }
1382
1383        // All pool entries failed or were cooled — try the configured fallback
1384        let fallback = self.config.effective_fallback();
1385        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1386        let fb_model_str = fb_model.as_str();
1387
1388        // Only try fallback if it wasn't already in the pool (avoid double-try)
1389        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1390
1391        if !fallback_already_tried
1392            && !crate::engine::runner::response::is_model_in_cooldown(&fb_agent, fb_model_str)
1393        {
1394            tracing::info!(
1395                agent = %fb_agent,
1396                model = %fb_model_str,
1397                "all pool entries exhausted — trying fallback router LLM"
1398            );
1399            let fb_model_opt = if fb_model_str.is_empty() {
1400                None
1401            } else {
1402                Some(fb_model_str)
1403            };
1404            match self
1405                .llm_router
1406                .route_with_llm_using(
1407                    task,
1408                    &llm_agents,
1409                    &self.config,
1410                    &mut self.last_agent,
1411                    repo,
1412                    &fb_agent,
1413                    fb_model_opt,
1414                )
1415                .await
1416            {
1417                Ok(result) => {
1418                    record_agent_success(&fb_agent, fb_model_str).await;
1419                    return Ok(result);
1420                }
1421                Err(e) => {
1422                    let err_msg = e.to_string();
1423                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1424                    if is_timeout {
1425                        // Cool down on timeout like pool entries (#3422), else a
1426                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1427                        crate::engine::runner::response::record_model_failure(
1428                            &fb_agent,
1429                            fb_model_str,
1430                        )
1431                        .await;
1432                        tracing::warn!(
1433                            agent = %fb_agent,
1434                            model = %fb_model_str,
1435                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1436                        );
1437                        return Err(e);
1438                    } else {
1439                        tracing::warn!(
1440                            agent = %fb_agent,
1441                            model = %fb_model_str,
1442                            error = %e,
1443                            "fallback router LLM also failed"
1444                        );
1445                        crate::engine::runner::response::record_model_failure(
1446                            &fb_agent,
1447                            fb_model_str,
1448                        )
1449                        .await;
1450                    }
1451                    last_err = Some(e);
1452                }
1453            }
1454        }
1455
1456        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1457    }
1458
1459    /// Record a rate limit event for an agent, reducing its routing weight.
1460    ///
1461    /// Called by the engine when an agent returns a 429/rate limit error.
1462    pub fn record_rate_limit(&mut self, agent: &str) {
1463        self.weights.record_rate_limit(agent);
1464    }
1465
1466    /// Record a successful task completion, restoring agent weight.
1467    pub fn record_success(&mut self, agent: &str) {
1468        self.weights.record_success(agent);
1469    }
1470
1471    /// Tick weight recovery for all agents.
1472    ///
1473    /// Called periodically by the engine to gradually restore weights
1474    /// as rate limit windows expire.
1475    pub fn tick_weight_recovery(&mut self) {
1476        self.weights.tick_recovery();
1477    }
1478
1479    /// Store routing result in the task store.
1480    pub async fn store_route_result(
1481        &self,
1482        task_id: &str,
1483        result: &RouteResult,
1484        store: &std::sync::Arc<TaskStore>,
1485        repo: &str,
1486    ) -> anyhow::Result<()> {
1487        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1488            store
1489                .set_fields(
1490                    store_id,
1491                    &[
1492                        ("agent", serde_json::json!(result.agent)),
1493                        ("complexity", serde_json::json!(result.complexity)),
1494                        ("route_reason", serde_json::json!(result.reason)),
1495                        (
1496                            "model",
1497                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1498                        ),
1499                    ],
1500                )
1501                .await?;
1502        }
1503        Ok(())
1504    }
1505}
1506
1507/// Retrieve routing result from the task store.
1508pub async fn get_route_result(
1509    store: &std::sync::Arc<TaskStore>,
1510    repo: &str,
1511    task_id: &str,
1512) -> Result<RouteResult, RouteResultError> {
1513    let store_id = store
1514        .resolve_task_id(repo, task_id)
1515        .await
1516        .map_err(RouteResultError::Other)?
1517        .ok_or_else(|| {
1518            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1519        })?;
1520    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1521
1522    let agent = task
1523        .agent
1524        .clone()
1525        .filter(|a| !a.is_empty())
1526        .ok_or_else(|| RouteResultError::NoAgent {
1527            task_id: task_id.to_string(),
1528        })?;
1529
1530    let complexity = if task.complexity.is_empty() {
1531        "medium".to_string()
1532    } else {
1533        task.complexity.clone()
1534    };
1535
1536    let reason = task.route_reason.clone();
1537    let model = task.model.clone().filter(|m| !m.is_empty());
1538
1539    // Validate that the stored model is valid for the resolved agent.
1540    // Agent-specific aliases (e.g. "opus") can leak across agents during
1541    // failover, so we verify against the router's model pools (#1604).
1542    let config = RouterConfig::from_config();
1543    let model = model.filter(|m| {
1544        let complexities = ["simple", "medium", "complex", "review"];
1545        // Collect all model pools configured for this agent across complexity tiers.
1546        let all_pools: Vec<Vec<String>> = complexities
1547            .iter()
1548            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1549            .collect();
1550        // If the agent has no model pools configured, we can't validate — pass through.
1551        if all_pools.is_empty() {
1552            return true;
1553        }
1554        let valid = all_pools.iter().any(|pool| pool.contains(m));
1555        if !valid {
1556            tracing::warn!(
1557                task_id,
1558                agent,
1559                model = %m,
1560                "discarding stale model: not in agent's model pools"
1561            );
1562        }
1563        valid
1564    });
1565
1566    // If a stale model was discarded, update the database to clear the model
1567    // and record a diagnostic last_error so task_runs.error is populated if
1568    // the subsequent run fails (fixes issue #2527).
1569    let original_model = task.model.clone().filter(|m| !m.is_empty());
1570    if original_model.is_some() && model.is_none() {
1571        let stale_model = original_model.as_deref().unwrap_or("unknown");
1572        let stale_err = format!("stale model discarded: {stale_model} not in {agent} model pools");
1573        if let Err(e) = store
1574            .set_fields(
1575                store_id,
1576                &[
1577                    ("model", serde_json::json!("")),
1578                    ("last_error", serde_json::json!(stale_err)),
1579                ],
1580            )
1581            .await
1582        {
1583            // Log the error so transient DB failures are visible and diagnosable.
1584            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1585        }
1586    }
1587
1588    Ok(RouteResult {
1589        agent,
1590        model,
1591        complexity,
1592        estimate: 0,
1593        reason,
1594        profile: AgentProfile::default(),
1595        selected_skills: vec![],
1596        warning: None,
1597    })
1598}
1599
1600#[cfg(test)]
1601mod tests {
1602    use super::config::DEFAULT_AGENTS;
1603    use super::llm::LlmRouteResponse;
1604    use super::strategies;
1605    use super::weights::{
1606        AgentWeights, RateLimitState, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY,
1607    };
1608    use super::*;
1609    use crate::backends::{ExternalId, ExternalTask};
1610    use crate::engine::cooldown::{
1611        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1612        record_model_failure, set_model_cooldown,
1613    };
1614    use crate::store::TaskStore;
1615    use serial_test::serial;
1616    use std::time::{Duration, Instant};
1617
1618    // Test-only delegates so tests can call router.parse_llm_response() and
1619    // router.check_routing_sanity() directly without referencing llm_router.
1620    impl Router {
1621        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1622            self.llm_router.parse_llm_response("claude", response)
1623        }
1624
1625        fn check_routing_sanity(
1626            &self,
1627            task: &ExternalTask,
1628            agent: &str,
1629            profile: &AgentProfile,
1630        ) -> Option<String> {
1631            self.llm_router.check_routing_sanity(task, agent, profile)
1632        }
1633    }
1634
1635    async fn test_store() -> std::sync::Arc<TaskStore> {
1636        std::sync::Arc::new(TaskStore::open_memory().await.unwrap())
1637    }
1638
1639    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1640        ExternalTask {
1641            id: ExternalId(id.to_string()),
1642            title: title.to_string(),
1643            body: "Test body".to_string(),
1644            state: "open".to_string(),
1645            labels,
1646            author: "test".to_string(),
1647            created_at: "2026-01-01T00:00:00Z".to_string(),
1648            updated_at: "2026-01-01T00:00:00Z".to_string(),
1649            url: format!("https://github.com/test/test/issues/{id}"),
1650        }
1651    }
1652
1653    #[serial(cooldown_state)]
1654    #[test]
1655    fn extract_agent_from_labels() {
1656        let mut config = RouterConfig::default();
1657        // Ensure a concrete model exists for the labeled agent so the label
1658        // override is honored by the router (routing requires a model).
1659        config
1660            .model_map
1661            .entry("medium".to_string())
1662            .or_default()
1663            .insert("claude".to_string(), vec!["haiku".to_string()]);
1664
1665        assert_eq!(
1666            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1667            Some("claude".to_string())
1668        );
1669        assert_eq!(
1670            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1671            Some("codex".to_string())
1672        );
1673        assert_eq!(
1674            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1675            Some("opencode".to_string())
1676        );
1677        assert_eq!(
1678            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1679            None
1680        );
1681        // Verify kimi and minimax are recognized from labels
1682        assert_eq!(
1683            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1684            Some("kimi".to_string())
1685        );
1686        assert_eq!(
1687            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1688            Some("minimax".to_string())
1689        );
1690    }
1691
1692    #[serial(cooldown_state)]
1693    #[tokio::test]
1694    async fn all_agents_cooled_error_names_degraded_agents() {
1695        crate::engine::cooldown::reset_global_state().await;
1696        mark_agent_degraded("claude");
1697        mark_agent_degraded("grok");
1698        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1699        let msg = err.to_string();
1700        assert!(
1701            msg.contains("degraded: claude, grok"),
1702            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1703        );
1704        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1705        clear_agent_degraded("claude");
1706        clear_agent_degraded("grok");
1707    }
1708
1709    #[serial(cooldown_state)]
1710    #[test]
1711    fn default_agents_constant() {
1712        assert_eq!(DEFAULT_AGENTS.len(), 7);
1713        assert!(DEFAULT_AGENTS.contains(&"claude"));
1714        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1715        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1716        assert!(DEFAULT_AGENTS.contains(&"glm"));
1717        assert!(DEFAULT_AGENTS.contains(&"grok"));
1718    }
1719
1720    #[serial(cooldown_state)]
1721    #[test]
1722    fn extract_complexity_from_labels() {
1723        assert_eq!(
1724            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1725            "simple"
1726        );
1727        assert_eq!(
1728            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1729            "medium"
1730        );
1731        assert_eq!(
1732            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1733            "complex"
1734        );
1735        assert_eq!(
1736            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1737            "medium"
1738        );
1739    }
1740
1741    #[serial(cooldown_state)]
1742    #[test]
1743    fn route_result_serialization() {
1744        let result = RouteResult {
1745            agent: "claude".to_string(),
1746            model: Some("sonnet".to_string()),
1747            complexity: "medium".to_string(),
1748            estimate: 5,
1749            reason: "test".to_string(),
1750            profile: AgentProfile {
1751                role: "backend".to_string(),
1752                skills: vec!["rust".to_string()],
1753                tools: vec!["git".to_string()],
1754                constraints: vec![],
1755            },
1756            selected_skills: vec!["gh".to_string()],
1757            warning: None,
1758        };
1759
1760        let json = serde_json::to_string(&result).unwrap();
1761        assert!(json.contains("claude"));
1762        assert!(json.contains("sonnet"));
1763
1764        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1765        assert_eq!(deserialized.agent, "claude");
1766        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1767    }
1768
1769    #[serial(cooldown_state)]
1770    #[test]
1771    fn router_config_default() {
1772        let mut config = RouterConfig::default();
1773        // Ensure a concrete model exists for the labeled agent so the label
1774        // override is honored by the router (routing requires a model).
1775        config
1776            .model_map
1777            .entry("medium".to_string())
1778            .or_default()
1779            .insert("claude".to_string(), vec!["haiku".to_string()]);
1780
1781        assert_eq!(config.mode, "round_robin");
1782        assert_eq!(config.router_agent, "claude");
1783        assert_eq!(config.router_model, "haiku");
1784        assert_eq!(config.fallback_executor, "codex");
1785        assert_eq!(config.max_route_attempts, 3);
1786        assert!(!config.allowed_tools.is_empty());
1787        assert!(!config.default_skills.is_empty());
1788
1789        // Verify configurable agents list includes all 7 agents
1790        assert_eq!(config.agents.len(), 7);
1791        assert!(config.agents.contains(&"claude".to_string()));
1792        assert!(config.agents.contains(&"codex".to_string()));
1793        assert!(config.agents.contains(&"opencode".to_string()));
1794        assert!(config.agents.contains(&"kimi".to_string()));
1795        assert!(config.agents.contains(&"minimax".to_string()));
1796        assert!(config.agents.contains(&"glm".to_string()));
1797        assert!(config.agents.contains(&"grok".to_string()));
1798    }
1799
1800    #[serial(cooldown_state)]
1801    #[test]
1802    fn model_map_lookup_returns_none_without_config() {
1803        // Default config has no hardcoded models — config.yml is the source of truth.
1804        let config = RouterConfig::default();
1805        assert!(config
1806            .model_for_complexity("claude", "simple", "")
1807            .is_none());
1808        assert!(config
1809            .model_for_complexity("claude", "medium", "")
1810            .is_none());
1811        assert!(config
1812            .model_for_complexity("claude", "complex", "")
1813            .is_none());
1814        assert!(config
1815            .model_for_complexity("opencode", "review", "")
1816            .is_none());
1817    }
1818
1819    #[serial(cooldown_state)]
1820    #[test]
1821    fn model_map_lookup_returns_configured_value() {
1822        // When model_map is populated (from config), it is returned correctly.
1823        let mut config = RouterConfig::default();
1824        config
1825            .model_map
1826            .entry("simple".to_string())
1827            .or_default()
1828            .insert("claude".to_string(), vec!["haiku".to_string()]);
1829        assert_eq!(
1830            config.model_for_complexity("claude", "simple", ""),
1831            Some("haiku".to_string())
1832        );
1833    }
1834
1835    #[serial(cooldown_state)]
1836    #[tokio::test]
1837    async fn model_pool_selection_skips_cooled() {
1838        crate::engine::cooldown::reset_global_state().await;
1839        use std::collections::HashMap;
1840
1841        let mut config = RouterConfig::default();
1842        // Set up a two-model pool for opencode/simple
1843        config
1844            .model_map
1845            .entry("simple".to_string())
1846            .or_default()
1847            .insert(
1848                "opencode".to_string(),
1849                vec!["model-a".to_string(), "model-b".to_string()],
1850            );
1851
1852        // Before any cooldown, both models are candidates
1853        let m = config.model_for_complexity("opencode", "simple", "task-1");
1854        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1855
1856        // Cool model-a
1857        record_model_failure("opencode", "model-a").await;
1858        assert!(is_model_in_cooldown("opencode", "model-a"));
1859
1860        // Now only model-b should be returned
1861        // (try many task_ids to rule out hash coincidence)
1862        for i in 0..20 {
1863            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1864            assert_eq!(
1865                result,
1866                Some("model-b".to_string()),
1867                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1868            );
1869        }
1870
1871        // Cool model-b too — now all models are cooled
1872        record_model_failure("opencode", "model-b").await;
1873        // All cooled → return None so the caller can fall back to a different agent
1874        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1875        assert_eq!(fallback, None);
1876        let _ = HashMap::<(), ()>::new(); // suppress unused import lint
1877    }
1878
1879    #[serial(cooldown_state)]
1880    #[test]
1881    fn model_pool_single_string_backward_compat() {
1882        // Single-item pools behave identically to the old string format.
1883        // Manually insert a single-item pool and verify it is returned correctly.
1884        let mut config = RouterConfig::default();
1885        config
1886            .model_map
1887            .entry("simple".to_string())
1888            .or_default()
1889            .insert("claude".to_string(), vec!["haiku".to_string()]);
1890        let m = config.model_for_complexity("claude", "simple", "any-task");
1891        assert_eq!(m, Some("haiku".to_string()));
1892    }
1893
1894    #[serial(cooldown_state)]
1895    #[test]
1896    fn parse_llm_response_direct_json() {
1897        let config = RouterConfig::default();
1898        let router = Router::new(config);
1899
1900        let response = r#"{
1901            "executor": "claude",
1902            "complexity": "complex",
1903            "reason": "requires architecture analysis",
1904            "profile": {
1905                "role": "architect",
1906                "skills": ["rust", "design"],
1907                "tools": ["git", "rg"],
1908                "constraints": []
1909            },
1910            "selected_skills": ["gh"]
1911        }"#;
1912
1913        let parsed = router.parse_llm_response(response).unwrap();
1914        assert_eq!(parsed.executor, "claude");
1915        assert_eq!(parsed.complexity, "complex");
1916        assert_eq!(parsed.reason, "requires architecture analysis");
1917        assert_eq!(parsed.profile.role, "architect");
1918    }
1919
1920    #[serial(cooldown_state)]
1921    #[test]
1922    fn parse_llm_response_markdown_fenced() {
1923        let config = RouterConfig::default();
1924        let router = Router::new(config);
1925
1926        let response = r#"Here's my analysis:
1927
1928```json
1929{
1930    "executor": "codex",
1931    "complexity": "medium",
1932    "reason": "coding task",
1933    "profile": {
1934        "role": "developer",
1935        "skills": ["coding"],
1936        "tools": [],
1937        "constraints": []
1938    },
1939    "selected_skills": []
1940}
1941```
1942
1943Hope that helps!"#;
1944
1945        let parsed = router.parse_llm_response(response).unwrap();
1946        assert_eq!(parsed.executor, "codex");
1947        assert_eq!(parsed.complexity, "medium");
1948    }
1949
1950    #[serial(cooldown_state)]
1951    #[test]
1952    fn parse_llm_response_claude_envelope() {
1953        let config = RouterConfig::default();
1954        let router = Router::new(config);
1955
1956        // Claude --output-format json wraps the response in an envelope
1957        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
1958
1959        let parsed = router.parse_llm_response(response).unwrap();
1960        assert_eq!(parsed.executor, "claude");
1961        assert_eq!(parsed.complexity, "complex");
1962        assert_eq!(parsed.reason, "requires deep analysis");
1963    }
1964
1965    #[serial(cooldown_state)]
1966    #[test]
1967    fn parse_llm_response_claude_envelope_with_code_block() {
1968        let config = RouterConfig::default();
1969        let router = Router::new(config);
1970
1971        // Claude may return the JSON inside a code block within the envelope
1972        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
1973        let envelope = serde_json::json!({
1974            "type": "result",
1975            "subtype": "success",
1976            "is_error": false,
1977            "result": inner,
1978            "usage": {"input_tokens": 50, "output_tokens": 30}
1979        });
1980
1981        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
1982        assert_eq!(parsed.executor, "codex");
1983        assert_eq!(parsed.complexity, "simple");
1984    }
1985
1986    #[serial(cooldown_state)]
1987    #[test]
1988    fn parse_llm_response_claude_envelope_error() {
1989        let config = RouterConfig::default();
1990        let router = Router::new(config);
1991
1992        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
1993
1994        let err = router.parse_llm_response(response).unwrap_err();
1995        assert!(err.to_string().contains("error"), "got: {err}");
1996    }
1997
1998    #[serial(cooldown_state)]
1999    #[test]
2000    fn parse_llm_response_empty() {
2001        let config = RouterConfig::default();
2002        let router = Router::new(config);
2003
2004        let err = router.parse_llm_response("").unwrap_err();
2005        assert!(err.to_string().contains("empty"), "got: {err}");
2006    }
2007
2008    /// Claude envelope where result is a JSON object, not a string.
2009    /// Some Claude versions may return the result as a nested object.
2010    #[serial(cooldown_state)]
2011    #[test]
2012    fn parse_llm_response_claude_envelope_result_as_object() {
2013        let config = RouterConfig::default();
2014        let router = Router::new(config);
2015
2016        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2017
2018        let parsed = router.parse_llm_response(response);
2019        assert!(
2020            parsed.is_ok(),
2021            "should handle result-as-object: {}",
2022            parsed.unwrap_err()
2023        );
2024        assert_eq!(parsed.unwrap().executor, "claude");
2025    }
2026
2027    /// LLM returns "agent" instead of "executor" — common hallucination.
2028    #[serial(cooldown_state)]
2029    #[test]
2030    fn parse_llm_response_agent_instead_of_executor() {
2031        let config = RouterConfig::default();
2032        let router = Router::new(config);
2033
2034        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2035
2036        let parsed = router.parse_llm_response(response);
2037        assert!(
2038            parsed.is_ok(),
2039            "should accept 'agent' alias: {}",
2040            parsed.unwrap_err()
2041        );
2042        assert_eq!(parsed.unwrap().executor, "claude");
2043    }
2044
2045    /// LLM returns minimal JSON without profile or reason.
2046    #[serial(cooldown_state)]
2047    #[test]
2048    fn parse_llm_response_minimal_json() {
2049        let config = RouterConfig::default();
2050        let router = Router::new(config);
2051
2052        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2053
2054        let parsed = router.parse_llm_response(response);
2055        assert!(
2056            parsed.is_ok(),
2057            "should accept minimal JSON: {}",
2058            parsed.unwrap_err()
2059        );
2060        assert_eq!(parsed.unwrap().executor, "codex");
2061    }
2062
2063    // ── Real fixture tests: route prompt → response → parsed result ──
2064
2065    /// Real Claude envelope with result as escaped JSON string.
2066    #[serial(cooldown_state)]
2067    #[test]
2068    fn fixture_route_response_string() {
2069        let config = RouterConfig::default();
2070        let router = Router::new(config);
2071
2072        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2073        let parsed = router.parse_llm_response(response).unwrap();
2074        assert_eq!(parsed.executor, "codex");
2075        assert_eq!(parsed.complexity, "medium");
2076        assert!(!parsed.reason.is_empty());
2077        assert!(!parsed.profile.role.is_empty());
2078    }
2079
2080    /// Real Claude envelope with result as JSON object (not string).
2081    /// This was the root cause of all router failures on 2026-02-27.
2082    #[serial(cooldown_state)]
2083    #[test]
2084    fn fixture_route_response_object() {
2085        let config = RouterConfig::default();
2086        let router = Router::new(config);
2087
2088        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2089        let parsed = router.parse_llm_response(response).unwrap();
2090        assert_eq!(parsed.executor, "claude");
2091        assert_eq!(parsed.complexity, "complex");
2092    }
2093
2094    /// Real Claude envelope with result as string containing markdown + JSON.
2095    #[serial(cooldown_state)]
2096    #[test]
2097    fn fixture_route_response_markdown() {
2098        let config = RouterConfig::default();
2099        let router = Router::new(config);
2100
2101        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2102        let parsed = router.parse_llm_response(response).unwrap();
2103        assert_eq!(parsed.executor, "codex");
2104        assert_eq!(parsed.complexity, "medium");
2105    }
2106
2107    /// Claude envelope with escaped JSON string containing newlines and markdown.
2108    #[serial(cooldown_state)]
2109    #[test]
2110    fn parse_llm_response_claude_envelope_with_prose() {
2111        let config = RouterConfig::default();
2112        let router = Router::new(config);
2113
2114        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2115        let envelope = serde_json::json!({
2116            "type": "result",
2117            "subtype": "success",
2118            "is_error": false,
2119            "result": inner,
2120            "usage": {"input_tokens": 100, "output_tokens": 80}
2121        });
2122
2123        let parsed = router.parse_llm_response(&envelope.to_string());
2124        assert!(
2125            parsed.is_ok(),
2126            "should handle prose + code block in envelope: {}",
2127            parsed.unwrap_err()
2128        );
2129        assert_eq!(parsed.unwrap().executor, "codex");
2130    }
2131
2132    #[serial(cooldown_state)]
2133    #[tokio::test]
2134    async fn route_round_robin_basic() {
2135        crate::engine::cooldown::reset_global_state().await;
2136        // Force at least one agent to be available for testing
2137        // In real usage, discover_agents finds installed CLIs
2138        let config = RouterConfig {
2139            mode: "round_robin".to_string(),
2140            ..Default::default()
2141        };
2142
2143        // Create router with mock available agents
2144        let agents = vec!["claude".to_string(), "codex".to_string()];
2145        let mut weights = AgentWeights::default();
2146        weights.ensure_agents(&agents);
2147        let router = Router {
2148            config,
2149            available_agents: agents,
2150            weights,
2151            llm_router: std::sync::Arc::new(LlmRouter::new()),
2152            ollama_router: None,
2153            rr_index: 0,
2154            last_agent: None,
2155            review_rr_index: 0,
2156            router_pool: vec![],
2157            pool_index: 0,
2158        };
2159
2160        let task = create_test_task("1", "Test task", vec![]);
2161        let result =
2162            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2163                .unwrap();
2164
2165        // Task 1 % 2 agents = agent at index 1 = codex
2166        assert_eq!(result.agent, "codex");
2167        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2168    }
2169
2170    #[serial(cooldown_state)]
2171    #[tokio::test]
2172    async fn route_uses_label_override() {
2173        crate::engine::cooldown::reset_global_state().await;
2174        let mut config = RouterConfig::default();
2175        // Ensure a concrete model exists so the label override can be honored
2176        // in the test environment where no external config.yml is present.
2177        config
2178            .model_map
2179            .entry("medium".to_string())
2180            .or_default()
2181            .insert("claude".to_string(), vec!["haiku".to_string()]);
2182        let agents = vec!["claude".to_string(), "codex".to_string()];
2183        let mut weights = AgentWeights::default();
2184        weights.ensure_agents(&agents);
2185        let mut router = Router {
2186            config,
2187            available_agents: agents,
2188            weights,
2189            llm_router: std::sync::Arc::new(LlmRouter::new()),
2190            ollama_router: None,
2191            rr_index: 0,
2192            last_agent: None,
2193            review_rr_index: 0,
2194            router_pool: vec![],
2195            pool_index: 0,
2196        };
2197
2198        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2199        let store = test_store().await;
2200
2201        // Should use label override, not LLM
2202        let result = router.route(&task, &store, "test/repo").await.unwrap();
2203        assert_eq!(result.agent, "claude");
2204        assert!(result.reason.contains("label"));
2205    }
2206
2207    #[serial(cooldown_state)]
2208    #[test]
2209    fn check_routing_sanity_warnings() {
2210        let config = RouterConfig::default();
2211        let router = Router::new(config);
2212
2213        // Backend task routed to claude should warn
2214        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2215        let profile = AgentProfile {
2216            role: "general".to_string(),
2217            skills: vec!["api".to_string()],
2218            tools: vec![],
2219            constraints: vec![],
2220        };
2221        let warning = router.check_routing_sanity(&task, "claude", &profile);
2222        assert!(warning.is_some());
2223        assert!(warning.unwrap().contains("backend"));
2224
2225        // Docs task routed to codex should warn
2226        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2227        let warning = router.check_routing_sanity(&task, "codex", &profile);
2228        assert!(warning.is_some());
2229        assert!(warning.unwrap().contains("docs"));
2230
2231        // Normal routing should not warn
2232        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2233        let warning = router.check_routing_sanity(&task, "codex", &profile);
2234        assert!(warning.is_none());
2235    }
2236
2237    #[serial(cooldown_state)]
2238    #[tokio::test]
2239    async fn router_reload_preserves_structure() {
2240        crate::engine::cooldown::reset_global_state().await;
2241        let config = RouterConfig::default();
2242        let agents = vec!["claude".to_string()];
2243        let mut weights = AgentWeights::default();
2244        weights.ensure_agents(&agents);
2245        let mut router = Router {
2246            config,
2247            available_agents: agents,
2248            weights,
2249            llm_router: std::sync::Arc::new(LlmRouter::new()),
2250            ollama_router: None,
2251            rr_index: 0,
2252            last_agent: None,
2253            review_rr_index: 0,
2254            router_pool: vec![],
2255            pool_index: 0,
2256        };
2257
2258        // Reload — should re-read config and remain valid
2259        router.reload_async().await;
2260
2261        // After reload, mode should be a valid value (llm, local, or round_robin)
2262        assert!(
2263            router.config.mode == "llm"
2264                || router.config.mode == "round_robin"
2265                || router.config.mode == "local",
2266            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2267            router.config.mode
2268        );
2269        // Fallback executor should always be set
2270        assert!(!router.config.fallback_executor.is_empty());
2271        // Tools should always be populated
2272        assert!(!router.config.allowed_tools.is_empty());
2273    }
2274
2275    // --- Weighted round-robin tests ---
2276
2277    #[serial(cooldown_state)]
2278    #[test]
2279    fn rate_limit_state_defaults_to_full_weight() {
2280        let state = RateLimitState::default();
2281        assert_eq!(state.weight, DEFAULT_WEIGHT);
2282        assert_eq!(state.consecutive_hits, 0);
2283        assert!(state.last_limited_at.is_none());
2284        assert!(!state.is_limited());
2285    }
2286
2287    #[serial(cooldown_state)]
2288    #[test]
2289    fn rate_limit_state_decays_on_hit() {
2290        let mut state = RateLimitState::default();
2291        state.record_rate_limit();
2292
2293        assert!(state.weight < DEFAULT_WEIGHT);
2294        assert_eq!(state.consecutive_hits, 1);
2295        assert!(state.last_limited_at.is_some());
2296        assert!(state.is_limited());
2297
2298        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2299        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2300        assert!((state.weight - expected).abs() < 1e-10);
2301    }
2302
2303    #[serial(cooldown_state)]
2304    #[test]
2305    fn rate_limit_state_never_drops_below_min() {
2306        let mut state = RateLimitState::default();
2307
2308        // Hit many times
2309        for _ in 0..100 {
2310            state.record_rate_limit();
2311        }
2312
2313        assert!(state.weight >= MIN_WEIGHT);
2314        assert_eq!(state.consecutive_hits, 100);
2315    }
2316
2317    #[serial(cooldown_state)]
2318    #[test]
2319    fn rate_limit_state_recovers_on_success() {
2320        let mut state = RateLimitState::default();
2321
2322        // Decay first
2323        state.record_rate_limit();
2324        state.record_rate_limit();
2325        let after_decay = state.weight;
2326
2327        // Record success
2328        state.record_success();
2329        assert!(state.weight > after_decay);
2330        assert_eq!(state.consecutive_hits, 0);
2331    }
2332
2333    #[serial(cooldown_state)]
2334    #[test]
2335    fn rate_limit_state_success_caps_at_default() {
2336        let mut state = RateLimitState::default();
2337
2338        // Already at full weight, success shouldn't exceed it
2339        state.record_success();
2340        assert_eq!(state.weight, DEFAULT_WEIGHT);
2341    }
2342
2343    #[serial(cooldown_state)]
2344    #[test]
2345    fn agent_weights_ensure_agents() {
2346        let mut weights = AgentWeights::default();
2347        let agents = vec!["claude".to_string(), "codex".to_string()];
2348        weights.ensure_agents(&agents);
2349
2350        assert_eq!(weights.states.len(), 2);
2351        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2352        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2353    }
2354
2355    #[serial(cooldown_state)]
2356    #[test]
2357    fn agent_weights_ensure_agents_preserves_existing() {
2358        let mut weights = AgentWeights::default();
2359        let agents = vec!["claude".to_string(), "codex".to_string()];
2360        weights.ensure_agents(&agents);
2361
2362        // Reduce claude's weight
2363        weights.record_rate_limit("claude");
2364        let claude_weight = weights.get_weight("claude");
2365        assert!(claude_weight < DEFAULT_WEIGHT);
2366
2367        // Ensure agents again — claude's weight should be preserved
2368        let agents2 = vec![
2369            "claude".to_string(),
2370            "codex".to_string(),
2371            "opencode".to_string(),
2372        ];
2373        weights.ensure_agents(&agents2);
2374
2375        assert_eq!(weights.get_weight("claude"), claude_weight);
2376        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2377    }
2378
2379    #[test]
2380    fn default_missing_weight_uses_minimum_of_configured() {
2381        use super::weights::default_missing_weight;
2382
2383        let mut configured = std::collections::HashMap::new();
2384        configured.insert("claude".to_string(), 0.2);
2385        configured.insert("codex".to_string(), 0.2);
2386        configured.insert("opencode".to_string(), 0.2);
2387        configured.insert("kimi".to_string(), 0.2);
2388
2389        // minimax omitted from router.weights — must not silently outrank
2390        // every explicitly configured agent (issue #3596).
2391        assert_eq!(default_missing_weight(&configured), 0.2);
2392    }
2393
2394    #[test]
2395    fn default_missing_weight_falls_back_when_none_configured() {
2396        use super::weights::default_missing_weight;
2397
2398        let configured = std::collections::HashMap::new();
2399        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2400    }
2401
2402    #[serial(cooldown_state)]
2403    #[test]
2404    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2405        let mut weights = AgentWeights::default();
2406        let mut configured = std::collections::HashMap::new();
2407        configured.insert("claude".to_string(), 0.2);
2408        configured.insert("codex".to_string(), 0.2);
2409        configured.insert("opencode".to_string(), 0.2);
2410        configured.insert("kimi".to_string(), 0.2);
2411
2412        let agents = vec![
2413            "claude".to_string(),
2414            "codex".to_string(),
2415            "opencode".to_string(),
2416            "kimi".to_string(),
2417            "minimax".to_string(),
2418        ];
2419        weights.ensure_agents_with_weights(&agents, &configured);
2420
2421        // minimax was omitted from router.weights but must not get the old
2422        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2423        assert_eq!(weights.get_weight("minimax"), 0.2);
2424        assert_eq!(weights.get_weight("claude"), 0.2);
2425    }
2426
2427    #[serial(cooldown_state)]
2428    #[test]
2429    fn agent_weights_record_rate_limit() {
2430        let mut weights = AgentWeights::default();
2431        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2432
2433        weights.record_rate_limit("claude");
2434
2435        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2436        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2437    }
2438
2439    #[serial(cooldown_state)]
2440    #[test]
2441    fn agent_weights_record_success() {
2442        let mut weights = AgentWeights::default();
2443        weights.ensure_agents(&["claude".to_string()]);
2444
2445        weights.record_rate_limit("claude");
2446        let after_limit = weights.get_weight("claude");
2447
2448        weights.record_success("claude");
2449        assert!(weights.get_weight("claude") > after_limit);
2450    }
2451
2452    #[serial(cooldown_state)]
2453    #[test]
2454    fn agent_weights_snapshot() {
2455        let mut weights = AgentWeights::default();
2456        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2457        weights.record_rate_limit("claude");
2458
2459        let snap = weights.snapshot();
2460        assert_eq!(snap.len(), 2);
2461
2462        // Snapshot is sorted alphabetically
2463        assert_eq!(snap[0].0, "claude");
2464        assert!(snap[0].1 < DEFAULT_WEIGHT);
2465        assert_eq!(snap[0].2, 1); // 1 hit
2466
2467        assert_eq!(snap[1].0, "codex");
2468        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2469        assert_eq!(snap[1].2, 0); // 0 hits
2470    }
2471
2472    #[serial(cooldown_state)]
2473    #[test]
2474    fn agent_weights_weighted_select_favors_higher_weight() {
2475        let mut weights = AgentWeights::default();
2476        let agents = vec!["claude".to_string(), "codex".to_string()];
2477        weights.ensure_agents(&agents);
2478
2479        // Heavily penalize claude
2480        for _ in 0..10 {
2481            weights.record_rate_limit("claude");
2482        }
2483
2484        // Run many selections and count
2485        let mut claude_count = 0;
2486        let mut codex_count = 0;
2487        for _ in 0..100 {
2488            match weights.weighted_select(&agents, "task-1") {
2489                Some(ref a) if a == "claude" => claude_count += 1,
2490                Some(ref a) if a == "codex" => codex_count += 1,
2491                _ => {}
2492            }
2493        }
2494
2495        // Codex should get significantly more selections
2496        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2497        assert!(
2498            codex_count > claude_count,
2499            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2500        );
2501    }
2502
2503    #[serial(cooldown_state)]
2504    #[test]
2505    fn agent_weights_weighted_select_empty_returns_none() {
2506        let weights = AgentWeights::default();
2507        assert!(weights.weighted_select(&[], "task-1").is_none());
2508    }
2509
2510    #[serial(cooldown_state)]
2511    #[test]
2512    fn agent_weights_batch_routing_produces_varied_selections() {
2513        // Regression test: when routing a batch of tasks in the same tick,
2514        // different task IDs should not all select the same agent.
2515        let mut weights = AgentWeights::default();
2516        let agents = vec!["claude".to_string(), "codex".to_string()];
2517        weights.ensure_agents(&agents);
2518
2519        // Simulate batch routing: 20 different task IDs called in rapid succession
2520        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2521        let selections: Vec<String> = task_ids
2522            .iter()
2523            .filter_map(|id| weights.weighted_select(&agents, id))
2524            .collect();
2525
2526        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2527        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2528        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2529        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2530        assert!(
2531            claude_count > 0 && codex_count > 0,
2532            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2533        );
2534    }
2535
2536    #[serial(cooldown_state)]
2537    #[test]
2538    fn agent_weights_weighted_select_single_agent() {
2539        let mut weights = AgentWeights::default();
2540        let agents = vec!["claude".to_string()];
2541        weights.ensure_agents(&agents);
2542
2543        let selected = weights.weighted_select(&agents, "task-1");
2544        assert_eq!(selected, Some("claude".to_string()));
2545    }
2546
2547    #[serial(cooldown_state)]
2548    #[test]
2549    fn agent_weights_tick_recovery_restores_weight() {
2550        let mut weights = AgentWeights::default();
2551        weights.ensure_agents(&["claude".to_string()]);
2552
2553        // Set up a rate limit that looks like it happened long ago
2554        let state = weights.states.get_mut("claude").unwrap();
2555        state.weight = 0.5;
2556        state.last_limited_at = Some(Instant::now() - RECOVERY_DELAY - Duration::from_secs(1));
2557        state.consecutive_hits = 2;
2558
2559        let before = weights.get_weight("claude");
2560        weights.tick_recovery();
2561        let after = weights.get_weight("claude");
2562
2563        assert!(after > before, "weight should increase after recovery tick");
2564    }
2565
2566    #[serial(cooldown_state)]
2567    #[test]
2568    fn agent_weights_tick_recovery_clears_on_full_restore() {
2569        let mut weights = AgentWeights::default();
2570        weights.ensure_agents(&["claude".to_string()]);
2571
2572        // Set weight just below full with an old limit
2573        let state = weights.states.get_mut("claude").unwrap();
2574        state.weight = DEFAULT_WEIGHT - 0.01;
2575        state.last_limited_at = Some(Instant::now() - RECOVERY_DELAY - Duration::from_secs(1));
2576        state.consecutive_hits = 1;
2577
2578        weights.tick_recovery();
2579
2580        let state = weights.states.get("claude").unwrap();
2581        assert_eq!(state.weight, DEFAULT_WEIGHT);
2582        assert!(state.last_limited_at.is_none());
2583        assert_eq!(state.consecutive_hits, 0);
2584    }
2585
2586    #[serial(cooldown_state)]
2587    #[tokio::test]
2588    async fn route_weighted_round_robin_basic() {
2589        crate::engine::cooldown::reset_global_state().await;
2590        let mut config = RouterConfig {
2591            weighted_round_robin: true,
2592            ..Default::default()
2593        };
2594        // Provide a concrete model for codex so label override can be used.
2595        config
2596            .model_map
2597            .entry("medium".to_string())
2598            .or_default()
2599            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2600
2601        let agents = vec!["claude".to_string(), "codex".to_string()];
2602        let mut weights = AgentWeights::default();
2603        weights.ensure_agents(&agents);
2604        let mut router = Router {
2605            config,
2606            available_agents: agents,
2607            weights,
2608            llm_router: std::sync::Arc::new(LlmRouter::new()),
2609            ollama_router: None,
2610            rr_index: 0,
2611            last_agent: None,
2612            review_rr_index: 0,
2613            router_pool: vec![],
2614            pool_index: 0,
2615        };
2616
2617        let task = create_test_task("1", "Test task", vec![]);
2618        let store = test_store().await;
2619        let result = router.route(&task, &store, "test/repo").await.unwrap();
2620
2621        // Should use weighted_round_robin
2622        assert!(result.reason.contains("weighted_round_robin"));
2623        assert!(
2624            result.agent == "claude" || result.agent == "codex",
2625            "agent should be claude or codex, got '{}'",
2626            result.agent
2627        );
2628    }
2629
2630    #[serial(cooldown_state)]
2631    #[tokio::test]
2632    async fn route_weighted_round_robin_respects_label_override() {
2633        crate::engine::cooldown::reset_global_state().await;
2634        let mut config = RouterConfig {
2635            weighted_round_robin: true,
2636            ..Default::default()
2637        };
2638        // Provide a concrete model for codex so label override can be used.
2639        config
2640            .model_map
2641            .entry("medium".to_string())
2642            .or_default()
2643            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2644
2645        let agents = vec!["claude".to_string(), "codex".to_string()];
2646        let mut weights = AgentWeights::default();
2647        weights.ensure_agents(&agents);
2648        let mut router = Router {
2649            config,
2650            available_agents: agents,
2651            weights,
2652            llm_router: std::sync::Arc::new(LlmRouter::new()),
2653            ollama_router: None,
2654            rr_index: 0,
2655            last_agent: None,
2656            review_rr_index: 0,
2657            router_pool: vec![],
2658            pool_index: 0,
2659        };
2660
2661        // Label override should take precedence over weighted routing
2662        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2663        let store = test_store().await;
2664        let result = router.route(&task, &store, "test/repo").await.unwrap();
2665        assert_eq!(result.agent, "codex");
2666        assert!(result.reason.contains("label"));
2667    }
2668
2669    #[serial(cooldown_state)]
2670    #[test]
2671    fn router_config_weighted_round_robin_default_false() {
2672        let config = RouterConfig::default();
2673        assert!(!config.weighted_round_robin);
2674    }
2675
2676    #[serial(cooldown_state)]
2677    #[test]
2678    fn router_record_and_recover_weights() {
2679        let config = RouterConfig {
2680            weighted_round_robin: true,
2681            ..Default::default()
2682        };
2683        let mut router = Router::new(config);
2684        // Override discovered agents for test
2685        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2686        router.weights.ensure_agents(&router.available_agents);
2687
2688        // Record rate limit
2689        router.record_rate_limit("claude");
2690        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2691        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2692
2693        // Record success for claude
2694        router.record_success("claude");
2695        let after_success = router.weights.get_weight("claude");
2696        assert!(after_success > MIN_WEIGHT);
2697    }
2698
2699    #[serial(cooldown_state)]
2700    #[test]
2701    fn review_rr_index_advances() {
2702        let mut config = RouterConfig::default();
2703        // Set up model_map so has_available_model_for_complexity returns true for test agents
2704        for agent in &["test_a", "test_b", "test_c"] {
2705            config
2706                .model_map
2707                .entry("medium".to_string())
2708                .or_default()
2709                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2710        }
2711        let agents = vec![
2712            "test_a".to_string(),
2713            "test_b".to_string(),
2714            "test_c".to_string(),
2715        ];
2716        let mut weights = AgentWeights::default();
2717        weights.ensure_agents(&agents);
2718        let mut router = Router {
2719            config,
2720            available_agents: agents,
2721            weights,
2722            llm_router: std::sync::Arc::new(LlmRouter::new()),
2723            ollama_router: None,
2724            rr_index: 0,
2725            last_agent: None,
2726            review_rr_index: 0,
2727            router_pool: vec![],
2728            pool_index: 0,
2729        };
2730
2731        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2732        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2733        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2734
2735        // All three agents should appear (order depends on start index)
2736        let mut seen = vec![a1, a2, a3];
2737        seen.sort();
2738        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2739    }
2740
2741    #[serial(cooldown_state)]
2742    #[test]
2743    fn review_rr_fallback_with_cooled_agent() {
2744        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2745        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2746        // candidates.get(idx) to return None even though candidates is non-empty.
2747        let mut config = RouterConfig::default();
2748        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2749        for agent in &["test_a", "test_b", "test_c"] {
2750            config
2751                .model_map
2752                .entry("medium".to_string())
2753                .or_default()
2754                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2755        }
2756        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2757        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2758        let mut weights = AgentWeights::default();
2759        weights.ensure_agents(&agents);
2760        let mut router = Router {
2761            config,
2762            available_agents: agents,
2763            weights,
2764            llm_router: std::sync::Arc::new(LlmRouter::new()),
2765            ollama_router: None,
2766            rr_index: 0,
2767            last_agent: None,
2768            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2769            review_rr_index: 2,
2770            router_pool: vec![],
2771            pool_index: 0,
2772        };
2773
2774        // All candidates excluded — should fall back to first candidate, not None
2775        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2776        assert!(
2777            result.is_some(),
2778            "next_round_robin_agent must not return None when candidates is non-empty"
2779        );
2780    }
2781
2782    #[serial(cooldown_state)]
2783    #[test]
2784    fn review_rr_index_uses_candidates_len_not_available_len() {
2785        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2786        // not available_agents.len(). When an agent has no model for the complexity,
2787        // candidates.len() < available_agents.len() and the old code produced a drifting
2788        // index that caused non-uniform distribution.
2789        let mut config = RouterConfig::default();
2790        // Only register models for test_a and test_b — test_c has no model, so it is
2791        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2792        for agent in &["test_a", "test_b"] {
2793            config
2794                .model_map
2795                .entry("medium".to_string())
2796                .or_default()
2797                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2798        }
2799        let agents = vec![
2800            "test_a".to_string(),
2801            "test_b".to_string(),
2802            "test_c".to_string(), // available but no model → filtered out of candidates
2803        ];
2804        let mut weights = AgentWeights::default();
2805        weights.ensure_agents(&agents);
2806        let mut router = Router {
2807            config,
2808            available_agents: agents,
2809            weights,
2810            llm_router: std::sync::Arc::new(LlmRouter::new()),
2811            ollama_router: None,
2812            rr_index: 0,
2813            last_agent: None,
2814            review_rr_index: 0,
2815            router_pool: vec![],
2816            pool_index: 0,
2817        };
2818
2819        // With the bug: index advances modulo 3 (available_agents.len()), so the
2820        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2821        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2822        // looks uniform by accident, but adding a 4th available agent breaks it.
2823        //
2824        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2825        // cycle. Verify this by checking review_rr_index after each call.
2826
2827        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2828        assert_eq!(
2829            router.review_rr_index, 1,
2830            "index should be 1 after first call"
2831        );
2832
2833        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2834        assert_eq!(
2835            router.review_rr_index, 0,
2836            "index should wrap to 0 after second call"
2837        );
2838
2839        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2840        assert_eq!(
2841            router.review_rr_index, 1,
2842            "index should be 1 after third call"
2843        );
2844
2845        // Agents must alternate uniformly
2846        assert_ne!(r1, r2, "consecutive calls should select different agents");
2847        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2848    }
2849
2850    #[serial(cooldown_state)]
2851    #[tokio::test]
2852    async fn last_agent_tracks_routing() {
2853        crate::engine::cooldown::reset_global_state().await;
2854        let mut config = RouterConfig {
2855            mode: "round_robin".to_string(),
2856            ..Default::default()
2857        };
2858        config
2859            .model_map
2860            .entry("medium".to_string())
2861            .or_default()
2862            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
2863        config
2864            .model_map
2865            .entry("medium".to_string())
2866            .or_default()
2867            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
2868        let agents = vec!["test_x".to_string(), "test_y".to_string()];
2869        let mut weights = AgentWeights::default();
2870        weights.ensure_agents(&agents);
2871        let mut router = Router {
2872            config,
2873            available_agents: agents,
2874            weights,
2875            llm_router: std::sync::Arc::new(LlmRouter::new()),
2876            ollama_router: None,
2877            rr_index: 0,
2878            last_agent: None,
2879            review_rr_index: 0,
2880            router_pool: vec![],
2881            pool_index: 0,
2882        };
2883
2884        assert!(router.last_agent.is_none());
2885
2886        let task = create_test_task("1", "Test routing", vec![]);
2887        let store = test_store().await;
2888        let result = router.route(&task, &store, "test/repo").await.unwrap();
2889
2890        assert!(router.last_agent.is_some());
2891        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
2892    }
2893
2894    // ── pool config ───────────────────────────────────────────────────────────
2895
2896    #[serial(cooldown_state)]
2897    #[test]
2898    fn parse_pool_entry_splits_on_first_colon() {
2899        let (agent, model) = super::parse_pool_entry("claude:haiku");
2900        assert_eq!(agent, "claude");
2901        assert_eq!(model, "haiku");
2902    }
2903
2904    #[serial(cooldown_state)]
2905    #[test]
2906    fn parse_pool_entry_model_with_slash() {
2907        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
2908        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
2909        assert_eq!(agent, "opencode");
2910        assert_eq!(model, "github-copilot/gpt-5-mini");
2911    }
2912
2913    #[serial(cooldown_state)]
2914    #[test]
2915    fn parse_pool_entry_no_colon() {
2916        let (agent, model) = super::parse_pool_entry("claude");
2917        assert_eq!(agent, "claude");
2918        assert_eq!(model, "");
2919    }
2920
2921    #[serial(cooldown_state)]
2922    #[test]
2923    fn effective_pool_uses_configured_pool() {
2924        let config = RouterConfig {
2925            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
2926            ..RouterConfig::default()
2927        };
2928        let pool = config.effective_pool();
2929        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
2930    }
2931
2932    #[serial(cooldown_state)]
2933    #[test]
2934    fn effective_pool_falls_back_to_router_agent_model() {
2935        let config = RouterConfig::default();
2936        // Default has no pool → derives from router_agent:router_model
2937        let pool = config.effective_pool();
2938        assert_eq!(pool.len(), 1);
2939        assert!(pool[0].starts_with(&config.router_agent));
2940        assert!(pool[0].contains(&config.router_model));
2941    }
2942
2943    #[serial(cooldown_state)]
2944    #[test]
2945    fn effective_fallback_uses_configured_fallback() {
2946        let config = RouterConfig {
2947            fallback: "claude:haiku".to_string(),
2948            ..RouterConfig::default()
2949        };
2950        assert_eq!(config.effective_fallback(), "claude:haiku");
2951    }
2952
2953    #[serial(cooldown_state)]
2954    #[test]
2955    fn effective_fallback_derives_from_router_agent_model() {
2956        let config = RouterConfig::default();
2957        let fallback = config.effective_fallback();
2958        assert!(fallback.starts_with(&config.router_agent));
2959        assert!(fallback.contains(&config.router_model));
2960    }
2961
2962    // ── pool expansion ────────────────────────────────────────────────────────
2963
2964    #[serial(cooldown_state)]
2965    #[test]
2966    fn expand_pool_basic_entries() {
2967        let config = RouterConfig {
2968            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
2969            ..RouterConfig::default()
2970        };
2971        let expanded = Router::expand_pool(&config);
2972        assert_eq!(expanded.len(), 2);
2973        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
2974        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
2975    }
2976
2977    #[serial(cooldown_state)]
2978    #[test]
2979    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
2980        let config = RouterConfig {
2981            pool: vec![],
2982            router_agent: "claude".to_string(),
2983            router_model: "haiku".to_string(),
2984            ..RouterConfig::default()
2985        };
2986        let expanded = Router::expand_pool(&config);
2987        assert_eq!(expanded.len(), 1);
2988        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
2989    }
2990
2991    #[serial(cooldown_state)]
2992    #[test]
2993    fn expand_pool_opencode_free_skipped_when_not_installed() {
2994        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
2995        // to nothing and the pool falls back to effective_fallback
2996        let config = RouterConfig {
2997            pool: vec!["opencode:free".to_string()],
2998            fallback: "claude:haiku".to_string(),
2999            ..RouterConfig::default()
3000        };
3001        let expanded = Router::expand_pool(&config);
3002        assert_eq!(expanded.len(), 1);
3003        assert_eq!(expanded[0].0, "claude");
3004        assert_eq!(expanded[0].1, "haiku");
3005    }
3006
3007    #[serial(cooldown_state)]
3008    #[test]
3009    fn expand_pool_single_entry_from_default() {
3010        let config = RouterConfig::default();
3011        let expanded = Router::expand_pool(&config);
3012        assert!(
3013            !expanded.is_empty(),
3014            "pool must never be empty after expansion"
3015        );
3016    }
3017
3018    // ── pool round-robin index ────────────────────────────────────────────────
3019
3020    #[serial(cooldown_state)]
3021    #[test]
3022    fn pool_index_initializes_to_zero() {
3023        let config = RouterConfig::default();
3024        let router = Router::new(config);
3025        assert_eq!(router.pool_index, 0);
3026    }
3027
3028    #[serial(cooldown_state)]
3029    #[test]
3030    fn router_pool_non_empty_after_construction() {
3031        let config = RouterConfig::default();
3032        let router = Router::new(config);
3033        assert!(!router.router_pool.is_empty());
3034    }
3035
3036    /**
3037     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3038     * `opencode serve --service` child behind, so a suite run leaked one server per test
3039     * process. The stub on PATH records any exec, so this fails if discovery comes back
3040     **/
3041    #[cfg(unix)]
3042    #[serial(cooldown_state)]
3043    #[test]
3044    fn building_a_router_never_runs_opencode() {
3045        use std::os::unix::fs::PermissionsExt;
3046
3047        struct PathRestore(Option<std::ffi::OsString>);
3048        impl Drop for PathRestore {
3049            fn drop(&mut self) {
3050                match &self.0 {
3051                    Some(p) => std::env::set_var("PATH", p),
3052                    None => std::env::remove_var("PATH"),
3053                }
3054                crate::cmd_cache::reset_for_test();
3055            }
3056        }
3057
3058        let dir = tempfile::tempdir().unwrap();
3059        let marker = dir.path().join("spawned");
3060        let bin_dir = dir.path().join("bin");
3061        std::fs::create_dir(&bin_dir).unwrap();
3062        let stub = bin_dir.join("opencode");
3063        std::fs::write(
3064            &stub,
3065            format!(
3066                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3067                marker.display()
3068            ),
3069        )
3070        .unwrap();
3071        std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap();
3072
3073        let prev_path = std::env::var_os("PATH");
3074        let mut new_path = bin_dir.as_os_str().to_os_string();
3075        new_path.push(":");
3076        if let Some(ref p) = prev_path {
3077            new_path.push(p);
3078        }
3079        std::env::set_var("PATH", &new_path);
3080        crate::cmd_cache::reset_for_test();
3081        let _restore = PathRestore(prev_path);
3082
3083        // A config that names opencode everywhere discovery reads from
3084        let mut config = RouterConfig {
3085            pool: vec!["opencode:free".to_string()],
3086            fallback: "opencode:free".to_string(),
3087            ..RouterConfig::default()
3088        };
3089        config
3090            .model_map
3091            .entry("simple".to_string())
3092            .or_default()
3093            .insert(
3094                "opencode".to_string(),
3095                vec!["canary-model-free".to_string()],
3096            );
3097
3098        let _router = Router::new(config.clone());
3099        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3100        let _pool = Router::expand_pool(&config);
3101        // Discovery refreshes on a background thread, give it room to exec
3102        std::thread::sleep(Duration::from_millis(300));
3103
3104        assert!(
3105            !marker.exists(),
3106            "building a router ran opencode: {}",
3107            std::fs::read_to_string(&marker).unwrap_or_default().trim()
3108        );
3109        assert!(
3110            opencode_children().is_empty(),
3111            "building a router left opencode processes: {:?}",
3112            opencode_children()
3113        );
3114    }
3115
3116    /// Commands of this process's direct children that mention opencode
3117    #[cfg(unix)]
3118    fn opencode_children() -> Vec<String> {
3119        let me = std::process::id().to_string();
3120        let output = std::process::Command::new("ps")
3121            .args(["-axo", "ppid=,command="])
3122            .output()
3123            .expect("ps");
3124        String::from_utf8_lossy(&output.stdout)
3125            .lines()
3126            .filter_map(|line| {
3127                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3128                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3129            })
3130            .collect()
3131    }
3132
3133    #[serial(cooldown_state)]
3134    #[tokio::test]
3135    async fn reload_resets_pool_index() {
3136        crate::engine::cooldown::reset_global_state().await;
3137        let config = RouterConfig::default();
3138        let mut router = Router::new(config);
3139        router.pool_index = 5;
3140        router.reload_async().await;
3141        assert_eq!(router.pool_index, 0);
3142    }
3143
3144    #[serial(cooldown_state)]
3145    #[test]
3146    fn pool_index_advances_after_failed_attempt() {
3147        let mut router = Router::new(RouterConfig::default());
3148        router.router_pool = vec![
3149            ("claude".to_string(), "haiku".to_string()),
3150            ("codex".to_string(), "gpt-5.2".to_string()),
3151            (
3152                "opencode".to_string(),
3153                "github-copilot/gpt-5-mini".to_string(),
3154            ),
3155        ];
3156        router.pool_index = 0;
3157
3158        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3159
3160        assert_eq!(router.pool_index, 1);
3161    }
3162
3163    #[serial(cooldown_state)]
3164    #[test]
3165    fn pool_index_wraps_after_last_attempt() {
3166        let mut router = Router::new(RouterConfig::default());
3167        router.router_pool = vec![
3168            ("claude".to_string(), "haiku".to_string()),
3169            ("codex".to_string(), "gpt-5.2".to_string()),
3170            (
3171                "opencode".to_string(),
3172                "github-copilot/gpt-5-mini".to_string(),
3173            ),
3174        ];
3175        router.pool_index = 2;
3176
3177        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3178
3179        assert_eq!(router.pool_index, 0);
3180    }
3181
3182    #[serial(cooldown_state)]
3183    #[test]
3184    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3185        let router = Router::new(RouterConfig::default());
3186        assert!(router.earliest_pool_cooldown().is_none());
3187    }
3188
3189    #[serial(cooldown_state)]
3190    #[tokio::test]
3191    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3192        crate::engine::cooldown::reset_global_state().await;
3193        // Use explicit pool with predictable model names to avoid opencode discovery.
3194        let config = RouterConfig {
3195            pool: vec![
3196                "claude:haiku-4-5-20251001".to_string(),
3197                "opencode:haiku-4-5-20251001".to_string(),
3198            ],
3199            ..Default::default()
3200        };
3201        let router = Router::new(config);
3202        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3203        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3204
3205        let earliest = router.earliest_pool_cooldown();
3206        assert!(earliest.is_some());
3207        let now = chrono::Utc::now().timestamp();
3208        let remaining = earliest.unwrap() - now;
3209        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3210        assert!((295..=300).contains(&remaining));
3211    }
3212
3213    #[serial(cooldown_state)]
3214    #[tokio::test]
3215    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3216        crate::engine::cooldown::reset_global_state().await;
3217        // Use explicit pool so we can predict what entries exist.
3218        let config = RouterConfig {
3219            pool: vec![
3220                "claude:pool-model-a".to_string(),
3221                "opencode:pool-model-b".to_string(),
3222            ],
3223            ..Default::default()
3224        };
3225        let router = Router::new(config);
3226        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3227        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3228        // Set cooldown on a pool entry (opencode:pool-model-b)
3229        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3230
3231        let earliest = router.earliest_pool_cooldown();
3232        assert!(earliest.is_some());
3233        let now = chrono::Utc::now().timestamp();
3234        let remaining = earliest.unwrap() - now;
3235        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3236        assert!((295..=300).contains(&remaining));
3237    }
3238
3239    // ---- Pre-emptive health check: degraded agent exclusion ----
3240
3241    #[serial(cooldown_state)]
3242    #[test]
3243    fn degraded_agent_excluded_from_routing() {
3244        let mut config = RouterConfig::default();
3245        config
3246            .model_map
3247            .entry("medium".to_string())
3248            .or_default()
3249            .insert(
3250                "test_degraded_routing".to_string(),
3251                vec!["model-a".to_string()],
3252            );
3253
3254        let mut router = Router::new(config);
3255        router.available_agents = vec!["test_degraded_routing".to_string()];
3256
3257        // Agent is routable before degradation
3258        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3259
3260        // Mark as degraded
3261        mark_agent_degraded("test_degraded_routing");
3262
3263        // Now excluded from routing
3264        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3265
3266        // available_agents_for_complexity returns empty
3267        assert!(
3268            router.available_agents_for_complexity("medium").is_empty(),
3269            "degraded agent should be excluded from available agents"
3270        );
3271
3272        // healthy_agent_count reflects exclusion
3273        assert_eq!(router.healthy_agent_count("medium"), 0);
3274
3275        // Cleanup
3276        clear_agent_degraded("test_degraded_routing");
3277    }
3278
3279    #[serial(cooldown_state)]
3280    #[test]
3281    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3282        let mut config = RouterConfig::default();
3283        for agent in &["agent_healthy", "agent_degraded"] {
3284            config
3285                .model_map
3286                .entry("medium".to_string())
3287                .or_default()
3288                .insert(agent.to_string(), vec!["model-x".to_string()]);
3289        }
3290
3291        let mut router = Router::new(config);
3292        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3293
3294        mark_agent_degraded("agent_degraded");
3295
3296        let candidates = router.available_agents_for_complexity("medium");
3297        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3298        assert_eq!(router.healthy_agent_count("medium"), 1);
3299
3300        // Cleanup
3301        clear_agent_degraded("agent_degraded");
3302    }
3303
3304    #[serial(cooldown_state)]
3305    #[tokio::test]
3306    async fn refresh_health_marks_degraded_from_rate_limits() {
3307        crate::engine::cooldown::reset_global_state().await;
3308        let store = test_store().await;
3309        let agent = "test_refresh_health_agent";
3310
3311        let mut config = RouterConfig::default();
3312        config
3313            .model_map
3314            .entry("medium".to_string())
3315            .or_default()
3316            .insert(agent.to_string(), vec!["model-z".to_string()]);
3317
3318        let mut router = Router::new(config);
3319        router.available_agents = vec![agent.to_string()];
3320
3321        // Insert rate limit events exceeding default threshold (3)
3322        for _ in 0..4 {
3323            store.record_rate_limit(agent, "rate", None).await.unwrap();
3324        }
3325
3326        router.refresh_health(&store).await;
3327
3328        assert!(
3329            is_agent_degraded(agent),
3330            "agent should be degraded after exceeding rate limit threshold"
3331        );
3332        assert!(
3333            !router.agent_is_routable(agent, "medium"),
3334            "degraded agent should not be routable"
3335        );
3336
3337        // Cleanup
3338        clear_agent_degraded(agent);
3339    }
3340
3341    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3342
3343    #[serial(cooldown_state)]
3344    #[tokio::test]
3345    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3346        crate::engine::cooldown::reset_global_state().await;
3347        let mut config = RouterConfig::default();
3348        // Agent's only model, shared across every complexity tier it resolves to.
3349        for tier in ["simple", "medium", "complex", "review"] {
3350            config
3351                .model_map
3352                .entry(tier.to_string())
3353                .or_default()
3354                .insert(
3355                    "test_all_cooled".to_string(),
3356                    vec!["only-model".to_string()],
3357                );
3358        }
3359
3360        let mut router = Router::new(config);
3361        router.available_agents = vec!["test_all_cooled".to_string()];
3362
3363        assert!(router.agent_has_any_available_model("test_all_cooled"));
3364
3365        record_model_failure("test_all_cooled", "only-model").await;
3366        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3367
3368        assert!(
3369            !router.agent_has_any_available_model("test_all_cooled"),
3370            "agent with its only model cooled across every tier should have no available model"
3371        );
3372    }
3373
3374    #[serial(cooldown_state)]
3375    #[tokio::test]
3376    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3377        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3378        // agent that has no agent-level cooldown but whose only resolvable model (across
3379        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3380        crate::engine::cooldown::reset_global_state().await;
3381        let mut config = RouterConfig::default();
3382        for tier in ["simple", "medium", "complex", "review"] {
3383            config
3384                .model_map
3385                .entry(tier.to_string())
3386                .or_default()
3387                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3388            config
3389                .model_map
3390                .get_mut(tier)
3391                .unwrap()
3392                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3393        }
3394
3395        let mut router = Router::new(config);
3396        router.available_agents = vec![
3397            "agent_cooled_model".to_string(),
3398            "agent_healthy".to_string(),
3399        ];
3400
3401        record_model_failure("agent_cooled_model", "opus").await;
3402        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3403        // No agent-level cooldown — the old agent-only filter would still offer it.
3404        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3405
3406        let uncooled_agents: Vec<String> = router
3407            .available_agents
3408            .iter()
3409            .filter(|a| {
3410                !is_agent_in_cooldown(a)
3411                    && !is_agent_degraded(a)
3412                    && router.agent_has_available_model_for_coding_tiers(a)
3413            })
3414            .cloned()
3415            .collect();
3416
3417        assert_eq!(
3418            uncooled_agents,
3419            vec!["agent_healthy".to_string()],
3420            "agent whose only model is cooled must be excluded from the LLM candidate list"
3421        );
3422    }
3423
3424    #[serial(cooldown_state)]
3425    #[tokio::test]
3426    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3427        // Regression test for #3521: an agent whose only healthy tier is `simple`
3428        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3429        // excluded from the LLM classification candidate list, otherwise the
3430        // classifier can pick it for a `medium`/`complex` task it can't serve,
3431        // wasting the classification call. Mirrors the real-world shape where
3432        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3433        crate::engine::cooldown::reset_global_state().await;
3434        let mut config = RouterConfig::default();
3435        config
3436            .model_map
3437            .entry("simple".to_string())
3438            .or_default()
3439            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3440        config
3441            .model_map
3442            .entry("medium".to_string())
3443            .or_default()
3444            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3445        config
3446            .model_map
3447            .entry("complex".to_string())
3448            .or_default()
3449            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3450        // review must be irrelevant to the coding-tier filter
3451        config
3452            .model_map
3453            .entry("review".to_string())
3454            .or_default()
3455            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3456        for tier in ["simple", "medium", "complex", "review"] {
3457            config
3458                .model_map
3459                .get_mut(tier)
3460                .unwrap()
3461                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3462        }
3463
3464        let mut router = Router::new(config);
3465        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3466
3467        record_model_failure("minimax_like", "opus").await;
3468        assert!(is_model_in_cooldown("minimax_like", "opus"));
3469        // No agent-level cooldown, and the simple tier is healthy, so the loose
3470        // "any tier" check still passes — this is exactly the leak #3511 missed.
3471        assert!(!is_agent_in_cooldown("minimax_like"));
3472        assert!(router.agent_has_any_available_model("minimax_like"));
3473        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3474
3475        let uncooled_agents: Vec<String> = router
3476            .available_agents
3477            .iter()
3478            .filter(|a| {
3479                !is_agent_in_cooldown(a)
3480                    && !is_agent_degraded(a)
3481                    && router.agent_has_available_model_for_coding_tiers(a)
3482            })
3483            .cloned()
3484            .collect();
3485
3486        assert_eq!(
3487            uncooled_agents,
3488            vec!["agent_healthy".to_string()],
3489            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3490        );
3491    }
3492}