Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22pub use config::{parse_pool_entry, RouterConfig};
23pub use weights::AgentWeights;
24
25fn fmt_degraded(degraded: &[String]) -> String {
26    if degraded.is_empty() {
27        String::new()
28    } else {
29        format!(" (degraded: {})", degraded.join(", "))
30    }
31}
32
33/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
34///
35/// This is a transient error that should not count against `max_route_attempts` since
36/// no agent was actually invoked. The caller should back off and retry later.
37#[derive(Debug, thiserror::Error)]
38pub enum AllAgentsCooledError {
39    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
40    AllCooled {
41        scope: String,
42        remaining_secs: Option<u64>,
43        degraded: Vec<String>,
44    },
45}
46
47impl AllAgentsCooledError {
48    /// Create a new error for the given scope, naming currently degraded agents
49    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
50        Self::AllCooled {
51            scope,
52            remaining_secs,
53            degraded: list_degraded_agents()
54                .into_iter()
55                .map(|(agent, _reason)| agent)
56                .collect(),
57        }
58    }
59
60    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
61
62    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
63    pub fn scope(&self) -> &str {
64        match self {
65            Self::AllCooled { scope, .. } => scope,
66        }
67    }
68
69    /// Remaining cooldown duration in seconds when available.
70    pub fn remaining_secs(&self) -> Option<u64> {
71        match self {
72            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
73        }
74    }
75
76    /// Agents currently marked degraded, shown by `orch cooldown list`
77    pub fn degraded(&self) -> &[String] {
78        match self {
79            Self::AllCooled { degraded, .. } => degraded,
80        }
81    }
82}
83
84/// Typed error returned by [`get_route_result`].
85#[derive(Debug, thiserror::Error)]
86pub enum RouteResultError {
87    /// The task has no agent field — it needs to be re-routed.
88    #[error("no agent configured for task {task_id}")]
89    NoAgent { task_id: String },
90    /// Any other failure (store lookup, SQL, etc.).
91    #[error(transparent)]
92    Other(#[from] anyhow::Error),
93}
94
95use crate::backends::ExternalTask;
96use crate::engine::cooldown::{
97    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
98    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
99};
100use crate::store::{get_task_field_direct, store_log_activity, TaskStore};
101use serde::{Deserialize, Serialize};
102use std::sync::Arc;
103
104use llm::{LlmRouter, TIMEOUT_PREFIX};
105
106/// Result of routing a task to an agent.
107#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
108pub struct RouteResult {
109    /// The selected agent: "claude", "codex", or "opencode"
110    pub agent: String,
111    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
112    pub model: Option<String>,
113    /// Complexity level: "simple", "medium", or "complex"
114    pub complexity: String,
115    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
116    pub estimate: u8,
117    /// Why this agent was selected
118    pub reason: String,
119    /// Specialized agent profile (skills, tools, constraints)
120    pub profile: AgentProfile,
121    /// Selected skill IDs from the catalog
122    pub selected_skills: Vec<String>,
123    /// Optional warning about routing decision
124    pub warning: Option<String>,
125}
126
127/// Specialized agent profile for a task.
128#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
129pub struct AgentProfile {
130    /// Role name, e.g., "backend specialist"
131    pub role: String,
132    /// Focus skills for this task
133    pub skills: Vec<String>,
134    /// Tools allowed for this task
135    pub tools: Vec<String>,
136    /// Constraints for this task
137    pub constraints: Vec<String>,
138}
139
140impl AgentProfile {
141    /// Build the default "general" profile from router config.
142    pub fn default_from_config(config: &RouterConfig) -> Self {
143        Self {
144            role: "general".to_string(),
145            skills: vec![],
146            tools: config.allowed_tools.clone(),
147            constraints: vec![],
148        }
149    }
150}
151
152impl RouteResult {
153    /// Build a standard route result from the three task-specific values.
154    ///
155    /// Fills `profile`, `selected_skills`, and `warning` with router-config
156    /// defaults so call sites only need to supply what varies per task.
157    pub fn from_config(
158        agent: String,
159        complexity: String,
160        reason: String,
161        config: &RouterConfig,
162        task_id: &str,
163    ) -> Self {
164        let model = config.model_for_complexity(&agent, &complexity, task_id);
165        Self {
166            agent,
167            model,
168            complexity,
169            estimate: 0,
170            reason,
171            profile: AgentProfile::default_from_config(config),
172            selected_skills: config.default_skills.clone(),
173            warning: None,
174        }
175    }
176}
177
178/// An opaque, cheaply-cloneable handle to the LLM routing subsystem.
179///
180/// Obtain one via [`Router::llm_router_handle`].  Holding this handle does
181/// **not** require holding the router's `RwLock`, so async operations can
182/// safely be called without starving write-lock waiters.
183pub struct LlmRouterHandle(std::sync::Arc<LlmRouter>);
184
185impl LlmRouterHandle {
186    /// Invalidate the skills catalog cache so the next routing call reloads
187    /// from disk.  Safe to await without holding any `RwLock` on `Router`.
188    pub async fn invalidate_skills_catalog(&self) {
189        self.0.invalidate_skills_catalog().await;
190    }
191}
192
193/// The agent router.
194pub struct Router {
195    /// Router configuration
196    pub config: RouterConfig,
197    /// Available agents discovered at runtime
198    pub available_agents: Vec<String>,
199    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
200    pub weights: AgentWeights,
201    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
202    /// without holding the router RwLock across async operations)
203    llm_router: std::sync::Arc<LlmRouter>,
204    /// Ollama router for local routing (contains reusable reqwest::Client)
205    ollama_router: Option<ollama::OllamaRouter>,
206    /// Round-robin index for task routing
207    pub(crate) rr_index: usize,
208    /// Last agent routed to (for distribution tracking)
209    pub(crate) last_agent: Option<String>,
210    /// Round-robin index for review agent selection
211    pub(crate) review_rr_index: usize,
212    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
213    /// `opencode:free` entries are expanded at construction time.
214    pub(crate) router_pool: Vec<(String, String)>,
215    /// Current round-robin index into router_pool
216    pub(crate) pool_index: usize,
217}
218
219impl Router {
220    /// Create a new router with the given configuration.
221    pub fn new(config: RouterConfig) -> Self {
222        let available_agents = Self::discover_agents(&config.agents);
223        let mut weights = AgentWeights {
224            base_weights: config.weights.clone(),
225            ..Default::default()
226        };
227        weights.ensure_agents_with_weights(&available_agents, &config.weights);
228        if !config.weights.is_empty() {
229            tracing::info!(
230                weights = ?config.weights,
231                "configured agent routing weights"
232            );
233        }
234        let router_pool = Self::expand_pool(&config);
235        tracing::info!(
236            pool = ?router_pool,
237            fallback = %config.effective_fallback(),
238            "router LLM pool initialized"
239        );
240        // Prime only when this config names opencode, so a default or claude-only
241        // router never runs `opencode models`
242        if config.uses_opencode() {
243            crate::engine::runner::agents::opencode::prime_free_model_cache();
244        }
245        let ollama_router = if config.mode == "local" {
246            Some(ollama::OllamaRouter::new(&config))
247        } else {
248            None
249        };
250        Self {
251            config,
252            available_agents,
253            weights,
254            llm_router: std::sync::Arc::new(LlmRouter::new()),
255            ollama_router,
256            rr_index: 0,
257            last_agent: None,
258            review_rr_index: 0,
259            router_pool,
260            pool_index: 0,
261        }
262    }
263
264    /// Create a router with default configuration loaded from files.
265    pub fn from_config() -> Self {
266        Self::new(RouterConfig::from_config())
267    }
268
269    /// Create a router with a forced agent list, bypassing CLI discovery.
270    /// Only for use in unit tests.
271    #[cfg(test)]
272    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
273        let mut weights = AgentWeights {
274            base_weights: config.weights.clone(),
275            ..Default::default()
276        };
277        weights.ensure_agents(&available_agents);
278        let router_pool = Self::expand_pool(&config);
279        Self {
280            config,
281            available_agents,
282            weights,
283            llm_router: std::sync::Arc::new(LlmRouter::new()),
284            ollama_router: None,
285            rr_index: 0,
286            last_agent: None,
287            review_rr_index: 0,
288            router_pool,
289            pool_index: 0,
290        }
291    }
292
293    /// Return a cloned handle to the LLM router subsystem.
294    ///
295    /// Callers that need to invoke async methods on the underlying `LlmRouter`
296    /// should clone this handle *before* releasing any `RwLock` read guard on
297    /// `Router`, then call the async method after the guard is dropped.  This
298    /// prevents holding the read lock across await points (which starves
299    /// write-lock waiters due to Tokio's write-preferring `RwLock`).
300    pub fn llm_router_handle(&self) -> LlmRouterHandle {
301        LlmRouterHandle(self.llm_router.clone())
302    }
303
304    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
305        debug_assert!(pool_len > 0, "router pool must not be empty");
306        self.pool_index = (idx + 1) % pool_len;
307    }
308
309    /// Discover available agent CLIs in PATH.
310    /// Checks all agents from the configured list.
311    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
312        let mut agents = Vec::new();
313        for agent in configured_agents {
314            if crate::cmd_cache::command_exists(agent) {
315                agents.push(agent.to_string());
316            }
317        }
318        agents
319    }
320
321    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
322    ///
323    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
324    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
325    /// - If `opencode` is not installed or the command fails, the entry is skipped.
326    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
327    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
328        let raw_pool = config.effective_pool();
329        let mut expanded: Vec<(String, String)> = Vec::new();
330
331        for entry in &raw_pool {
332            if entry == "opencode:free" {
333                let free_models = Self::discover_free_opencode_models();
334                if free_models.is_empty() {
335                    tracing::debug!("opencode:free expanded to nothing — skipping");
336                } else {
337                    tracing::debug!(models = ?free_models, "opencode:free expanded");
338                    for model in free_models {
339                        expanded.push(("opencode".to_string(), model));
340                    }
341                }
342            } else {
343                let (agent, model) = parse_pool_entry(entry);
344                expanded.push((agent, model));
345            }
346        }
347
348        if expanded.is_empty() {
349            // All entries were opencode:free and opencode isn't installed — use fallback
350            let (agent, model) = parse_pool_entry(&config.effective_fallback());
351            tracing::debug!(
352                agent = %agent, model = %model,
353                "pool empty after expansion, using fallback as sole pool entry"
354            );
355            expanded.push((agent, model));
356        }
357
358        expanded
359    }
360
361    /// Async version of expand_pool which may query the opencode CLI without
362    /// blocking the Tokio runtime.
363    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
364        let raw_pool = config.effective_pool();
365        let mut expanded: Vec<(String, String)> = Vec::new();
366
367        for entry in &raw_pool {
368            if entry == "opencode:free" {
369                let free_models = Self::discover_free_opencode_models_async().await;
370                if free_models.is_empty() {
371                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
372                } else {
373                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
374                    for model in free_models {
375                        expanded.push(("opencode".to_string(), model));
376                    }
377                }
378            } else {
379                let (agent, model) = parse_pool_entry(entry);
380                expanded.push((agent, model));
381            }
382        }
383
384        if expanded.is_empty() {
385            // All entries were opencode:free and opencode isn't installed — use fallback
386            let (agent, model) = parse_pool_entry(&config.effective_fallback());
387            tracing::debug!(
388                agent = %agent, model = %model,
389                "pool empty after async expansion, using fallback as sole pool entry"
390            );
391            expanded.push((agent, model));
392        }
393
394        expanded
395    }
396
397    /// Async reload that uses async pool expansion to avoid blocking the runtime
398    /// when discovering opencode free models.
399    pub async fn reload_async(&mut self) {
400        let new_config = RouterConfig::from_config();
401        let new_agents = Self::discover_agents(&new_config.agents);
402        let new_pool = Self::expand_pool_async(&new_config).await;
403        tracing::info!(
404            mode = %new_config.mode,
405            agents = ?new_agents,
406            fallback = %new_config.fallback_executor,
407            weighted_rr = new_config.weighted_round_robin,
408            pool = ?new_pool,
409            "router reloaded (async)"
410        );
411        self.config = new_config;
412        self.available_agents = new_agents.clone();
413        // Ensure new agents have weight entries (preserves existing weights)
414        self.weights.ensure_agents(&new_agents);
415        self.router_pool = new_pool;
416        self.pool_index = 0;
417        // Reinitialize Ollama router if mode changed to local
418        self.ollama_router = if self.config.mode == "local" {
419            Some(ollama::OllamaRouter::new(&self.config))
420        } else {
421            None
422        };
423    }
424
425    fn discover_free_opencode_models() -> Vec<String> {
426        crate::engine::runner::agents::opencode::discover_free_opencode_models()
427    }
428
429    async fn discover_free_opencode_models_async() -> Vec<String> {
430        tokio::task::spawn_blocking(Self::discover_free_opencode_models)
431            .await
432            .unwrap_or_default()
433    }
434
435    /// Run the pre-emptive health check, refreshing the degraded-agent set.
436    ///
437    /// Queries the `rate_limits` table for recent events and combines with
438    /// cooldown state to mark agents as degraded before routing attempts them.
439    pub async fn refresh_health(&self, store: &std::sync::Arc<TaskStore>) {
440        // KV is the source of truth for cooldowns. Reconcile the in-memory map
441        // from KV every tick so external mutations (e.g. `orch cooldown clear`
442        // from a separate CLI process) actually take effect on the running
443        // service. Without this, the engine keeps stale entries until they
444        // expire on their own, producing the "all agents cooled" log with
445        // `orch cooldown list` simultaneously reporting "No active cooldowns".
446        sync_from_kv(store).await;
447
448        let agents = self.available_agents.clone();
449        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
450        refresh_degraded_agents(
451            store,
452            &agents,
453            &model_checker,
454            config::health_check_window_hours(),
455            config::degraded_rate_limit_threshold(),
456        )
457        .await;
458    }
459
460    /// Check if an agent is available.
461    pub fn is_agent_available(&self, agent: &str) -> bool {
462        self.available_agents.contains(&agent.to_string())
463    }
464
465    /// True if the agent has at least one uncooled model across any complexity
466    /// tier. Used where the eventual complexity isn't known yet (e.g. before
467    /// LLM classification), so we can't check a single tier.
468    fn agent_has_any_available_model(&self, agent: &str) -> bool {
469        ["simple", "medium", "complex", "review"]
470            .iter()
471            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
472    }
473
474    /// True if the agent has an available (uncooled) model for the `medium`
475    /// and `complex` tiers — the tiers real coding tasks are classified into
476    /// by the LLM classifier. Used to pre-filter the candidate list handed to
477    /// that classifier so an agent whose healthy tiers are only `simple` isn't
478    /// offered: the classifier can't be constrained to `simple`, so offering
479    /// such an agent produces a near-certain wasted classification call whenever
480    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
481    /// outcome of initial classification and is deliberately excluded.
482    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
483        ["medium", "complex"]
484            .iter()
485            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
486    }
487
488    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
489        if !self.is_agent_available(agent) {
490            return false;
491        }
492        if is_agent_in_cooldown(agent) {
493            tracing::debug!(agent, "agent skipped: in cooldown");
494            return false;
495        }
496        if is_agent_degraded(agent) {
497            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
498            return false;
499        }
500        // If weighted routing is used, consider the agent degraded when its
501        // weight has decayed below the threshold. This avoids
502        // proactively routing to agents that recently hit many rate limits.
503        if self.config.weighted_round_robin {
504            let weight = self.weights.get_weight(agent);
505            if weight < config::SKIP_LIMITED_THRESHOLD {
506                tracing::debug!(
507                    agent,
508                    weight,
509                    "agent considered degraded by weight threshold, skipping"
510                );
511                return false;
512            }
513        }
514        if !self
515            .config
516            .has_available_model_for_complexity(agent, complexity)
517        {
518            tracing::debug!(agent, complexity, "agent skipped: no available model");
519            return false;
520        }
521        true
522    }
523
524    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
525        self.available_agents
526            .iter()
527            .filter(|agent| self.agent_is_routable(agent, complexity))
528            .cloned()
529            .collect()
530    }
531
532    /// Count of healthy (routable) agents for a given complexity.
533    /// Returns the number of agents that are available, not in cooldown, and have an available model.
534    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
535        self.available_agents_for_complexity(complexity).len()
536    }
537
538    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
539    /// Uses "simple" complexity as the baseline since it has the widest model support.
540    pub fn is_degraded(&self, threshold: usize) -> bool {
541        self.healthy_agent_count("simple") < threshold
542    }
543
544    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
545        let mut earliest: Option<i64> = None;
546
547        for agent in &self.available_agents {
548            if let Some(until) = cooldown_until(agent) {
549                earliest = Some(earliest.map_or(until, |current| current.min(until)));
550            }
551
552            if let Some(comp) = complexity {
553                if let Some(pool) = self.config.model_pool_for_complexity(agent, comp) {
554                    for model in pool {
555                        let key = format!("{agent}:{model}");
556                        if let Some(until) = cooldown_until(&key) {
557                            earliest = Some(earliest.map_or(until, |current| current.min(until)));
558                        }
559                    }
560                }
561            }
562        }
563
564        earliest
565    }
566
567    /// Find the earliest cooldown expiration among the router LLM pool entries.
568    ///
569    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
570    /// or `None` if no pool model has a cooldown.
571    fn earliest_pool_cooldown(&self) -> Option<i64> {
572        let mut earliest: Option<i64> = None;
573        for (agent, model) in &self.router_pool {
574            let key = format!("{agent}:{model}");
575            if let Some(until) = cooldown_until(&key) {
576                earliest = Some(earliest.map_or(until, |current| current.min(until)));
577            }
578        }
579        earliest
580    }
581
582    async fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
583        // If any cooldowns are present, return an error immediately so the
584        // caller (the tick loop) can skip this task and retry on the next tick.
585        // Sleeping inside the router blocks the entire tick loop for the
586        // cooldown duration which starves other tasks. The engine tick already
587        // retries every `tick_interval` (default 10s), so allow the tick loop
588        // to drive retries instead of sleeping here.
589        let now = chrono::Utc::now().timestamp();
590        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
591            // All agents are degraded but none have a timed cooldown entry
592            // (e.g. over-threshold rate limiting without explicit cooldown duration).
593            // Log at warn and return an error so the tick skips this task and retries after backoff.
594            tracing::warn!(
595        scope = complexity.unwrap_or("all agents"),
596        "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
597    );
598            // Return an error similar to the cooldown case to trigger backoff in the tick loop
599            return Err(AllAgentsCooledError::new(
600                complexity.unwrap_or("all agents").to_string(),
601                None,
602            )
603            .into());
604        };
605
606        let remaining = earliest.saturating_sub(now);
607        if remaining > 0 {
608            let scope_str = complexity.unwrap_or("all agents").to_string();
609            tracing::warn!(
610                remaining_secs = remaining,
611                scope = %scope_str,
612                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
613            );
614            // Return a domain error indicating all candidates are cooled so the
615            // caller can handle it (tick will skip this task and retry later).
616            return Err(AllAgentsCooledError::new(scope_str, Some(remaining as u64)).into());
617        }
618
619        Ok(())
620    }
621
622    /// Get the first available agent.
623    /// Pick next agent via round-robin (for review or other non-task routing).
624    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
625    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
626    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
627        // Use available_agents_for_complexity which already filters using agent_is_routable,
628        // eliminating duplicate cooldown/degraded/model checks.
629        let candidates = self.available_agents_for_complexity(complexity);
630        if candidates.is_empty() {
631            return None;
632        }
633        let idx = self.review_rr_index;
634        let n = candidates.len();
635
636        // Try to find an agent that isn't excluded
637        let agent = (0..n)
638            .map(|offset| &candidates[(idx + offset) % n])
639            .find(|a| !exclude.contains(&a.as_str()))
640            .cloned()
641            // Fallback: use the first available candidate (even if excluded)
642            .or_else(|| candidates.first().cloned())?;
643
644        self.review_rr_index = (idx + 1) % candidates.len().max(1);
645        Some(agent)
646    }
647
648    /// If the selected agent matches `no_code_last_agent` for this task, filter it
649    /// out of the candidates and return the filtered list. Used to prevent the
650    /// same agent from being selected twice in a row after a no-code failure (#2410).
651    async fn skip_no_code_agent(
652        &self,
653        store: &std::sync::Arc<TaskStore>,
654        repo: &str,
655        task_id: &str,
656        candidates: &[String],
657    ) -> Option<Vec<String>> {
658        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
659            .await
660            .unwrap_or_else(|| {
661                tracing::warn!(
662                    task_id,
663                    "no_code_last_agent DB read returned None — loop detection may be impaired"
664                );
665                String::new()
666            });
667        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
668            return None;
669        }
670        let filtered: Vec<String> = candidates
671            .iter()
672            .filter(|a| a.as_str() != no_code_agent)
673            .cloned()
674            .collect();
675        if filtered.is_empty() {
676            tracing::warn!(
677                task_id,
678                no_code_agent = %no_code_agent,
679                "all available agents are the no-code agent — cannot skip"
680            );
681            return None;
682        }
683        tracing::info!(
684            task_id,
685            skipped = %no_code_agent,
686            remaining = ?filtered,
687            "skipping no-code agent, routing to alternative"
688        );
689        Some(filtered)
690    }
691
692    /// Route a task to the best agent.
693    ///
694    /// Routing logic (in priority order):
695    /// 1. Check for `agent:*` label — use that agent directly
696    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
697    /// 3. If round_robin mode, cycle through agents (stateful)
698    /// 4. Call LLM classifier for intelligent routing
699    /// 5. After max_route_attempts LLM failures, fall back to round-robin
700    pub async fn route(
701        &mut self,
702        task: &ExternalTask,
703        store: &std::sync::Arc<TaskStore>,
704        repo: &str,
705    ) -> anyhow::Result<RouteResult> {
706        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
707        // before the routing loop, not per-task. Do not re-add it here.
708
709        // Hold the task before any classification, label override, or fallback.
710        // A waiting task must not increment route_attempts.
711        match store.resolve_id_by_external(repo, &task.id.0).await {
712            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
713                Ok(Some(blockers)) => {
714                    return Err(crate::store::WaitingOnDependency { blockers }.into());
715                }
716                Ok(None) => {}
717                Err(err) => return Err(err),
718            },
719            Ok(None) => {}
720            Err(err) => return Err(err),
721        }
722
723        loop {
724            // 1. Check store agent field first (set by failover, authoritative over labels)
725            // Use a lightweight lookup to avoid deserializing the full Task row when
726            // only the `agent` column is needed. This avoids two full-row loads
727            // per routing pass (resolve_task_id + get) and significantly reduces
728            // CPU work on busy instances.
729            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
730                Ok(Some(agent)) => {
731                    // Router enforces that the agent string matches a configured agent
732                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
733                        Some(agent)
734                    } else {
735                        None
736                    }
737                }
738                Ok(None) => None,
739                Err(e) => {
740                    tracing::warn!(
741                        repo,
742                        task_id = %task.id.0,
743                        err = %e,
744                        "failed to fetch stored agent, falling back to label/LLM routing"
745                    );
746                    None
747                }
748            };
749
750            // 2. Fall back to explicit agent label
751            let resolved_agent = store_agent.or_else(|| {
752                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
753            });
754
755            if let Some(agent) = resolved_agent {
756                let complexity = strategies::extract_complexity_from_labels(&task.labels);
757                // If agent appears routable (not in agent-wide cooldown and has
758                // available models or no model_map configured), check whether
759                // we actually have a concrete model to dispatch. If the
760                // resolved label maps to no model (None), we fall through to
761                // the standard routing logic so the router LLM can pick a
762                // proper agent/model, instead of dispatching with an empty
763                // model which causes some agents (e.g. opencode) to exit
764                // silently.
765                if self.agent_is_routable(&agent, &complexity) {
766                    let model = self
767                        .config
768                        .model_for_complexity(&agent, &complexity, &task.id.0);
769
770                    // Only dispatch label overrides when we have a concrete
771                    // model to send to the executor. Dispatching with an
772                    // empty model led to silent exits for some agents
773                    // (notably opencode). If no concrete model is available
774                    // for the labeled agent, fall through to the standard
775                    // routing logic so the LLM or round-robin can pick an
776                    // appropriate agent/model.
777                    if model.is_some() {
778                        tracing::debug!(
779                            task_id = %task.id.0,
780                            agent = %agent,
781                            complexity = %complexity,
782                            "routed via label"
783                        );
784                        let mut result = RouteResult::from_config(
785                            agent.clone(),
786                            complexity.clone(),
787                            format!("label agent:{agent}"),
788                            &self.config,
789                            &task.id.0,
790                        );
791                        // Label routing uses agent-specific role and the
792                        // resolved model (may differ from from_config's default).
793                        result.profile.role = format!("{} specialist", agent);
794                        result.model = model.clone();
795                        self.log_route_activity(store, repo, &task.id.0, &result, None)
796                            .await;
797                        return Ok(result);
798                    } else {
799                        // No concrete model found for the labeled agent — do
800                        // not honor the label override. Fall through to the
801                        // standard routing logic so another agent or the LLM
802                        // can be chosen with a valid model.
803                        tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
804                    }
805                }
806
807                let candidates = self.available_agents_for_complexity(&complexity);
808                if candidates.is_empty() {
809                    self.wait_for_cooldown(Some(&complexity)).await?;
810                    continue;
811                }
812                // Label target is cooled or lacked model; fall through to standard routing.
813            }
814
815            let complexity = strategies::extract_complexity_from_labels(&task.labels);
816
817            // Check if any agents have available models for this complexity
818            let mut candidates = self.available_agents_for_complexity(&complexity);
819            if candidates.is_empty() {
820                self.wait_for_cooldown(Some(&complexity)).await?;
821                continue;
822            }
823
824            // Skip the no-code agent if one is recorded for this task (#2410).
825            // If all available agents are filtered out, we fall through to
826            // LLM routing with no constraints (LLM will pick the same agent but
827            // the response_handler will block it immediately via same-agent check).
828            if let Some(filtered) = self
829                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
830                .await
831            {
832                candidates = filtered;
833            }
834
835            // 2. Weighted round-robin — capacity-based selection
836            if self.config.weighted_round_robin {
837                let result = strategies::route_via_weighted_round_robin(
838                    &candidates,
839                    &self.weights,
840                    &self.config,
841                    task,
842                    &mut self.last_agent,
843                )?;
844                self.log_route_activity(store, repo, &task.id.0, &result, None)
845                    .await;
846                return Ok(result);
847            }
848
849            // 3. Round-robin mode — use stateful round-robin
850            if self.config.mode == "round_robin" {
851                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
852                let result = strategies::route_via_round_robin_stateful(
853                    &candidates,
854                    &self.config,
855                    task,
856                    &mut self.rr_index,
857                    &mut self.last_agent,
858                )?;
859                self.log_route_activity(store, repo, &task.id.0, &result, None)
860                    .await;
861                return Ok(result);
862            }
863
864            // 4. Local Ollama routing
865            if self.config.mode == "local" {
866                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
867                let ollama_router = self
868                    .ollama_router
869                    .as_ref()
870                    .expect("ollama_router must be initialized in Router::new when mode=local");
871                match ollama_router.route(task, &self.config).await {
872                    Ok(result) => {
873                        self.log_route_activity(store, repo, &task.id.0, &result, None)
874                            .await;
875                        return Ok(result);
876                    }
877                    Err(e) => {
878                        tracing::warn!(
879                            task_id = %task.id.0,
880                            error = %e,
881                            "Ollama routing failed, falling back to round-robin"
882                        );
883                        let result = strategies::route_via_round_robin_stateful(
884                            &candidates,
885                            &self.config,
886                            task,
887                            &mut self.rr_index,
888                            &mut self.last_agent,
889                        )?;
890                        self.log_route_activity(store, repo, &task.id.0, &result, None)
891                            .await;
892                        return Ok(result);
893                    }
894                }
895            }
896
897            // 5. LLM-based routing with retry tracking
898            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
899
900            if route_attempts >= self.config.max_route_attempts {
901                tracing::warn!(
902                    task_id = %task.id.0,
903                    attempts = route_attempts,
904                    max = self.config.max_route_attempts,
905                    "max LLM route attempts reached, falling back to round-robin"
906                );
907                let candidates = self.available_agents_for_complexity(&complexity);
908                if candidates.is_empty() {
909                    self.wait_for_cooldown(Some(&complexity)).await?;
910                    continue;
911                }
912                let result = strategies::route_via_round_robin_stateful(
913                    &candidates,
914                    &self.config,
915                    task,
916                    &mut self.rr_index,
917                    &mut self.last_agent,
918                )?;
919                self.log_route_activity(store, repo, &task.id.0, &result, None)
920                    .await;
921                return Ok(result);
922            }
923
924            // Log routing start (before await)
925            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
926
927            match self.route_with_llm(task, repo).await {
928                Ok(result) => {
929                    let candidates = self.available_agents_for_complexity(&result.complexity);
930                    if candidates.is_empty() {
931                        self.wait_for_cooldown(Some(&result.complexity)).await?;
932                        continue;
933                    }
934
935                    // If the LLM selected the no-code agent, filter it out and
936                    // pick an alternative (#2410). Re-use the existing fallback logic.
937                    let no_code_agent =
938                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
939                             .await
940                             .unwrap_or_else(|| {
941                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
942                                 String::new()
943                             });
944                    if result.agent == no_code_agent {
945                        // Pick the first agent that isn't the no-code agent.
946                        let fallback_agent = candidates
947                            .iter()
948                            .find(|a| a.as_str() != no_code_agent)
949                            .cloned()
950                            .unwrap_or_else(|| candidates[0].clone());
951                        if fallback_agent.as_str() == no_code_agent {
952                            tracing::warn!(
953                                task_id = %task.id.0,
954                                no_code_agent = %no_code_agent,
955                                complexity = %result.complexity,
956                                "all available agents are the no-code agent — waiting for another agent"
957                            );
958                            self.wait_for_cooldown(Some(&result.complexity)).await?;
959                            continue;
960                        }
961                        tracing::info!(
962                            task_id = %task.id.0,
963                            llm_selected = %result.agent,
964                            skipped = %no_code_agent,
965                            fallback = %fallback_agent,
966                            "LLM selected the no-code agent — routing to alternative"
967                        );
968                        let fallback_model = self.config.model_for_complexity(
969                            &fallback_agent,
970                            &result.complexity,
971                            &task.id.0,
972                        );
973
974                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
975                        let result = RouteResult {
976                            agent: fallback_agent.clone(),
977                            model: fallback_model,
978                            complexity: result.complexity.clone(),
979                            estimate: result.estimate,
980                            reason: format!(
981                                "LLM selected no-code agent {}; routed to {}",
982                                result.agent, fallback_agent
983                            ),
984                            profile: result.profile.clone(),
985                            selected_skills: result.selected_skills.clone(),
986                            warning: Some(format!(
987                                "LLM selected no-code agent {}; used fallback {}",
988                                result.agent, fallback_agent
989                            )),
990                        };
991                        self.log_route_activity(store, repo, &task.id.0, &result, None)
992                            .await;
993                        return Ok(result);
994                    }
995
996                    let model_cooled = result
997                        .model
998                        .as_deref()
999                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1000                    if !candidates.contains(&result.agent) || model_cooled {
1001                        let fallback_agent = candidates
1002                            .iter()
1003                            .find(|agent| *agent != &result.agent)
1004                            .cloned()
1005                            .unwrap_or_else(|| candidates[0].clone());
1006                        let fallback_model = self.config.model_for_complexity(
1007                            &fallback_agent,
1008                            &result.complexity,
1009                            &task.id.0,
1010                        );
1011
1012                        // If the fallback model is None and the fallback agent is the
1013                        // same as the LLM-selected agent, it means all models for the
1014                        // only available agent are cooled. In that case, wait for
1015                        // cooldowns instead of returning a RouteResult with
1016                        // model: None which would be dispatched incorrectly.
1017                        if fallback_model.is_none() && fallback_agent == result.agent {
1018                            tracing::warn!(
1019                                task_id = %task.id.0,
1020                                agent = %result.agent,
1021                                complexity = %result.complexity,
1022                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1023                            );
1024                            self.wait_for_cooldown(Some(&result.complexity)).await?;
1025                            continue;
1026                        }
1027
1028                        // Distinguish logging and reasons when we actually switch
1029                        // agents vs when we keep the same agent but pick a
1030                        // different (non-cooled) model.
1031                        if fallback_agent == result.agent {
1032                            tracing::warn!(
1033                                task_id = %task.id.0,
1034                                agent = %result.agent,
1035                                complexity = %result.complexity,
1036                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1037                            );
1038                        } else {
1039                            tracing::warn!(
1040                                task_id = %task.id.0,
1041                                agent = %result.agent,
1042                                fallback = %fallback_agent,
1043                                complexity = %result.complexity,
1044                                "LLM selected cooled agent/model; rerouting to available agent"
1045                            );
1046                        }
1047
1048                        // Reset attempts on success
1049                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1050
1051                        let reason = if fallback_agent == result.agent {
1052                            format!(
1053                                "LLM selected cooled model; using same agent {} with different model",
1054                                fallback_agent
1055                            )
1056                        } else {
1057                            format!(
1058                                "LLM selected cooled agent/model; rerouted to {}",
1059                                fallback_agent
1060                            )
1061                        };
1062
1063                        let result = RouteResult {
1064                            agent: fallback_agent.clone(),
1065                            model: fallback_model,
1066                            complexity: result.complexity.clone(),
1067                            estimate: result.estimate,
1068                            reason: reason.clone(),
1069                            profile: result.profile.clone(),
1070                            selected_skills: result.selected_skills.clone(),
1071                            warning: Some(if fallback_agent == result.agent {
1072                                "LLM-selected model was cooled; selected alternative model for same agent"
1073                                    .to_string()
1074                            } else {
1075                                "LLM-selected agent/model was cooled; rerouted to available agent"
1076                                    .to_string()
1077                            }),
1078                        };
1079                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1080                            .await;
1081                        return Ok(result);
1082                    }
1083
1084                    // Reset attempts on success
1085                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1086                    tracing::info!(
1087                        task_id = %task.id.0,
1088                        agent = %result.agent,
1089                        complexity = %result.complexity,
1090                        "routed via LLM"
1091                    );
1092                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1093                        .await;
1094                    return Ok(result);
1095                }
1096                Err(e) => {
1097                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1098                        let scope = err.scope();
1099                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1100                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1101                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1102                        // (no complexity-specific pool lookup, which would fail for "router pool").
1103                        let scope_opt = match scope {
1104                            "all agents" | "router pool" => None,
1105                            _ => Some(scope),
1106                        };
1107                        self.wait_for_cooldown(scope_opt).await?;
1108                        continue;
1109                    }
1110
1111                    let new_attempts = route_attempts + 1;
1112                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1113                        .await;
1114                    tracing::warn!(
1115                        task_id = %task.id.0,
1116                        error = %e,
1117                        error_chain = ?e,
1118                        attempt = new_attempts,
1119                        max = self.config.max_route_attempts,
1120                        "LLM routing failed"
1121                    );
1122
1123                    if new_attempts >= self.config.max_route_attempts {
1124                        tracing::info!(
1125                            task_id = %task.id.0,
1126                            "falling back to round-robin after {} failed attempts",
1127                            new_attempts
1128                        );
1129                        let candidates = self.available_agents_for_complexity(&complexity);
1130                        if candidates.is_empty() {
1131                            self.wait_for_cooldown(Some(&complexity)).await?;
1132                            continue;
1133                        }
1134                        let result = strategies::route_via_round_robin_stateful(
1135                            &candidates,
1136                            &self.config,
1137                            task,
1138                            &mut self.rr_index,
1139                            &mut self.last_agent,
1140                        )?;
1141                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1142                            .await;
1143                        return Ok(result);
1144                    }
1145
1146                    let candidates = self.available_agents_for_complexity(&complexity);
1147                    if candidates.is_empty() {
1148                        self.wait_for_cooldown(Some(&complexity)).await?;
1149                        continue;
1150                    }
1151                    let result = strategies::route_via_fallback(
1152                        &candidates,
1153                        &self.config,
1154                        task,
1155                        Some(&self.weights),
1156                    )?;
1157                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1158                        .await;
1159                    return Ok(result);
1160                }
1161            }
1162        }
1163    }
1164
1165    /// Get the number of LLM routing attempts for a task from the store.
1166    async fn get_route_attempts(
1167        &self,
1168        task_id: &str,
1169        store: &std::sync::Arc<TaskStore>,
1170        repo: &str,
1171    ) -> u32 {
1172        match store.resolve_task_id(repo, task_id).await {
1173            Ok(Some(store_id)) => store
1174                .get(store_id)
1175                .await
1176                .map(|t| t.route_attempts as u32)
1177                .unwrap_or(0),
1178            _ => 0,
1179        }
1180    }
1181
1182    /// Set the number of LLM routing attempts for a task in the store.
1183    async fn set_route_attempts(
1184        &self,
1185        task_id: &str,
1186        attempts: u32,
1187        store: &std::sync::Arc<TaskStore>,
1188        repo: &str,
1189    ) {
1190        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1191            if let Err(e) = store
1192                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1193                .await
1194            {
1195                tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1196            }
1197        }
1198    }
1199
1200    /// Log a route event to task activity timeline.
1201    async fn log_route_activity(
1202        &self,
1203        store: &std::sync::Arc<TaskStore>,
1204        repo: &str,
1205        task_id: &str,
1206        result: &RouteResult,
1207        from_agent: Option<&str>,
1208    ) {
1209        let details = serde_json::json!({
1210            "reason": result.reason,
1211            "complexity": result.complexity,
1212            "skills": result.selected_skills,
1213            "role": result.profile.role,
1214            "warning": result.warning,
1215        });
1216        let event_type = if from_agent.is_some() {
1217            "rerouted"
1218        } else {
1219            "routed"
1220        };
1221        store_log_activity(
1222            &Some(Arc::clone(store)),
1223            repo,
1224            task_id,
1225            event_type,
1226            None,
1227            Some("routed"),
1228            Some(&result.agent),
1229            result.model.as_deref(),
1230            Some(&details),
1231        )
1232        .await;
1233    }
1234
1235    /// Route using LLM classification with pool round-robin.
1236    ///
1237    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1238    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1239    /// returns the last error (the caller falls back to round-robin agent selection).
1240    async fn route_with_llm(
1241        &mut self,
1242        task: &ExternalTask,
1243        repo: &str,
1244    ) -> anyhow::Result<RouteResult> {
1245        // Filter out cooled and degraded agents, and agents without an available
1246        // model for the coding tiers (`medium`/`complex`) that real tasks are
1247        // classified into, so the LLM only sees candidates that could actually
1248        // be dispatched for whatever complexity it picks. An agent whose only
1249        // healthy tier is `simple` is excluded too: the classifier can't be
1250        // constrained to `simple`, so leaving such an agent in would waste the
1251        // call whenever it picks `medium`/`complex` (issue #3521).
1252        // Fall back to the full list if all agents are cooled/degraded.
1253        // Cloned to avoid borrow conflict with &mut self in the loop.
1254        let uncooled_agents: Vec<String> = self
1255            .available_agents
1256            .iter()
1257            .filter(|a| {
1258                !is_agent_in_cooldown(a)
1259                    && !is_agent_degraded(a)
1260                    && self.agent_has_available_model_for_coding_tiers(a)
1261            })
1262            .cloned()
1263            .collect();
1264        if uncooled_agents.is_empty() {
1265            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1266        }
1267        let llm_agents = uncooled_agents;
1268
1269        let pool = self.router_pool.clone();
1270        let n = pool.len();
1271        let start = self.pool_index;
1272        let mut last_err: Option<anyhow::Error> = None;
1273        let mut attempted_any = false;
1274
1275        // Try pool entries in round-robin order, skipping cooled ones
1276        for i in 0..n {
1277            let idx = (start + i) % n;
1278            let (agent, model) = &pool[idx];
1279            let model_str = model.as_str();
1280
1281            if crate::engine::runner::response::is_model_in_cooldown(agent, model_str) {
1282                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1283                continue;
1284            }
1285
1286            // At least one pool entry was available to attempt
1287            attempted_any = true;
1288
1289            let model_opt = if model_str.is_empty() {
1290                None
1291            } else {
1292                Some(model_str)
1293            };
1294
1295            match self
1296                .llm_router
1297                .route_with_llm_using(
1298                    task,
1299                    &llm_agents,
1300                    &self.config,
1301                    &mut self.last_agent,
1302                    repo,
1303                    agent,
1304                    model_opt,
1305                )
1306                .await
1307            {
1308                Ok(result) => {
1309                    // Advance index so the next call starts at the next pool entry
1310                    self.advance_pool_index_after_attempt(idx, n);
1311                    tracing::debug!(
1312                        agent,
1313                        model = model_str,
1314                        pool_idx = idx,
1315                        "router LLM pool entry succeeded"
1316                    );
1317                    record_agent_success(agent, model_str).await;
1318                    return Ok(result);
1319                }
1320                Err(e) => {
1321                    let err_msg = e.to_string();
1322                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1323                    if is_timeout {
1324                        // Advance pool index so the next tick starts at the next entry,
1325                        // then return immediately. Continuing to try further pool entries
1326                        // in the same tick would accumulate multiple timeout windows and
1327                        // stall the tick loop for minutes (issue #3187).
1328                        //
1329                        // Also record a standard cooldown, same as the non-timeout branch,
1330                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1331                        // on future ticks instead of burning the full timeout window again
1332                        // every time round-robin reaches it (issue #3422).
1333                        self.advance_pool_index_after_attempt(idx, n);
1334                        tracing::warn!(
1335                            agent,
1336                            model = model_str,
1337                            next_pool_idx = self.pool_index,
1338                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1339                        );
1340                        crate::engine::runner::response::record_model_failure(agent, model_str)
1341                            .await;
1342                        return Err(e);
1343                    } else {
1344                        tracing::warn!(
1345                            agent,
1346                            model = model_str,
1347                            error = %e,
1348                            "pool entry failed, recording cooldown and trying next"
1349                        );
1350                        crate::engine::runner::response::record_model_failure(agent, model_str)
1351                            .await;
1352                    }
1353                    last_err = Some(e);
1354                    self.advance_pool_index_after_attempt(idx, n);
1355                }
1356            }
1357        }
1358
1359        // If every pool entry was pre-cooled (nothing was attempted), do not
1360        // immediately exhaust the pool again — fail fast and let the tick loop
1361        // retry on the next cycle instead of cascading failures.
1362        // However, we should still try the fallback if configured.
1363        if !attempted_any {
1364            let now = chrono::Utc::now().timestamp();
1365            let earliest = self.earliest_pool_cooldown();
1366            if let Some(until) = earliest {
1367                // Pool entries exist but are all cooled - we'll try fallback below
1368                let remaining = until.saturating_sub(now);
1369                tracing::warn!(
1370                    remaining_secs = remaining,
1371                    pool = ?pool,
1372                    "all router LLM pool entries on cooldown — will try fallback"
1373                );
1374                // Continue to fallback logic instead of returning early
1375            } else {
1376                // No pool cooldowns found either — this means the pool was empty
1377                // or every entry was filtered out for some other reason.
1378                return Err(last_err
1379                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1380            }
1381        }
1382
1383        // All pool entries failed or were cooled — try the configured fallback
1384        let fallback = self.config.effective_fallback();
1385        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1386        let fb_model_str = fb_model.as_str();
1387
1388        // Only try fallback if it wasn't already in the pool (avoid double-try)
1389        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1390
1391        if !fallback_already_tried
1392            && !crate::engine::runner::response::is_model_in_cooldown(&fb_agent, fb_model_str)
1393        {
1394            tracing::info!(
1395                agent = %fb_agent,
1396                model = %fb_model_str,
1397                "all pool entries exhausted — trying fallback router LLM"
1398            );
1399            let fb_model_opt = if fb_model_str.is_empty() {
1400                None
1401            } else {
1402                Some(fb_model_str)
1403            };
1404            match self
1405                .llm_router
1406                .route_with_llm_using(
1407                    task,
1408                    &llm_agents,
1409                    &self.config,
1410                    &mut self.last_agent,
1411                    repo,
1412                    &fb_agent,
1413                    fb_model_opt,
1414                )
1415                .await
1416            {
1417                Ok(result) => {
1418                    record_agent_success(&fb_agent, fb_model_str).await;
1419                    return Ok(result);
1420                }
1421                Err(e) => {
1422                    let err_msg = e.to_string();
1423                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1424                    if is_timeout {
1425                        // Cool down on timeout like pool entries (#3422), else a
1426                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1427                        crate::engine::runner::response::record_model_failure(
1428                            &fb_agent,
1429                            fb_model_str,
1430                        )
1431                        .await;
1432                        tracing::warn!(
1433                            agent = %fb_agent,
1434                            model = %fb_model_str,
1435                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1436                        );
1437                        return Err(e);
1438                    } else {
1439                        tracing::warn!(
1440                            agent = %fb_agent,
1441                            model = %fb_model_str,
1442                            error = %e,
1443                            "fallback router LLM also failed"
1444                        );
1445                        crate::engine::runner::response::record_model_failure(
1446                            &fb_agent,
1447                            fb_model_str,
1448                        )
1449                        .await;
1450                    }
1451                    last_err = Some(e);
1452                }
1453            }
1454        }
1455
1456        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1457    }
1458
1459    /// Record a rate limit event for an agent, reducing its routing weight.
1460    ///
1461    /// Called by the engine when an agent returns a 429/rate limit error.
1462    pub fn record_rate_limit(&mut self, agent: &str) {
1463        self.weights.record_rate_limit(agent);
1464    }
1465
1466    /// Record a successful task completion, restoring agent weight.
1467    pub fn record_success(&mut self, agent: &str) {
1468        self.weights.record_success(agent);
1469    }
1470
1471    /// Tick weight recovery for all agents.
1472    ///
1473    /// Called periodically by the engine to gradually restore weights
1474    /// as rate limit windows expire.
1475    pub fn tick_weight_recovery(&mut self) {
1476        self.weights.tick_recovery();
1477    }
1478
1479    /// Store routing result in the task store.
1480    pub async fn store_route_result(
1481        &self,
1482        task_id: &str,
1483        result: &RouteResult,
1484        store: &std::sync::Arc<TaskStore>,
1485        repo: &str,
1486    ) -> anyhow::Result<()> {
1487        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1488            store
1489                .set_fields(
1490                    store_id,
1491                    &[
1492                        ("agent", serde_json::json!(result.agent)),
1493                        ("complexity", serde_json::json!(result.complexity)),
1494                        ("route_reason", serde_json::json!(result.reason)),
1495                        (
1496                            "model",
1497                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1498                        ),
1499                    ],
1500                )
1501                .await?;
1502        }
1503        Ok(())
1504    }
1505}
1506
1507/// Retrieve routing result from the task store.
1508pub async fn get_route_result(
1509    store: &std::sync::Arc<TaskStore>,
1510    repo: &str,
1511    task_id: &str,
1512) -> Result<RouteResult, RouteResultError> {
1513    let store_id = store
1514        .resolve_task_id(repo, task_id)
1515        .await
1516        .map_err(RouteResultError::Other)?
1517        .ok_or_else(|| {
1518            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1519        })?;
1520    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1521
1522    let agent = task
1523        .agent
1524        .clone()
1525        .filter(|a| !a.is_empty())
1526        .ok_or_else(|| RouteResultError::NoAgent {
1527            task_id: task_id.to_string(),
1528        })?;
1529
1530    let complexity = if task.complexity.is_empty() {
1531        "medium".to_string()
1532    } else {
1533        task.complexity.clone()
1534    };
1535
1536    let reason = task.route_reason.clone();
1537    let model = task.model.clone().filter(|m| !m.is_empty());
1538
1539    // Validate that the stored model is valid for the resolved agent.
1540    // Agent-specific aliases (e.g. "opus") can leak across agents during
1541    // failover, so we verify against the router's model pools (#1604).
1542    let config = RouterConfig::from_config();
1543    let model = model.filter(|m| {
1544        let complexities = ["simple", "medium", "complex", "review"];
1545        // Collect all model pools configured for this agent across complexity tiers.
1546        let all_pools: Vec<Vec<String>> = complexities
1547            .iter()
1548            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1549            .collect();
1550        // If the agent has no model pools configured, we can't validate — pass through.
1551        if all_pools.is_empty() {
1552            return true;
1553        }
1554        let valid = all_pools.iter().any(|pool| pool.contains(m));
1555        if !valid {
1556            tracing::warn!(
1557                task_id,
1558                agent,
1559                model = %m,
1560                "discarding stale model: not in agent's model pools"
1561            );
1562        }
1563        valid
1564    });
1565
1566    // A stale model is replaced by the agent's model for the task's complexity, an empty
1567    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1568    let original_model = task.model.clone().filter(|m| !m.is_empty());
1569    let mut model = model;
1570    if original_model.is_some() && model.is_none() {
1571        model = config.model_for_complexity(&agent, &complexity, task_id);
1572        let stale_model = original_model.as_deref().unwrap_or("unknown");
1573        let stale_err = format!(
1574            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1575            model.as_deref().unwrap_or("the agent default")
1576        );
1577        if let Err(e) = store
1578            .set_fields(
1579                store_id,
1580                &[
1581                    (
1582                        "model",
1583                        serde_json::json!(model.clone().unwrap_or_default()),
1584                    ),
1585                    ("last_error", serde_json::json!(stale_err)),
1586                ],
1587            )
1588            .await
1589        {
1590            // Log the error so transient DB failures are visible and diagnosable.
1591            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1592        }
1593    }
1594
1595    Ok(RouteResult {
1596        agent,
1597        model,
1598        complexity,
1599        estimate: 0,
1600        reason,
1601        profile: AgentProfile::default(),
1602        selected_skills: vec![],
1603        warning: None,
1604    })
1605}
1606
1607#[cfg(test)]
1608mod tests {
1609    use super::config::DEFAULT_AGENTS;
1610    use super::llm::LlmRouteResponse;
1611    use super::strategies;
1612    use super::weights::{
1613        AgentWeights, RateLimitState, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY,
1614    };
1615    use super::*;
1616    use crate::backends::{ExternalId, ExternalTask};
1617    use crate::engine::cooldown::{
1618        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1619        record_model_failure, set_model_cooldown,
1620    };
1621    use crate::store::TaskStore;
1622    use serial_test::serial;
1623    use std::time::{Duration, Instant};
1624
1625    #[tokio::test]
1626    #[serial_test::serial(orch_home)]
1627    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1628        let home = tempfile::tempdir().unwrap();
1629        std::fs::write(
1630            home.path().join("config.yml"),
1631            "model_map:\n  medium:\n    claude: sonnet\n",
1632        )
1633        .unwrap();
1634        std::env::set_var("ORCH_HOME", home.path());
1635
1636        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1637        let store_id = store
1638            .create(&crate::store::NewTask {
1639                external_id: Some("7".to_string()),
1640                repo: "owner/repo".to_string(),
1641                origin: "github".to_string(),
1642                title: "stale model".to_string(),
1643                ..Default::default()
1644            })
1645            .await
1646            .unwrap();
1647        store
1648            .set_fields(
1649                store_id,
1650                &[
1651                    ("agent", serde_json::json!("claude")),
1652                    ("model", serde_json::json!("claude-removed-model")),
1653                    ("complexity", serde_json::json!("medium")),
1654                ],
1655            )
1656            .await
1657            .unwrap();
1658
1659        let route = get_route_result(&store, "owner/repo", "7").await;
1660        let stored = store.get(store_id).await.unwrap().model;
1661        std::env::remove_var("ORCH_HOME");
1662
1663        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1664        assert_eq!(stored.as_deref(), Some("sonnet"));
1665    }
1666
1667    // Test-only delegates so tests can call router.parse_llm_response() and
1668    // router.check_routing_sanity() directly without referencing llm_router.
1669    impl Router {
1670        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1671            self.llm_router.parse_llm_response("claude", response)
1672        }
1673
1674        fn check_routing_sanity(
1675            &self,
1676            task: &ExternalTask,
1677            agent: &str,
1678            profile: &AgentProfile,
1679        ) -> Option<String> {
1680            self.llm_router.check_routing_sanity(task, agent, profile)
1681        }
1682    }
1683
1684    async fn test_store() -> std::sync::Arc<TaskStore> {
1685        std::sync::Arc::new(TaskStore::open_memory().await.unwrap())
1686    }
1687
1688    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1689        ExternalTask {
1690            id: ExternalId(id.to_string()),
1691            title: title.to_string(),
1692            body: "Test body".to_string(),
1693            state: "open".to_string(),
1694            labels,
1695            author: "test".to_string(),
1696            created_at: "2026-01-01T00:00:00Z".to_string(),
1697            updated_at: "2026-01-01T00:00:00Z".to_string(),
1698            url: format!("https://github.com/test/test/issues/{id}"),
1699        }
1700    }
1701
1702    #[serial(cooldown_state)]
1703    #[test]
1704    fn extract_agent_from_labels() {
1705        let mut config = RouterConfig::default();
1706        // Ensure a concrete model exists for the labeled agent so the label
1707        // override is honored by the router (routing requires a model).
1708        config
1709            .model_map
1710            .entry("medium".to_string())
1711            .or_default()
1712            .insert("claude".to_string(), vec!["haiku".to_string()]);
1713
1714        assert_eq!(
1715            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1716            Some("claude".to_string())
1717        );
1718        assert_eq!(
1719            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1720            Some("codex".to_string())
1721        );
1722        assert_eq!(
1723            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1724            Some("opencode".to_string())
1725        );
1726        assert_eq!(
1727            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1728            None
1729        );
1730        // Verify kimi and minimax are recognized from labels
1731        assert_eq!(
1732            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1733            Some("kimi".to_string())
1734        );
1735        assert_eq!(
1736            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1737            Some("minimax".to_string())
1738        );
1739    }
1740
1741    #[serial(cooldown_state)]
1742    #[tokio::test]
1743    async fn all_agents_cooled_error_names_degraded_agents() {
1744        crate::engine::cooldown::reset_global_state().await;
1745        mark_agent_degraded("claude");
1746        mark_agent_degraded("grok");
1747        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1748        let msg = err.to_string();
1749        assert!(
1750            msg.contains("degraded: claude, grok"),
1751            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1752        );
1753        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1754        clear_agent_degraded("claude");
1755        clear_agent_degraded("grok");
1756    }
1757
1758    #[serial(cooldown_state)]
1759    #[test]
1760    fn default_agents_constant() {
1761        assert_eq!(DEFAULT_AGENTS.len(), 7);
1762        assert!(DEFAULT_AGENTS.contains(&"claude"));
1763        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1764        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1765        assert!(DEFAULT_AGENTS.contains(&"glm"));
1766        assert!(DEFAULT_AGENTS.contains(&"grok"));
1767    }
1768
1769    #[serial(cooldown_state)]
1770    #[test]
1771    fn extract_complexity_from_labels() {
1772        assert_eq!(
1773            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1774            "simple"
1775        );
1776        assert_eq!(
1777            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1778            "medium"
1779        );
1780        assert_eq!(
1781            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1782            "complex"
1783        );
1784        assert_eq!(
1785            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1786            "medium"
1787        );
1788    }
1789
1790    #[serial(cooldown_state)]
1791    #[test]
1792    fn route_result_serialization() {
1793        let result = RouteResult {
1794            agent: "claude".to_string(),
1795            model: Some("sonnet".to_string()),
1796            complexity: "medium".to_string(),
1797            estimate: 5,
1798            reason: "test".to_string(),
1799            profile: AgentProfile {
1800                role: "backend".to_string(),
1801                skills: vec!["rust".to_string()],
1802                tools: vec!["git".to_string()],
1803                constraints: vec![],
1804            },
1805            selected_skills: vec!["gh".to_string()],
1806            warning: None,
1807        };
1808
1809        let json = serde_json::to_string(&result).unwrap();
1810        assert!(json.contains("claude"));
1811        assert!(json.contains("sonnet"));
1812
1813        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1814        assert_eq!(deserialized.agent, "claude");
1815        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1816    }
1817
1818    #[serial(cooldown_state)]
1819    #[test]
1820    fn router_config_default() {
1821        let mut config = RouterConfig::default();
1822        // Ensure a concrete model exists for the labeled agent so the label
1823        // override is honored by the router (routing requires a model).
1824        config
1825            .model_map
1826            .entry("medium".to_string())
1827            .or_default()
1828            .insert("claude".to_string(), vec!["haiku".to_string()]);
1829
1830        assert_eq!(config.mode, "round_robin");
1831        assert_eq!(config.router_agent, "claude");
1832        assert_eq!(config.router_model, "haiku");
1833        assert_eq!(config.fallback_executor, "codex");
1834        assert_eq!(config.max_route_attempts, 3);
1835        assert!(!config.allowed_tools.is_empty());
1836        assert!(!config.default_skills.is_empty());
1837
1838        // Verify configurable agents list includes all 7 agents
1839        assert_eq!(config.agents.len(), 7);
1840        assert!(config.agents.contains(&"claude".to_string()));
1841        assert!(config.agents.contains(&"codex".to_string()));
1842        assert!(config.agents.contains(&"opencode".to_string()));
1843        assert!(config.agents.contains(&"kimi".to_string()));
1844        assert!(config.agents.contains(&"minimax".to_string()));
1845        assert!(config.agents.contains(&"glm".to_string()));
1846        assert!(config.agents.contains(&"grok".to_string()));
1847    }
1848
1849    #[serial(cooldown_state)]
1850    #[test]
1851    fn model_map_lookup_returns_none_without_config() {
1852        // Default config has no hardcoded models — config.yml is the source of truth.
1853        let config = RouterConfig::default();
1854        assert!(config
1855            .model_for_complexity("claude", "simple", "")
1856            .is_none());
1857        assert!(config
1858            .model_for_complexity("claude", "medium", "")
1859            .is_none());
1860        assert!(config
1861            .model_for_complexity("claude", "complex", "")
1862            .is_none());
1863        assert!(config
1864            .model_for_complexity("opencode", "review", "")
1865            .is_none());
1866    }
1867
1868    #[serial(cooldown_state)]
1869    #[test]
1870    fn model_map_lookup_returns_configured_value() {
1871        // When model_map is populated (from config), it is returned correctly.
1872        let mut config = RouterConfig::default();
1873        config
1874            .model_map
1875            .entry("simple".to_string())
1876            .or_default()
1877            .insert("claude".to_string(), vec!["haiku".to_string()]);
1878        assert_eq!(
1879            config.model_for_complexity("claude", "simple", ""),
1880            Some("haiku".to_string())
1881        );
1882    }
1883
1884    #[serial(cooldown_state)]
1885    #[tokio::test]
1886    async fn model_pool_selection_skips_cooled() {
1887        crate::engine::cooldown::reset_global_state().await;
1888        use std::collections::HashMap;
1889
1890        let mut config = RouterConfig::default();
1891        // Set up a two-model pool for opencode/simple
1892        config
1893            .model_map
1894            .entry("simple".to_string())
1895            .or_default()
1896            .insert(
1897                "opencode".to_string(),
1898                vec!["model-a".to_string(), "model-b".to_string()],
1899            );
1900
1901        // Before any cooldown, both models are candidates
1902        let m = config.model_for_complexity("opencode", "simple", "task-1");
1903        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1904
1905        // Cool model-a
1906        record_model_failure("opencode", "model-a").await;
1907        assert!(is_model_in_cooldown("opencode", "model-a"));
1908
1909        // Now only model-b should be returned
1910        // (try many task_ids to rule out hash coincidence)
1911        for i in 0..20 {
1912            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1913            assert_eq!(
1914                result,
1915                Some("model-b".to_string()),
1916                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1917            );
1918        }
1919
1920        // Cool model-b too — now all models are cooled
1921        record_model_failure("opencode", "model-b").await;
1922        // All cooled → return None so the caller can fall back to a different agent
1923        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1924        assert_eq!(fallback, None);
1925        let _ = HashMap::<(), ()>::new(); // suppress unused import lint
1926    }
1927
1928    #[serial(cooldown_state)]
1929    #[test]
1930    fn model_pool_single_string_backward_compat() {
1931        // Single-item pools behave identically to the old string format.
1932        // Manually insert a single-item pool and verify it is returned correctly.
1933        let mut config = RouterConfig::default();
1934        config
1935            .model_map
1936            .entry("simple".to_string())
1937            .or_default()
1938            .insert("claude".to_string(), vec!["haiku".to_string()]);
1939        let m = config.model_for_complexity("claude", "simple", "any-task");
1940        assert_eq!(m, Some("haiku".to_string()));
1941    }
1942
1943    #[serial(cooldown_state)]
1944    #[test]
1945    fn parse_llm_response_direct_json() {
1946        let config = RouterConfig::default();
1947        let router = Router::new(config);
1948
1949        let response = r#"{
1950            "executor": "claude",
1951            "complexity": "complex",
1952            "reason": "requires architecture analysis",
1953            "profile": {
1954                "role": "architect",
1955                "skills": ["rust", "design"],
1956                "tools": ["git", "rg"],
1957                "constraints": []
1958            },
1959            "selected_skills": ["gh"]
1960        }"#;
1961
1962        let parsed = router.parse_llm_response(response).unwrap();
1963        assert_eq!(parsed.executor, "claude");
1964        assert_eq!(parsed.complexity, "complex");
1965        assert_eq!(parsed.reason, "requires architecture analysis");
1966        assert_eq!(parsed.profile.role, "architect");
1967    }
1968
1969    #[serial(cooldown_state)]
1970    #[test]
1971    fn parse_llm_response_markdown_fenced() {
1972        let config = RouterConfig::default();
1973        let router = Router::new(config);
1974
1975        let response = r#"Here's my analysis:
1976
1977```json
1978{
1979    "executor": "codex",
1980    "complexity": "medium",
1981    "reason": "coding task",
1982    "profile": {
1983        "role": "developer",
1984        "skills": ["coding"],
1985        "tools": [],
1986        "constraints": []
1987    },
1988    "selected_skills": []
1989}
1990```
1991
1992Hope that helps!"#;
1993
1994        let parsed = router.parse_llm_response(response).unwrap();
1995        assert_eq!(parsed.executor, "codex");
1996        assert_eq!(parsed.complexity, "medium");
1997    }
1998
1999    #[serial(cooldown_state)]
2000    #[test]
2001    fn parse_llm_response_claude_envelope() {
2002        let config = RouterConfig::default();
2003        let router = Router::new(config);
2004
2005        // Claude --output-format json wraps the response in an envelope
2006        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2007
2008        let parsed = router.parse_llm_response(response).unwrap();
2009        assert_eq!(parsed.executor, "claude");
2010        assert_eq!(parsed.complexity, "complex");
2011        assert_eq!(parsed.reason, "requires deep analysis");
2012    }
2013
2014    #[serial(cooldown_state)]
2015    #[test]
2016    fn parse_llm_response_claude_envelope_with_code_block() {
2017        let config = RouterConfig::default();
2018        let router = Router::new(config);
2019
2020        // Claude may return the JSON inside a code block within the envelope
2021        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2022        let envelope = serde_json::json!({
2023            "type": "result",
2024            "subtype": "success",
2025            "is_error": false,
2026            "result": inner,
2027            "usage": {"input_tokens": 50, "output_tokens": 30}
2028        });
2029
2030        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2031        assert_eq!(parsed.executor, "codex");
2032        assert_eq!(parsed.complexity, "simple");
2033    }
2034
2035    #[serial(cooldown_state)]
2036    #[test]
2037    fn parse_llm_response_claude_envelope_error() {
2038        let config = RouterConfig::default();
2039        let router = Router::new(config);
2040
2041        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2042
2043        let err = router.parse_llm_response(response).unwrap_err();
2044        assert!(err.to_string().contains("error"), "got: {err}");
2045    }
2046
2047    #[serial(cooldown_state)]
2048    #[test]
2049    fn parse_llm_response_empty() {
2050        let config = RouterConfig::default();
2051        let router = Router::new(config);
2052
2053        let err = router.parse_llm_response("").unwrap_err();
2054        assert!(err.to_string().contains("empty"), "got: {err}");
2055    }
2056
2057    /// Claude envelope where result is a JSON object, not a string.
2058    /// Some Claude versions may return the result as a nested object.
2059    #[serial(cooldown_state)]
2060    #[test]
2061    fn parse_llm_response_claude_envelope_result_as_object() {
2062        let config = RouterConfig::default();
2063        let router = Router::new(config);
2064
2065        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2066
2067        let parsed = router.parse_llm_response(response);
2068        assert!(
2069            parsed.is_ok(),
2070            "should handle result-as-object: {}",
2071            parsed.unwrap_err()
2072        );
2073        assert_eq!(parsed.unwrap().executor, "claude");
2074    }
2075
2076    /// LLM returns "agent" instead of "executor" — common hallucination.
2077    #[serial(cooldown_state)]
2078    #[test]
2079    fn parse_llm_response_agent_instead_of_executor() {
2080        let config = RouterConfig::default();
2081        let router = Router::new(config);
2082
2083        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2084
2085        let parsed = router.parse_llm_response(response);
2086        assert!(
2087            parsed.is_ok(),
2088            "should accept 'agent' alias: {}",
2089            parsed.unwrap_err()
2090        );
2091        assert_eq!(parsed.unwrap().executor, "claude");
2092    }
2093
2094    /// LLM returns minimal JSON without profile or reason.
2095    #[serial(cooldown_state)]
2096    #[test]
2097    fn parse_llm_response_minimal_json() {
2098        let config = RouterConfig::default();
2099        let router = Router::new(config);
2100
2101        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2102
2103        let parsed = router.parse_llm_response(response);
2104        assert!(
2105            parsed.is_ok(),
2106            "should accept minimal JSON: {}",
2107            parsed.unwrap_err()
2108        );
2109        assert_eq!(parsed.unwrap().executor, "codex");
2110    }
2111
2112    // ── Real fixture tests: route prompt → response → parsed result ──
2113
2114    /// Real Claude envelope with result as escaped JSON string.
2115    #[serial(cooldown_state)]
2116    #[test]
2117    fn fixture_route_response_string() {
2118        let config = RouterConfig::default();
2119        let router = Router::new(config);
2120
2121        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2122        let parsed = router.parse_llm_response(response).unwrap();
2123        assert_eq!(parsed.executor, "codex");
2124        assert_eq!(parsed.complexity, "medium");
2125        assert!(!parsed.reason.is_empty());
2126        assert!(!parsed.profile.role.is_empty());
2127    }
2128
2129    /// Real Claude envelope with result as JSON object (not string).
2130    /// This was the root cause of all router failures on 2026-02-27.
2131    #[serial(cooldown_state)]
2132    #[test]
2133    fn fixture_route_response_object() {
2134        let config = RouterConfig::default();
2135        let router = Router::new(config);
2136
2137        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2138        let parsed = router.parse_llm_response(response).unwrap();
2139        assert_eq!(parsed.executor, "claude");
2140        assert_eq!(parsed.complexity, "complex");
2141    }
2142
2143    /// Real Claude envelope with result as string containing markdown + JSON.
2144    #[serial(cooldown_state)]
2145    #[test]
2146    fn fixture_route_response_markdown() {
2147        let config = RouterConfig::default();
2148        let router = Router::new(config);
2149
2150        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2151        let parsed = router.parse_llm_response(response).unwrap();
2152        assert_eq!(parsed.executor, "codex");
2153        assert_eq!(parsed.complexity, "medium");
2154    }
2155
2156    /// Claude envelope with escaped JSON string containing newlines and markdown.
2157    #[serial(cooldown_state)]
2158    #[test]
2159    fn parse_llm_response_claude_envelope_with_prose() {
2160        let config = RouterConfig::default();
2161        let router = Router::new(config);
2162
2163        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2164        let envelope = serde_json::json!({
2165            "type": "result",
2166            "subtype": "success",
2167            "is_error": false,
2168            "result": inner,
2169            "usage": {"input_tokens": 100, "output_tokens": 80}
2170        });
2171
2172        let parsed = router.parse_llm_response(&envelope.to_string());
2173        assert!(
2174            parsed.is_ok(),
2175            "should handle prose + code block in envelope: {}",
2176            parsed.unwrap_err()
2177        );
2178        assert_eq!(parsed.unwrap().executor, "codex");
2179    }
2180
2181    #[serial(cooldown_state)]
2182    #[tokio::test]
2183    async fn route_round_robin_basic() {
2184        crate::engine::cooldown::reset_global_state().await;
2185        // Force at least one agent to be available for testing
2186        // In real usage, discover_agents finds installed CLIs
2187        let config = RouterConfig {
2188            mode: "round_robin".to_string(),
2189            ..Default::default()
2190        };
2191
2192        // Create router with mock available agents
2193        let agents = vec!["claude".to_string(), "codex".to_string()];
2194        let mut weights = AgentWeights::default();
2195        weights.ensure_agents(&agents);
2196        let router = Router {
2197            config,
2198            available_agents: agents,
2199            weights,
2200            llm_router: std::sync::Arc::new(LlmRouter::new()),
2201            ollama_router: None,
2202            rr_index: 0,
2203            last_agent: None,
2204            review_rr_index: 0,
2205            router_pool: vec![],
2206            pool_index: 0,
2207        };
2208
2209        let task = create_test_task("1", "Test task", vec![]);
2210        let result =
2211            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2212                .unwrap();
2213
2214        // Task 1 % 2 agents = agent at index 1 = codex
2215        assert_eq!(result.agent, "codex");
2216        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2217    }
2218
2219    #[serial(cooldown_state)]
2220    #[tokio::test]
2221    async fn route_uses_label_override() {
2222        crate::engine::cooldown::reset_global_state().await;
2223        let mut config = RouterConfig::default();
2224        // Ensure a concrete model exists so the label override can be honored
2225        // in the test environment where no external config.yml is present.
2226        config
2227            .model_map
2228            .entry("medium".to_string())
2229            .or_default()
2230            .insert("claude".to_string(), vec!["haiku".to_string()]);
2231        let agents = vec!["claude".to_string(), "codex".to_string()];
2232        let mut weights = AgentWeights::default();
2233        weights.ensure_agents(&agents);
2234        let mut router = Router {
2235            config,
2236            available_agents: agents,
2237            weights,
2238            llm_router: std::sync::Arc::new(LlmRouter::new()),
2239            ollama_router: None,
2240            rr_index: 0,
2241            last_agent: None,
2242            review_rr_index: 0,
2243            router_pool: vec![],
2244            pool_index: 0,
2245        };
2246
2247        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2248        let store = test_store().await;
2249
2250        // Should use label override, not LLM
2251        let result = router.route(&task, &store, "test/repo").await.unwrap();
2252        assert_eq!(result.agent, "claude");
2253        assert!(result.reason.contains("label"));
2254    }
2255
2256    #[serial(cooldown_state)]
2257    #[test]
2258    fn check_routing_sanity_warnings() {
2259        let config = RouterConfig::default();
2260        let router = Router::new(config);
2261
2262        // Backend task routed to claude should warn
2263        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2264        let profile = AgentProfile {
2265            role: "general".to_string(),
2266            skills: vec!["api".to_string()],
2267            tools: vec![],
2268            constraints: vec![],
2269        };
2270        let warning = router.check_routing_sanity(&task, "claude", &profile);
2271        assert!(warning.is_some());
2272        assert!(warning.unwrap().contains("backend"));
2273
2274        // Docs task routed to codex should warn
2275        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2276        let warning = router.check_routing_sanity(&task, "codex", &profile);
2277        assert!(warning.is_some());
2278        assert!(warning.unwrap().contains("docs"));
2279
2280        // Normal routing should not warn
2281        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2282        let warning = router.check_routing_sanity(&task, "codex", &profile);
2283        assert!(warning.is_none());
2284    }
2285
2286    #[serial(cooldown_state)]
2287    #[tokio::test]
2288    async fn router_reload_preserves_structure() {
2289        crate::engine::cooldown::reset_global_state().await;
2290        let config = RouterConfig::default();
2291        let agents = vec!["claude".to_string()];
2292        let mut weights = AgentWeights::default();
2293        weights.ensure_agents(&agents);
2294        let mut router = Router {
2295            config,
2296            available_agents: agents,
2297            weights,
2298            llm_router: std::sync::Arc::new(LlmRouter::new()),
2299            ollama_router: None,
2300            rr_index: 0,
2301            last_agent: None,
2302            review_rr_index: 0,
2303            router_pool: vec![],
2304            pool_index: 0,
2305        };
2306
2307        // Reload — should re-read config and remain valid
2308        router.reload_async().await;
2309
2310        // After reload, mode should be a valid value (llm, local, or round_robin)
2311        assert!(
2312            router.config.mode == "llm"
2313                || router.config.mode == "round_robin"
2314                || router.config.mode == "local",
2315            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2316            router.config.mode
2317        );
2318        // Fallback executor should always be set
2319        assert!(!router.config.fallback_executor.is_empty());
2320        // Tools should always be populated
2321        assert!(!router.config.allowed_tools.is_empty());
2322    }
2323
2324    // --- Weighted round-robin tests ---
2325
2326    #[serial(cooldown_state)]
2327    #[test]
2328    fn rate_limit_state_defaults_to_full_weight() {
2329        let state = RateLimitState::default();
2330        assert_eq!(state.weight, DEFAULT_WEIGHT);
2331        assert_eq!(state.consecutive_hits, 0);
2332        assert!(state.last_limited_at.is_none());
2333        assert!(!state.is_limited());
2334    }
2335
2336    #[serial(cooldown_state)]
2337    #[test]
2338    fn rate_limit_state_decays_on_hit() {
2339        let mut state = RateLimitState::default();
2340        state.record_rate_limit();
2341
2342        assert!(state.weight < DEFAULT_WEIGHT);
2343        assert_eq!(state.consecutive_hits, 1);
2344        assert!(state.last_limited_at.is_some());
2345        assert!(state.is_limited());
2346
2347        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2348        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2349        assert!((state.weight - expected).abs() < 1e-10);
2350    }
2351
2352    #[serial(cooldown_state)]
2353    #[test]
2354    fn rate_limit_state_never_drops_below_min() {
2355        let mut state = RateLimitState::default();
2356
2357        // Hit many times
2358        for _ in 0..100 {
2359            state.record_rate_limit();
2360        }
2361
2362        assert!(state.weight >= MIN_WEIGHT);
2363        assert_eq!(state.consecutive_hits, 100);
2364    }
2365
2366    #[serial(cooldown_state)]
2367    #[test]
2368    fn rate_limit_state_recovers_on_success() {
2369        let mut state = RateLimitState::default();
2370
2371        // Decay first
2372        state.record_rate_limit();
2373        state.record_rate_limit();
2374        let after_decay = state.weight;
2375
2376        // Record success
2377        state.record_success();
2378        assert!(state.weight > after_decay);
2379        assert_eq!(state.consecutive_hits, 0);
2380    }
2381
2382    #[serial(cooldown_state)]
2383    #[test]
2384    fn rate_limit_state_success_caps_at_default() {
2385        let mut state = RateLimitState::default();
2386
2387        // Already at full weight, success shouldn't exceed it
2388        state.record_success();
2389        assert_eq!(state.weight, DEFAULT_WEIGHT);
2390    }
2391
2392    #[serial(cooldown_state)]
2393    #[test]
2394    fn agent_weights_ensure_agents() {
2395        let mut weights = AgentWeights::default();
2396        let agents = vec!["claude".to_string(), "codex".to_string()];
2397        weights.ensure_agents(&agents);
2398
2399        assert_eq!(weights.states.len(), 2);
2400        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2401        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2402    }
2403
2404    #[serial(cooldown_state)]
2405    #[test]
2406    fn agent_weights_ensure_agents_preserves_existing() {
2407        let mut weights = AgentWeights::default();
2408        let agents = vec!["claude".to_string(), "codex".to_string()];
2409        weights.ensure_agents(&agents);
2410
2411        // Reduce claude's weight
2412        weights.record_rate_limit("claude");
2413        let claude_weight = weights.get_weight("claude");
2414        assert!(claude_weight < DEFAULT_WEIGHT);
2415
2416        // Ensure agents again — claude's weight should be preserved
2417        let agents2 = vec![
2418            "claude".to_string(),
2419            "codex".to_string(),
2420            "opencode".to_string(),
2421        ];
2422        weights.ensure_agents(&agents2);
2423
2424        assert_eq!(weights.get_weight("claude"), claude_weight);
2425        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2426    }
2427
2428    #[test]
2429    fn default_missing_weight_uses_minimum_of_configured() {
2430        use super::weights::default_missing_weight;
2431
2432        let mut configured = std::collections::HashMap::new();
2433        configured.insert("claude".to_string(), 0.2);
2434        configured.insert("codex".to_string(), 0.2);
2435        configured.insert("opencode".to_string(), 0.2);
2436        configured.insert("kimi".to_string(), 0.2);
2437
2438        // minimax omitted from router.weights — must not silently outrank
2439        // every explicitly configured agent (issue #3596).
2440        assert_eq!(default_missing_weight(&configured), 0.2);
2441    }
2442
2443    #[test]
2444    fn default_missing_weight_falls_back_when_none_configured() {
2445        use super::weights::default_missing_weight;
2446
2447        let configured = std::collections::HashMap::new();
2448        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2449    }
2450
2451    #[serial(cooldown_state)]
2452    #[test]
2453    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2454        let mut weights = AgentWeights::default();
2455        let mut configured = std::collections::HashMap::new();
2456        configured.insert("claude".to_string(), 0.2);
2457        configured.insert("codex".to_string(), 0.2);
2458        configured.insert("opencode".to_string(), 0.2);
2459        configured.insert("kimi".to_string(), 0.2);
2460
2461        let agents = vec![
2462            "claude".to_string(),
2463            "codex".to_string(),
2464            "opencode".to_string(),
2465            "kimi".to_string(),
2466            "minimax".to_string(),
2467        ];
2468        weights.ensure_agents_with_weights(&agents, &configured);
2469
2470        // minimax was omitted from router.weights but must not get the old
2471        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2472        assert_eq!(weights.get_weight("minimax"), 0.2);
2473        assert_eq!(weights.get_weight("claude"), 0.2);
2474    }
2475
2476    #[serial(cooldown_state)]
2477    #[test]
2478    fn agent_weights_record_rate_limit() {
2479        let mut weights = AgentWeights::default();
2480        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2481
2482        weights.record_rate_limit("claude");
2483
2484        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2485        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2486    }
2487
2488    #[serial(cooldown_state)]
2489    #[test]
2490    fn agent_weights_record_success() {
2491        let mut weights = AgentWeights::default();
2492        weights.ensure_agents(&["claude".to_string()]);
2493
2494        weights.record_rate_limit("claude");
2495        let after_limit = weights.get_weight("claude");
2496
2497        weights.record_success("claude");
2498        assert!(weights.get_weight("claude") > after_limit);
2499    }
2500
2501    #[serial(cooldown_state)]
2502    #[test]
2503    fn agent_weights_snapshot() {
2504        let mut weights = AgentWeights::default();
2505        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2506        weights.record_rate_limit("claude");
2507
2508        let snap = weights.snapshot();
2509        assert_eq!(snap.len(), 2);
2510
2511        // Snapshot is sorted alphabetically
2512        assert_eq!(snap[0].0, "claude");
2513        assert!(snap[0].1 < DEFAULT_WEIGHT);
2514        assert_eq!(snap[0].2, 1); // 1 hit
2515
2516        assert_eq!(snap[1].0, "codex");
2517        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2518        assert_eq!(snap[1].2, 0); // 0 hits
2519    }
2520
2521    #[serial(cooldown_state)]
2522    #[test]
2523    fn agent_weights_weighted_select_favors_higher_weight() {
2524        let mut weights = AgentWeights::default();
2525        let agents = vec!["claude".to_string(), "codex".to_string()];
2526        weights.ensure_agents(&agents);
2527
2528        // Heavily penalize claude
2529        for _ in 0..10 {
2530            weights.record_rate_limit("claude");
2531        }
2532
2533        // Run many selections and count
2534        let mut claude_count = 0;
2535        let mut codex_count = 0;
2536        for _ in 0..100 {
2537            match weights.weighted_select(&agents, "task-1") {
2538                Some(ref a) if a == "claude" => claude_count += 1,
2539                Some(ref a) if a == "codex" => codex_count += 1,
2540                _ => {}
2541            }
2542        }
2543
2544        // Codex should get significantly more selections
2545        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2546        assert!(
2547            codex_count > claude_count,
2548            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2549        );
2550    }
2551
2552    #[serial(cooldown_state)]
2553    #[test]
2554    fn agent_weights_weighted_select_empty_returns_none() {
2555        let weights = AgentWeights::default();
2556        assert!(weights.weighted_select(&[], "task-1").is_none());
2557    }
2558
2559    #[serial(cooldown_state)]
2560    #[test]
2561    fn agent_weights_batch_routing_produces_varied_selections() {
2562        // Regression test: when routing a batch of tasks in the same tick,
2563        // different task IDs should not all select the same agent.
2564        let mut weights = AgentWeights::default();
2565        let agents = vec!["claude".to_string(), "codex".to_string()];
2566        weights.ensure_agents(&agents);
2567
2568        // Simulate batch routing: 20 different task IDs called in rapid succession
2569        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2570        let selections: Vec<String> = task_ids
2571            .iter()
2572            .filter_map(|id| weights.weighted_select(&agents, id))
2573            .collect();
2574
2575        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2576        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2577        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2578        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2579        assert!(
2580            claude_count > 0 && codex_count > 0,
2581            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2582        );
2583    }
2584
2585    #[serial(cooldown_state)]
2586    #[test]
2587    fn agent_weights_weighted_select_single_agent() {
2588        let mut weights = AgentWeights::default();
2589        let agents = vec!["claude".to_string()];
2590        weights.ensure_agents(&agents);
2591
2592        let selected = weights.weighted_select(&agents, "task-1");
2593        assert_eq!(selected, Some("claude".to_string()));
2594    }
2595
2596    #[serial(cooldown_state)]
2597    #[test]
2598    fn agent_weights_tick_recovery_restores_weight() {
2599        let mut weights = AgentWeights::default();
2600        weights.ensure_agents(&["claude".to_string()]);
2601
2602        // Set up a rate limit that looks like it happened long ago
2603        let state = weights.states.get_mut("claude").unwrap();
2604        state.weight = 0.5;
2605        state.last_limited_at = Some(Instant::now() - RECOVERY_DELAY - Duration::from_secs(1));
2606        state.consecutive_hits = 2;
2607
2608        let before = weights.get_weight("claude");
2609        weights.tick_recovery();
2610        let after = weights.get_weight("claude");
2611
2612        assert!(after > before, "weight should increase after recovery tick");
2613    }
2614
2615    #[serial(cooldown_state)]
2616    #[test]
2617    fn agent_weights_tick_recovery_clears_on_full_restore() {
2618        let mut weights = AgentWeights::default();
2619        weights.ensure_agents(&["claude".to_string()]);
2620
2621        // Set weight just below full with an old limit
2622        let state = weights.states.get_mut("claude").unwrap();
2623        state.weight = DEFAULT_WEIGHT - 0.01;
2624        state.last_limited_at = Some(Instant::now() - RECOVERY_DELAY - Duration::from_secs(1));
2625        state.consecutive_hits = 1;
2626
2627        weights.tick_recovery();
2628
2629        let state = weights.states.get("claude").unwrap();
2630        assert_eq!(state.weight, DEFAULT_WEIGHT);
2631        assert!(state.last_limited_at.is_none());
2632        assert_eq!(state.consecutive_hits, 0);
2633    }
2634
2635    #[serial(cooldown_state)]
2636    #[tokio::test]
2637    async fn route_weighted_round_robin_basic() {
2638        crate::engine::cooldown::reset_global_state().await;
2639        let mut config = RouterConfig {
2640            weighted_round_robin: true,
2641            ..Default::default()
2642        };
2643        // Provide a concrete model for codex so label override can be used.
2644        config
2645            .model_map
2646            .entry("medium".to_string())
2647            .or_default()
2648            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2649
2650        let agents = vec!["claude".to_string(), "codex".to_string()];
2651        let mut weights = AgentWeights::default();
2652        weights.ensure_agents(&agents);
2653        let mut router = Router {
2654            config,
2655            available_agents: agents,
2656            weights,
2657            llm_router: std::sync::Arc::new(LlmRouter::new()),
2658            ollama_router: None,
2659            rr_index: 0,
2660            last_agent: None,
2661            review_rr_index: 0,
2662            router_pool: vec![],
2663            pool_index: 0,
2664        };
2665
2666        let task = create_test_task("1", "Test task", vec![]);
2667        let store = test_store().await;
2668        let result = router.route(&task, &store, "test/repo").await.unwrap();
2669
2670        // Should use weighted_round_robin
2671        assert!(result.reason.contains("weighted_round_robin"));
2672        assert!(
2673            result.agent == "claude" || result.agent == "codex",
2674            "agent should be claude or codex, got '{}'",
2675            result.agent
2676        );
2677    }
2678
2679    #[serial(cooldown_state)]
2680    #[tokio::test]
2681    async fn route_weighted_round_robin_respects_label_override() {
2682        crate::engine::cooldown::reset_global_state().await;
2683        let mut config = RouterConfig {
2684            weighted_round_robin: true,
2685            ..Default::default()
2686        };
2687        // Provide a concrete model for codex so label override can be used.
2688        config
2689            .model_map
2690            .entry("medium".to_string())
2691            .or_default()
2692            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2693
2694        let agents = vec!["claude".to_string(), "codex".to_string()];
2695        let mut weights = AgentWeights::default();
2696        weights.ensure_agents(&agents);
2697        let mut router = Router {
2698            config,
2699            available_agents: agents,
2700            weights,
2701            llm_router: std::sync::Arc::new(LlmRouter::new()),
2702            ollama_router: None,
2703            rr_index: 0,
2704            last_agent: None,
2705            review_rr_index: 0,
2706            router_pool: vec![],
2707            pool_index: 0,
2708        };
2709
2710        // Label override should take precedence over weighted routing
2711        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2712        let store = test_store().await;
2713        let result = router.route(&task, &store, "test/repo").await.unwrap();
2714        assert_eq!(result.agent, "codex");
2715        assert!(result.reason.contains("label"));
2716    }
2717
2718    #[serial(cooldown_state)]
2719    #[test]
2720    fn router_config_weighted_round_robin_default_false() {
2721        let config = RouterConfig::default();
2722        assert!(!config.weighted_round_robin);
2723    }
2724
2725    #[serial(cooldown_state)]
2726    #[test]
2727    fn router_record_and_recover_weights() {
2728        let config = RouterConfig {
2729            weighted_round_robin: true,
2730            ..Default::default()
2731        };
2732        let mut router = Router::new(config);
2733        // Override discovered agents for test
2734        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2735        router.weights.ensure_agents(&router.available_agents);
2736
2737        // Record rate limit
2738        router.record_rate_limit("claude");
2739        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2740        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2741
2742        // Record success for claude
2743        router.record_success("claude");
2744        let after_success = router.weights.get_weight("claude");
2745        assert!(after_success > MIN_WEIGHT);
2746    }
2747
2748    #[serial(cooldown_state)]
2749    #[test]
2750    fn review_rr_index_advances() {
2751        let mut config = RouterConfig::default();
2752        // Set up model_map so has_available_model_for_complexity returns true for test agents
2753        for agent in &["test_a", "test_b", "test_c"] {
2754            config
2755                .model_map
2756                .entry("medium".to_string())
2757                .or_default()
2758                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2759        }
2760        let agents = vec![
2761            "test_a".to_string(),
2762            "test_b".to_string(),
2763            "test_c".to_string(),
2764        ];
2765        let mut weights = AgentWeights::default();
2766        weights.ensure_agents(&agents);
2767        let mut router = Router {
2768            config,
2769            available_agents: agents,
2770            weights,
2771            llm_router: std::sync::Arc::new(LlmRouter::new()),
2772            ollama_router: None,
2773            rr_index: 0,
2774            last_agent: None,
2775            review_rr_index: 0,
2776            router_pool: vec![],
2777            pool_index: 0,
2778        };
2779
2780        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2781        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2782        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2783
2784        // All three agents should appear (order depends on start index)
2785        let mut seen = vec![a1, a2, a3];
2786        seen.sort();
2787        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2788    }
2789
2790    #[serial(cooldown_state)]
2791    #[test]
2792    fn review_rr_fallback_with_cooled_agent() {
2793        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2794        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2795        // candidates.get(idx) to return None even though candidates is non-empty.
2796        let mut config = RouterConfig::default();
2797        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2798        for agent in &["test_a", "test_b", "test_c"] {
2799            config
2800                .model_map
2801                .entry("medium".to_string())
2802                .or_default()
2803                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2804        }
2805        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2806        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2807        let mut weights = AgentWeights::default();
2808        weights.ensure_agents(&agents);
2809        let mut router = Router {
2810            config,
2811            available_agents: agents,
2812            weights,
2813            llm_router: std::sync::Arc::new(LlmRouter::new()),
2814            ollama_router: None,
2815            rr_index: 0,
2816            last_agent: None,
2817            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2818            review_rr_index: 2,
2819            router_pool: vec![],
2820            pool_index: 0,
2821        };
2822
2823        // All candidates excluded — should fall back to first candidate, not None
2824        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2825        assert!(
2826            result.is_some(),
2827            "next_round_robin_agent must not return None when candidates is non-empty"
2828        );
2829    }
2830
2831    #[serial(cooldown_state)]
2832    #[test]
2833    fn review_rr_index_uses_candidates_len_not_available_len() {
2834        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2835        // not available_agents.len(). When an agent has no model for the complexity,
2836        // candidates.len() < available_agents.len() and the old code produced a drifting
2837        // index that caused non-uniform distribution.
2838        let mut config = RouterConfig::default();
2839        // Only register models for test_a and test_b — test_c has no model, so it is
2840        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2841        for agent in &["test_a", "test_b"] {
2842            config
2843                .model_map
2844                .entry("medium".to_string())
2845                .or_default()
2846                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2847        }
2848        let agents = vec![
2849            "test_a".to_string(),
2850            "test_b".to_string(),
2851            "test_c".to_string(), // available but no model → filtered out of candidates
2852        ];
2853        let mut weights = AgentWeights::default();
2854        weights.ensure_agents(&agents);
2855        let mut router = Router {
2856            config,
2857            available_agents: agents,
2858            weights,
2859            llm_router: std::sync::Arc::new(LlmRouter::new()),
2860            ollama_router: None,
2861            rr_index: 0,
2862            last_agent: None,
2863            review_rr_index: 0,
2864            router_pool: vec![],
2865            pool_index: 0,
2866        };
2867
2868        // With the bug: index advances modulo 3 (available_agents.len()), so the
2869        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2870        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2871        // looks uniform by accident, but adding a 4th available agent breaks it.
2872        //
2873        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2874        // cycle. Verify this by checking review_rr_index after each call.
2875
2876        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2877        assert_eq!(
2878            router.review_rr_index, 1,
2879            "index should be 1 after first call"
2880        );
2881
2882        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2883        assert_eq!(
2884            router.review_rr_index, 0,
2885            "index should wrap to 0 after second call"
2886        );
2887
2888        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2889        assert_eq!(
2890            router.review_rr_index, 1,
2891            "index should be 1 after third call"
2892        );
2893
2894        // Agents must alternate uniformly
2895        assert_ne!(r1, r2, "consecutive calls should select different agents");
2896        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2897    }
2898
2899    #[serial(cooldown_state)]
2900    #[tokio::test]
2901    async fn last_agent_tracks_routing() {
2902        crate::engine::cooldown::reset_global_state().await;
2903        let mut config = RouterConfig {
2904            mode: "round_robin".to_string(),
2905            ..Default::default()
2906        };
2907        config
2908            .model_map
2909            .entry("medium".to_string())
2910            .or_default()
2911            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
2912        config
2913            .model_map
2914            .entry("medium".to_string())
2915            .or_default()
2916            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
2917        let agents = vec!["test_x".to_string(), "test_y".to_string()];
2918        let mut weights = AgentWeights::default();
2919        weights.ensure_agents(&agents);
2920        let mut router = Router {
2921            config,
2922            available_agents: agents,
2923            weights,
2924            llm_router: std::sync::Arc::new(LlmRouter::new()),
2925            ollama_router: None,
2926            rr_index: 0,
2927            last_agent: None,
2928            review_rr_index: 0,
2929            router_pool: vec![],
2930            pool_index: 0,
2931        };
2932
2933        assert!(router.last_agent.is_none());
2934
2935        let task = create_test_task("1", "Test routing", vec![]);
2936        let store = test_store().await;
2937        let result = router.route(&task, &store, "test/repo").await.unwrap();
2938
2939        assert!(router.last_agent.is_some());
2940        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
2941    }
2942
2943    // ── pool config ───────────────────────────────────────────────────────────
2944
2945    #[serial(cooldown_state)]
2946    #[test]
2947    fn parse_pool_entry_splits_on_first_colon() {
2948        let (agent, model) = super::parse_pool_entry("claude:haiku");
2949        assert_eq!(agent, "claude");
2950        assert_eq!(model, "haiku");
2951    }
2952
2953    #[serial(cooldown_state)]
2954    #[test]
2955    fn parse_pool_entry_model_with_slash() {
2956        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
2957        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
2958        assert_eq!(agent, "opencode");
2959        assert_eq!(model, "github-copilot/gpt-5-mini");
2960    }
2961
2962    #[serial(cooldown_state)]
2963    #[test]
2964    fn parse_pool_entry_no_colon() {
2965        let (agent, model) = super::parse_pool_entry("claude");
2966        assert_eq!(agent, "claude");
2967        assert_eq!(model, "");
2968    }
2969
2970    #[serial(cooldown_state)]
2971    #[test]
2972    fn effective_pool_uses_configured_pool() {
2973        let config = RouterConfig {
2974            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
2975            ..RouterConfig::default()
2976        };
2977        let pool = config.effective_pool();
2978        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
2979    }
2980
2981    #[serial(cooldown_state)]
2982    #[test]
2983    fn effective_pool_falls_back_to_router_agent_model() {
2984        let config = RouterConfig::default();
2985        // Default has no pool → derives from router_agent:router_model
2986        let pool = config.effective_pool();
2987        assert_eq!(pool.len(), 1);
2988        assert!(pool[0].starts_with(&config.router_agent));
2989        assert!(pool[0].contains(&config.router_model));
2990    }
2991
2992    #[serial(cooldown_state)]
2993    #[test]
2994    fn effective_fallback_uses_configured_fallback() {
2995        let config = RouterConfig {
2996            fallback: "claude:haiku".to_string(),
2997            ..RouterConfig::default()
2998        };
2999        assert_eq!(config.effective_fallback(), "claude:haiku");
3000    }
3001
3002    #[serial(cooldown_state)]
3003    #[test]
3004    fn effective_fallback_derives_from_router_agent_model() {
3005        let config = RouterConfig::default();
3006        let fallback = config.effective_fallback();
3007        assert!(fallback.starts_with(&config.router_agent));
3008        assert!(fallback.contains(&config.router_model));
3009    }
3010
3011    // ── pool expansion ────────────────────────────────────────────────────────
3012
3013    #[serial(cooldown_state)]
3014    #[test]
3015    fn expand_pool_basic_entries() {
3016        let config = RouterConfig {
3017            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3018            ..RouterConfig::default()
3019        };
3020        let expanded = Router::expand_pool(&config);
3021        assert_eq!(expanded.len(), 2);
3022        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3023        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3024    }
3025
3026    #[serial(cooldown_state)]
3027    #[test]
3028    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3029        let config = RouterConfig {
3030            pool: vec![],
3031            router_agent: "claude".to_string(),
3032            router_model: "haiku".to_string(),
3033            ..RouterConfig::default()
3034        };
3035        let expanded = Router::expand_pool(&config);
3036        assert_eq!(expanded.len(), 1);
3037        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3038    }
3039
3040    #[serial(cooldown_state)]
3041    #[test]
3042    fn expand_pool_opencode_free_skipped_when_not_installed() {
3043        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3044        // to nothing and the pool falls back to effective_fallback
3045        let config = RouterConfig {
3046            pool: vec!["opencode:free".to_string()],
3047            fallback: "claude:haiku".to_string(),
3048            ..RouterConfig::default()
3049        };
3050        let expanded = Router::expand_pool(&config);
3051        assert_eq!(expanded.len(), 1);
3052        assert_eq!(expanded[0].0, "claude");
3053        assert_eq!(expanded[0].1, "haiku");
3054    }
3055
3056    #[serial(cooldown_state)]
3057    #[test]
3058    fn expand_pool_single_entry_from_default() {
3059        let config = RouterConfig::default();
3060        let expanded = Router::expand_pool(&config);
3061        assert!(
3062            !expanded.is_empty(),
3063            "pool must never be empty after expansion"
3064        );
3065    }
3066
3067    // ── pool round-robin index ────────────────────────────────────────────────
3068
3069    #[serial(cooldown_state)]
3070    #[test]
3071    fn pool_index_initializes_to_zero() {
3072        let config = RouterConfig::default();
3073        let router = Router::new(config);
3074        assert_eq!(router.pool_index, 0);
3075    }
3076
3077    #[serial(cooldown_state)]
3078    #[test]
3079    fn router_pool_non_empty_after_construction() {
3080        let config = RouterConfig::default();
3081        let router = Router::new(config);
3082        assert!(!router.router_pool.is_empty());
3083    }
3084
3085    /**
3086     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3087     * `opencode serve --service` child behind, so a suite run leaked one server per test
3088     * process. The stub on PATH records any exec, so this fails if discovery comes back
3089     **/
3090    #[cfg(unix)]
3091    #[serial(cooldown_state)]
3092    #[test]
3093    fn building_a_router_never_runs_opencode() {
3094        use std::os::unix::fs::PermissionsExt;
3095
3096        struct PathRestore(Option<std::ffi::OsString>);
3097        impl Drop for PathRestore {
3098            fn drop(&mut self) {
3099                match &self.0 {
3100                    Some(p) => std::env::set_var("PATH", p),
3101                    None => std::env::remove_var("PATH"),
3102                }
3103                crate::cmd_cache::reset_for_test();
3104            }
3105        }
3106
3107        let dir = tempfile::tempdir().unwrap();
3108        let marker = dir.path().join("spawned");
3109        let bin_dir = dir.path().join("bin");
3110        std::fs::create_dir(&bin_dir).unwrap();
3111        let stub = bin_dir.join("opencode");
3112        std::fs::write(
3113            &stub,
3114            format!(
3115                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3116                marker.display()
3117            ),
3118        )
3119        .unwrap();
3120        std::fs::set_permissions(&stub, std::fs::Permissions::from_mode(0o755)).unwrap();
3121
3122        let prev_path = std::env::var_os("PATH");
3123        let mut new_path = bin_dir.as_os_str().to_os_string();
3124        new_path.push(":");
3125        if let Some(ref p) = prev_path {
3126            new_path.push(p);
3127        }
3128        std::env::set_var("PATH", &new_path);
3129        crate::cmd_cache::reset_for_test();
3130        let _restore = PathRestore(prev_path);
3131
3132        // A config that names opencode everywhere discovery reads from
3133        let mut config = RouterConfig {
3134            pool: vec!["opencode:free".to_string()],
3135            fallback: "opencode:free".to_string(),
3136            ..RouterConfig::default()
3137        };
3138        config
3139            .model_map
3140            .entry("simple".to_string())
3141            .or_default()
3142            .insert(
3143                "opencode".to_string(),
3144                vec!["canary-model-free".to_string()],
3145            );
3146
3147        let _router = Router::new(config.clone());
3148        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3149        let _pool = Router::expand_pool(&config);
3150        // Discovery refreshes on a background thread, give it room to exec
3151        std::thread::sleep(Duration::from_millis(300));
3152
3153        assert!(
3154            !marker.exists(),
3155            "building a router ran opencode: {}",
3156            std::fs::read_to_string(&marker).unwrap_or_default().trim()
3157        );
3158        assert!(
3159            opencode_children().is_empty(),
3160            "building a router left opencode processes: {:?}",
3161            opencode_children()
3162        );
3163    }
3164
3165    /// Commands of this process's direct children that mention opencode
3166    #[cfg(unix)]
3167    fn opencode_children() -> Vec<String> {
3168        let me = std::process::id().to_string();
3169        let output = std::process::Command::new("ps")
3170            .args(["-axo", "ppid=,command="])
3171            .output()
3172            .expect("ps");
3173        String::from_utf8_lossy(&output.stdout)
3174            .lines()
3175            .filter_map(|line| {
3176                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3177                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3178            })
3179            .collect()
3180    }
3181
3182    #[serial(cooldown_state)]
3183    #[tokio::test]
3184    async fn reload_resets_pool_index() {
3185        crate::engine::cooldown::reset_global_state().await;
3186        let config = RouterConfig::default();
3187        let mut router = Router::new(config);
3188        router.pool_index = 5;
3189        router.reload_async().await;
3190        assert_eq!(router.pool_index, 0);
3191    }
3192
3193    #[serial(cooldown_state)]
3194    #[test]
3195    fn pool_index_advances_after_failed_attempt() {
3196        let mut router = Router::new(RouterConfig::default());
3197        router.router_pool = vec![
3198            ("claude".to_string(), "haiku".to_string()),
3199            ("codex".to_string(), "gpt-5.2".to_string()),
3200            (
3201                "opencode".to_string(),
3202                "github-copilot/gpt-5-mini".to_string(),
3203            ),
3204        ];
3205        router.pool_index = 0;
3206
3207        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3208
3209        assert_eq!(router.pool_index, 1);
3210    }
3211
3212    #[serial(cooldown_state)]
3213    #[test]
3214    fn pool_index_wraps_after_last_attempt() {
3215        let mut router = Router::new(RouterConfig::default());
3216        router.router_pool = vec![
3217            ("claude".to_string(), "haiku".to_string()),
3218            ("codex".to_string(), "gpt-5.2".to_string()),
3219            (
3220                "opencode".to_string(),
3221                "github-copilot/gpt-5-mini".to_string(),
3222            ),
3223        ];
3224        router.pool_index = 2;
3225
3226        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3227
3228        assert_eq!(router.pool_index, 0);
3229    }
3230
3231    #[serial(cooldown_state)]
3232    #[test]
3233    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3234        let router = Router::new(RouterConfig::default());
3235        assert!(router.earliest_pool_cooldown().is_none());
3236    }
3237
3238    #[serial(cooldown_state)]
3239    #[tokio::test]
3240    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3241        crate::engine::cooldown::reset_global_state().await;
3242        // Use explicit pool with predictable model names to avoid opencode discovery.
3243        let config = RouterConfig {
3244            pool: vec![
3245                "claude:haiku-4-5-20251001".to_string(),
3246                "opencode:haiku-4-5-20251001".to_string(),
3247            ],
3248            ..Default::default()
3249        };
3250        let router = Router::new(config);
3251        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3252        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3253
3254        let earliest = router.earliest_pool_cooldown();
3255        assert!(earliest.is_some());
3256        let now = chrono::Utc::now().timestamp();
3257        let remaining = earliest.unwrap() - now;
3258        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3259        assert!((295..=300).contains(&remaining));
3260    }
3261
3262    #[serial(cooldown_state)]
3263    #[tokio::test]
3264    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3265        crate::engine::cooldown::reset_global_state().await;
3266        // Use explicit pool so we can predict what entries exist.
3267        let config = RouterConfig {
3268            pool: vec![
3269                "claude:pool-model-a".to_string(),
3270                "opencode:pool-model-b".to_string(),
3271            ],
3272            ..Default::default()
3273        };
3274        let router = Router::new(config);
3275        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3276        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3277        // Set cooldown on a pool entry (opencode:pool-model-b)
3278        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3279
3280        let earliest = router.earliest_pool_cooldown();
3281        assert!(earliest.is_some());
3282        let now = chrono::Utc::now().timestamp();
3283        let remaining = earliest.unwrap() - now;
3284        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3285        assert!((295..=300).contains(&remaining));
3286    }
3287
3288    // ---- Pre-emptive health check: degraded agent exclusion ----
3289
3290    #[serial(cooldown_state)]
3291    #[test]
3292    fn degraded_agent_excluded_from_routing() {
3293        let mut config = RouterConfig::default();
3294        config
3295            .model_map
3296            .entry("medium".to_string())
3297            .or_default()
3298            .insert(
3299                "test_degraded_routing".to_string(),
3300                vec!["model-a".to_string()],
3301            );
3302
3303        let mut router = Router::new(config);
3304        router.available_agents = vec!["test_degraded_routing".to_string()];
3305
3306        // Agent is routable before degradation
3307        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3308
3309        // Mark as degraded
3310        mark_agent_degraded("test_degraded_routing");
3311
3312        // Now excluded from routing
3313        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3314
3315        // available_agents_for_complexity returns empty
3316        assert!(
3317            router.available_agents_for_complexity("medium").is_empty(),
3318            "degraded agent should be excluded from available agents"
3319        );
3320
3321        // healthy_agent_count reflects exclusion
3322        assert_eq!(router.healthy_agent_count("medium"), 0);
3323
3324        // Cleanup
3325        clear_agent_degraded("test_degraded_routing");
3326    }
3327
3328    #[serial(cooldown_state)]
3329    #[test]
3330    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3331        let mut config = RouterConfig::default();
3332        for agent in &["agent_healthy", "agent_degraded"] {
3333            config
3334                .model_map
3335                .entry("medium".to_string())
3336                .or_default()
3337                .insert(agent.to_string(), vec!["model-x".to_string()]);
3338        }
3339
3340        let mut router = Router::new(config);
3341        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3342
3343        mark_agent_degraded("agent_degraded");
3344
3345        let candidates = router.available_agents_for_complexity("medium");
3346        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3347        assert_eq!(router.healthy_agent_count("medium"), 1);
3348
3349        // Cleanup
3350        clear_agent_degraded("agent_degraded");
3351    }
3352
3353    #[serial(cooldown_state)]
3354    #[tokio::test]
3355    async fn refresh_health_marks_degraded_from_rate_limits() {
3356        crate::engine::cooldown::reset_global_state().await;
3357        let store = test_store().await;
3358        let agent = "test_refresh_health_agent";
3359
3360        let mut config = RouterConfig::default();
3361        config
3362            .model_map
3363            .entry("medium".to_string())
3364            .or_default()
3365            .insert(agent.to_string(), vec!["model-z".to_string()]);
3366
3367        let mut router = Router::new(config);
3368        router.available_agents = vec![agent.to_string()];
3369
3370        // Insert rate limit events exceeding default threshold (3)
3371        for _ in 0..4 {
3372            store.record_rate_limit(agent, "rate", None).await.unwrap();
3373        }
3374
3375        router.refresh_health(&store).await;
3376
3377        assert!(
3378            is_agent_degraded(agent),
3379            "agent should be degraded after exceeding rate limit threshold"
3380        );
3381        assert!(
3382            !router.agent_is_routable(agent, "medium"),
3383            "degraded agent should not be routable"
3384        );
3385
3386        // Cleanup
3387        clear_agent_degraded(agent);
3388    }
3389
3390    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3391
3392    #[serial(cooldown_state)]
3393    #[tokio::test]
3394    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3395        crate::engine::cooldown::reset_global_state().await;
3396        let mut config = RouterConfig::default();
3397        // Agent's only model, shared across every complexity tier it resolves to.
3398        for tier in ["simple", "medium", "complex", "review"] {
3399            config
3400                .model_map
3401                .entry(tier.to_string())
3402                .or_default()
3403                .insert(
3404                    "test_all_cooled".to_string(),
3405                    vec!["only-model".to_string()],
3406                );
3407        }
3408
3409        let mut router = Router::new(config);
3410        router.available_agents = vec!["test_all_cooled".to_string()];
3411
3412        assert!(router.agent_has_any_available_model("test_all_cooled"));
3413
3414        record_model_failure("test_all_cooled", "only-model").await;
3415        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3416
3417        assert!(
3418            !router.agent_has_any_available_model("test_all_cooled"),
3419            "agent with its only model cooled across every tier should have no available model"
3420        );
3421    }
3422
3423    #[serial(cooldown_state)]
3424    #[tokio::test]
3425    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3426        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3427        // agent that has no agent-level cooldown but whose only resolvable model (across
3428        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3429        crate::engine::cooldown::reset_global_state().await;
3430        let mut config = RouterConfig::default();
3431        for tier in ["simple", "medium", "complex", "review"] {
3432            config
3433                .model_map
3434                .entry(tier.to_string())
3435                .or_default()
3436                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3437            config
3438                .model_map
3439                .get_mut(tier)
3440                .unwrap()
3441                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3442        }
3443
3444        let mut router = Router::new(config);
3445        router.available_agents = vec![
3446            "agent_cooled_model".to_string(),
3447            "agent_healthy".to_string(),
3448        ];
3449
3450        record_model_failure("agent_cooled_model", "opus").await;
3451        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3452        // No agent-level cooldown — the old agent-only filter would still offer it.
3453        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3454
3455        let uncooled_agents: Vec<String> = router
3456            .available_agents
3457            .iter()
3458            .filter(|a| {
3459                !is_agent_in_cooldown(a)
3460                    && !is_agent_degraded(a)
3461                    && router.agent_has_available_model_for_coding_tiers(a)
3462            })
3463            .cloned()
3464            .collect();
3465
3466        assert_eq!(
3467            uncooled_agents,
3468            vec!["agent_healthy".to_string()],
3469            "agent whose only model is cooled must be excluded from the LLM candidate list"
3470        );
3471    }
3472
3473    #[serial(cooldown_state)]
3474    #[tokio::test]
3475    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3476        // Regression test for #3521: an agent whose only healthy tier is `simple`
3477        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3478        // excluded from the LLM classification candidate list, otherwise the
3479        // classifier can pick it for a `medium`/`complex` task it can't serve,
3480        // wasting the classification call. Mirrors the real-world shape where
3481        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3482        crate::engine::cooldown::reset_global_state().await;
3483        let mut config = RouterConfig::default();
3484        config
3485            .model_map
3486            .entry("simple".to_string())
3487            .or_default()
3488            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3489        config
3490            .model_map
3491            .entry("medium".to_string())
3492            .or_default()
3493            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3494        config
3495            .model_map
3496            .entry("complex".to_string())
3497            .or_default()
3498            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3499        // review must be irrelevant to the coding-tier filter
3500        config
3501            .model_map
3502            .entry("review".to_string())
3503            .or_default()
3504            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3505        for tier in ["simple", "medium", "complex", "review"] {
3506            config
3507                .model_map
3508                .get_mut(tier)
3509                .unwrap()
3510                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3511        }
3512
3513        let mut router = Router::new(config);
3514        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3515
3516        record_model_failure("minimax_like", "opus").await;
3517        assert!(is_model_in_cooldown("minimax_like", "opus"));
3518        // No agent-level cooldown, and the simple tier is healthy, so the loose
3519        // "any tier" check still passes — this is exactly the leak #3511 missed.
3520        assert!(!is_agent_in_cooldown("minimax_like"));
3521        assert!(router.agent_has_any_available_model("minimax_like"));
3522        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3523
3524        let uncooled_agents: Vec<String> = router
3525            .available_agents
3526            .iter()
3527            .filter(|a| {
3528                !is_agent_in_cooldown(a)
3529                    && !is_agent_degraded(a)
3530                    && router.agent_has_available_model_for_coding_tiers(a)
3531            })
3532            .cloned()
3533            .collect();
3534
3535        assert_eq!(
3536            uncooled_agents,
3537            vec!["agent_healthy".to_string()],
3538            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3539        );
3540    }
3541}