Skip to main content

orch/engine/router/
mod.rs

1//! Agent router — selects the best agent and model for each task.
2//!
3//! The router uses LLM-based classification to route tasks to the best agent
4//! (claude, codex, opencode, kimi, or minimax) based on task content, labels,
5//! and configured routing rules. It also generates a specialized agent profile.
6//!
7//! Routing logic (in priority order):
8//! 1. Check for `agent:*` label on task — use that agent directly
9//! 2. If weighted_round_robin enabled, select by capacity-weighted probability
10//! 3. If round_robin mode, cycle through agents (stateful, skips last-used)
11//! 4. Call LLM classifier for intelligent routing
12//! 5. After N LLM failures, fall back to round-robin
13//! 6. Track last routed agent to distribute load across agents
14
15pub mod config;
16mod llm;
17mod ollama;
18mod selection;
19mod strategies;
20pub mod weights;
21
22use crate::backends::ExternalTask;
23use crate::cmd_cache;
24use crate::engine::cooldown;
25use crate::engine::cooldown::{
26    cooldown_until, is_agent_degraded, is_agent_in_cooldown, is_model_in_cooldown,
27    list_degraded_agents, record_agent_success, refresh_degraded_agents, sync_from_kv,
28};
29use crate::engine::runner::agents::opencode;
30use crate::engine::runner::response;
31use crate::store::WaitingOnDependency;
32use crate::store::{TaskStore, get_task_field_direct, store_log_activity};
33pub use config::{RouterConfig, parse_pool_entry};
34use llm::{LlmRouter, TIMEOUT_PREFIX};
35use serde::{Deserialize, Serialize};
36use std::fmt;
37use std::fmt::Debug;
38use std::fmt::Formatter;
39use std::sync::Arc;
40use tokio::task;
41pub use weights::AgentWeights;
42
43fn fmt_degraded(degraded: &[String]) -> String {
44    if degraded.is_empty() {
45        String::new()
46    } else {
47        format!(" (degraded: {})", degraded.join(", "))
48    }
49}
50
51/// Error returned when all agents/models are in cooldown and routing is temporarily unavailable.
52///
53/// This is a transient error that should not count against `max_route_attempts` since
54/// no agent was actually invoked. The caller should back off and retry later.
55#[derive(Debug, thiserror::Error)]
56pub enum AllAgentsCooledError {
57    #[error("all agents/models in cooldown for {scope}{}", fmt_degraded(.degraded))]
58    AllCooled {
59        scope: String,
60        remaining_secs: Option<u64>,
61        degraded: Vec<String>,
62    },
63}
64
65impl AllAgentsCooledError {
66    /// Create a new error for the given scope, naming currently degraded agents
67    pub fn new(scope: String, remaining_secs: Option<u64>) -> Self {
68        Self::AllCooled {
69            scope,
70            remaining_secs,
71            degraded: list_degraded_agents()
72                .into_iter()
73                .map(|(agent, _reason)| agent)
74                .collect(),
75        }
76    }
77
78    // LLM bypass helpers moved to `impl Router` so they can access Router fields.
79
80    /// Get the scope of the cooldown (e.g., "medium", "all agents", "router pool").
81    pub fn scope(&self) -> &str {
82        match self {
83            Self::AllCooled { scope, .. } => scope,
84        }
85    }
86
87    /// Remaining cooldown duration in seconds when available.
88    pub fn remaining_secs(&self) -> Option<u64> {
89        match self {
90            Self::AllCooled { remaining_secs, .. } => *remaining_secs,
91        }
92    }
93
94    /// Agents currently marked degraded, shown by `orch cooldown list`
95    pub fn degraded(&self) -> &[String] {
96        match self {
97            Self::AllCooled { degraded, .. } => degraded,
98        }
99    }
100}
101
102/// Typed error returned by [`get_route_result`].
103#[derive(Debug, thiserror::Error)]
104pub enum RouteResultError {
105    /// The task has no agent field — it needs to be re-routed.
106    #[error("no agent configured for task {task_id}")]
107    NoAgent { task_id: String },
108    /// Any other failure (store lookup, SQL, etc.).
109    #[error(transparent)]
110    Other(#[from] anyhow::Error),
111}
112
113/// Result of routing a task to an agent.
114#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)]
115pub struct RouteResult {
116    /// The selected agent: "claude", "codex", or "opencode"
117    pub agent: String,
118    /// Optional model suggestion, e.g., "claude-sonnet-4-6", "claude-opus-4-6", "o3"
119    pub model: Option<String>,
120    /// Complexity level: "simple", "medium", or "complex"
121    pub complexity: String,
122    /// Fibonacci effort estimate (1, 2, 3, 5, 8, 13, or 21). 0 means not provided.
123    pub estimate: u8,
124    /// Why this agent was selected
125    pub reason: String,
126    /// Specialized agent profile (role, tools, constraints)
127    pub profile: AgentProfile,
128    /// Optional warning about routing decision
129    pub warning: Option<String>,
130}
131
132/// Specialized agent profile for a task.
133#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Default)]
134pub struct AgentProfile {
135    /// Role name, e.g., "backend specialist"
136    pub role: String,
137    /// Tools allowed for this task
138    pub tools: Vec<String>,
139    /// Constraints for this task
140    pub constraints: Vec<String>,
141}
142
143impl AgentProfile {
144    /// Build the default "general" profile from router config.
145    pub fn default_from_config(config: &RouterConfig) -> Self {
146        Self {
147            role: "general".to_string(),
148            tools: config.allowed_tools.clone(),
149            constraints: vec![],
150        }
151    }
152}
153
154impl RouteResult {
155    /// The model for the task's tier, or one from the nearest tier with a free model. The warning
156    /// names the tier used when it differs, so the switch shows in the route activity
157    pub(crate) fn resolve_model(
158        config: &RouterConfig,
159        agent: &str,
160        complexity: &str,
161        task_id: &str,
162    ) -> (Option<String>, Option<String>) {
163        match config.model_for_complexity_or_adjacent(agent, complexity, task_id) {
164            Some((model, tier)) if tier != complexity => {
165                let warning =
166                    format!("no available {complexity} model for {agent}; using {tier} tier");
167                (Some(model), Some(warning))
168            }
169            Some((model, _)) => (Some(model), None),
170            None => (None, None),
171        }
172    }
173
174    /// Build a standard route result from the three task-specific values.
175    ///
176    /// Fills `profile` and `warning` with router-config defaults so call sites
177    /// only need to supply what varies per task.
178    pub fn from_config(
179        agent: String,
180        complexity: String,
181        reason: String,
182        config: &RouterConfig,
183        task_id: &str,
184    ) -> Self {
185        let (model, warning) = Self::resolve_model(config, &agent, &complexity, task_id);
186        Self {
187            agent,
188            model,
189            complexity,
190            estimate: 0,
191            reason,
192            profile: AgentProfile::default_from_config(config),
193            warning,
194        }
195    }
196}
197
198/// The agent router.
199#[expect(
200    clippy::struct_field_names,
201    reason = "the router holds sub-routers whose names say which backend they use"
202)]
203pub struct Router {
204    /// Router configuration
205    pub config: RouterConfig,
206    /// Available agents discovered at runtime
207    pub available_agents: Vec<String>,
208    /// Per-agent rate limit weights (used when weighted_round_robin is enabled)
209    pub weights: AgentWeights,
210    /// LLM routing subsystem (wrapped in Arc so callers can clone a handle
211    /// without holding the router RwLock across async operations)
212    llm_router: Arc<LlmRouter>,
213    /// Ollama router for local routing (contains reusable reqwest::Client)
214    ollama_router: Option<ollama::OllamaRouter>,
215    /// Round-robin index for task routing
216    pub(crate) rr_index: usize,
217    /// Last agent routed to (for distribution tracking)
218    pub(crate) last_agent: Option<String>,
219    /// Round-robin index for review agent selection
220    pub(crate) review_rr_index: usize,
221    /// Expanded pool of (agent, model) pairs for router LLM round-robin.
222    /// `opencode:free` entries are expanded at construction time.
223    pub(crate) router_pool: Vec<(String, String)>,
224    /// Current round-robin index into router_pool
225    pub(crate) pool_index: usize,
226}
227
228impl Debug for Router {
229    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
230        f.debug_struct("Router")
231            .field("config", &self.config)
232            .field("available_agents", &self.available_agents)
233            .field("weights", &self.weights)
234            .field("ollama_router", &self.ollama_router)
235            .field("rr_index", &self.rr_index)
236            .field("last_agent", &self.last_agent)
237            .field("review_rr_index", &self.review_rr_index)
238            .field("router_pool", &self.router_pool)
239            .field("pool_index", &self.pool_index)
240            .finish_non_exhaustive()
241    }
242}
243
244impl Router {
245    /// Create a new router with the given configuration.
246    pub fn new(config: RouterConfig) -> Self {
247        let available_agents = Self::discover_agents(&config.agents);
248        let mut weights = AgentWeights {
249            base_weights: config.weights.clone(),
250            ..Default::default()
251        };
252        weights.ensure_agents_with_weights(&available_agents, &config.weights);
253        if !config.weights.is_empty() {
254            tracing::info!(
255                weights = ?config.weights,
256                "configured agent routing weights"
257            );
258        }
259        let router_pool = Self::expand_pool(&config);
260        tracing::info!(
261            pool = ?router_pool,
262            fallback = %config.effective_fallback(),
263            "router LLM pool initialized"
264        );
265        // Prime only when this config names opencode, so a default or claude-only
266        // router never runs `opencode models`
267        if config.uses_opencode() {
268            opencode::prime_free_model_cache();
269        }
270        let ollama_router = if config.mode == "local" {
271            Some(ollama::OllamaRouter::new(&config))
272        } else {
273            None
274        };
275        Self {
276            config,
277            available_agents,
278            weights,
279            llm_router: Arc::new(LlmRouter::new()),
280            ollama_router,
281            rr_index: 0,
282            last_agent: None,
283            review_rr_index: 0,
284            router_pool,
285            pool_index: 0,
286        }
287    }
288
289    /// Create a router with default configuration loaded from files.
290    pub fn from_config() -> Self {
291        Self::new(RouterConfig::from_config())
292    }
293
294    /// Create a router with a forced agent list, bypassing CLI discovery.
295    /// Only for use in unit tests.
296    #[cfg(test)]
297    pub(crate) fn new_for_test(config: RouterConfig, available_agents: Vec<String>) -> Self {
298        let mut weights = AgentWeights {
299            base_weights: config.weights.clone(),
300            ..Default::default()
301        };
302        weights.ensure_agents(&available_agents);
303        let router_pool = Self::expand_pool(&config);
304        Self {
305            config,
306            available_agents,
307            weights,
308            llm_router: Arc::new(LlmRouter::new()),
309            ollama_router: None,
310            rr_index: 0,
311            last_agent: None,
312            review_rr_index: 0,
313            router_pool,
314            pool_index: 0,
315        }
316    }
317
318    fn advance_pool_index_after_attempt(&mut self, idx: usize, pool_len: usize) {
319        debug_assert!(pool_len > 0, "router pool must not be empty");
320        self.pool_index = (idx + 1) % pool_len;
321    }
322
323    /// Discover available agent CLIs in PATH.
324    /// Checks all agents from the configured list.
325    fn discover_agents(configured_agents: &[String]) -> Vec<String> {
326        let mut agents = Vec::new();
327        for agent in configured_agents {
328            if cmd_cache::command_exists(agent) {
329                agents.push(agent.clone());
330            }
331        }
332        agents
333    }
334
335    /// Expand the router LLM pool, resolving `opencode:free` to discovered free models.
336    ///
337    /// - Entries other than `opencode:free` are parsed as `agent:model` and added as-is.
338    /// - `opencode:free` is expanded by running `opencode models | grep free` at startup.
339    /// - If `opencode` is not installed or the command fails, the entry is skipped.
340    /// - If the expanded pool is empty, falls back to a single entry from `effective_fallback()`.
341    fn expand_pool(config: &RouterConfig) -> Vec<(String, String)> {
342        let raw_pool = config.effective_pool();
343        let mut expanded: Vec<(String, String)> = Vec::new();
344
345        for entry in &raw_pool {
346            if entry == "opencode:free" {
347                let free_models = Self::discover_free_opencode_models();
348                if free_models.is_empty() {
349                    tracing::debug!("opencode:free expanded to nothing — skipping");
350                } else {
351                    tracing::debug!(models = ?free_models, "opencode:free expanded");
352                    for model in free_models {
353                        expanded.push(("opencode".to_string(), model));
354                    }
355                }
356            } else {
357                let (agent, model) = parse_pool_entry(entry);
358                expanded.push((agent, model));
359            }
360        }
361
362        if expanded.is_empty() {
363            // All entries were opencode:free and opencode isn't installed — use fallback
364            let (agent, model) = parse_pool_entry(&config.effective_fallback());
365            tracing::debug!(
366                agent = %agent, model = %model,
367                "pool empty after expansion, using fallback as sole pool entry"
368            );
369            expanded.push((agent, model));
370        }
371
372        expanded
373    }
374
375    /// Async version of expand_pool which may query the opencode CLI without
376    /// blocking the Tokio runtime.
377    async fn expand_pool_async(config: &RouterConfig) -> Vec<(String, String)> {
378        let raw_pool = config.effective_pool();
379        let mut expanded: Vec<(String, String)> = Vec::new();
380
381        for entry in &raw_pool {
382            if entry == "opencode:free" {
383                let free_models = Self::discover_free_opencode_models_async().await;
384                if free_models.is_empty() {
385                    tracing::debug!("opencode:free expanded to nothing — skipping (async)");
386                } else {
387                    tracing::debug!(models = ?free_models, "opencode:free expanded (async)");
388                    for model in free_models {
389                        expanded.push(("opencode".to_string(), model));
390                    }
391                }
392            } else {
393                let (agent, model) = parse_pool_entry(entry);
394                expanded.push((agent, model));
395            }
396        }
397
398        if expanded.is_empty() {
399            // All entries were opencode:free and opencode isn't installed — use fallback
400            let (agent, model) = parse_pool_entry(&config.effective_fallback());
401            tracing::debug!(
402                agent = %agent, model = %model,
403                "pool empty after async expansion, using fallback as sole pool entry"
404            );
405            expanded.push((agent, model));
406        }
407
408        expanded
409    }
410
411    /// Async reload that uses async pool expansion to avoid blocking the runtime
412    /// when discovering opencode free models.
413    pub async fn reload_async(&mut self) {
414        let new_config = RouterConfig::from_config();
415        let new_agents = Self::discover_agents(&new_config.agents);
416        let new_pool = Self::expand_pool_async(&new_config).await;
417        tracing::info!(
418            mode = %new_config.mode,
419            agents = ?new_agents,
420            fallback = %new_config.fallback_executor,
421            weighted_rr = new_config.weighted_round_robin,
422            pool = ?new_pool,
423            "router reloaded (async)"
424        );
425        self.config = new_config;
426        self.available_agents.clone_from(&new_agents);
427        // Ensure new agents have weight entries (preserves existing weights)
428        self.weights.ensure_agents(&new_agents);
429        self.router_pool = new_pool;
430        self.pool_index = 0;
431        // Reinitialize Ollama router if mode changed to local
432        self.ollama_router = if self.config.mode == "local" {
433            Some(ollama::OllamaRouter::new(&self.config))
434        } else {
435            None
436        };
437    }
438
439    fn discover_free_opencode_models() -> Vec<String> {
440        opencode::discover_free_opencode_models()
441    }
442
443    async fn discover_free_opencode_models_async() -> Vec<String> {
444        task::spawn_blocking(Self::discover_free_opencode_models)
445            .await
446            .unwrap_or_default()
447    }
448
449    /// Run the pre-emptive health check, refreshing the degraded-agent set.
450    ///
451    /// Queries the `rate_limits` table for recent events and combines with
452    /// cooldown state to mark agents as degraded before routing attempts them.
453    pub async fn refresh_health(&self, store: &Arc<TaskStore>) {
454        // KV is the source of truth for cooldowns. Reconcile the in-memory map
455        // from KV every tick so external mutations (e.g. `orch cooldown clear`
456        // from a separate CLI process) actually take effect on the running
457        // service. Without this, the engine keeps stale entries until they
458        // expire on their own, producing the "all agents cooled" log with
459        // `orch cooldown list` simultaneously reporting "No active cooldowns".
460        sync_from_kv(store).await;
461
462        let agents = self.available_agents.clone();
463        let model_checker = |agent: &str| -> bool { self.agent_has_any_available_model(agent) };
464        refresh_degraded_agents(
465            store,
466            &agents,
467            &model_checker,
468            config::health_check_window_hours(),
469            config::degraded_rate_limit_threshold(),
470        )
471        .await;
472    }
473
474    /// Check if an agent is available.
475    pub fn is_agent_available(&self, agent: &str) -> bool {
476        self.available_agents.contains(&agent.to_string())
477    }
478
479    /// True if the agent has at least one uncooled model across any complexity
480    /// tier. Used where the eventual complexity isn't known yet (e.g. before
481    /// LLM classification), so we can't check a single tier.
482    fn agent_has_any_available_model(&self, agent: &str) -> bool {
483        ["simple", "medium", "complex", "review"]
484            .iter()
485            .any(|comp| self.config.has_available_model_for_complexity(agent, comp))
486    }
487
488    /// True if the agent has an available (uncooled) model for the `medium`
489    /// and `complex` tiers — the tiers real coding tasks are classified into
490    /// by the LLM classifier. Used to pre-filter the candidate list handed to
491    /// that classifier so an agent whose healthy tiers are only `simple` isn't
492    /// offered: the classifier can't be constrained to `simple`, so offering
493    /// such an agent produces a near-certain wasted classification call whenever
494    /// it picks `medium`/`complex` (issue #3521). `review` is not a candidate
495    /// outcome of initial classification and is deliberately excluded.
496    fn agent_has_available_model_for_coding_tiers(&self, agent: &str) -> bool {
497        ["medium", "complex"]
498            .iter()
499            .all(|comp| self.config.has_available_model_for_complexity(agent, comp))
500    }
501
502    fn agent_is_routable(&self, agent: &str, complexity: &str) -> bool {
503        if !self.is_agent_available(agent) {
504            return false;
505        }
506        if is_agent_in_cooldown(agent) {
507            tracing::debug!(agent, "agent skipped: in cooldown");
508            return false;
509        }
510        if is_agent_degraded(agent) {
511            tracing::debug!(agent, "agent skipped: degraded (pre-emptive health check)");
512            return false;
513        }
514        // If weighted routing is used, consider the agent degraded when its
515        // weight has decayed below the threshold. This avoids
516        // proactively routing to agents that recently hit many rate limits.
517        if self.config.weighted_round_robin {
518            let weight = self.weights.get_weight(agent);
519            if weight < config::SKIP_LIMITED_THRESHOLD {
520                tracing::debug!(
521                    agent,
522                    weight,
523                    "agent considered degraded by weight threshold, skipping"
524                );
525                return false;
526            }
527        }
528        if !self
529            .config
530            .has_available_model_in_any_tier(agent, complexity)
531        {
532            tracing::debug!(agent, complexity, "agent skipped: no available model");
533            return false;
534        }
535        true
536    }
537
538    fn available_agents_for_complexity(&self, complexity: &str) -> Vec<String> {
539        self.available_agents
540            .iter()
541            .filter(|agent| self.agent_is_routable(agent, complexity))
542            .cloned()
543            .collect()
544    }
545
546    /// Count of healthy (routable) agents for a given complexity.
547    /// Returns the number of agents that are available, not in cooldown, and have an available model.
548    pub fn healthy_agent_count(&self, complexity: &str) -> usize {
549        self.available_agents_for_complexity(complexity).len()
550    }
551
552    /// Check if the system is in degraded mode (fewer than threshold healthy agents).
553    /// Uses "simple" complexity as the baseline since it has the widest model support.
554    pub fn is_degraded(&self, threshold: usize) -> bool {
555        self.healthy_agent_count("simple") < threshold
556    }
557
558    pub fn earliest_cooldown_until(&self, complexity: Option<&str>) -> Option<i64> {
559        let mut earliest: Option<i64> = None;
560
561        for agent in &self.available_agents {
562            if let Some(until) = cooldown_until(agent) {
563                earliest = Some(earliest.map_or(until, |current| current.min(until)));
564            }
565
566            if let Some(comp) = complexity
567                && let Some(pool) = self.config.model_pool_for_complexity(agent, comp)
568            {
569                for model in pool {
570                    let key = format!("{agent}:{model}");
571                    if let Some(until) = cooldown_until(&key) {
572                        earliest = Some(earliest.map_or(until, |current| current.min(until)));
573                    }
574                }
575            }
576        }
577
578        earliest
579    }
580
581    /// Find the earliest cooldown expiration among the router LLM pool entries.
582    ///
583    /// Returns `Some(earliest_ts)` if at least one pool model is on cooldown,
584    /// or `None` if no pool model has a cooldown.
585    fn earliest_pool_cooldown(&self) -> Option<i64> {
586        let mut earliest: Option<i64> = None;
587        for (agent, model) in &self.router_pool {
588            let key = format!("{agent}:{model}");
589            if let Some(until) = cooldown_until(&key) {
590                earliest = Some(earliest.map_or(until, |current| current.min(until)));
591            }
592        }
593        earliest
594    }
595
596    fn wait_for_cooldown(&self, complexity: Option<&str>) -> anyhow::Result<()> {
597        // If any cooldowns are present, return an error immediately so the
598        // caller (the tick loop) can skip this task and retry on the next tick.
599        // Sleeping inside the router blocks the entire tick loop for the
600        // cooldown duration which starves other tasks. The engine tick already
601        // retries every `tick_interval` (default 10s), so allow the tick loop
602        // to drive retries instead of sleeping here.
603        let now = chrono::Utc::now().timestamp();
604        let Some(earliest) = self.earliest_cooldown_until(complexity) else {
605            // All agents are degraded but none have a timed cooldown entry
606            // (e.g. over-threshold rate limiting without explicit cooldown duration).
607            // Log at warn and return an error so the tick skips this task and retries after backoff.
608            tracing::warn!(
609                scope = complexity.unwrap_or("all agents"),
610                "all agents degraded (no cooldown timestamp available) — skipping task, will retry after backoff"
611            );
612            // Return an error similar to the cooldown case to trigger backoff in the tick loop
613            return Err(AllAgentsCooledError::new(
614                complexity.unwrap_or("all agents").to_string(),
615                None,
616            )
617            .into());
618        };
619
620        let remaining = earliest.saturating_sub(now);
621        if remaining > 0 {
622            let scope_str = complexity.unwrap_or("all agents").to_string();
623            tracing::warn!(
624                remaining_secs = remaining,
625                scope = %scope_str,
626                "all agents/models cooled — routing is unavailable until cooldown expires; failing fast to let tick retry"
627            );
628            // Return a domain error indicating all candidates are cooled so the
629            // caller can handle it (tick will skip this task and retry later).
630            return Err(AllAgentsCooledError::new(scope_str, u64::try_from(remaining).ok()).into());
631        }
632
633        Ok(())
634    }
635
636    /// Get the first available agent.
637    /// Pick next agent via round-robin (for review or other non-task routing).
638    /// Pick the next review agent, optionally excluding one (e.g. the task's original agent).
639    /// Falls back to the first candidate whenever every candidate is excluded (#3645).
640    pub fn next_round_robin_agent(&mut self, exclude: &[&str], complexity: &str) -> Option<String> {
641        // Use available_agents_for_complexity which already filters using agent_is_routable,
642        // eliminating duplicate cooldown/degraded/model checks.
643        let candidates = self.available_agents_for_complexity(complexity);
644        if candidates.is_empty() {
645            return None;
646        }
647        let idx = self.review_rr_index;
648        let n = candidates.len();
649
650        // Try to find an agent that isn't excluded
651        let agent = (0..n)
652            .map(|offset| &candidates[(idx + offset) % n])
653            .find(|a| !exclude.contains(&a.as_str()))
654            .cloned()
655            // Fallback: use the first available candidate (even if excluded)
656            .or_else(|| candidates.first().cloned())?;
657
658        self.review_rr_index = (idx + 1) % candidates.len().max(1);
659        Some(agent)
660    }
661
662    /// If the selected agent matches `no_code_last_agent` for this task, filter it
663    /// out of the candidates and return the filtered list. Used to prevent the
664    /// same agent from being selected twice in a row after a no-code failure (#2410).
665    async fn skip_no_code_agent(
666        &self,
667        store: &Arc<TaskStore>,
668        repo: &str,
669        task_id: &str,
670        candidates: &[String],
671    ) -> Option<Vec<String>> {
672        let no_code_agent = get_task_field_direct(store, repo, task_id, "no_code_last_agent")
673            .await
674            .unwrap_or_else(|| {
675                tracing::warn!(
676                    task_id,
677                    "no_code_last_agent DB read returned None — loop detection may be impaired"
678                );
679                String::new()
680            });
681        if no_code_agent.is_empty() || !candidates.iter().any(|a| a == &no_code_agent) {
682            return None;
683        }
684        let filtered: Vec<String> = candidates
685            .iter()
686            .filter(|a| a.as_str() != no_code_agent)
687            .cloned()
688            .collect();
689        if filtered.is_empty() {
690            tracing::warn!(
691                task_id,
692                no_code_agent = %no_code_agent,
693                "all available agents are the no-code agent — cannot skip"
694            );
695            return None;
696        }
697        tracing::info!(
698            task_id,
699            skipped = %no_code_agent,
700            remaining = ?filtered,
701            "skipping no-code agent, routing to alternative"
702        );
703        Some(filtered)
704    }
705
706    /// Route a task to the best agent.
707    ///
708    /// Routing logic (in priority order):
709    /// 1. Check for `agent:*` label — use that agent directly
710    /// 2. If weighted_round_robin enabled, select by capacity-weighted probability
711    /// 3. If round_robin mode, cycle through agents (stateful)
712    /// 4. Call LLM classifier for intelligent routing
713    /// 5. After max_route_attempts LLM failures, fall back to round-robin
714    pub async fn route(
715        &mut self,
716        task: &ExternalTask,
717        store: &Arc<TaskStore>,
718        repo: &str,
719    ) -> anyhow::Result<RouteResult> {
720        // NOTE: refresh_health() is called once per tick in tick_route_tasks()
721        // before the routing loop, not per-task. Do not re-add it here.
722
723        // Hold the task before any classification, label override, or fallback.
724        // A waiting task must not increment route_attempts.
725        match store.resolve_id_by_external(repo, &task.id.0).await {
726            Ok(Some(id)) => match store.block_claim_if_waiting(id).await {
727                Ok(Some(blockers)) => {
728                    return Err(WaitingOnDependency { blockers }.into());
729                }
730                Ok(None) => {}
731                Err(err) => return Err(err),
732            },
733            Ok(None) => {}
734            Err(err) => return Err(err),
735        }
736
737        loop {
738            // 1. Check store agent field first (set by failover, authoritative over labels)
739            // Use a lightweight lookup to avoid deserializing the full Task row when
740            // only the `agent` column is needed. This avoids two full-row loads
741            // per routing pass (resolve_task_id + get) and significantly reduces
742            // CPU work on busy instances.
743            let store_agent = match store.get_agent_by_external_id(repo, &task.id.0).await {
744                Ok(Some(agent)) => {
745                    // Router enforces that the agent string matches a configured agent
746                    if !agent.is_empty() && self.config.agents.iter().any(|cfg| cfg == &agent) {
747                        Some(agent)
748                    } else {
749                        None
750                    }
751                }
752                Ok(None) => None,
753                Err(e) => {
754                    tracing::warn!(
755                        repo,
756                        task_id = %task.id.0,
757                        err = %e,
758                        "failed to fetch stored agent, falling back to label/LLM routing"
759                    );
760                    None
761                }
762            };
763
764            // 2. Fall back to explicit agent label
765            let resolved_agent = store_agent.or_else(|| {
766                strategies::extract_agent_from_labels(&self.config.agents, &task.labels)
767            });
768
769            if let Some(agent) = resolved_agent {
770                let complexity = strategies::extract_complexity_from_labels(&task.labels);
771                // If agent appears routable (not in agent-wide cooldown and has
772                // available models or no model_map configured), check whether
773                // we actually have a concrete model to dispatch. If the
774                // resolved label maps to no model (None), we fall through to
775                // the standard routing logic so the router LLM can pick a
776                // proper agent/model, instead of dispatching with an empty
777                // model which causes some agents (e.g. opencode) to exit
778                // silently.
779                if self.agent_is_routable(&agent, &complexity) {
780                    let (model, tier_warning) =
781                        RouteResult::resolve_model(&self.config, &agent, &complexity, &task.id.0);
782
783                    // Only dispatch label overrides when we have a concrete
784                    // model to send to the executor. Dispatching with an
785                    // empty model led to silent exits for some agents
786                    // (notably opencode). If no concrete model is available
787                    // for the labeled agent, fall through to the standard
788                    // routing logic so the LLM or round-robin can pick an
789                    // appropriate agent/model.
790                    if model.is_some() {
791                        tracing::debug!(
792                            task_id = %task.id.0,
793                            agent = %agent,
794                            complexity = %complexity,
795                            "routed via label"
796                        );
797                        let mut result = RouteResult::from_config(
798                            agent.clone(),
799                            complexity.clone(),
800                            format!("label agent:{agent}"),
801                            &self.config,
802                            &task.id.0,
803                        );
804                        // Label routing uses agent-specific role and the
805                        // resolved model (may differ from from_config's default).
806                        result.profile.role = format!("{agent} specialist");
807                        result.model = model.clone();
808                        result.warning = tier_warning;
809                        self.log_route_activity(store, repo, &task.id.0, &result, None)
810                            .await;
811                        return Ok(result);
812                    }
813                    // No concrete model found for the labeled agent — do
814                    // not honor the label override. Fall through to the
815                    // standard routing logic so another agent or the LLM
816                    // can be chosen with a valid model.
817                    tracing::warn!(task_id = %task.id.0, agent = %agent, "label agent has no available model — falling through to LLM routing");
818                }
819
820                let candidates = self.available_agents_for_complexity(&complexity);
821                if candidates.is_empty() {
822                    self.wait_for_cooldown(Some(&complexity))?;
823                    continue;
824                }
825                // Label target is cooled or lacked model; fall through to standard routing.
826            }
827
828            let complexity = strategies::extract_complexity_from_labels(&task.labels);
829
830            // Check if any agents have available models for this complexity
831            let mut candidates = self.available_agents_for_complexity(&complexity);
832            if candidates.is_empty() {
833                self.wait_for_cooldown(Some(&complexity))?;
834                continue;
835            }
836
837            // Skip the no-code agent if one is recorded for this task (#2410).
838            // If all available agents are filtered out, we fall through to
839            // LLM routing with no constraints (LLM will pick the same agent but
840            // the response_handler will block it immediately via same-agent check).
841            if let Some(filtered) = self
842                .skip_no_code_agent(store, repo, &task.id.0, &candidates)
843                .await
844            {
845                candidates = filtered;
846            }
847
848            // 2. Weighted round-robin — capacity-based selection
849            if self.config.weighted_round_robin {
850                let result = strategies::route_via_weighted_round_robin(
851                    &candidates,
852                    &self.weights,
853                    &self.config,
854                    task,
855                    &mut self.last_agent,
856                )?;
857                self.log_route_activity(store, repo, &task.id.0, &result, None)
858                    .await;
859                return Ok(result);
860            }
861
862            // 3. Round-robin mode — use stateful round-robin
863            if self.config.mode == "round_robin" {
864                tracing::debug!(task_id = %task.id.0, "routing via round-robin mode");
865                let result = strategies::route_via_round_robin_stateful(
866                    &candidates,
867                    &self.config,
868                    task,
869                    &mut self.rr_index,
870                    &mut self.last_agent,
871                )?;
872                self.log_route_activity(store, repo, &task.id.0, &result, None)
873                    .await;
874                return Ok(result);
875            }
876
877            // 4. Local Ollama routing
878            if self.config.mode == "local" {
879                tracing::debug!(task_id = %task.id.0, ollama_url = %self.config.ollama_url, "routing via local Ollama");
880                let ollama_router = self
881                    .ollama_router
882                    .as_ref()
883                    .expect("ollama_router must be initialized in Router::new when mode=local");
884                match ollama_router.route(task, &self.config).await {
885                    Ok(result) => {
886                        self.log_route_activity(store, repo, &task.id.0, &result, None)
887                            .await;
888                        return Ok(result);
889                    }
890                    Err(e) => {
891                        tracing::warn!(
892                            task_id = %task.id.0,
893                            error = %e,
894                            "Ollama routing failed, falling back to round-robin"
895                        );
896                        let result = strategies::route_via_round_robin_stateful(
897                            &candidates,
898                            &self.config,
899                            task,
900                            &mut self.rr_index,
901                            &mut self.last_agent,
902                        )?;
903                        self.log_route_activity(store, repo, &task.id.0, &result, None)
904                            .await;
905                        return Ok(result);
906                    }
907                }
908            }
909
910            // 5. LLM-based routing with retry tracking
911            let route_attempts = self.get_route_attempts(&task.id.0, store, repo).await;
912
913            if route_attempts >= self.config.max_route_attempts {
914                tracing::warn!(
915                    task_id = %task.id.0,
916                    attempts = route_attempts,
917                    max = self.config.max_route_attempts,
918                    "max LLM route attempts reached, falling back to round-robin"
919                );
920                let candidates = self.available_agents_for_complexity(&complexity);
921                if candidates.is_empty() {
922                    self.wait_for_cooldown(Some(&complexity))?;
923                    continue;
924                }
925                let result = strategies::route_via_round_robin_stateful(
926                    &candidates,
927                    &self.config,
928                    task,
929                    &mut self.rr_index,
930                    &mut self.last_agent,
931                )?;
932                self.log_route_activity(store, repo, &task.id.0, &result, None)
933                    .await;
934                return Ok(result);
935            }
936
937            // Log routing start (before await)
938            tracing::debug!(task_id = %task.id.0, "starting LLM routing");
939
940            match self.route_with_llm(task, repo).await {
941                Ok(result) => {
942                    let candidates = self.available_agents_for_complexity(&result.complexity);
943                    if candidates.is_empty() {
944                        self.wait_for_cooldown(Some(&result.complexity))?;
945                        continue;
946                    }
947
948                    // If the LLM selected the no-code agent, filter it out and
949                    // pick an alternative (#2410). Re-use the existing fallback logic.
950                    let no_code_agent =
951                         get_task_field_direct(store, repo, &task.id.0, "no_code_last_agent")
952                             .await
953                             .unwrap_or_else(|| {
954                                 tracing::warn!(task_id = %task.id.0, "no_code_last_agent DB read returned None — loop detection may be impaired");
955                                 String::new()
956                             });
957                    if result.agent == no_code_agent {
958                        // Pick the first agent that isn't the no-code agent.
959                        let fallback_agent = candidates
960                            .iter()
961                            .find(|a| a.as_str() != no_code_agent)
962                            .cloned()
963                            .unwrap_or_else(|| candidates[0].clone());
964                        if fallback_agent.as_str() == no_code_agent {
965                            tracing::warn!(
966                                task_id = %task.id.0,
967                                no_code_agent = %no_code_agent,
968                                complexity = %result.complexity,
969                                "all available agents are the no-code agent — waiting for another agent"
970                            );
971                            self.wait_for_cooldown(Some(&result.complexity))?;
972                            continue;
973                        }
974                        tracing::info!(
975                            task_id = %task.id.0,
976                            llm_selected = %result.agent,
977                            skipped = %no_code_agent,
978                            fallback = %fallback_agent,
979                            "LLM selected the no-code agent — routing to alternative"
980                        );
981                        let (fallback_model, _) = RouteResult::resolve_model(
982                            &self.config,
983                            &fallback_agent,
984                            &result.complexity,
985                            &task.id.0,
986                        );
987
988                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
989                        let result = RouteResult {
990                            agent: fallback_agent.clone(),
991                            model: fallback_model,
992                            complexity: result.complexity.clone(),
993                            estimate: result.estimate,
994                            reason: format!(
995                                "LLM selected no-code agent {}; routed to {}",
996                                result.agent, fallback_agent
997                            ),
998                            profile: result.profile.clone(),
999                            warning: Some(format!(
1000                                "LLM selected no-code agent {}; used fallback {}",
1001                                result.agent, fallback_agent
1002                            )),
1003                        };
1004                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1005                            .await;
1006                        return Ok(result);
1007                    }
1008
1009                    let model_cooled = result
1010                        .model
1011                        .as_deref()
1012                        .is_some_and(|model| is_model_in_cooldown(&result.agent, model));
1013                    if !candidates.contains(&result.agent) || model_cooled {
1014                        let fallback_agent = candidates
1015                            .iter()
1016                            .find(|agent| *agent != &result.agent)
1017                            .cloned()
1018                            .unwrap_or_else(|| candidates[0].clone());
1019                        let (fallback_model, _) = RouteResult::resolve_model(
1020                            &self.config,
1021                            &fallback_agent,
1022                            &result.complexity,
1023                            &task.id.0,
1024                        );
1025
1026                        // If the fallback model is None and the fallback agent is the
1027                        // same as the LLM-selected agent, it means all models for the
1028                        // only available agent are cooled. In that case, wait for
1029                        // cooldowns instead of returning a RouteResult with
1030                        // model: None which would be dispatched incorrectly.
1031                        if fallback_model.is_none() && fallback_agent == result.agent {
1032                            tracing::warn!(
1033                                task_id = %task.id.0,
1034                                agent = %result.agent,
1035                                complexity = %result.complexity,
1036                                "LLM-selected model(s) are cooled for the only available agent; waiting for cooldown"
1037                            );
1038                            self.wait_for_cooldown(Some(&result.complexity))?;
1039                            continue;
1040                        }
1041
1042                        // Distinguish logging and reasons when we actually switch
1043                        // agents vs when we keep the same agent but pick a
1044                        // different (non-cooled) model.
1045                        if fallback_agent == result.agent {
1046                            tracing::warn!(
1047                                task_id = %task.id.0,
1048                                agent = %result.agent,
1049                                complexity = %result.complexity,
1050                                "LLM-selected model was cooled; selecting an alternative model for the same agent"
1051                            );
1052                        } else {
1053                            tracing::warn!(
1054                                task_id = %task.id.0,
1055                                agent = %result.agent,
1056                                fallback = %fallback_agent,
1057                                complexity = %result.complexity,
1058                                "LLM selected cooled agent/model; rerouting to available agent"
1059                            );
1060                        }
1061
1062                        // Reset attempts on success
1063                        self.set_route_attempts(&task.id.0, 0, store, repo).await;
1064
1065                        let reason = if fallback_agent == result.agent {
1066                            format!(
1067                                "LLM selected cooled model; using same agent {fallback_agent} with different model"
1068                            )
1069                        } else {
1070                            format!("LLM selected cooled agent/model; rerouted to {fallback_agent}")
1071                        };
1072
1073                        let result = RouteResult {
1074                            agent: fallback_agent.clone(),
1075                            model: fallback_model,
1076                            complexity: result.complexity.clone(),
1077                            estimate: result.estimate,
1078                            reason: reason.clone(),
1079                            profile: result.profile.clone(),
1080                            warning: Some(if fallback_agent == result.agent {
1081                                "LLM-selected model was cooled; selected alternative model for same agent"
1082                                    .to_string()
1083                            } else {
1084                                "LLM-selected agent/model was cooled; rerouted to available agent"
1085                                    .to_string()
1086                            }),
1087                        };
1088                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1089                            .await;
1090                        return Ok(result);
1091                    }
1092
1093                    // Reset attempts on success
1094                    self.set_route_attempts(&task.id.0, 0, store, repo).await;
1095                    tracing::info!(
1096                        task_id = %task.id.0,
1097                        agent = %result.agent,
1098                        complexity = %result.complexity,
1099                        "routed via LLM"
1100                    );
1101                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1102                        .await;
1103                    return Ok(result);
1104                }
1105                Err(e) => {
1106                    if let Some(err) = e.downcast_ref::<AllAgentsCooledError>() {
1107                        let scope = err.scope();
1108                        tracing::warn!(scope = %scope, "router cooldown gate tripped");
1109                        // "router pool" scope means all router LLM pool entries were pre-cooled.
1110                        // Pass None so wait_for_cooldown queries all agent+model cooldowns generically
1111                        // (no complexity-specific pool lookup, which would fail for "router pool").
1112                        let scope_opt = match scope {
1113                            "all agents" | "router pool" => None,
1114                            _ => Some(scope),
1115                        };
1116                        self.wait_for_cooldown(scope_opt)?;
1117                        continue;
1118                    }
1119
1120                    let new_attempts = route_attempts + 1;
1121                    self.set_route_attempts(&task.id.0, new_attempts, store, repo)
1122                        .await;
1123                    tracing::warn!(
1124                        task_id = %task.id.0,
1125                        error = %e,
1126                        error_chain = ?e,
1127                        attempt = new_attempts,
1128                        max = self.config.max_route_attempts,
1129                        "LLM routing failed"
1130                    );
1131
1132                    if new_attempts >= self.config.max_route_attempts {
1133                        tracing::info!(
1134                            task_id = %task.id.0,
1135                            "falling back to round-robin after {} failed attempts",
1136                            new_attempts
1137                        );
1138                        let candidates = self.available_agents_for_complexity(&complexity);
1139                        if candidates.is_empty() {
1140                            self.wait_for_cooldown(Some(&complexity))?;
1141                            continue;
1142                        }
1143                        let result = strategies::route_via_round_robin_stateful(
1144                            &candidates,
1145                            &self.config,
1146                            task,
1147                            &mut self.rr_index,
1148                            &mut self.last_agent,
1149                        )?;
1150                        self.log_route_activity(store, repo, &task.id.0, &result, None)
1151                            .await;
1152                        return Ok(result);
1153                    }
1154
1155                    let candidates = self.available_agents_for_complexity(&complexity);
1156                    if candidates.is_empty() {
1157                        self.wait_for_cooldown(Some(&complexity))?;
1158                        continue;
1159                    }
1160                    let result = strategies::route_via_fallback(
1161                        &candidates,
1162                        &self.config,
1163                        task,
1164                        Some(&self.weights),
1165                    )?;
1166                    self.log_route_activity(store, repo, &task.id.0, &result, None)
1167                        .await;
1168                    return Ok(result);
1169                }
1170            }
1171        }
1172    }
1173
1174    /// Get the number of LLM routing attempts for a task from the store.
1175    async fn get_route_attempts(&self, task_id: &str, store: &Arc<TaskStore>, repo: &str) -> u32 {
1176        match store.resolve_task_id(repo, task_id).await {
1177            Ok(Some(store_id)) => store
1178                .get(store_id)
1179                .await
1180                .map_or(0, |t| u32::try_from(t.route_attempts).unwrap_or(0)),
1181            _ => 0,
1182        }
1183    }
1184
1185    /// Set the number of LLM routing attempts for a task in the store.
1186    async fn set_route_attempts(
1187        &self,
1188        task_id: &str,
1189        attempts: u32,
1190        store: &Arc<TaskStore>,
1191        repo: &str,
1192    ) {
1193        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await
1194            && let Err(e) = store
1195                .set_fields(store_id, &[("route_attempts", serde_json::json!(attempts))])
1196                .await
1197        {
1198            tracing::warn!(task_id, error = %e, "failed to store route_attempts");
1199        }
1200    }
1201
1202    /// Log a route event to task activity timeline.
1203    async fn log_route_activity(
1204        &self,
1205        store: &Arc<TaskStore>,
1206        repo: &str,
1207        task_id: &str,
1208        result: &RouteResult,
1209        from_agent: Option<&str>,
1210    ) {
1211        let details = serde_json::json!({
1212            "reason": result.reason,
1213            "complexity": result.complexity,
1214            "role": result.profile.role,
1215            "warning": result.warning,
1216        });
1217        let event_type = if from_agent.is_some() {
1218            "rerouted"
1219        } else {
1220            "routed"
1221        };
1222        store_log_activity(
1223            &Some(Arc::clone(store)),
1224            repo,
1225            task_id,
1226            event_type,
1227            None,
1228            Some("routed"),
1229            Some(&result.agent),
1230            result.model.as_deref(),
1231            Some(&details),
1232        )
1233        .await;
1234    }
1235
1236    /// Route using LLM classification with pool round-robin.
1237    ///
1238    /// Iterates through `router_pool` (skipping cooled entries) until one succeeds.
1239    /// On full exhaustion, tries the configured fallback. If the fallback also fails,
1240    /// returns the last error (the caller falls back to round-robin agent selection).
1241    async fn route_with_llm(
1242        &mut self,
1243        task: &ExternalTask,
1244        repo: &str,
1245    ) -> anyhow::Result<RouteResult> {
1246        // Filter out cooled and degraded agents, and agents without an available
1247        // model for the coding tiers (`medium`/`complex`) that real tasks are
1248        // classified into, so the LLM only sees candidates that could actually
1249        // be dispatched for whatever complexity it picks. An agent whose only
1250        // healthy tier is `simple` is excluded too: the classifier can't be
1251        // constrained to `simple`, so leaving such an agent in would waste the
1252        // call whenever it picks `medium`/`complex` (issue #3521).
1253        // Fall back to the full list if all agents are cooled/degraded.
1254        // Cloned to avoid borrow conflict with &mut self in the loop.
1255        let uncooled_agents: Vec<String> = self
1256            .available_agents
1257            .iter()
1258            .filter(|a| {
1259                !is_agent_in_cooldown(a)
1260                    && !is_agent_degraded(a)
1261                    && self.agent_has_available_model_for_coding_tiers(a)
1262            })
1263            .cloned()
1264            .collect();
1265        if uncooled_agents.is_empty() {
1266            return Err(AllAgentsCooledError::new("all agents".to_string(), None).into());
1267        }
1268        let llm_agents = uncooled_agents;
1269
1270        let pool = self.router_pool.clone();
1271        let n = pool.len();
1272        let start = self.pool_index;
1273        let mut last_err: Option<anyhow::Error> = None;
1274        let mut attempted_any = false;
1275
1276        // Try pool entries in round-robin order, skipping cooled ones
1277        for i in 0..n {
1278            let idx = (start + i) % n;
1279            let (agent, model) = &pool[idx];
1280            let model_str = model.as_str();
1281
1282            if response::is_model_in_cooldown(agent, model_str) {
1283                tracing::debug!(agent, model = model_str, "pool entry on cooldown, skipping");
1284                continue;
1285            }
1286
1287            // At least one pool entry was available to attempt
1288            attempted_any = true;
1289
1290            let model_opt = if model_str.is_empty() {
1291                None
1292            } else {
1293                Some(model_str)
1294            };
1295
1296            match self
1297                .llm_router
1298                .route_with_llm_using(
1299                    task,
1300                    &llm_agents,
1301                    &self.config,
1302                    &mut self.last_agent,
1303                    repo,
1304                    agent,
1305                    model_opt,
1306                )
1307                .await
1308            {
1309                Ok(result) => {
1310                    // Advance index so the next call starts at the next pool entry
1311                    self.advance_pool_index_after_attempt(idx, n);
1312                    tracing::debug!(
1313                        agent,
1314                        model = model_str,
1315                        pool_idx = idx,
1316                        "router LLM pool entry succeeded"
1317                    );
1318                    record_agent_success(agent, model_str).await;
1319                    return Ok(result);
1320                }
1321                Err(e) => {
1322                    let err_msg = e.to_string();
1323                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1324                    if is_timeout {
1325                        // Advance pool index so the next tick starts at the next entry,
1326                        // then return immediately. Continuing to try further pool entries
1327                        // in the same tick would accumulate multiple timeout windows and
1328                        // stall the tick loop for minutes (issue #3187).
1329                        //
1330                        // Also record a standard cooldown, same as the non-timeout branch,
1331                        // so a chronically-timing-out entry is skipped by is_model_in_cooldown
1332                        // on future ticks instead of burning the full timeout window again
1333                        // every time round-robin reaches it (issue #3422).
1334                        self.advance_pool_index_after_attempt(idx, n);
1335                        tracing::warn!(
1336                            agent,
1337                            model = model_str,
1338                            next_pool_idx = self.pool_index,
1339                            "router LLM pool entry timed out — recording cooldown, will retry next tick"
1340                        );
1341                        response::record_model_failure(agent, model_str).await;
1342                        return Err(e);
1343                    }
1344                    tracing::warn!(
1345                        agent,
1346                        model = model_str,
1347                        error = %e,
1348                        "pool entry failed, recording cooldown and trying next"
1349                    );
1350                    if !cooldown::record_credit_exhaustion_from_message(
1351                        agent,
1352                        Some(model_str),
1353                        &err_msg,
1354                    )
1355                    .await
1356                    {
1357                        response::record_model_failure(agent, model_str).await;
1358                    }
1359                    last_err = Some(e);
1360                    self.advance_pool_index_after_attempt(idx, n);
1361                }
1362            }
1363        }
1364
1365        // If every pool entry was pre-cooled (nothing was attempted), do not
1366        // immediately exhaust the pool again — fail fast and let the tick loop
1367        // retry on the next cycle instead of cascading failures.
1368        // However, we should still try the fallback if configured.
1369        if !attempted_any {
1370            let now = chrono::Utc::now().timestamp();
1371            let earliest = self.earliest_pool_cooldown();
1372            if let Some(until) = earliest {
1373                // Pool entries exist but are all cooled - we'll try fallback below
1374                let remaining = until.saturating_sub(now);
1375                tracing::warn!(
1376                    remaining_secs = remaining,
1377                    pool = ?pool,
1378                    "all router LLM pool entries on cooldown — will try fallback"
1379                );
1380                // Continue to fallback logic instead of returning early
1381            } else {
1382                // No pool cooldowns found either — this means the pool was empty
1383                // or every entry was filtered out for some other reason.
1384                return Err(last_err
1385                    .unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")));
1386            }
1387        }
1388
1389        // All pool entries failed or were cooled — try the configured fallback
1390        let fallback = self.config.effective_fallback();
1391        let (fb_agent, fb_model) = parse_pool_entry(&fallback);
1392        let fb_model_str = fb_model.as_str();
1393
1394        // Only try fallback if it wasn't already in the pool (avoid double-try)
1395        let fallback_already_tried = pool.iter().any(|(a, m)| a == &fb_agent && m == &fb_model);
1396
1397        if !fallback_already_tried && !response::is_model_in_cooldown(&fb_agent, fb_model_str) {
1398            tracing::info!(
1399                agent = %fb_agent,
1400                model = %fb_model_str,
1401                "all pool entries exhausted — trying fallback router LLM"
1402            );
1403            let fb_model_opt = if fb_model_str.is_empty() {
1404                None
1405            } else {
1406                Some(fb_model_str)
1407            };
1408            match self
1409                .llm_router
1410                .route_with_llm_using(
1411                    task,
1412                    &llm_agents,
1413                    &self.config,
1414                    &mut self.last_agent,
1415                    repo,
1416                    &fb_agent,
1417                    fb_model_opt,
1418                )
1419                .await
1420            {
1421                Ok(result) => {
1422                    record_agent_success(&fb_agent, fb_model_str).await;
1423                    return Ok(result);
1424                }
1425                Err(e) => {
1426                    let err_msg = e.to_string();
1427                    let is_timeout = err_msg.contains(TIMEOUT_PREFIX);
1428                    if is_timeout {
1429                        // Cool down on timeout like pool entries (#3422), else a
1430                        // chronically-timing-out fallback burns the full timeout window every tick (#3616)
1431                        response::record_model_failure(&fb_agent, fb_model_str).await;
1432                        tracing::warn!(
1433                            agent = %fb_agent,
1434                            model = %fb_model_str,
1435                            "fallback router LLM timed out — returning immediately to unblock tick loop"
1436                        );
1437                        return Err(e);
1438                    }
1439                    tracing::warn!(
1440                        agent = %fb_agent,
1441                        model = %fb_model_str,
1442                        error = %e,
1443                        "fallback router LLM also failed"
1444                    );
1445                    response::record_model_failure(&fb_agent, fb_model_str).await;
1446                    last_err = Some(e);
1447                }
1448            }
1449        }
1450
1451        Err(last_err.unwrap_or_else(|| anyhow::anyhow!("all router LLM pool entries exhausted")))
1452    }
1453
1454    /// Record a rate limit event for an agent, reducing its routing weight.
1455    ///
1456    /// Called by the engine when an agent returns a 429/rate limit error.
1457    pub fn record_rate_limit(&mut self, agent: &str) {
1458        self.weights.record_rate_limit(agent);
1459    }
1460
1461    /// Record a successful task completion, restoring agent weight.
1462    pub fn record_success(&mut self, agent: &str) {
1463        self.weights.record_success(agent);
1464    }
1465
1466    /// Tick weight recovery for all agents.
1467    ///
1468    /// Called periodically by the engine to gradually restore weights
1469    /// as rate limit windows expire.
1470    pub fn tick_weight_recovery(&mut self) {
1471        self.weights.tick_recovery();
1472    }
1473
1474    /// Store routing result in the task store.
1475    pub async fn store_route_result(
1476        &self,
1477        task_id: &str,
1478        result: &RouteResult,
1479        store: &Arc<TaskStore>,
1480        repo: &str,
1481    ) -> anyhow::Result<()> {
1482        if let Ok(Some(store_id)) = store.resolve_task_id(repo, task_id).await {
1483            store
1484                .set_fields(
1485                    store_id,
1486                    &[
1487                        ("agent", serde_json::json!(result.agent)),
1488                        ("complexity", serde_json::json!(result.complexity)),
1489                        ("route_reason", serde_json::json!(result.reason)),
1490                        (
1491                            "model",
1492                            serde_json::json!(result.model.as_deref().unwrap_or("")),
1493                        ),
1494                    ],
1495                )
1496                .await?;
1497        }
1498        Ok(())
1499    }
1500}
1501
1502/// Retrieve routing result from the task store.
1503pub async fn get_route_result(
1504    store: &Arc<TaskStore>,
1505    repo: &str,
1506    task_id: &str,
1507) -> Result<RouteResult, RouteResultError> {
1508    let store_id = store
1509        .resolve_task_id(repo, task_id)
1510        .await
1511        .map_err(RouteResultError::Other)?
1512        .ok_or_else(|| {
1513            RouteResultError::Other(anyhow::anyhow!("task {task_id} not found in store"))
1514        })?;
1515    let task = store.get(store_id).await.map_err(RouteResultError::Other)?;
1516
1517    let agent = task
1518        .agent
1519        .clone()
1520        .filter(|a| !a.is_empty())
1521        .ok_or_else(|| RouteResultError::NoAgent {
1522            task_id: task_id.to_string(),
1523        })?;
1524
1525    let complexity = if task.complexity.is_empty() {
1526        "medium".to_string()
1527    } else {
1528        task.complexity.clone()
1529    };
1530
1531    let reason = task.route_reason.clone();
1532    let model = task.model.clone().filter(|m| !m.is_empty());
1533
1534    // Validate that the stored model is valid for the resolved agent.
1535    // Agent-specific aliases (e.g. "opus") can leak across agents during
1536    // failover, so we verify against the router's model pools (#1604).
1537    let config = RouterConfig::from_config();
1538    let model = model.filter(|m| {
1539        let complexities = ["simple", "medium", "complex", "review"];
1540        // Collect all model pools configured for this agent across complexity tiers.
1541        let all_pools: Vec<Vec<String>> = complexities
1542            .iter()
1543            .filter_map(|comp| config.model_pool_for_complexity(&agent, comp))
1544            .collect();
1545        // If the agent has no model pools configured, we can't validate — pass through.
1546        if all_pools.is_empty() {
1547            return true;
1548        }
1549        let valid = all_pools.iter().any(|pool| pool.contains(m));
1550        if !valid {
1551            tracing::warn!(
1552                task_id,
1553                agent,
1554                model = %m,
1555                "discarding stale model: not in agent's model pools"
1556            );
1557        }
1558        valid
1559    });
1560
1561    // A stale model is replaced by the agent's model for the task's complexity, an empty
1562    // model would reach the agent as `--model ""`. last_error keeps the reason (#2527)
1563    let original_model = task.model.clone().filter(|m| !m.is_empty());
1564    let mut model = model;
1565    if original_model.is_some() && model.is_none() {
1566        model = config.model_for_complexity(&agent, &complexity, task_id);
1567        let stale_model = original_model.as_deref().unwrap_or("unknown");
1568        let stale_err = format!(
1569            "stale model discarded: {stale_model} not in {agent} model pools, using {}",
1570            model.as_deref().unwrap_or("the agent default")
1571        );
1572        if let Err(e) = store
1573            .set_fields(
1574                store_id,
1575                &[
1576                    (
1577                        "model",
1578                        serde_json::json!(model.clone().unwrap_or_default()),
1579                    ),
1580                    ("last_error", serde_json::json!(stale_err)),
1581                ],
1582            )
1583            .await
1584        {
1585            // Log the error so transient DB failures are visible and diagnosable.
1586            tracing::warn!(task_id = %task_id, error = %e, "failed to clear stale model from store — will retry on next dispatch");
1587        }
1588    }
1589
1590    Ok(RouteResult {
1591        agent,
1592        model,
1593        complexity,
1594        estimate: 0,
1595        reason,
1596        profile: AgentProfile::default(),
1597        warning: None,
1598    })
1599}
1600
1601#[cfg(test)]
1602#[expect(
1603    clippy::float_cmp,
1604    reason = "the weights are stored constants, so they compare exactly"
1605)]
1606mod tests {
1607    use super::config::DEFAULT_AGENTS;
1608    use super::llm::LlmRouteResponse;
1609    use super::strategies;
1610    use super::weights::default_missing_weight;
1611    use super::weights::{
1612        AgentWeights, DEFAULT_WEIGHT, MIN_WEIGHT, RATE_LIMIT_DECAY, RECOVERY_DELAY, RateLimitState,
1613    };
1614    use super::*;
1615    use crate::backends::{ExternalId, ExternalTask};
1616    use crate::engine::cooldown::{
1617        clear_agent_degraded, is_agent_degraded, is_model_in_cooldown, mark_agent_degraded,
1618        record_model_failure, set_model_cooldown,
1619    };
1620    use crate::store::NewTask;
1621    use crate::store::TaskStore;
1622    use serial_test::serial;
1623    use std::collections::HashMap;
1624    use std::env;
1625    #[cfg(unix)]
1626    use std::ffi::OsString;
1627    use std::fs;
1628    #[cfg(unix)]
1629    use std::fs::Permissions;
1630    #[cfg(unix)]
1631    use std::os::unix::fs::PermissionsExt;
1632    #[cfg(unix)]
1633    use std::process;
1634    #[cfg(unix)]
1635    use std::process::Command;
1636    #[cfg(unix)]
1637    use std::thread;
1638    use std::time::{Duration, Instant};
1639
1640    #[tokio::test]
1641    #[serial_test::serial(orch_home)]
1642    async fn get_route_result_replaces_a_stale_model_with_the_complexity_model() {
1643        let home = tempfile::tempdir().unwrap();
1644        fs::write(
1645            home.path().join("config.yml"),
1646            "model_map:\n  medium:\n    claude: sonnet\n",
1647        )
1648        .unwrap();
1649        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1650        unsafe { env::set_var("ORCH_HOME", home.path()) };
1651
1652        let store = Arc::new(TaskStore::open_memory().await.unwrap());
1653        let store_id = store
1654            .create(&NewTask {
1655                external_id: Some("7".to_string()),
1656                repo: "owner/repo".to_string(),
1657                origin: "github".to_string(),
1658                title: "stale model".to_string(),
1659                ..Default::default()
1660            })
1661            .await
1662            .unwrap();
1663        store
1664            .set_fields(
1665                store_id,
1666                &[
1667                    ("agent", serde_json::json!("claude")),
1668                    ("model", serde_json::json!("claude-removed-model")),
1669                    ("complexity", serde_json::json!("medium")),
1670                ],
1671            )
1672            .await
1673            .unwrap();
1674
1675        let route = get_route_result(&store, "owner/repo", "7").await;
1676        let stored = store.get(store_id).await.unwrap().model;
1677        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
1678        unsafe { env::remove_var("ORCH_HOME") };
1679
1680        assert_eq!(route.unwrap().model.as_deref(), Some("sonnet"));
1681        assert_eq!(stored.as_deref(), Some("sonnet"));
1682    }
1683
1684    // Test-only delegate so tests can call router.parse_llm_response() directly
1685    // without referencing llm_router.
1686    impl Router {
1687        #[expect(
1688            clippy::unused_self,
1689            reason = "the test helper keeps the router method syntax"
1690        )]
1691        fn parse_llm_response(&self, response: &str) -> anyhow::Result<LlmRouteResponse> {
1692            LlmRouter::parse_llm_response("claude", response)
1693        }
1694    }
1695
1696    async fn test_store() -> Arc<TaskStore> {
1697        Arc::new(TaskStore::open_memory().await.unwrap())
1698    }
1699
1700    fn create_test_task(id: &str, title: &str, labels: Vec<String>) -> ExternalTask {
1701        ExternalTask {
1702            id: ExternalId(id.to_string()),
1703            title: title.to_string(),
1704            body: "Test body".to_string(),
1705            state: "open".to_string(),
1706            labels,
1707            author: "test".to_string(),
1708            created_at: "2026-01-01T00:00:00Z".to_string(),
1709            updated_at: "2026-01-01T00:00:00Z".to_string(),
1710            url: format!("https://github.com/test/test/issues/{id}"),
1711        }
1712    }
1713
1714    #[serial(cooldown_state)]
1715    #[test]
1716    fn extract_agent_from_labels() {
1717        let mut config = RouterConfig::default();
1718        // Ensure a concrete model exists for the labeled agent so the label
1719        // override is honored by the router (routing requires a model).
1720        config
1721            .model_map
1722            .entry("medium".to_string())
1723            .or_default()
1724            .insert("claude".to_string(), vec!["haiku".to_string()]);
1725
1726        assert_eq!(
1727            strategies::extract_agent_from_labels(&config.agents, &["agent:claude".to_string()]),
1728            Some("claude".to_string())
1729        );
1730        assert_eq!(
1731            strategies::extract_agent_from_labels(&config.agents, &["agent:codex".to_string()]),
1732            Some("codex".to_string())
1733        );
1734        assert_eq!(
1735            strategies::extract_agent_from_labels(&config.agents, &["agent:opencode".to_string()]),
1736            Some("opencode".to_string())
1737        );
1738        assert_eq!(
1739            strategies::extract_agent_from_labels(&config.agents, &["status:new".to_string()]),
1740            None
1741        );
1742        // Verify kimi and minimax are recognized from labels
1743        assert_eq!(
1744            strategies::extract_agent_from_labels(&config.agents, &["agent:kimi".to_string()]),
1745            Some("kimi".to_string())
1746        );
1747        assert_eq!(
1748            strategies::extract_agent_from_labels(&config.agents, &["agent:minimax".to_string()]),
1749            Some("minimax".to_string())
1750        );
1751    }
1752
1753    #[serial(cooldown_state)]
1754    #[tokio::test]
1755    async fn all_agents_cooled_error_names_degraded_agents() {
1756        cooldown::reset_global_state().await;
1757        mark_agent_degraded("claude");
1758        mark_agent_degraded("grok");
1759        let err = AllAgentsCooledError::new("all agents".to_string(), None);
1760        let msg = err.to_string();
1761        assert!(
1762            msg.contains("degraded: claude, grok"),
1763            "ALL AGENTS COOLED must name degraded agents, got {msg}"
1764        );
1765        assert_eq!(err.degraded(), &["claude".to_string(), "grok".to_string()]);
1766        clear_agent_degraded("claude");
1767        clear_agent_degraded("grok");
1768    }
1769
1770    #[serial(cooldown_state)]
1771    #[test]
1772    fn default_agents_constant() {
1773        assert_eq!(DEFAULT_AGENTS.len(), 7);
1774        assert!(DEFAULT_AGENTS.contains(&"claude"));
1775        assert!(DEFAULT_AGENTS.contains(&"kimi"));
1776        assert!(DEFAULT_AGENTS.contains(&"minimax"));
1777        assert!(DEFAULT_AGENTS.contains(&"glm"));
1778        assert!(DEFAULT_AGENTS.contains(&"grok"));
1779    }
1780
1781    #[serial(cooldown_state)]
1782    #[test]
1783    fn extract_complexity_from_labels() {
1784        assert_eq!(
1785            strategies::extract_complexity_from_labels(&["complexity:simple".to_string()]),
1786            "simple"
1787        );
1788        assert_eq!(
1789            strategies::extract_complexity_from_labels(&["complexity:medium".to_string()]),
1790            "medium"
1791        );
1792        assert_eq!(
1793            strategies::extract_complexity_from_labels(&["complexity:complex".to_string()]),
1794            "complex"
1795        );
1796        assert_eq!(
1797            strategies::extract_complexity_from_labels(&["status:new".to_string()]),
1798            "medium"
1799        );
1800    }
1801
1802    #[serial(cooldown_state)]
1803    #[test]
1804    fn route_result_serialization() {
1805        let result = RouteResult {
1806            agent: "claude".to_string(),
1807            model: Some("sonnet".to_string()),
1808            complexity: "medium".to_string(),
1809            estimate: 5,
1810            reason: "test".to_string(),
1811            profile: AgentProfile {
1812                role: "backend".to_string(),
1813                tools: vec!["git".to_string()],
1814                constraints: vec![],
1815            },
1816            warning: None,
1817        };
1818
1819        let json = serde_json::to_string(&result).unwrap();
1820        assert!(json.contains("claude"));
1821        assert!(json.contains("sonnet"));
1822
1823        let deserialized: RouteResult = serde_json::from_str(&json).unwrap();
1824        assert_eq!(deserialized.agent, "claude");
1825        assert_eq!(deserialized.model, Some("sonnet".to_string()));
1826    }
1827
1828    #[serial(cooldown_state)]
1829    #[test]
1830    fn router_config_default() {
1831        let mut config = RouterConfig::default();
1832        // Ensure a concrete model exists for the labeled agent so the label
1833        // override is honored by the router (routing requires a model).
1834        config
1835            .model_map
1836            .entry("medium".to_string())
1837            .or_default()
1838            .insert("claude".to_string(), vec!["haiku".to_string()]);
1839
1840        assert_eq!(config.mode, "round_robin");
1841        assert_eq!(config.router_agent, "claude");
1842        assert_eq!(
1843            config.router_model, "",
1844            "no router.model means the router agent's CLI default"
1845        );
1846        assert_eq!(config.fallback_executor, "codex");
1847        assert_eq!(config.max_route_attempts, 3);
1848        assert_ne!(config.allowed_tools, [] as [String; 0]);
1849
1850        // Verify configurable agents list includes all 7 agents
1851        assert_eq!(config.agents.len(), 7);
1852        assert!(config.agents.contains(&"claude".to_string()));
1853        assert!(config.agents.contains(&"codex".to_string()));
1854        assert!(config.agents.contains(&"opencode".to_string()));
1855        assert!(config.agents.contains(&"kimi".to_string()));
1856        assert!(config.agents.contains(&"minimax".to_string()));
1857        assert!(config.agents.contains(&"glm".to_string()));
1858        assert!(config.agents.contains(&"grok".to_string()));
1859    }
1860
1861    #[serial(cooldown_state)]
1862    #[test]
1863    fn model_map_lookup_returns_none_without_config() {
1864        // Default config has no hardcoded models — config.yml is the source of truth.
1865        let config = RouterConfig::default();
1866        assert!(
1867            config
1868                .model_for_complexity("claude", "simple", "")
1869                .is_none()
1870        );
1871        assert!(
1872            config
1873                .model_for_complexity("claude", "medium", "")
1874                .is_none()
1875        );
1876        assert!(
1877            config
1878                .model_for_complexity("claude", "complex", "")
1879                .is_none()
1880        );
1881        assert!(
1882            config
1883                .model_for_complexity("opencode", "review", "")
1884                .is_none()
1885        );
1886    }
1887
1888    #[serial(cooldown_state)]
1889    #[test]
1890    fn model_map_lookup_returns_configured_value() {
1891        // When model_map is populated (from config), it is returned correctly.
1892        let mut config = RouterConfig::default();
1893        config
1894            .model_map
1895            .entry("simple".to_string())
1896            .or_default()
1897            .insert("claude".to_string(), vec!["haiku".to_string()]);
1898        assert_eq!(
1899            config.model_for_complexity("claude", "simple", ""),
1900            Some("haiku".to_string())
1901        );
1902    }
1903
1904    #[serial(cooldown_state)]
1905    #[tokio::test]
1906    async fn model_pool_selection_skips_cooled() {
1907        cooldown::reset_global_state().await;
1908
1909        let mut config = RouterConfig::default();
1910        // Set up a two-model pool for opencode/simple
1911        config
1912            .model_map
1913            .entry("simple".to_string())
1914            .or_default()
1915            .insert(
1916                "opencode".to_string(),
1917                vec!["model-a".to_string(), "model-b".to_string()],
1918            );
1919
1920        // Before any cooldown, both models are candidates
1921        let m = config.model_for_complexity("opencode", "simple", "task-1");
1922        assert!(m == Some("model-a".to_string()) || m == Some("model-b".to_string()));
1923
1924        // Cool model-a
1925        record_model_failure("opencode", "model-a").await;
1926        assert!(is_model_in_cooldown("opencode", "model-a"));
1927
1928        // Now only model-b should be returned
1929        // (try many task_ids to rule out hash coincidence)
1930        for i in 0..20 {
1931            let result = config.model_for_complexity("opencode", "simple", &i.to_string());
1932            assert_eq!(
1933                result,
1934                Some("model-b".to_string()),
1935                "expected model-b when model-a is cooled, got {result:?} for task_id={i}"
1936            );
1937        }
1938
1939        // Cool model-b too — now all models are cooled
1940        record_model_failure("opencode", "model-b").await;
1941        // All cooled → return None so the caller can fall back to a different agent
1942        let fallback = config.model_for_complexity("opencode", "simple", "task-fallback");
1943        assert_eq!(fallback, None);
1944    }
1945
1946    #[serial(cooldown_state)]
1947    #[test]
1948    fn model_pool_single_string_backward_compat() {
1949        // Single-item pools behave identically to the old string format.
1950        // Manually insert a single-item pool and verify it is returned correctly.
1951        let mut config = RouterConfig::default();
1952        config
1953            .model_map
1954            .entry("simple".to_string())
1955            .or_default()
1956            .insert("claude".to_string(), vec!["haiku".to_string()]);
1957        let m = config.model_for_complexity("claude", "simple", "any-task");
1958        assert_eq!(m, Some("haiku".to_string()));
1959    }
1960
1961    #[serial(cooldown_state)]
1962    #[test]
1963    fn parse_llm_response_direct_json() {
1964        let config = RouterConfig::default();
1965        let router = Router::new(config);
1966
1967        let response = r#"{
1968            "executor": "claude",
1969            "complexity": "complex",
1970            "reason": "requires architecture analysis",
1971            "profile": {
1972                "role": "architect",
1973                "skills": ["rust", "design"],
1974                "tools": ["git", "rg"],
1975                "constraints": []
1976            },
1977            "selected_skills": ["gh"]
1978        }"#;
1979
1980        let parsed = router.parse_llm_response(response).unwrap();
1981        assert_eq!(parsed.executor, "claude");
1982        assert_eq!(parsed.complexity, "complex");
1983        assert_eq!(parsed.reason, "requires architecture analysis");
1984        assert_eq!(parsed.profile.role, "architect");
1985    }
1986
1987    #[serial(cooldown_state)]
1988    #[test]
1989    fn parse_llm_response_markdown_fenced() {
1990        let config = RouterConfig::default();
1991        let router = Router::new(config);
1992
1993        let response = r#"Here's my analysis:
1994
1995```json
1996{
1997    "executor": "codex",
1998    "complexity": "medium",
1999    "reason": "coding task",
2000    "profile": {
2001        "role": "developer",
2002        "skills": ["coding"],
2003        "tools": [],
2004        "constraints": []
2005    },
2006    "selected_skills": []
2007}
2008```
2009
2010Hope that helps!"#;
2011
2012        let parsed = router.parse_llm_response(response).unwrap();
2013        assert_eq!(parsed.executor, "codex");
2014        assert_eq!(parsed.complexity, "medium");
2015    }
2016
2017    #[serial(cooldown_state)]
2018    #[test]
2019    fn parse_llm_response_claude_envelope() {
2020        let config = RouterConfig::default();
2021        let router = Router::new(config);
2022
2023        // Claude --output-format json wraps the response in an envelope
2024        let response = r#"{"type":"result","subtype":"success","is_error":false,"duration_ms":1500,"result":"{\"executor\":\"claude\",\"complexity\":\"complex\",\"reason\":\"requires deep analysis\",\"profile\":{\"role\":\"architect\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}","usage":{"input_tokens":100,"output_tokens":50}}"#;
2025
2026        let parsed = router.parse_llm_response(response).unwrap();
2027        assert_eq!(parsed.executor, "claude");
2028        assert_eq!(parsed.complexity, "complex");
2029        assert_eq!(parsed.reason, "requires deep analysis");
2030    }
2031
2032    #[serial(cooldown_state)]
2033    #[test]
2034    fn parse_llm_response_claude_envelope_with_code_block() {
2035        let config = RouterConfig::default();
2036        let router = Router::new(config);
2037
2038        // Claude may return the JSON inside a code block within the envelope
2039        let inner = "```json\n{\"executor\":\"codex\",\"complexity\":\"simple\",\"reason\":\"simple fix\",\"profile\":{\"role\":\"developer\",\"skills\":[],\"tools\":[],\"constraints\":[]},\"selected_skills\":[]}\n```";
2040        let envelope = serde_json::json!({
2041            "type": "result",
2042            "subtype": "success",
2043            "is_error": false,
2044            "result": inner,
2045            "usage": {"input_tokens": 50, "output_tokens": 30}
2046        });
2047
2048        let parsed = router.parse_llm_response(&envelope.to_string()).unwrap();
2049        assert_eq!(parsed.executor, "codex");
2050        assert_eq!(parsed.complexity, "simple");
2051    }
2052
2053    #[serial(cooldown_state)]
2054    #[test]
2055    fn parse_llm_response_claude_envelope_error() {
2056        let config = RouterConfig::default();
2057        let router = Router::new(config);
2058
2059        let response = r#"{"type":"result","subtype":"error","is_error":true,"result":"rate limit exceeded","usage":{}}"#;
2060
2061        let err = router.parse_llm_response(response).unwrap_err();
2062        assert!(err.to_string().contains("error"), "got: {err}");
2063    }
2064
2065    #[serial(cooldown_state)]
2066    #[test]
2067    fn parse_llm_response_empty() {
2068        let config = RouterConfig::default();
2069        let router = Router::new(config);
2070
2071        let err = router.parse_llm_response("").unwrap_err();
2072        assert!(err.to_string().contains("empty"), "got: {err}");
2073    }
2074
2075    /// Claude envelope where result is a JSON object, not a string.
2076    /// Some Claude versions may return the result as a nested object.
2077    #[serial(cooldown_state)]
2078    #[test]
2079    fn parse_llm_response_claude_envelope_result_as_object() {
2080        let config = RouterConfig::default();
2081        let router = Router::new(config);
2082
2083        let response = r#"{"type":"result","subtype":"success","is_error":false,"result":{"executor":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]},"usage":{"input_tokens":100,"output_tokens":50}}"#;
2084
2085        let parsed = router.parse_llm_response(response);
2086        assert!(
2087            parsed.is_ok(),
2088            "should handle result-as-object: {}",
2089            parsed.unwrap_err()
2090        );
2091        assert_eq!(parsed.unwrap().executor, "claude");
2092    }
2093
2094    /// LLM returns "agent" instead of "executor" — common hallucination.
2095    #[serial(cooldown_state)]
2096    #[test]
2097    fn parse_llm_response_agent_instead_of_executor() {
2098        let config = RouterConfig::default();
2099        let router = Router::new(config);
2100
2101        let response = r#"{"agent":"claude","complexity":"medium","reason":"multi-file feature","profile":{"role":"developer","skills":[],"tools":[],"constraints":[]},"selected_skills":[]}"#;
2102
2103        let parsed = router.parse_llm_response(response);
2104        assert!(
2105            parsed.is_ok(),
2106            "should accept 'agent' alias: {}",
2107            parsed.unwrap_err()
2108        );
2109        assert_eq!(parsed.unwrap().executor, "claude");
2110    }
2111
2112    /// LLM returns minimal JSON without profile or reason.
2113    #[serial(cooldown_state)]
2114    #[test]
2115    fn parse_llm_response_minimal_json() {
2116        let config = RouterConfig::default();
2117        let router = Router::new(config);
2118
2119        let response = r#"{"executor":"codex","complexity":"simple"}"#;
2120
2121        let parsed = router.parse_llm_response(response);
2122        assert!(
2123            parsed.is_ok(),
2124            "should accept minimal JSON: {}",
2125            parsed.unwrap_err()
2126        );
2127        assert_eq!(parsed.unwrap().executor, "codex");
2128    }
2129
2130    // ── Real fixture tests: route prompt → response → parsed result ──
2131
2132    /// Real Claude envelope with result as escaped JSON string.
2133    #[serial(cooldown_state)]
2134    #[test]
2135    fn fixture_route_response_string() {
2136        let config = RouterConfig::default();
2137        let router = Router::new(config);
2138
2139        let response = include_str!("../../../tests/fixtures/route-response-string.json");
2140        let parsed = router.parse_llm_response(response).unwrap();
2141        assert_eq!(parsed.executor, "codex");
2142        assert_eq!(parsed.complexity, "medium");
2143        assert_ne!(parsed.reason, "");
2144        assert_ne!(parsed.profile.role, "");
2145    }
2146
2147    /// Real Claude envelope with result as JSON object (not string).
2148    /// This was the root cause of all router failures on 2026-02-27.
2149    #[serial(cooldown_state)]
2150    #[test]
2151    fn fixture_route_response_object() {
2152        let config = RouterConfig::default();
2153        let router = Router::new(config);
2154
2155        let response = include_str!("../../../tests/fixtures/route-response-object.json");
2156        let parsed = router.parse_llm_response(response).unwrap();
2157        assert_eq!(parsed.executor, "claude");
2158        assert_eq!(parsed.complexity, "complex");
2159    }
2160
2161    /// Real Claude envelope with result as string containing markdown + JSON.
2162    #[serial(cooldown_state)]
2163    #[test]
2164    fn fixture_route_response_markdown() {
2165        let config = RouterConfig::default();
2166        let router = Router::new(config);
2167
2168        let response = include_str!("../../../tests/fixtures/route-response-markdown.json");
2169        let parsed = router.parse_llm_response(response).unwrap();
2170        assert_eq!(parsed.executor, "codex");
2171        assert_eq!(parsed.complexity, "medium");
2172    }
2173
2174    /// Claude envelope with escaped JSON string containing newlines and markdown.
2175    #[serial(cooldown_state)]
2176    #[test]
2177    fn parse_llm_response_claude_envelope_with_prose() {
2178        let config = RouterConfig::default();
2179        let router = Router::new(config);
2180
2181        let inner = "Based on the task description, here's my routing decision:\n\n```json\n{\"executor\":\"codex\",\"complexity\":\"medium\",\"reason\":\"standard feature work\",\"profile\":{\"role\":\"developer\",\"skills\":[\"rust\"],\"tools\":[\"git\",\"cargo\"],\"constraints\":[]},\"selected_skills\":[]}\n```\n\nThis task involves adding slash commands which is standard development work.";
2182        let envelope = serde_json::json!({
2183            "type": "result",
2184            "subtype": "success",
2185            "is_error": false,
2186            "result": inner,
2187            "usage": {"input_tokens": 100, "output_tokens": 80}
2188        });
2189
2190        let parsed = router.parse_llm_response(&envelope.to_string());
2191        assert!(
2192            parsed.is_ok(),
2193            "should handle prose + code block in envelope: {}",
2194            parsed.unwrap_err()
2195        );
2196        assert_eq!(parsed.unwrap().executor, "codex");
2197    }
2198
2199    #[serial(cooldown_state)]
2200    #[tokio::test]
2201    async fn route_round_robin_basic() {
2202        cooldown::reset_global_state().await;
2203        // Force at least one agent to be available for testing
2204        // In real usage, discover_agents finds installed CLIs
2205        let config = RouterConfig {
2206            mode: "round_robin".to_string(),
2207            ..Default::default()
2208        };
2209
2210        // Create router with mock available agents
2211        let agents = vec!["claude".to_string(), "codex".to_string()];
2212        let mut weights = AgentWeights::default();
2213        weights.ensure_agents(&agents);
2214        let router = Router {
2215            config,
2216            available_agents: agents,
2217            weights,
2218            llm_router: Arc::new(LlmRouter::new()),
2219            ollama_router: None,
2220            rr_index: 0,
2221            last_agent: None,
2222            review_rr_index: 0,
2223            router_pool: vec![],
2224            pool_index: 0,
2225        };
2226
2227        let task = create_test_task("1", "Test task", vec![]);
2228        let result =
2229            strategies::route_via_round_robin(&router.available_agents, &router.config, &task)
2230                .unwrap();
2231
2232        // Task 1 % 2 agents = agent at index 1 = codex
2233        assert_eq!(result.agent, "codex");
2234        assert_eq!(result.reason, "round_robin (task 1 % 2 agents)");
2235    }
2236
2237    #[serial(cooldown_state)]
2238    #[tokio::test]
2239    async fn route_uses_label_override() {
2240        cooldown::reset_global_state().await;
2241        let mut config = RouterConfig::default();
2242        // Ensure a concrete model exists so the label override can be honored
2243        // in the test environment where no external config.yml is present.
2244        config
2245            .model_map
2246            .entry("medium".to_string())
2247            .or_default()
2248            .insert("claude".to_string(), vec!["haiku".to_string()]);
2249        let agents = vec!["claude".to_string(), "codex".to_string()];
2250        let mut weights = AgentWeights::default();
2251        weights.ensure_agents(&agents);
2252        let mut router = Router {
2253            config,
2254            available_agents: agents,
2255            weights,
2256            llm_router: Arc::new(LlmRouter::new()),
2257            ollama_router: None,
2258            rr_index: 0,
2259            last_agent: None,
2260            review_rr_index: 0,
2261            router_pool: vec![],
2262            pool_index: 0,
2263        };
2264
2265        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2266        let store = test_store().await;
2267
2268        // Should use label override, not LLM
2269        let result = router.route(&task, &store, "test/repo").await.unwrap();
2270        assert_eq!(result.agent, "claude");
2271        assert!(result.reason.contains("label"));
2272    }
2273
2274    #[serial(cooldown_state)]
2275    #[tokio::test]
2276    async fn route_uses_stored_agent_over_label() {
2277        cooldown::reset_global_state().await;
2278        let mut config = RouterConfig::default();
2279        let medium = config.model_map.entry("medium".to_string()).or_default();
2280        medium.insert("claude".to_string(), vec!["haiku".to_string()]);
2281        medium.insert("codex".to_string(), vec!["gpt-test".to_string()]);
2282        let agents = vec!["claude".to_string(), "codex".to_string()];
2283        let mut router = Router::new_for_test(config, agents);
2284
2285        let store = test_store().await;
2286        let store_id = store
2287            .create(&NewTask {
2288                external_id: Some("1".to_string()),
2289                repo: "test/repo".to_string(),
2290                origin: "github".to_string(),
2291                title: "Test".to_string(),
2292                ..Default::default()
2293            })
2294            .await
2295            .unwrap();
2296        store
2297            .set_fields(store_id, &[("agent", serde_json::json!("codex"))])
2298            .await
2299            .unwrap();
2300
2301        let task = create_test_task("1", "Test", vec!["agent:claude".to_string()]);
2302        let result = router.route(&task, &store, "test/repo").await.unwrap();
2303        assert_eq!(result.agent, "codex");
2304    }
2305
2306    #[serial(cooldown_state)]
2307    #[test]
2308    fn check_routing_sanity_warnings() {
2309        // Backend task routed to claude should warn
2310        let task = create_test_task("1", "Fix API", vec!["backend".to_string()]);
2311        let profile = AgentProfile {
2312            role: "general".to_string(),
2313            tools: vec![],
2314            constraints: vec![],
2315        };
2316        let warning = LlmRouter::check_routing_sanity(&task, "claude", &profile);
2317        assert!(warning.is_some());
2318        assert!(warning.unwrap().contains("backend"));
2319
2320        // Docs task routed to codex should warn
2321        let task = create_test_task("1", "Update README", vec!["docs".to_string()]);
2322        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2323        assert!(warning.is_some());
2324        assert!(warning.unwrap().contains("docs"));
2325
2326        // Normal routing should not warn
2327        let task = create_test_task("1", "Fix bug", vec!["bug".to_string()]);
2328        let warning = LlmRouter::check_routing_sanity(&task, "codex", &profile);
2329        assert!(warning.is_none());
2330    }
2331
2332    #[serial(cooldown_state)]
2333    #[tokio::test]
2334    async fn router_reload_preserves_structure() {
2335        cooldown::reset_global_state().await;
2336        let config = RouterConfig::default();
2337        let agents = vec!["claude".to_string()];
2338        let mut weights = AgentWeights::default();
2339        weights.ensure_agents(&agents);
2340        let mut router = Router {
2341            config,
2342            available_agents: agents,
2343            weights,
2344            llm_router: Arc::new(LlmRouter::new()),
2345            ollama_router: None,
2346            rr_index: 0,
2347            last_agent: None,
2348            review_rr_index: 0,
2349            router_pool: vec![],
2350            pool_index: 0,
2351        };
2352
2353        // Reload — should re-read config and remain valid
2354        router.reload_async().await;
2355
2356        // After reload, mode should be a valid value (llm, local, or round_robin)
2357        assert!(
2358            router.config.mode == "llm"
2359                || router.config.mode == "round_robin"
2360                || router.config.mode == "local",
2361            "mode should be 'llm', 'local', or 'round_robin', got '{}'",
2362            router.config.mode
2363        );
2364        // Fallback executor should always be set
2365        assert_ne!(router.config.fallback_executor, "");
2366        // Tools should always be populated
2367        assert_ne!(router.config.allowed_tools, [] as [String; 0]);
2368    }
2369
2370    // --- Weighted round-robin tests ---
2371
2372    #[serial(cooldown_state)]
2373    #[test]
2374    fn rate_limit_state_defaults_to_full_weight() {
2375        let state = RateLimitState::default();
2376        assert_eq!(state.weight, DEFAULT_WEIGHT);
2377        assert_eq!(state.consecutive_hits, 0);
2378        assert!(state.last_limited_at.is_none());
2379        assert!(!state.is_limited());
2380    }
2381
2382    #[serial(cooldown_state)]
2383    #[test]
2384    fn rate_limit_state_decays_on_hit() {
2385        let mut state = RateLimitState::default();
2386        state.record_rate_limit();
2387
2388        assert!(state.weight < DEFAULT_WEIGHT);
2389        assert_eq!(state.consecutive_hits, 1);
2390        assert!(state.last_limited_at.is_some());
2391        assert!(state.is_limited());
2392
2393        // Weight should be DEFAULT_WEIGHT * RATE_LIMIT_DECAY
2394        let expected = DEFAULT_WEIGHT * RATE_LIMIT_DECAY;
2395        assert!((state.weight - expected).abs() < 1e-10);
2396    }
2397
2398    #[serial(cooldown_state)]
2399    #[test]
2400    fn rate_limit_state_never_drops_below_min() {
2401        let mut state = RateLimitState::default();
2402
2403        // Hit many times
2404        for _ in 0..100 {
2405            state.record_rate_limit();
2406        }
2407
2408        assert!(state.weight >= MIN_WEIGHT);
2409        assert_eq!(state.consecutive_hits, 100);
2410    }
2411
2412    #[serial(cooldown_state)]
2413    #[test]
2414    fn rate_limit_state_recovers_on_success() {
2415        let mut state = RateLimitState::default();
2416
2417        // Decay first
2418        state.record_rate_limit();
2419        state.record_rate_limit();
2420        let after_decay = state.weight;
2421
2422        // Record success
2423        state.record_success();
2424        assert!(state.weight > after_decay);
2425        assert_eq!(state.consecutive_hits, 0);
2426    }
2427
2428    #[serial(cooldown_state)]
2429    #[test]
2430    fn rate_limit_state_success_caps_at_default() {
2431        let mut state = RateLimitState::default();
2432
2433        // Already at full weight, success shouldn't exceed it
2434        state.record_success();
2435        assert_eq!(state.weight, DEFAULT_WEIGHT);
2436    }
2437
2438    #[serial(cooldown_state)]
2439    #[test]
2440    fn agent_weights_ensure_agents() {
2441        let mut weights = AgentWeights::default();
2442        let agents = vec!["claude".to_string(), "codex".to_string()];
2443        weights.ensure_agents(&agents);
2444
2445        assert_eq!(weights.states.len(), 2);
2446        assert_eq!(weights.get_weight("claude"), DEFAULT_WEIGHT);
2447        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2448    }
2449
2450    #[serial(cooldown_state)]
2451    #[test]
2452    fn agent_weights_ensure_agents_preserves_existing() {
2453        let mut weights = AgentWeights::default();
2454        let agents = vec!["claude".to_string(), "codex".to_string()];
2455        weights.ensure_agents(&agents);
2456
2457        // Reduce claude's weight
2458        weights.record_rate_limit("claude");
2459        let claude_weight = weights.get_weight("claude");
2460        assert!(claude_weight < DEFAULT_WEIGHT);
2461
2462        // Ensure agents again — claude's weight should be preserved
2463        let agents2 = vec![
2464            "claude".to_string(),
2465            "codex".to_string(),
2466            "opencode".to_string(),
2467        ];
2468        weights.ensure_agents(&agents2);
2469
2470        assert_eq!(weights.get_weight("claude"), claude_weight);
2471        assert_eq!(weights.get_weight("opencode"), DEFAULT_WEIGHT);
2472    }
2473
2474    #[test]
2475    fn default_missing_weight_uses_minimum_of_configured() {
2476        let mut configured = HashMap::new();
2477        configured.insert("claude".to_string(), 0.2);
2478        configured.insert("codex".to_string(), 0.2);
2479        configured.insert("opencode".to_string(), 0.2);
2480        configured.insert("kimi".to_string(), 0.2);
2481
2482        // minimax omitted from router.weights — must not silently outrank
2483        // every explicitly configured agent (issue #3596).
2484        assert_eq!(default_missing_weight(&configured), 0.2);
2485    }
2486
2487    #[test]
2488    fn default_missing_weight_falls_back_when_none_configured() {
2489        let configured = HashMap::new();
2490        assert_eq!(default_missing_weight(&configured), DEFAULT_WEIGHT);
2491    }
2492
2493    #[serial(cooldown_state)]
2494    #[test]
2495    fn ensure_agents_with_weights_missing_agent_does_not_outrank_configured() {
2496        let mut weights = AgentWeights::default();
2497        let mut configured = HashMap::new();
2498        configured.insert("claude".to_string(), 0.2);
2499        configured.insert("codex".to_string(), 0.2);
2500        configured.insert("opencode".to_string(), 0.2);
2501        configured.insert("kimi".to_string(), 0.2);
2502
2503        let agents = vec![
2504            "claude".to_string(),
2505            "codex".to_string(),
2506            "opencode".to_string(),
2507            "kimi".to_string(),
2508            "minimax".to_string(),
2509        ];
2510        weights.ensure_agents_with_weights(&agents, &configured);
2511
2512        // minimax was omitted from router.weights but must not get the old
2513        // DEFAULT_WEIGHT of 1.0, which would outrank every configured agent.
2514        assert_eq!(weights.get_weight("minimax"), 0.2);
2515        assert_eq!(weights.get_weight("claude"), 0.2);
2516    }
2517
2518    #[serial(cooldown_state)]
2519    #[test]
2520    fn agent_weights_record_rate_limit() {
2521        let mut weights = AgentWeights::default();
2522        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2523
2524        weights.record_rate_limit("claude");
2525
2526        assert!(weights.get_weight("claude") < DEFAULT_WEIGHT);
2527        assert_eq!(weights.get_weight("codex"), DEFAULT_WEIGHT);
2528    }
2529
2530    #[serial(cooldown_state)]
2531    #[test]
2532    fn agent_weights_record_success() {
2533        let mut weights = AgentWeights::default();
2534        weights.ensure_agents(&["claude".to_string()]);
2535
2536        weights.record_rate_limit("claude");
2537        let after_limit = weights.get_weight("claude");
2538
2539        weights.record_success("claude");
2540        assert!(weights.get_weight("claude") > after_limit);
2541    }
2542
2543    #[serial(cooldown_state)]
2544    #[test]
2545    fn agent_weights_snapshot() {
2546        let mut weights = AgentWeights::default();
2547        weights.ensure_agents(&["claude".to_string(), "codex".to_string()]);
2548        weights.record_rate_limit("claude");
2549
2550        let snap = weights.snapshot();
2551        assert_eq!(snap.len(), 2);
2552
2553        // Snapshot is sorted alphabetically
2554        assert_eq!(snap[0].0, "claude");
2555        assert!(snap[0].1 < DEFAULT_WEIGHT);
2556        assert_eq!(snap[0].2, 1); // 1 hit
2557
2558        assert_eq!(snap[1].0, "codex");
2559        assert_eq!(snap[1].1, DEFAULT_WEIGHT);
2560        assert_eq!(snap[1].2, 0); // 0 hits
2561    }
2562
2563    #[serial(cooldown_state)]
2564    #[test]
2565    fn agent_weights_weighted_select_favors_higher_weight() {
2566        let mut weights = AgentWeights::default();
2567        let agents = vec!["claude".to_string(), "codex".to_string()];
2568        weights.ensure_agents(&agents);
2569
2570        // Heavily penalize claude
2571        for _ in 0..10 {
2572            weights.record_rate_limit("claude");
2573        }
2574
2575        // Run many selections and count
2576        let mut claude_count = 0;
2577        let mut codex_count = 0;
2578        for _ in 0..100 {
2579            match weights.weighted_select(&agents, "task-1") {
2580                Some(ref a) if a == "claude" => claude_count += 1,
2581                Some(ref a) if a == "codex" => codex_count += 1,
2582                _ => {}
2583            }
2584        }
2585
2586        // Codex should get significantly more selections
2587        // (claude weight is near MIN_WEIGHT, codex is at 1.0)
2588        assert!(
2589            codex_count > claude_count,
2590            "codex ({codex_count}) should get more selections than claude ({claude_count})"
2591        );
2592    }
2593
2594    #[serial(cooldown_state)]
2595    #[test]
2596    fn agent_weights_weighted_select_empty_returns_none() {
2597        let weights = AgentWeights::default();
2598        assert!(weights.weighted_select(&[], "task-1").is_none());
2599    }
2600
2601    #[serial(cooldown_state)]
2602    #[test]
2603    fn agent_weights_batch_routing_produces_varied_selections() {
2604        // Regression test: when routing a batch of tasks in the same tick,
2605        // different task IDs should not all select the same agent.
2606        let mut weights = AgentWeights::default();
2607        let agents = vec!["claude".to_string(), "codex".to_string()];
2608        weights.ensure_agents(&agents);
2609
2610        // Simulate batch routing: 20 different task IDs called in rapid succession
2611        let task_ids: Vec<String> = (1..=20).map(|i| format!("task-{i}")).collect();
2612        let selections: Vec<String> = task_ids
2613            .iter()
2614            .filter_map(|id| weights.weighted_select(&agents, id))
2615            .collect();
2616
2617        // With 20 tasks and 2 agents of equal weight, we should see both agents selected.
2618        // The probability of all 20 selecting the same agent by chance is (0.5)^19 < 1e-6.
2619        let claude_count = selections.iter().filter(|a| a.as_str() == "claude").count();
2620        let codex_count = selections.iter().filter(|a| a.as_str() == "codex").count();
2621        assert!(
2622            claude_count > 0 && codex_count > 0,
2623            "batch routing should distribute across agents; got claude={claude_count} codex={codex_count}"
2624        );
2625    }
2626
2627    #[serial(cooldown_state)]
2628    #[test]
2629    fn agent_weights_weighted_select_single_agent() {
2630        let mut weights = AgentWeights::default();
2631        let agents = vec!["claude".to_string()];
2632        weights.ensure_agents(&agents);
2633
2634        let selected = weights.weighted_select(&agents, "task-1");
2635        assert_eq!(selected, Some("claude".to_string()));
2636    }
2637
2638    #[serial(cooldown_state)]
2639    #[test]
2640    fn agent_weights_tick_recovery_restores_weight() {
2641        let mut weights = AgentWeights::default();
2642        weights.ensure_agents(&["claude".to_string()]);
2643
2644        // Set up a rate limit that looks like it happened long ago
2645        let state = weights.states.get_mut("claude").unwrap();
2646        state.weight = 0.5;
2647        state.last_limited_at = Some(
2648            Instant::now()
2649                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2650                .expect("the clock is past the recovery delay"),
2651        );
2652        state.consecutive_hits = 2;
2653
2654        let before = weights.get_weight("claude");
2655        weights.tick_recovery();
2656        let after = weights.get_weight("claude");
2657
2658        assert!(after > before, "weight should increase after recovery tick");
2659    }
2660
2661    #[serial(cooldown_state)]
2662    #[test]
2663    fn agent_weights_tick_recovery_clears_on_full_restore() {
2664        let mut weights = AgentWeights::default();
2665        weights.ensure_agents(&["claude".to_string()]);
2666
2667        // Set weight just below full with an old limit
2668        let state = weights.states.get_mut("claude").unwrap();
2669        state.weight = DEFAULT_WEIGHT - 0.01;
2670        state.last_limited_at = Some(
2671            Instant::now()
2672                .checked_sub(RECOVERY_DELAY + Duration::from_secs(1))
2673                .expect("the clock is past the recovery delay"),
2674        );
2675        state.consecutive_hits = 1;
2676
2677        weights.tick_recovery();
2678
2679        let state = weights.states.get("claude").unwrap();
2680        assert_eq!(state.weight, DEFAULT_WEIGHT);
2681        assert!(state.last_limited_at.is_none());
2682        assert_eq!(state.consecutive_hits, 0);
2683    }
2684
2685    #[serial(cooldown_state)]
2686    #[tokio::test]
2687    async fn route_weighted_round_robin_basic() {
2688        cooldown::reset_global_state().await;
2689        let mut config = RouterConfig {
2690            weighted_round_robin: true,
2691            ..Default::default()
2692        };
2693        // Provide a concrete model for codex so label override can be used.
2694        config
2695            .model_map
2696            .entry("medium".to_string())
2697            .or_default()
2698            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2699
2700        let agents = vec!["claude".to_string(), "codex".to_string()];
2701        let mut weights = AgentWeights::default();
2702        weights.ensure_agents(&agents);
2703        let mut router = Router {
2704            config,
2705            available_agents: agents,
2706            weights,
2707            llm_router: Arc::new(LlmRouter::new()),
2708            ollama_router: None,
2709            rr_index: 0,
2710            last_agent: None,
2711            review_rr_index: 0,
2712            router_pool: vec![],
2713            pool_index: 0,
2714        };
2715
2716        let task = create_test_task("1", "Test task", vec![]);
2717        let store = test_store().await;
2718        let result = router.route(&task, &store, "test/repo").await.unwrap();
2719
2720        // Should use weighted_round_robin
2721        assert!(result.reason.contains("weighted_round_robin"));
2722        assert!(
2723            result.agent == "claude" || result.agent == "codex",
2724            "agent should be claude or codex, got '{}'",
2725            result.agent
2726        );
2727    }
2728
2729    #[serial(cooldown_state)]
2730    #[tokio::test]
2731    async fn route_weighted_round_robin_respects_label_override() {
2732        cooldown::reset_global_state().await;
2733        let mut config = RouterConfig {
2734            weighted_round_robin: true,
2735            ..Default::default()
2736        };
2737        // Provide a concrete model for codex so label override can be used.
2738        config
2739            .model_map
2740            .entry("medium".to_string())
2741            .or_default()
2742            .insert("codex".to_string(), vec!["gpt-5.2".to_string()]);
2743
2744        let agents = vec!["claude".to_string(), "codex".to_string()];
2745        let mut weights = AgentWeights::default();
2746        weights.ensure_agents(&agents);
2747        let mut router = Router {
2748            config,
2749            available_agents: agents,
2750            weights,
2751            llm_router: Arc::new(LlmRouter::new()),
2752            ollama_router: None,
2753            rr_index: 0,
2754            last_agent: None,
2755            review_rr_index: 0,
2756            router_pool: vec![],
2757            pool_index: 0,
2758        };
2759
2760        // Label override should take precedence over weighted routing
2761        let task = create_test_task("1", "Test task", vec!["agent:codex".to_string()]);
2762        let store = test_store().await;
2763        let result = router.route(&task, &store, "test/repo").await.unwrap();
2764        assert_eq!(result.agent, "codex");
2765        assert!(result.reason.contains("label"));
2766    }
2767
2768    #[serial(cooldown_state)]
2769    #[tokio::test]
2770    async fn route_takes_the_adjacent_tier_model_when_the_own_tier_is_cooled() {
2771        cooldown::reset_global_state().await;
2772        let agent = "tierroute_agent";
2773        let mut config = RouterConfig {
2774            agents: vec![agent.to_string()],
2775            ..Default::default()
2776        };
2777        config.model_map.clear();
2778        for (tier, model) in [("medium", "tr-m"), ("complex", "tr-c")] {
2779            config
2780                .model_map
2781                .entry(tier.to_string())
2782                .or_default()
2783                .insert(agent.to_string(), vec![model.to_string()]);
2784        }
2785        cooldown::record_model_failure(agent, "tr-m").await;
2786        let mut router = Router::new_for_test(config, vec![agent.to_string()]);
2787
2788        let task = create_test_task(
2789            "1",
2790            "Test task",
2791            vec![format!("agent:{agent}"), "complexity:medium".to_string()],
2792        );
2793        let store = test_store().await;
2794        let result = router.route(&task, &store, "test/repo").await.unwrap();
2795        assert_eq!(result.agent, agent);
2796        assert_eq!(result.model.as_deref(), Some("tr-c"));
2797        assert_eq!(result.complexity, "medium");
2798        assert!(result.warning.unwrap().contains("complex tier"));
2799    }
2800
2801    #[serial(cooldown_state)]
2802    #[test]
2803    fn router_config_weighted_round_robin_default_false() {
2804        let config = RouterConfig::default();
2805        assert!(!config.weighted_round_robin);
2806    }
2807
2808    #[serial(cooldown_state)]
2809    #[test]
2810    fn router_record_and_recover_weights() {
2811        let config = RouterConfig {
2812            weighted_round_robin: true,
2813            ..Default::default()
2814        };
2815        let mut router = Router::new(config);
2816        // Override discovered agents for test
2817        router.available_agents = vec!["claude".to_string(), "codex".to_string()];
2818        router.weights.ensure_agents(&router.available_agents);
2819
2820        // Record rate limit
2821        router.record_rate_limit("claude");
2822        assert!(router.weights.get_weight("claude") < DEFAULT_WEIGHT);
2823        assert_eq!(router.weights.get_weight("codex"), DEFAULT_WEIGHT);
2824
2825        // Record success for claude
2826        router.record_success("claude");
2827        let after_success = router.weights.get_weight("claude");
2828        assert!(after_success > MIN_WEIGHT);
2829    }
2830
2831    #[serial(cooldown_state)]
2832    #[test]
2833    fn review_rr_index_advances() {
2834        let mut config = RouterConfig::default();
2835        // Set up model_map so has_available_model_for_complexity returns true for test agents
2836        for agent in &["test_a", "test_b", "test_c"] {
2837            config
2838                .model_map
2839                .entry("medium".to_string())
2840                .or_default()
2841                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2842        }
2843        let agents = vec![
2844            "test_a".to_string(),
2845            "test_b".to_string(),
2846            "test_c".to_string(),
2847        ];
2848        let mut weights = AgentWeights::default();
2849        weights.ensure_agents(&agents);
2850        let mut router = Router {
2851            config,
2852            available_agents: agents,
2853            weights,
2854            llm_router: Arc::new(LlmRouter::new()),
2855            ollama_router: None,
2856            rr_index: 0,
2857            last_agent: None,
2858            review_rr_index: 0,
2859            router_pool: vec![],
2860            pool_index: 0,
2861        };
2862
2863        let a1 = router.next_round_robin_agent(&[], "medium").unwrap();
2864        let a2 = router.next_round_robin_agent(&[], "medium").unwrap();
2865        let a3 = router.next_round_robin_agent(&[], "medium").unwrap();
2866
2867        // All three agents should appear (order depends on start index)
2868        let mut seen = vec![a1, a2, a3];
2869        seen.sort();
2870        assert_eq!(seen, vec!["test_a", "test_b", "test_c"]);
2871    }
2872
2873    #[serial(cooldown_state)]
2874    #[test]
2875    fn review_rr_fallback_with_cooled_agent() {
2876        // Regression test for #2428: when one agent is in cooldown (candidates.len() <
2877        // available_agents.len()), review_rr_index can exceed candidates.len()-1 causing
2878        // candidates.get(idx) to return None even though candidates is non-empty.
2879        let mut config = RouterConfig::default();
2880        // Register 3 agents in model_map but only 2 will be "available" (simulate cooldown)
2881        for agent in &["test_a", "test_b", "test_c"] {
2882            config
2883                .model_map
2884                .entry("medium".to_string())
2885                .or_default()
2886                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2887        }
2888        // Only 2 candidates available (test_c is in cooldown / not in available_agents)
2889        let agents = vec!["test_a".to_string(), "test_b".to_string()];
2890        let mut weights = AgentWeights::default();
2891        weights.ensure_agents(&agents);
2892        let mut router = Router {
2893            config,
2894            available_agents: agents,
2895            weights,
2896            llm_router: Arc::new(LlmRouter::new()),
2897            ollama_router: None,
2898            rr_index: 0,
2899            last_agent: None,
2900            // Start index at 2, which is >= candidates.len() (2), triggering the bug
2901            review_rr_index: 2,
2902            router_pool: vec![],
2903            pool_index: 0,
2904        };
2905
2906        // All candidates excluded — should fall back to first candidate, not None
2907        let result = router.next_round_robin_agent(&["test_a", "test_b"], "medium");
2908        assert!(
2909            result.is_some(),
2910            "next_round_robin_agent must not return None when candidates is non-empty"
2911        );
2912    }
2913
2914    #[serial(cooldown_state)]
2915    #[test]
2916    fn review_rr_index_uses_candidates_len_not_available_len() {
2917        // Regression test for #2449: review_rr_index must advance modulo candidates.len(),
2918        // not available_agents.len(). When an agent has no model for the complexity,
2919        // candidates.len() < available_agents.len() and the old code produced a drifting
2920        // index that caused non-uniform distribution.
2921        let mut config = RouterConfig::default();
2922        // Only register models for test_a and test_b — test_c has no model, so it is
2923        // filtered out by has_available_model_for_complexity → candidates.len() == 2.
2924        for agent in &["test_a", "test_b"] {
2925            config
2926                .model_map
2927                .entry("medium".to_string())
2928                .or_default()
2929                .insert((*agent).to_string(), vec!["test-model".to_string()]);
2930        }
2931        let agents = vec![
2932            "test_a".to_string(),
2933            "test_b".to_string(),
2934            "test_c".to_string(), // available but no model → filtered out of candidates
2935        ];
2936        let mut weights = AgentWeights::default();
2937        weights.ensure_agents(&agents);
2938        let mut router = Router {
2939            config,
2940            available_agents: agents,
2941            weights,
2942            llm_router: Arc::new(LlmRouter::new()),
2943            ollama_router: None,
2944            rr_index: 0,
2945            last_agent: None,
2946            review_rr_index: 0,
2947            router_pool: vec![],
2948            pool_index: 0,
2949        };
2950
2951        // With the bug: index advances modulo 3 (available_agents.len()), so the
2952        // sequence would be idx=0→1, idx=1→2, idx=2→0, meaning selection is
2953        // candidates[0%2]=test_a, candidates[1%2]=test_b, candidates[2%2]=test_a —
2954        // looks uniform by accident, but adding a 4th available agent breaks it.
2955        //
2956        // The fix: index advances modulo 2 (candidates.len()), giving a clean 0→1→0→1
2957        // cycle. Verify this by checking review_rr_index after each call.
2958
2959        let r1 = router.next_round_robin_agent(&[], "medium").unwrap();
2960        assert_eq!(
2961            router.review_rr_index, 1,
2962            "index should be 1 after first call"
2963        );
2964
2965        let r2 = router.next_round_robin_agent(&[], "medium").unwrap();
2966        assert_eq!(
2967            router.review_rr_index, 0,
2968            "index should wrap to 0 after second call"
2969        );
2970
2971        let r3 = router.next_round_robin_agent(&[], "medium").unwrap();
2972        assert_eq!(
2973            router.review_rr_index, 1,
2974            "index should be 1 after third call"
2975        );
2976
2977        // Agents must alternate uniformly
2978        assert_ne!(r1, r2, "consecutive calls should select different agents");
2979        assert_eq!(r1, r3, "third call should match first (clean cycle)");
2980    }
2981
2982    #[serial(cooldown_state)]
2983    #[tokio::test]
2984    async fn last_agent_tracks_routing() {
2985        cooldown::reset_global_state().await;
2986        let mut config = RouterConfig {
2987            mode: "round_robin".to_string(),
2988            ..Default::default()
2989        };
2990        config
2991            .model_map
2992            .entry("medium".to_string())
2993            .or_default()
2994            .insert("test_x".to_string(), vec!["test-model-x".to_string()]);
2995        config
2996            .model_map
2997            .entry("medium".to_string())
2998            .or_default()
2999            .insert("test_y".to_string(), vec!["test-model-y".to_string()]);
3000        let agents = vec!["test_x".to_string(), "test_y".to_string()];
3001        let mut weights = AgentWeights::default();
3002        weights.ensure_agents(&agents);
3003        let mut router = Router {
3004            config,
3005            available_agents: agents,
3006            weights,
3007            llm_router: Arc::new(LlmRouter::new()),
3008            ollama_router: None,
3009            rr_index: 0,
3010            last_agent: None,
3011            review_rr_index: 0,
3012            router_pool: vec![],
3013            pool_index: 0,
3014        };
3015
3016        assert!(router.last_agent.is_none());
3017
3018        let task = create_test_task("1", "Test routing", vec![]);
3019        let store = test_store().await;
3020        let result = router.route(&task, &store, "test/repo").await.unwrap();
3021
3022        assert!(router.last_agent.is_some());
3023        assert_eq!(router.last_agent.as_deref(), Some(result.agent.as_str()));
3024    }
3025
3026    // ── pool config ───────────────────────────────────────────────────────────
3027
3028    #[serial(cooldown_state)]
3029    #[test]
3030    fn parse_pool_entry_splits_on_first_colon() {
3031        let (agent, model) = super::parse_pool_entry("claude:haiku");
3032        assert_eq!(agent, "claude");
3033        assert_eq!(model, "haiku");
3034    }
3035
3036    #[serial(cooldown_state)]
3037    #[test]
3038    fn parse_pool_entry_model_with_slash() {
3039        // Models like "github-copilot/gpt-5-mini" must not be split on the slash
3040        let (agent, model) = super::parse_pool_entry("opencode:github-copilot/gpt-5-mini");
3041        assert_eq!(agent, "opencode");
3042        assert_eq!(model, "github-copilot/gpt-5-mini");
3043    }
3044
3045    #[serial(cooldown_state)]
3046    #[test]
3047    fn parse_pool_entry_no_colon() {
3048        let (agent, model) = super::parse_pool_entry("claude");
3049        assert_eq!(agent, "claude");
3050        assert_eq!(model, "");
3051    }
3052
3053    #[serial(cooldown_state)]
3054    #[test]
3055    fn effective_pool_uses_configured_pool() {
3056        let config = RouterConfig {
3057            pool: vec!["kimi:k2p5".to_string(), "claude:haiku".to_string()],
3058            ..RouterConfig::default()
3059        };
3060        let pool = config.effective_pool();
3061        assert_eq!(pool, vec!["kimi:k2p5", "claude:haiku"]);
3062    }
3063
3064    #[serial(cooldown_state)]
3065    #[test]
3066    fn effective_pool_falls_back_to_router_agent_model() {
3067        let config = RouterConfig::default();
3068        // Default has no pool → derives from router_agent:router_model
3069        let pool = config.effective_pool();
3070        assert_eq!(pool.len(), 1);
3071        assert!(pool[0].starts_with(&config.router_agent));
3072        assert!(pool[0].contains(&config.router_model));
3073    }
3074
3075    #[serial(cooldown_state)]
3076    #[test]
3077    fn effective_fallback_uses_configured_fallback() {
3078        let config = RouterConfig {
3079            fallback: "claude:haiku".to_string(),
3080            ..RouterConfig::default()
3081        };
3082        assert_eq!(config.effective_fallback(), "claude:haiku");
3083    }
3084
3085    #[serial(cooldown_state)]
3086    #[test]
3087    fn effective_fallback_derives_from_router_agent_model() {
3088        let config = RouterConfig::default();
3089        let fallback = config.effective_fallback();
3090        assert!(fallback.starts_with(&config.router_agent));
3091        assert!(fallback.contains(&config.router_model));
3092    }
3093
3094    // ── pool expansion ────────────────────────────────────────────────────────
3095
3096    #[serial(cooldown_state)]
3097    #[test]
3098    fn expand_pool_basic_entries() {
3099        let config = RouterConfig {
3100            pool: vec!["claude:haiku".to_string(), "kimi:k2p5".to_string()],
3101            ..RouterConfig::default()
3102        };
3103        let expanded = Router::expand_pool(&config);
3104        assert_eq!(expanded.len(), 2);
3105        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3106        assert_eq!(expanded[1], ("kimi".to_string(), "k2p5".to_string()));
3107    }
3108
3109    #[serial(cooldown_state)]
3110    #[test]
3111    fn expand_pool_uses_effective_pool_when_pool_is_empty() {
3112        let config = RouterConfig {
3113            pool: vec![],
3114            router_agent: "claude".to_string(),
3115            router_model: "haiku".to_string(),
3116            ..RouterConfig::default()
3117        };
3118        let expanded = Router::expand_pool(&config);
3119        assert_eq!(expanded.len(), 1);
3120        assert_eq!(expanded[0], ("claude".to_string(), "haiku".to_string()));
3121    }
3122
3123    #[serial(cooldown_state)]
3124    #[test]
3125    fn expand_pool_opencode_free_skipped_when_not_installed() {
3126        // Test/debug builds skip live `opencode models`, so `opencode:free` expands
3127        // to nothing and the pool falls back to effective_fallback
3128        let config = RouterConfig {
3129            pool: vec!["opencode:free".to_string()],
3130            fallback: "claude:haiku".to_string(),
3131            ..RouterConfig::default()
3132        };
3133        let expanded = Router::expand_pool(&config);
3134        assert_eq!(expanded.len(), 1);
3135        assert_eq!(expanded[0].0, "claude");
3136        assert_eq!(expanded[0].1, "haiku");
3137    }
3138
3139    #[serial(cooldown_state)]
3140    #[test]
3141    fn expand_pool_single_entry_from_default() {
3142        let config = RouterConfig::default();
3143        let expanded = Router::expand_pool(&config);
3144        assert!(
3145            !expanded.is_empty(),
3146            "pool must never be empty after expansion"
3147        );
3148    }
3149
3150    // ── pool round-robin index ────────────────────────────────────────────────
3151
3152    #[serial(cooldown_state)]
3153    #[test]
3154    fn pool_index_initializes_to_zero() {
3155        let config = RouterConfig::default();
3156        let router = Router::new(config);
3157        assert_eq!(router.pool_index, 0);
3158    }
3159
3160    #[serial(cooldown_state)]
3161    #[test]
3162    fn router_pool_non_empty_after_construction() {
3163        let config = RouterConfig::default();
3164        let router = Router::new(config);
3165        assert_ne!(router.router_pool, [] as [(String, String); 0]);
3166    }
3167
3168    /**
3169     * Router construction used to run `opencode models`, and opencode 2.2.1 leaves an
3170     * `opencode serve --service` child behind, so a suite run leaked one server per test
3171     * process. The stub on PATH records any exec, so this fails if discovery comes back
3172     **/
3173    #[cfg(unix)]
3174    #[serial(cooldown_state)]
3175    #[test]
3176    fn building_a_router_never_runs_opencode() {
3177        struct PathRestore(Option<OsString>);
3178        impl Drop for PathRestore {
3179            fn drop(&mut self) {
3180                match &self.0 {
3181                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3182                    Some(p) => unsafe { env::set_var("PATH", p) },
3183                    // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3184                    None => unsafe { env::remove_var("PATH") },
3185                }
3186                cmd_cache::reset_for_test();
3187            }
3188        }
3189
3190        let dir = tempfile::tempdir().unwrap();
3191        let marker = dir.path().join("spawned");
3192        let bin_dir = dir.path().join("bin");
3193        fs::create_dir(&bin_dir).unwrap();
3194        let stub = bin_dir.join("opencode");
3195        fs::write(
3196            &stub,
3197            format!(
3198                "#!/bin/sh\necho \"$@\" > '{}'\necho canary-model-free\n",
3199                marker.display()
3200            ),
3201        )
3202        .unwrap();
3203        fs::set_permissions(&stub, Permissions::from_mode(0o755)).unwrap();
3204
3205        let prev_path = env::var_os("PATH");
3206        let mut new_path = bin_dir.as_os_str().to_os_string();
3207        new_path.push(":");
3208        if let Some(ref p) = prev_path {
3209            new_path.push(p);
3210        }
3211        // SAFETY: nextest runs each test in its own process, so no other test touches the environment
3212        unsafe { env::set_var("PATH", &new_path) };
3213        cmd_cache::reset_for_test();
3214        let _restore = PathRestore(prev_path);
3215
3216        // A config that names opencode everywhere discovery reads from
3217        let mut config = RouterConfig {
3218            pool: vec!["opencode:free".to_string()],
3219            fallback: "opencode:free".to_string(),
3220            ..RouterConfig::default()
3221        };
3222        config
3223            .model_map
3224            .entry("simple".to_string())
3225            .or_default()
3226            .insert(
3227                "opencode".to_string(),
3228                vec!["canary-model-free".to_string()],
3229            );
3230
3231        let _router = Router::new(config.clone());
3232        let _test_router = Router::new_for_test(config.clone(), vec!["opencode".to_string()]);
3233        let _pool = Router::expand_pool(&config);
3234        // Discovery refreshes on a background thread, give it room to exec
3235        thread::sleep(Duration::from_millis(300));
3236
3237        assert!(
3238            !marker.exists(),
3239            "building a router ran opencode: {}",
3240            fs::read_to_string(&marker).unwrap_or_default().trim()
3241        );
3242        assert!(
3243            opencode_children().is_empty(),
3244            "building a router left opencode processes: {:?}",
3245            opencode_children()
3246        );
3247    }
3248
3249    /// Commands of this process's direct children that mention opencode
3250    #[cfg(unix)]
3251    fn opencode_children() -> Vec<String> {
3252        let me = process::id().to_string();
3253        let output = Command::new("ps")
3254            .args(["-axo", "ppid=,command="])
3255            .output()
3256            .expect("ps");
3257        String::from_utf8_lossy(&output.stdout)
3258            .lines()
3259            .filter_map(|line| {
3260                let (ppid, cmd) = line.trim().split_once(char::is_whitespace)?;
3261                (ppid == me && cmd.contains("opencode")).then(|| cmd.to_string())
3262            })
3263            .collect()
3264    }
3265
3266    #[serial(cooldown_state)]
3267    #[tokio::test]
3268    async fn reload_resets_pool_index() {
3269        cooldown::reset_global_state().await;
3270        let config = RouterConfig::default();
3271        let mut router = Router::new(config);
3272        router.pool_index = 5;
3273        router.reload_async().await;
3274        assert_eq!(router.pool_index, 0);
3275    }
3276
3277    #[serial(cooldown_state)]
3278    #[test]
3279    fn pool_index_advances_after_failed_attempt() {
3280        let mut router = Router::new(RouterConfig::default());
3281        router.router_pool = vec![
3282            ("claude".to_string(), "haiku".to_string()),
3283            ("codex".to_string(), "gpt-5.2".to_string()),
3284            (
3285                "opencode".to_string(),
3286                "github-copilot/gpt-5-mini".to_string(),
3287            ),
3288        ];
3289        router.pool_index = 0;
3290
3291        router.advance_pool_index_after_attempt(0, router.router_pool.len());
3292
3293        assert_eq!(router.pool_index, 1);
3294    }
3295
3296    #[serial(cooldown_state)]
3297    #[test]
3298    fn pool_index_wraps_after_last_attempt() {
3299        let mut router = Router::new(RouterConfig::default());
3300        router.router_pool = vec![
3301            ("claude".to_string(), "haiku".to_string()),
3302            ("codex".to_string(), "gpt-5.2".to_string()),
3303            (
3304                "opencode".to_string(),
3305                "github-copilot/gpt-5-mini".to_string(),
3306            ),
3307        ];
3308        router.pool_index = 2;
3309
3310        router.advance_pool_index_after_attempt(2, router.router_pool.len());
3311
3312        assert_eq!(router.pool_index, 0);
3313    }
3314
3315    #[serial(cooldown_state)]
3316    #[test]
3317    fn earliest_pool_cooldown_returns_none_when_no_cooldowns() {
3318        let router = Router::new(RouterConfig::default());
3319        assert!(router.earliest_pool_cooldown().is_none());
3320    }
3321
3322    #[serial(cooldown_state)]
3323    #[tokio::test]
3324    async fn earliest_pool_cooldown_returns_earliest_when_set() {
3325        cooldown::reset_global_state().await;
3326        // Use explicit pool with predictable model names to avoid opencode discovery.
3327        let config = RouterConfig {
3328            pool: vec![
3329                "claude:haiku-4-5-20251001".to_string(),
3330                "opencode:haiku-4-5-20251001".to_string(),
3331            ],
3332            ..Default::default()
3333        };
3334        let router = Router::new(config);
3335        set_model_cooldown("claude", "haiku-4-5-20251001", 300, "silence_detected").await; // 5 min
3336        set_model_cooldown("opencode", "haiku-4-5-20251001", 600, "silence_detected").await; // 10 min
3337
3338        let earliest = router.earliest_pool_cooldown();
3339        assert!(earliest.is_some());
3340        let now = chrono::Utc::now().timestamp();
3341        let remaining = earliest.unwrap() - now;
3342        // Should be close to 300 (5 min, the minimum) — allow 5s tolerance for test runtime
3343        assert!((295..=300).contains(&remaining));
3344    }
3345
3346    #[serial(cooldown_state)]
3347    #[tokio::test]
3348    async fn earliest_pool_cooldown_skips_non_pool_entries() {
3349        cooldown::reset_global_state().await;
3350        // Use explicit pool so we can predict what entries exist.
3351        let config = RouterConfig {
3352            pool: vec![
3353                "claude:pool-model-a".to_string(),
3354                "opencode:pool-model-b".to_string(),
3355            ],
3356            ..Default::default()
3357        };
3358        let router = Router::new(config);
3359        // Set cooldown on a non-pool model — should not affect earliest_pool_cooldown
3360        set_model_cooldown("claude", "opus", 999, "silence_detected").await;
3361        // Set cooldown on a pool entry (opencode:pool-model-b)
3362        set_model_cooldown("opencode", "pool-model-b", 300, "silence_detected").await;
3363
3364        let earliest = router.earliest_pool_cooldown();
3365        assert!(earliest.is_some());
3366        let now = chrono::Utc::now().timestamp();
3367        let remaining = earliest.unwrap() - now;
3368        // Should be ~300 (the pool-model-b cooldown), NOT 999 (claude:opus is not in pool)
3369        assert!((295..=300).contains(&remaining));
3370    }
3371
3372    // ---- Pre-emptive health check: degraded agent exclusion ----
3373
3374    #[serial(cooldown_state)]
3375    #[test]
3376    fn degraded_agent_excluded_from_routing() {
3377        let mut config = RouterConfig::default();
3378        config
3379            .model_map
3380            .entry("medium".to_string())
3381            .or_default()
3382            .insert(
3383                "test_degraded_routing".to_string(),
3384                vec!["model-a".to_string()],
3385            );
3386
3387        let mut router = Router::new(config);
3388        router.available_agents = vec!["test_degraded_routing".to_string()];
3389
3390        // Agent is routable before degradation
3391        assert!(router.agent_is_routable("test_degraded_routing", "medium"));
3392
3393        // Mark as degraded
3394        mark_agent_degraded("test_degraded_routing");
3395
3396        // Now excluded from routing
3397        assert!(!router.agent_is_routable("test_degraded_routing", "medium"));
3398
3399        // available_agents_for_complexity returns empty
3400        assert!(
3401            router.available_agents_for_complexity("medium").is_empty(),
3402            "degraded agent should be excluded from available agents"
3403        );
3404
3405        // healthy_agent_count reflects exclusion
3406        assert_eq!(router.healthy_agent_count("medium"), 0);
3407
3408        // Cleanup
3409        clear_agent_degraded("test_degraded_routing");
3410    }
3411
3412    #[serial(cooldown_state)]
3413    #[test]
3414    fn healthy_agents_skips_degraded_but_keeps_healthy() {
3415        let mut config = RouterConfig::default();
3416        for agent in &["agent_healthy", "agent_degraded"] {
3417            config
3418                .model_map
3419                .entry("medium".to_string())
3420                .or_default()
3421                .insert(agent.to_string(), vec!["model-x".to_string()]);
3422        }
3423
3424        let mut router = Router::new(config);
3425        router.available_agents = vec!["agent_healthy".to_string(), "agent_degraded".to_string()];
3426
3427        mark_agent_degraded("agent_degraded");
3428
3429        let candidates = router.available_agents_for_complexity("medium");
3430        assert_eq!(candidates, vec!["agent_healthy".to_string()]);
3431        assert_eq!(router.healthy_agent_count("medium"), 1);
3432
3433        // Cleanup
3434        clear_agent_degraded("agent_degraded");
3435    }
3436
3437    #[serial(cooldown_state)]
3438    #[tokio::test]
3439    async fn refresh_health_marks_degraded_from_rate_limits() {
3440        cooldown::reset_global_state().await;
3441        let store = test_store().await;
3442        let agent = "test_refresh_health_agent";
3443
3444        let mut config = RouterConfig::default();
3445        config
3446            .model_map
3447            .entry("medium".to_string())
3448            .or_default()
3449            .insert(agent.to_string(), vec!["model-z".to_string()]);
3450
3451        let mut router = Router::new(config);
3452        router.available_agents = vec![agent.to_string()];
3453
3454        // Insert rate limit events exceeding default threshold (3)
3455        for _ in 0..4 {
3456            store.record_rate_limit(agent, "rate", None).await.unwrap();
3457        }
3458
3459        router.refresh_health(&store).await;
3460
3461        assert!(
3462            is_agent_degraded(agent),
3463            "agent should be degraded after exceeding rate limit threshold"
3464        );
3465        assert!(
3466            !router.agent_is_routable(agent, "medium"),
3467            "degraded agent should not be routable"
3468        );
3469
3470        // Cleanup
3471        clear_agent_degraded(agent);
3472    }
3473
3474    // ---- route_with_llm candidate filter: agents with all models cooled (#3509) ----
3475
3476    #[serial(cooldown_state)]
3477    #[tokio::test]
3478    async fn agent_has_any_available_model_false_when_only_model_cooled() {
3479        cooldown::reset_global_state().await;
3480        let mut config = RouterConfig::default();
3481        // Agent's only model, shared across every complexity tier it resolves to.
3482        for tier in ["simple", "medium", "complex", "review"] {
3483            config
3484                .model_map
3485                .entry(tier.to_string())
3486                .or_default()
3487                .insert(
3488                    "test_all_cooled".to_string(),
3489                    vec!["only-model".to_string()],
3490                );
3491        }
3492
3493        let mut router = Router::new(config);
3494        router.available_agents = vec!["test_all_cooled".to_string()];
3495
3496        assert!(router.agent_has_any_available_model("test_all_cooled"));
3497
3498        record_model_failure("test_all_cooled", "only-model").await;
3499        assert!(is_model_in_cooldown("test_all_cooled", "only-model"));
3500
3501        assert!(
3502            !router.agent_has_any_available_model("test_all_cooled"),
3503            "agent with its only model cooled across every tier should have no available model"
3504        );
3505    }
3506
3507    #[serial(cooldown_state)]
3508    #[tokio::test]
3509    async fn llm_candidate_excludes_agent_with_all_models_cooled() {
3510        // Regression test for #3509: route_with_llm's candidate filter must exclude an
3511        // agent that has no agent-level cooldown but whose only resolvable model (across
3512        // every complexity tier) is cooled, so it is never handed to the LLM classifier.
3513        cooldown::reset_global_state().await;
3514        let mut config = RouterConfig::default();
3515        for tier in ["simple", "medium", "complex", "review"] {
3516            config
3517                .model_map
3518                .entry(tier.to_string())
3519                .or_default()
3520                .insert("agent_cooled_model".to_string(), vec!["opus".to_string()]);
3521            config
3522                .model_map
3523                .get_mut(tier)
3524                .unwrap()
3525                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3526        }
3527
3528        let mut router = Router::new(config);
3529        router.available_agents = vec![
3530            "agent_cooled_model".to_string(),
3531            "agent_healthy".to_string(),
3532        ];
3533
3534        record_model_failure("agent_cooled_model", "opus").await;
3535        assert!(is_model_in_cooldown("agent_cooled_model", "opus"));
3536        // No agent-level cooldown — the old agent-only filter would still offer it.
3537        assert!(!is_agent_in_cooldown("agent_cooled_model"));
3538
3539        let uncooled_agents: Vec<String> = router
3540            .available_agents
3541            .iter()
3542            .filter(|a| {
3543                !is_agent_in_cooldown(a)
3544                    && !is_agent_degraded(a)
3545                    && router.agent_has_available_model_for_coding_tiers(a)
3546            })
3547            .cloned()
3548            .collect();
3549
3550        assert_eq!(
3551            uncooled_agents,
3552            vec!["agent_healthy".to_string()],
3553            "agent whose only model is cooled must be excluded from the LLM candidate list"
3554        );
3555    }
3556
3557    #[serial(cooldown_state)]
3558    #[tokio::test]
3559    async fn llm_candidate_excludes_agent_with_only_simple_tier_uncooled() {
3560        // Regression test for #3521: an agent whose only healthy tier is `simple`
3561        // (its `medium`/`complex` tiers resolve to the same cooled model) must be
3562        // excluded from the LLM classification candidate list, otherwise the
3563        // classifier can pick it for a `medium`/`complex` task it can't serve,
3564        // wasting the classification call. Mirrors the real-world shape where
3565        // `simple` → sonnet (uncooled) and `medium`/`complex` → opus (cooled).
3566        cooldown::reset_global_state().await;
3567        let mut config = RouterConfig::default();
3568        config
3569            .model_map
3570            .entry("simple".to_string())
3571            .or_default()
3572            .insert("minimax_like".to_string(), vec!["sonnet".to_string()]);
3573        config
3574            .model_map
3575            .entry("medium".to_string())
3576            .or_default()
3577            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3578        config
3579            .model_map
3580            .entry("complex".to_string())
3581            .or_default()
3582            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3583        // review must be irrelevant to the coding-tier filter
3584        config
3585            .model_map
3586            .entry("review".to_string())
3587            .or_default()
3588            .insert("minimax_like".to_string(), vec!["opus".to_string()]);
3589        for tier in ["simple", "medium", "complex", "review"] {
3590            config
3591                .model_map
3592                .get_mut(tier)
3593                .unwrap()
3594                .insert("agent_healthy".to_string(), vec!["model-x".to_string()]);
3595        }
3596
3597        let mut router = Router::new(config);
3598        router.available_agents = vec!["minimax_like".to_string(), "agent_healthy".to_string()];
3599
3600        record_model_failure("minimax_like", "opus").await;
3601        assert!(is_model_in_cooldown("minimax_like", "opus"));
3602        // No agent-level cooldown, and the simple tier is healthy, so the loose
3603        // "any tier" check still passes — this is exactly the leak #3511 missed.
3604        assert!(!is_agent_in_cooldown("minimax_like"));
3605        assert!(router.agent_has_any_available_model("minimax_like"));
3606        assert!(!router.agent_has_available_model_for_coding_tiers("minimax_like"));
3607
3608        let uncooled_agents: Vec<String> = router
3609            .available_agents
3610            .iter()
3611            .filter(|a| {
3612                !is_agent_in_cooldown(a)
3613                    && !is_agent_degraded(a)
3614                    && router.agent_has_available_model_for_coding_tiers(a)
3615            })
3616            .cloned()
3617            .collect();
3618
3619        assert_eq!(
3620            uncooled_agents,
3621            vec!["agent_healthy".to_string()],
3622            "agent with only its simple tier uncooled must be excluded from the LLM candidate list"
3623        );
3624    }
3625}