Skip to main content

orch/engine/router/
weights.rs

1//! Per-agent routing weights and rate-limit recovery.
2//!
3//! `RateLimitState` tracks a single agent's current routing weight and
4//! recovers it over time after rate-limit events.
5//!
6//! `AgentWeights` aggregates per-agent states and provides weighted
7//! probabilistic selection via `weighted_select`.
8
9use std::collections::HashMap;
10use std::time::{Duration, Instant};
11
12use super::selection::{hash_task_id, simple_hash_fraction_for, simple_hash_index_for};
13
14/// Default weight for agents with full capacity.
15pub(super) const DEFAULT_WEIGHT: f64 = 1.0;
16
17/// Minimum weight — an agent never drops below this (still gets occasional tasks).
18pub(super) const MIN_WEIGHT: f64 = 0.05;
19
20/// How much to reduce weight on each rate limit hit (multiplicative decay).
21pub(super) const RATE_LIMIT_DECAY: f64 = 0.3;
22
23/// Weight to use for an agent that is absent from a configured `router.weights`
24/// map. Rather than the constant `DEFAULT_WEIGHT` (1.0) — which would silently
25/// outrank every agent the operator *did* configure — an omitted agent falls
26/// back to the minimum of the explicitly configured weights. An empty map
27/// means no weights are configured at all, so `DEFAULT_WEIGHT` applies to
28/// everyone equally.
29pub(super) fn default_missing_weight(configured_weights: &HashMap<String, f64>) -> f64 {
30    configured_weights
31        .values()
32        .copied()
33        .fold(None, |min, w| Some(min.map_or(w, |m: f64| m.min(w))))
34        .unwrap_or(DEFAULT_WEIGHT)
35}
36
37/// Generate jitter duration for recovery delay based on agent name.
38/// Uses a simple hash to create deterministic but varied jitter per agent.
39fn generate_recovery_jitter(agent: &str) -> Duration {
40    let max_secs = RECOVERY_JITTER_MAX.as_secs();
41    let jitter_secs = hash_task_id(agent) % max_secs;
42    Duration::from_secs(jitter_secs)
43}
44
45/// Duration after which a rate-limited agent starts recovering weight.
46pub(super) const RECOVERY_DELAY: Duration = Duration::from_secs(60);
47
48/// Maximum jitter added to recovery delay to stagger agent recoveries.
49/// This prevents thundering herd where all agents recover simultaneously.
50pub(super) const RECOVERY_JITTER_MAX: Duration = Duration::from_secs(30);
51
52/// Per-tick weight recovery amount (additive, applied each routing call).
53pub(super) const RECOVERY_RATE: f64 = 0.1;
54
55/// Rate limit state for a single agent.
56#[derive(Debug, Clone)]
57pub struct RateLimitState {
58    /// Current routing weight (0.0..=1.0). Higher = more tasks.
59    pub weight: f64,
60    /// When the last rate limit error was recorded.
61    pub last_limited_at: Option<Instant>,
62    /// Jitter added to recovery delay to stagger recoveries (avoid thundering herd).
63    pub recovery_jitter: Duration,
64    /// How many consecutive rate limit hits.
65    pub consecutive_hits: u32,
66    /// The configured base weight this agent should recover toward.
67    pub base_weight: f64,
68}
69
70impl Default for RateLimitState {
71    fn default() -> Self {
72        Self {
73            weight: DEFAULT_WEIGHT,
74            last_limited_at: None,
75            recovery_jitter: Duration::ZERO,
76            consecutive_hits: 0,
77            base_weight: DEFAULT_WEIGHT,
78        }
79    }
80}
81
82impl RateLimitState {
83    /// Record a rate limit event — decay the weight.
84    /// Note: Jitter is set at the AgentWeights level using the agent name.
85    pub fn record_rate_limit(&mut self) {
86        self.consecutive_hits += 1;
87        self.weight = (self.weight * RATE_LIMIT_DECAY).max(MIN_WEIGHT);
88        self.last_limited_at = Some(Instant::now());
89    }
90
91    /// Record a successful completion — bump weight back toward base_weight.
92    pub fn record_success(&mut self) {
93        self.consecutive_hits = 0;
94        self.weight = (self.weight + RECOVERY_RATE).min(self.base_weight);
95    }
96
97    /// Tick recovery: if enough time has passed since the last limit (plus jitter), gradually restore.
98    pub fn maybe_recover(&mut self) {
99        if let Some(last) = self.last_limited_at {
100            let recovery_threshold = RECOVERY_DELAY + self.recovery_jitter;
101            if last.elapsed() >= recovery_threshold {
102                self.weight = (self.weight + RECOVERY_RATE).min(self.base_weight);
103                if self.weight >= self.base_weight {
104                    self.last_limited_at = None;
105                    self.recovery_jitter = Duration::ZERO;
106                    self.consecutive_hits = 0;
107                }
108            }
109        }
110    }
111
112    /// Is this agent currently rate-limited (weight below its base weight)?
113    pub fn is_limited(&self) -> bool {
114        self.weight < self.base_weight
115    }
116}
117
118/// Tracks per-agent weights for weighted round-robin routing.
119#[derive(Debug, Clone, Default)]
120pub struct AgentWeights {
121    pub states: HashMap<String, RateLimitState>,
122    /// Configured base weights per agent (from config.yml).
123    /// Used as the recovery ceiling instead of DEFAULT_WEIGHT.
124    pub base_weights: HashMap<String, f64>,
125}
126
127impl AgentWeights {
128    /// Ensure all available agents have an entry, using configured base weights.
129    pub fn ensure_agents(&mut self, agents: &[String]) {
130        for agent in agents {
131            self.states.entry(agent.clone()).or_default();
132        }
133    }
134
135    /// Ensure all available agents have an entry, applying configured base weights.
136    ///
137    /// Agents with an explicit weight in `config.yml` use that as their base.
138    /// Agents without a configured weight default to the minimum of the
139    /// explicitly configured weights (see `default_missing_weight`), so an
140    /// omission never outranks an agent the operator actually configured.
141    pub fn ensure_agents_with_weights(
142        &mut self,
143        agents: &[String],
144        configured_weights: &HashMap<String, f64>,
145    ) {
146        for agent in agents {
147            let base = configured_weights
148                .get(agent)
149                .copied()
150                .unwrap_or_else(|| default_missing_weight(configured_weights));
151            let state = self
152                .states
153                .entry(agent.clone())
154                .or_insert_with(|| RateLimitState {
155                    weight: base,
156                    base_weight: base,
157                    ..Default::default()
158                });
159            state.base_weight = base;
160            // Update base weight if agent was already initialized (e.g. after config reload)
161            // but only if the agent isn't currently rate-limited (don't override decay)
162            if !state.is_limited() {
163                state.weight = base;
164            }
165        }
166    }
167
168    /// Record a rate limit event for an agent.
169    pub fn record_rate_limit(&mut self, agent: &str) {
170        let state = self.states.entry(agent.to_string()).or_default();
171        state.record_rate_limit();
172        // Generate jitter to stagger recovery times across agents
173        state.recovery_jitter = generate_recovery_jitter(agent);
174        tracing::info!(
175            agent,
176            weight = state.weight,
177            hits = state.consecutive_hits,
178            jitter_secs = state.recovery_jitter.as_secs(),
179            "agent weight reduced (rate limit)"
180        );
181    }
182
183    /// Record a successful task completion for an agent.
184    pub fn record_success(&mut self, agent: &str) {
185        self.states
186            .entry(agent.to_string())
187            .or_default()
188            .record_success();
189    }
190
191    /// Tick recovery for all agents.
192    pub fn tick_recovery(&mut self) {
193        for (agent, state) in &mut self.states {
194            let was_limited = state.is_limited();
195            state.maybe_recover();
196            if was_limited && !state.is_limited() {
197                tracing::info!(agent, "agent weight fully recovered");
198            }
199        }
200    }
201
202    /// Select an agent by weighted probability from the given list.
203    ///
204    /// Uses a simple weighted random selection: each agent's probability is
205    /// proportional to its weight. If all weights are zero (shouldn't happen
206    /// due to MIN_WEIGHT), falls back to uniform selection.
207    pub fn weighted_select(&self, agents: &[String], task_id: &str) -> Option<String> {
208        if agents.is_empty() {
209            return None;
210        }
211
212        let weights: Vec<f64> = agents
213            .iter()
214            .map(|a| {
215                self.states.get(a).map_or_else(
216                    || {
217                        self.base_weights
218                            .get(a)
219                            .copied()
220                            .unwrap_or_else(|| default_missing_weight(&self.base_weights))
221                    },
222                    |s| s.weight,
223                )
224            })
225            .collect();
226
227        let total: f64 = weights.iter().sum();
228        if total <= 0.0 {
229            // Safety fallback: uniform random
230            let idx = simple_hash_index_for(agents.len(), task_id);
231            return Some(agents[idx].clone());
232        }
233
234        // Deterministic-ish selection using a hash of the current time
235        // to avoid requiring rand crate. Good enough for load distribution.
236        let pick = simple_hash_fraction_for(task_id) * total;
237        let mut cumulative = 0.0;
238        for (i, w) in weights.iter().enumerate() {
239            cumulative += w;
240            if pick < cumulative {
241                return Some(agents[i].clone());
242            }
243        }
244
245        // Rounding edge case — return last agent (agents is guaranteed non-empty by caller)
246        debug_assert!(!agents.is_empty(), "agents must not be empty");
247        Some(agents[agents.len() - 1].clone())
248    }
249
250    /// Get the current weight for an agent.
251    pub fn get_weight(&self, agent: &str) -> f64 {
252        self.states.get(agent).map_or_else(
253            || {
254                self.base_weights
255                    .get(agent)
256                    .copied()
257                    .unwrap_or_else(|| default_missing_weight(&self.base_weights))
258            },
259            |s| s.weight,
260        )
261    }
262
263    /// Get a snapshot of all agent weights (for logging/debugging).
264    pub fn snapshot(&self) -> Vec<(String, f64, u32)> {
265        let mut snap: Vec<_> = self
266            .states
267            .iter()
268            .map(|(a, s)| (a.clone(), s.weight, s.consecutive_hits))
269            .collect();
270        snap.sort_by(|a, b| a.0.cmp(&b.0));
271        snap
272    }
273}
274
275#[cfg(test)]
276mod tests {
277    use super::*;
278
279    #[test]
280    fn recovery_jitter_is_the_byte_hash_modulo_the_cap() {
281        assert_eq!(generate_recovery_jitter(""), Duration::from_secs(0));
282        assert_eq!(generate_recovery_jitter("a"), Duration::from_secs(7));
283        assert_eq!(
284            generate_recovery_jitter("claude"),
285            generate_recovery_jitter("claude")
286        );
287        assert!(generate_recovery_jitter("claude") < RECOVERY_JITTER_MAX);
288    }
289}