roder_core/
reliability.rs

1use roder_api::reliability::{
2    ReliabilityErrorClass, ReliabilityLimitDecision, ReliabilityLimitKind, ReliabilityRequestPolicy,
3};
4use roder_api::transcript::ToolResultRecord;
5
6#[derive(Debug, Clone, PartialEq, Eq)]
7pub struct RuntimeReliabilityConfig {
8    pub max_consecutive_tool_failures: u32,
9    pub max_tool_failures_per_turn: u32,
10    pub max_model_calls_per_turn: u32,
11    /// When true, hitting `max_consecutive_tool_failures` requests a turn
12    /// continuation (the runtime resets the consecutive counter and nudges the
13    /// model to keep going) instead of stopping the turn. Loop persistence for
14    /// eval-style runs where a couple of failed tool calls should not end an
15    /// otherwise-productive turn. Progress stays bounded by
16    /// `max_tool_failures_per_turn` and `max_model_calls_per_turn`, which still
17    /// stop the turn. Defaults to `false` so interactive/non-interactive UX is
18    /// unchanged; the eval profile enables it.
19    pub continue_on_failure_limit: bool,
20    /// Number of times a non-interactive/eval turn may be nudged to keep working
21    /// when the model returns a final message with no tool calls. `0` disables
22    /// the nudge (default) so interactive completions end promptly.
23    pub empty_tool_call_nudges: u32,
24    pub provider_retry_max_attempts: u32,
25    pub provider_retry_initial_backoff_ms: u64,
26    pub provider_retry_backoff_factor: u32,
27    pub provider_retry_status_codes: Vec<u16>,
28    pub retry_empty_provider_body: bool,
29}
30
31impl Default for RuntimeReliabilityConfig {
32    fn default() -> Self {
33        Self {
34            max_consecutive_tool_failures: 5,
35            max_tool_failures_per_turn: 128,
36            max_model_calls_per_turn: 512,
37            continue_on_failure_limit: false,
38            empty_tool_call_nudges: 0,
39            provider_retry_max_attempts: 3,
40            provider_retry_initial_backoff_ms: 1_000,
41            provider_retry_backoff_factor: 2,
42            provider_retry_status_codes: vec![429, 500, 502, 503, 504],
43            retry_empty_provider_body: true,
44        }
45    }
46}
47
48impl From<RuntimeReliabilityConfig> for ReliabilityRequestPolicy {
49    fn from(config: RuntimeReliabilityConfig) -> Self {
50        Self {
51            provider_retry_max_attempts: config.provider_retry_max_attempts,
52            provider_retry_initial_backoff_ms: config.provider_retry_initial_backoff_ms,
53            provider_retry_backoff_factor: config.provider_retry_backoff_factor,
54            retry_empty_provider_body: config.retry_empty_provider_body,
55            provider_retry_status_codes: config.provider_retry_status_codes,
56        }
57    }
58}
59
60#[derive(Debug, Default, Clone, PartialEq, Eq)]
61pub(crate) struct TurnReliabilityState {
62    model_calls: u32,
63    consecutive_tool_failures: u32,
64    tool_failures: u32,
65}
66
67#[derive(Debug, Clone, PartialEq, Eq)]
68pub(crate) struct ReliabilityLimitHit {
69    pub error_class: ReliabilityErrorClass,
70    pub limit_kind: ReliabilityLimitKind,
71    pub decision: ReliabilityLimitDecision,
72    pub current: u32,
73    pub limit: u32,
74    pub message: String,
75}
76
77pub(crate) fn provider_stream_retry_cause(message: &str) -> Option<&'static str> {
78    let lower = message.to_ascii_lowercase();
79    if lower.contains("error decoding response body") {
80        return Some("stream_decode_error");
81    }
82    if lower.contains("stream closed before response.completed") {
83        return Some("stream_closed_before_completed");
84    }
85    if lower.contains("stream closed before message_stop") {
86        return Some("stream_closed_before_message_stop");
87    }
88    None
89}
90
91impl TurnReliabilityState {
92    pub(crate) fn record_model_call(
93        &mut self,
94        cfg: &RuntimeReliabilityConfig,
95        interactive: bool,
96    ) -> Option<ReliabilityLimitHit> {
97        self.model_calls = self.model_calls.saturating_add(1);
98        if self.model_calls > cfg.max_model_calls_per_turn {
99            // The per-turn model-call ceiling is the hard loop guard: it always
100            // stops the turn regardless of profile. Interactive turns still emit
101            // the softer `RequestContinuation` decision for observability parity.
102            return Some(limit_hit(
103                ReliabilityErrorClass::ProviderError,
104                ReliabilityLimitKind::ModelCallsPerTurn,
105                self.model_calls,
106                cfg.max_model_calls_per_turn,
107                continuation_decision(interactive),
108                "model call limit reached",
109            ));
110        }
111        None
112    }
113
114    pub(crate) fn record_tool_results(
115        &mut self,
116        cfg: &RuntimeReliabilityConfig,
117        results: &[ToolResultRecord],
118        interactive: bool,
119    ) -> Option<ReliabilityLimitHit> {
120        for result in results {
121            if result.is_error {
122                self.tool_failures = self.tool_failures.saturating_add(1);
123                self.consecutive_tool_failures = self.consecutive_tool_failures.saturating_add(1);
124            } else {
125                self.consecutive_tool_failures = 0;
126            }
127        }
128        if self.consecutive_tool_failures >= cfg.max_consecutive_tool_failures {
129            // Interactive turns already recover from this limit; `continue_on_failure_limit`
130            // extends that recovery to non-interactive/eval turns so a short burst of
131            // failed tool calls does not end an otherwise-productive turn.
132            let decision = continuation_decision(interactive || cfg.continue_on_failure_limit);
133            return Some(limit_hit(
134                ReliabilityErrorClass::InvalidArguments,
135                ReliabilityLimitKind::ConsecutiveToolFailures,
136                self.consecutive_tool_failures,
137                cfg.max_consecutive_tool_failures,
138                decision,
139                "consecutive tool failure limit reached",
140            ));
141        }
142        if self.tool_failures >= cfg.max_tool_failures_per_turn {
143            // The per-turn total is the hard failure ceiling: always stop so a
144            // continuation loop cannot spin forever on a broken tool.
145            return Some(limit_hit(
146                ReliabilityErrorClass::InvalidArguments,
147                ReliabilityLimitKind::ToolFailuresPerTurn,
148                self.tool_failures,
149                cfg.max_tool_failures_per_turn,
150                ReliabilityLimitDecision::StopTurn,
151                "tool failure limit reached",
152            ));
153        }
154        None
155    }
156
157    /// Clears the consecutive-failure counter after a continuation so the very
158    /// next tool round does not immediately re-trip the limit. The per-turn total
159    /// (`tool_failures`) is intentionally preserved so the hard ceiling still applies.
160    pub(crate) fn reset_consecutive_failures(&mut self) {
161        self.consecutive_tool_failures = 0;
162    }
163
164    pub(crate) fn tool_failure_count(&self) -> u32 {
165        self.tool_failures
166    }
167}
168
169fn continuation_decision(request_continuation: bool) -> ReliabilityLimitDecision {
170    if request_continuation {
171        ReliabilityLimitDecision::RequestContinuation
172    } else {
173        ReliabilityLimitDecision::StopTurn
174    }
175}
176
177fn limit_hit(
178    error_class: ReliabilityErrorClass,
179    limit_kind: ReliabilityLimitKind,
180    current: u32,
181    limit: u32,
182    decision: ReliabilityLimitDecision,
183    message: &str,
184) -> ReliabilityLimitHit {
185    ReliabilityLimitHit {
186        error_class,
187        limit_kind,
188        decision,
189        current,
190        limit,
191        message: format!("{message}: {current}/{limit}"),
192    }
193}
194
195#[cfg(test)]
196mod tests {
197    use super::*;
198
199    fn result(id: &str, is_error: bool) -> ToolResultRecord {
200        ToolResultRecord {
201            id: id.to_string(),
202            name: Some("test".to_string()),
203            result: if is_error { "error" } else { "ok" }.to_string(),
204            display_payload: None,
205            is_error,
206        }
207    }
208
209    #[test]
210    fn reliability_limits_reset_consecutive_failures_after_success() {
211        let cfg = RuntimeReliabilityConfig {
212            max_consecutive_tool_failures: 2,
213            max_tool_failures_per_turn: 128,
214            ..RuntimeReliabilityConfig::default()
215        };
216        let mut state = TurnReliabilityState::default();
217
218        assert!(
219            state
220                .record_tool_results(&cfg, &[result("first", true)], false)
221                .is_none()
222        );
223        assert!(
224            state
225                .record_tool_results(&cfg, &[result("success", false)], false)
226                .is_none()
227        );
228        assert!(
229            state
230                .record_tool_results(&cfg, &[result("second", true)], false)
231                .is_none()
232        );
233        let limit = state
234            .record_tool_results(&cfg, &[result("third", true)], false)
235            .unwrap();
236        assert_eq!(
237            limit.limit_kind,
238            ReliabilityLimitKind::ConsecutiveToolFailures
239        );
240        assert_eq!(limit.current, 2);
241    }
242
243    #[test]
244    fn consecutive_failure_limit_stops_non_interactive_turn_by_default() {
245        let cfg = RuntimeReliabilityConfig {
246            max_consecutive_tool_failures: 2,
247            ..RuntimeReliabilityConfig::default()
248        };
249        let mut state = TurnReliabilityState::default();
250        state.record_tool_results(&cfg, &[result("first", true)], false);
251        let limit = state
252            .record_tool_results(&cfg, &[result("second", true)], false)
253            .unwrap();
254        assert_eq!(
255            limit.limit_kind,
256            ReliabilityLimitKind::ConsecutiveToolFailures
257        );
258        assert_eq!(limit.decision, ReliabilityLimitDecision::StopTurn);
259    }
260
261    #[test]
262    fn consecutive_failure_limit_requests_continuation_when_knob_enabled() {
263        let cfg = RuntimeReliabilityConfig {
264            max_consecutive_tool_failures: 2,
265            continue_on_failure_limit: true,
266            ..RuntimeReliabilityConfig::default()
267        };
268        let mut state = TurnReliabilityState::default();
269        state.record_tool_results(&cfg, &[result("first", true)], false);
270        let limit = state
271            .record_tool_results(&cfg, &[result("second", true)], false)
272            .unwrap();
273        assert_eq!(
274            limit.decision,
275            ReliabilityLimitDecision::RequestContinuation
276        );
277
278        // Resetting the consecutive counter after a continuation prevents an
279        // immediate re-trip while preserving the per-turn total.
280        state.reset_consecutive_failures();
281        assert!(
282            state
283                .record_tool_results(&cfg, &[result("third", true)], false)
284                .is_none()
285        );
286    }
287
288    #[test]
289    fn tool_failures_per_turn_always_stops_even_with_continuation_knob() {
290        let cfg = RuntimeReliabilityConfig {
291            max_consecutive_tool_failures: 100,
292            max_tool_failures_per_turn: 2,
293            continue_on_failure_limit: true,
294            ..RuntimeReliabilityConfig::default()
295        };
296        let mut state = TurnReliabilityState::default();
297        state.record_tool_results(&cfg, &[result("first", true)], false);
298        let limit = state
299            .record_tool_results(&cfg, &[result("second", true)], false)
300            .unwrap();
301        assert_eq!(limit.limit_kind, ReliabilityLimitKind::ToolFailuresPerTurn);
302        assert_eq!(limit.decision, ReliabilityLimitDecision::StopTurn);
303    }
304
305    #[test]
306    fn default_model_call_limit_allows_long_agentic_turns() {
307        assert_eq!(
308            RuntimeReliabilityConfig::default().max_model_calls_per_turn,
309            512
310        );
311    }
312
313    #[test]
314    fn provider_stream_retry_cause_classifies_transient_stream_failures() {
315        assert_eq!(
316            provider_stream_retry_cause("error decoding response body"),
317            Some("stream_decode_error")
318        );
319        assert_eq!(
320            provider_stream_retry_cause("stream closed before response.completed"),
321            Some("stream_closed_before_completed")
322        );
323        assert_eq!(
324            provider_stream_retry_cause("Anthropic stream closed before message_stop"),
325            Some("stream_closed_before_message_stop")
326        );
327        assert_eq!(provider_stream_retry_cause("invalid request body"), None);
328    }
329}
roder_core/reliability.rs

roder_core/
reliability.rs
