reddb_server/runtime/ai/
provider_failover.rs

1//! `ProviderFailover` — pure ordered-list failover kernel for ASK.
2//!
3//! Issue #404 (PRD #391): when a user asks `ASK '...'` the runtime picks
4//! a provider from `ask.providers.fallback = ['groq', 'openai',
5//! 'anthropic']` (or per-query `USING 'a,b,c'`) and walks the list in
6//! order until one succeeds. Failover triggers on **retryable** outcomes
7//! — transport errors, 5xx, and timeouts. Authoritative errors like
8//! 4xx auth failures or content-policy refusals short-circuit: we do
9//! not paper over a bad key by silently switching vendors.
10//!
11//! Deep module: no I/O, no async, no clock. The caller supplies an
12//! attempt function `FnMut(&str) -> Result<R, AttemptError>` and we
13//! drive the loop. This keeps the kernel trivially testable with
14//! synchronous stubs and lets the eventual wiring slice plug in real
15//! HTTP transports without changing the policy logic.
16//!
17//! ## Why "retryable" is a closed set
18//!
19//! Failover is risky: if the second provider produces a different
20//! answer than the first, the user sees nondeterminism for what was
21//! supposed to be a deterministic ASK (#400). We only fail over when
22//! the first provider could not have produced *any* answer:
23//!
24//! - **Transport** — DNS, TCP, TLS, dropped connection. No response
25//!   bytes received, so no answer was committed.
26//! - **5xx** — provider acknowledged the request but admitted failure.
27//!   By HTTP convention, the resource is in an unknown/transient bad
28//!   state; safe to retry on a sibling.
29//! - **Timeout** — request exceeded the deadline. From our side the
30//!   call is over; whether the provider eventually completed is moot.
31//!
32//! Everything else — 4xx, malformed response, content-filter refusal,
33//! non-retryable provider-specific codes — is reported as-is. The
34//! caller turns those into the user-visible error.
35//!
36//! ## Preservation of determinism inputs
37//!
38//! `seed`, `temperature`, and `strict` are part of the request the
39//! caller passes to the attempt fn. The kernel is generic over the
40//! request payload, so by construction every attempt sees the same
41//! parameters. We do not "fix up" requests between attempts.
42//!
43//! ## Outcome shape
44//!
45//! On success: `(provider, response, prior_errors)`. We surface
46//! prior_errors so the audit log can record that, e.g., groq 502'd
47//! before openai answered — that's signal for capacity planning even
48//! when the user got a good answer.
49//!
50//! On total failure: `AllProvidersFailed { attempts }` where each entry
51//! is `(provider, AttemptError)`. The HTTP layer maps this to 503 per
52//! the acceptance criteria.
53
54use std::fmt;
55use std::time::Duration;
56
57/// A classification of one attempt's failure.
58///
59/// `Transport`, `Status5xx`, and `Timeout` are retryable — the failover
60/// loop moves to the next provider. `NonRetryable` aborts the loop
61/// and is returned to the caller wrapped in the outcome.
62#[derive(Debug, Clone, PartialEq)]
63pub enum AttemptError {
64    /// Network-level failure before/while receiving a response.
65    /// Examples: DNS resolution failure, connection refused, TLS error,
66    /// socket reset mid-stream. `String` is a short human description
67    /// suitable for audit.
68    Transport(String),
69    /// Provider returned a 5xx response. Carries the actual status code
70    /// (e.g. 502, 503, 504) and a short body excerpt.
71    Status5xx { code: u16, body: String },
72    /// Per-request deadline elapsed before completion.
73    Timeout(Duration),
74    /// Authoritative error that must NOT trigger failover. Examples:
75    /// 4xx auth failure (wrong API key), 4xx quota exhausted on the
76    /// account level, content-policy refusal, malformed response we
77    /// cannot recover from. The kernel returns immediately when it
78    /// sees this — there is no value in asking another provider when
79    /// the request itself is bad.
80    NonRetryable(String),
81}
82
83impl AttemptError {
84    /// Whether this error should trigger advancement to the next provider.
85    pub fn is_retryable(&self) -> bool {
86        matches!(
87            self,
88            AttemptError::Transport(_) | AttemptError::Status5xx { .. } | AttemptError::Timeout(_)
89        )
90    }
91}
92
93impl fmt::Display for AttemptError {
94    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
95        match self {
96            AttemptError::Transport(msg) => write!(f, "transport: {msg}"),
97            AttemptError::Status5xx { code, body } => write!(f, "http {code}: {body}"),
98            AttemptError::Timeout(d) => write!(f, "timeout after {}ms", d.as_millis()),
99            AttemptError::NonRetryable(msg) => write!(f, "non-retryable: {msg}"),
100        }
101    }
102}
103
104/// Successful failover result. `prior_errors` lists every retryable
105/// failure we walked through to get here — useful for audit but not
106/// for user output.
107#[derive(Debug, Clone, PartialEq)]
108pub struct FailoverSuccess<R> {
109    pub provider: String,
110    pub response: R,
111    pub prior_errors: Vec<(String, AttemptError)>,
112}
113
114/// All-providers-exhausted result. The HTTP layer maps this to 503 per
115/// the acceptance criteria; `attempts` becomes the visible list of
116/// providers that were tried and how each one failed.
117#[derive(Debug, Clone, PartialEq)]
118pub struct FailoverExhausted {
119    pub attempts: Vec<(String, AttemptError)>,
120}
121
122impl fmt::Display for FailoverExhausted {
123    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
124        write!(f, "all providers failed:")?;
125        for (provider, err) in &self.attempts {
126            write!(f, " [{provider}: {err}]")?;
127        }
128        Ok(())
129    }
130}
131
132/// Walk `providers` in order. For each, invoke `attempt`. The first
133/// `Ok` short-circuits and is returned with the trail of prior
134/// retryable errors. A `NonRetryable` short-circuits to
135/// `Err(FailoverExhausted)` containing the attempts up to and including
136/// the non-retryable one — we do not pretend more providers were tried.
137/// Retryable failures advance to the next provider.
138///
139/// Empty `providers` returns `Err(FailoverExhausted { attempts: [] })`.
140/// The HTTP layer should treat that as a config error, not a 503; the
141/// kernel does not encode that distinction.
142pub fn run<R, F>(providers: &[&str], mut attempt: F) -> Result<FailoverSuccess<R>, FailoverExhausted>
143where
144    F: FnMut(&str) -> Result<R, AttemptError>,
145{
146    let mut prior: Vec<(String, AttemptError)> = Vec::new();
147
148    for provider in providers {
149        match attempt(provider) {
150            Ok(response) => {
151                return Ok(FailoverSuccess {
152                    provider: (*provider).to_string(),
153                    response,
154                    prior_errors: prior,
155                });
156            }
157            Err(err) => {
158                let retryable = err.is_retryable();
159                prior.push(((*provider).to_string(), err));
160                if !retryable {
161                    return Err(FailoverExhausted { attempts: prior });
162                }
163            }
164        }
165    }
166
167    Err(FailoverExhausted { attempts: prior })
168}
169
170/// Parse a `USING 'a,b,c'` override into an ordered, deduped list of
171/// non-empty provider names. Surrounding whitespace is trimmed. Empty
172/// segments are dropped. Order of first occurrence wins on dedup —
173/// the user's intent is honored, not silently reordered.
174///
175/// Returns `None` if the parse yields zero providers; the caller falls
176/// back to the global `ask.providers.fallback` setting.
177pub fn parse_using_clause(raw: &str) -> Option<Vec<String>> {
178    let mut out: Vec<String> = Vec::new();
179    for segment in raw.split(',') {
180        let name = segment.trim();
181        if name.is_empty() {
182            continue;
183        }
184        if !out.iter().any(|existing| existing == name) {
185            out.push(name.to_string());
186        }
187    }
188    if out.is_empty() { None } else { Some(out) }
189}
190
191#[cfg(test)]
192mod tests {
193    use super::*;
194    use std::cell::RefCell;
195
196    // --- AttemptError classification ------------------------------------
197
198    #[test]
199    fn transport_is_retryable() {
200        assert!(AttemptError::Transport("dns".into()).is_retryable());
201    }
202
203    #[test]
204    fn status_5xx_is_retryable() {
205        assert!(
206            AttemptError::Status5xx {
207                code: 502,
208                body: "bad gateway".into()
209            }
210            .is_retryable()
211        );
212    }
213
214    #[test]
215    fn timeout_is_retryable() {
216        assert!(AttemptError::Timeout(Duration::from_secs(30)).is_retryable());
217    }
218
219    #[test]
220    fn non_retryable_is_not_retryable() {
221        assert!(!AttemptError::NonRetryable("401 unauthorized".into()).is_retryable());
222    }
223
224    // --- run() success paths --------------------------------------------
225
226    #[test]
227    fn first_provider_succeeds_no_prior_errors() {
228        let providers = ["groq", "openai", "anthropic"];
229        let result = run(&providers, |p| Ok::<_, AttemptError>(format!("answer from {p}")));
230        let ok = result.expect("should succeed");
231        assert_eq!(ok.provider, "groq");
232        assert_eq!(ok.response, "answer from groq");
233        assert!(ok.prior_errors.is_empty());
234    }
235
236    #[test]
237    fn second_provider_succeeds_after_5xx() {
238        // Acceptance: integration test with two stub providers where
239        // the first errors and the second succeeds.
240        let providers = ["groq", "openai"];
241        let calls = RefCell::new(0u32);
242        let result = run(&providers, |p| {
243            *calls.borrow_mut() += 1;
244            if p == "groq" {
245                Err(AttemptError::Status5xx {
246                    code: 502,
247                    body: "bad gateway".into(),
248                })
249            } else {
250                Ok(format!("answer from {p}"))
251            }
252        });
253        let ok = result.expect("should succeed");
254        assert_eq!(ok.provider, "openai");
255        assert_eq!(ok.response, "answer from openai");
256        assert_eq!(*calls.borrow(), 2);
257        assert_eq!(ok.prior_errors.len(), 1);
258        assert_eq!(ok.prior_errors[0].0, "groq");
259    }
260
261    #[test]
262    fn third_provider_succeeds_after_transport_and_timeout() {
263        let providers = ["groq", "openai", "anthropic"];
264        let result = run(&providers, |p| match p {
265            "groq" => Err(AttemptError::Transport("connection reset".into())),
266            "openai" => Err(AttemptError::Timeout(Duration::from_secs(30))),
267            _ => Ok(format!("answer from {p}")),
268        });
269        let ok = result.expect("should succeed");
270        assert_eq!(ok.provider, "anthropic");
271        assert_eq!(ok.prior_errors.len(), 2);
272        assert!(matches!(ok.prior_errors[0].1, AttemptError::Transport(_)));
273        assert!(matches!(ok.prior_errors[1].1, AttemptError::Timeout(_)));
274    }
275
276    // --- run() failure paths --------------------------------------------
277
278    #[test]
279    fn all_retryable_failures_exhausts_with_full_attempt_list() {
280        let providers = ["groq", "openai", "anthropic"];
281        let result = run::<String, _>(&providers, |p| {
282            Err(AttemptError::Status5xx {
283                code: 503,
284                body: format!("{p} unavailable"),
285            })
286        });
287        let exhausted = result.expect_err("should exhaust");
288        assert_eq!(exhausted.attempts.len(), 3);
289        assert_eq!(exhausted.attempts[0].0, "groq");
290        assert_eq!(exhausted.attempts[1].0, "openai");
291        assert_eq!(exhausted.attempts[2].0, "anthropic");
292    }
293
294    #[test]
295    fn non_retryable_short_circuits_without_trying_remaining() {
296        // 401 from the first provider must NOT be papered over by
297        // silently switching to the next vendor. The user sees the
298        // auth error directly.
299        let providers = ["groq", "openai", "anthropic"];
300        let calls = RefCell::new(0u32);
301        let result = run::<String, _>(&providers, |p| {
302            *calls.borrow_mut() += 1;
303            if p == "groq" {
304                Err(AttemptError::NonRetryable("401 unauthorized".into()))
305            } else {
306                panic!("must not call sibling providers after non-retryable")
307            }
308        });
309        let exhausted = result.expect_err("should short-circuit");
310        assert_eq!(*calls.borrow(), 1);
311        assert_eq!(exhausted.attempts.len(), 1);
312        assert_eq!(exhausted.attempts[0].0, "groq");
313        assert!(matches!(
314            exhausted.attempts[0].1,
315            AttemptError::NonRetryable(_)
316        ));
317    }
318
319    #[test]
320    fn non_retryable_after_retryable_preserves_full_trail() {
321        // 502 from groq, then 401 from openai — the audit log should
322        // see both, and anthropic must not be called.
323        let providers = ["groq", "openai", "anthropic"];
324        let calls = RefCell::new(Vec::<String>::new());
325        let result = run::<String, _>(&providers, |p| {
326            calls.borrow_mut().push(p.to_string());
327            match p {
328                "groq" => Err(AttemptError::Status5xx {
329                    code: 502,
330                    body: "bad".into(),
331                }),
332                "openai" => Err(AttemptError::NonRetryable("401".into())),
333                _ => panic!("anthropic must not be called"),
334            }
335        });
336        let exhausted = result.expect_err("should fail");
337        assert_eq!(*calls.borrow(), vec!["groq", "openai"]);
338        assert_eq!(exhausted.attempts.len(), 2);
339    }
340
341    #[test]
342    fn empty_provider_list_returns_empty_exhausted() {
343        let providers: [&str; 0] = [];
344        let result = run::<String, _>(&providers, |_| panic!("must not be called"));
345        let exhausted = result.expect_err("empty list yields exhausted");
346        assert!(exhausted.attempts.is_empty());
347    }
348
349    // --- determinism preservation ---------------------------------------
350
351    #[test]
352    fn attempt_fn_is_invoked_with_identical_inputs() {
353        // The kernel does not modify any per-request state between
354        // attempts. We verify by capturing a request payload struct
355        // and asserting equality across calls.
356        #[derive(Clone, PartialEq, Debug)]
357        struct Req {
358            seed: u64,
359            temperature: f32,
360            strict: bool,
361        }
362        let req = Req {
363            seed: 42,
364            temperature: 0.0,
365            strict: true,
366        };
367        let providers = ["groq", "openai"];
368        let seen = RefCell::new(Vec::<Req>::new());
369        let _ = run::<(), _>(&providers, |_| {
370            seen.borrow_mut().push(req.clone());
371            Err(AttemptError::Transport("retry".into()))
372        });
373        let seen = seen.borrow();
374        assert_eq!(seen.len(), 2);
375        assert_eq!(seen[0], seen[1]);
376    }
377
378    // --- USING clause parsing -------------------------------------------
379
380    #[test]
381    fn parse_using_simple() {
382        assert_eq!(
383            parse_using_clause("groq,openai"),
384            Some(vec!["groq".into(), "openai".into()])
385        );
386    }
387
388    #[test]
389    fn parse_using_trims_whitespace() {
390        assert_eq!(
391            parse_using_clause("  groq , openai , anthropic  "),
392            Some(vec!["groq".into(), "openai".into(), "anthropic".into()])
393        );
394    }
395
396    #[test]
397    fn parse_using_drops_empty_segments() {
398        assert_eq!(
399            parse_using_clause("groq,,openai,"),
400            Some(vec!["groq".into(), "openai".into()])
401        );
402    }
403
404    #[test]
405    fn parse_using_dedupes_preserving_first_occurrence() {
406        assert_eq!(
407            parse_using_clause("groq,openai,groq"),
408            Some(vec!["groq".into(), "openai".into()])
409        );
410    }
411
412    #[test]
413    fn parse_using_empty_returns_none() {
414        assert_eq!(parse_using_clause(""), None);
415        assert_eq!(parse_using_clause(" , , "), None);
416    }
417
418    #[test]
419    fn parse_using_single_provider() {
420        assert_eq!(parse_using_clause("groq"), Some(vec!["groq".into()]));
421    }
422
423    // --- Display impls (audit-facing) -----------------------------------
424
425    #[test]
426    fn exhausted_display_lists_each_attempt() {
427        let exhausted = FailoverExhausted {
428            attempts: vec![
429                ("groq".into(), AttemptError::Transport("dns".into())),
430                (
431                    "openai".into(),
432                    AttemptError::Status5xx {
433                        code: 502,
434                        body: "bad".into(),
435                    },
436                ),
437            ],
438        };
439        let s = format!("{exhausted}");
440        assert!(s.contains("groq"));
441        assert!(s.contains("openai"));
442        assert!(s.contains("502"));
443    }
444}
reddb_server/runtime/ai/provider_failover.rs

reddb_server/runtime/ai/
provider_failover.rs
