Skip to main content

udb_client/
error.rs

1//! Typed errors and retry policy.
2//!
3//! The broker attaches a structured [`ErrorDetail`] to failures in the
4//! `udb-error-detail-bin` trailer. Without decoding it a caller sees only a gRPC
5//! code and a message, and has to pattern-match on error strings to learn whether
6//! a failure is retryable, which capability was missing, or which field was
7//! rejected — exactly the things the broker already said precisely.
8
9use std::time::Duration;
10
11use tonic::{Code, Status};
12
13use crate::proto::udb::entity::v1::ErrorDetail;
14
15/// The metadata key the broker uses. The `-bin` suffix tells gRPC the value is
16/// raw bytes rather than ASCII, so tonic base64-decodes it for us.
17pub const ERROR_DETAIL_TRAILER: &str = "udb-error-detail-bin";
18
19/// A `Status` plus the broker's structured detail, when it sent one.
20#[derive(Debug, Clone)]
21pub struct UdbError {
22    pub status: Status,
23    pub detail: Option<ErrorDetail>,
24}
25
26impl UdbError {
27    /// Decode the trailer, if present and well-formed.
28    ///
29    /// A malformed trailer is treated as absent rather than as a new failure: the
30    /// call already failed and the caller needs the original status, not a
31    /// decoding complaint layered over it.
32    pub fn from_status(status: Status) -> Self {
33        let detail = status
34            .metadata()
35            .get_bin(ERROR_DETAIL_TRAILER)
36            .and_then(|v| v.to_bytes().ok())
37            .and_then(|bytes| <ErrorDetail as prost::Message>::decode(bytes).ok());
38        Self { status, detail }
39    }
40
41    pub fn code(&self) -> Code {
42        self.status.code()
43    }
44
45    pub fn message(&self) -> &str {
46        self.status.message()
47    }
48
49    /// Whether the BROKER said this is retryable.
50    ///
51    /// Trusted over any code-based guess: the broker knows whether it got far
52    /// enough to have side effects, and a client cannot infer that from
53    /// `UNAVAILABLE` alone.
54    pub fn is_retryable(&self) -> bool {
55        match &self.detail {
56            Some(d) => d.retryable,
57            // No detail: fall back to codes that cannot have applied a mutation.
58            None => matches!(
59                self.status.code(),
60                Code::Unavailable | Code::ResourceExhausted
61            ),
62        }
63    }
64
65    /// How long the broker asked us to wait, if it said.
66    pub fn retry_after(&self) -> Option<Duration> {
67        let ms = self.detail.as_ref()?.retry_after_ms;
68        (ms > 0).then(|| Duration::from_millis(ms as u64))
69    }
70
71    /// The capability the deployment is missing, for a capability refusal.
72    pub fn capability_required(&self) -> Option<&str> {
73        let c = self.detail.as_ref()?.capability_required.as_str();
74        (!c.is_empty()).then_some(c)
75    }
76
77    /// The correlation id to quote in a bug report or a support ticket.
78    pub fn correlation_id(&self) -> Option<&str> {
79        let c = self.detail.as_ref()?.correlation_id.as_str();
80        (!c.is_empty()).then_some(c)
81    }
82
83    /// The broker's typed refusal reason (`ErrorDetail.reason`, e.g.
84    /// `UDB_TOKEN_EXPIRED`), when it sent one. Branch on this, never on the
85    /// message text.
86    pub fn reason(&self) -> Option<&str> {
87        let r = self.detail.as_ref()?.reason.as_str();
88        (!r.is_empty()).then_some(r)
89    }
90
91    /// The broker refused the bearer because it expired (`UDB_TOKEN_EXPIRED`):
92    /// refresh it or sign in again. A delegated job token cannot be refreshed:
93    /// the job exchanges a fresh user token or stops.
94    pub fn is_token_expired(&self) -> bool {
95        self.reason() == Some("UDB_TOKEN_EXPIRED")
96    }
97
98    /// The broker refused the bearer because it was revoked or is no longer
99    /// backed by live state (`UDB_TOKEN_REVOKED`): do not retry with it.
100    pub fn is_token_revoked(&self) -> bool {
101        self.reason() == Some("UDB_TOKEN_REVOKED")
102    }
103
104    /// Per-field rejections, for a validation failure.
105    pub fn field_violations(&self) -> &[crate::proto::udb::entity::v1::ErrorFieldViolation] {
106        self.detail
107            .as_ref()
108            .map(|d| d.field_violations.as_slice())
109            .unwrap_or_default()
110    }
111}
112
113impl std::fmt::Display for UdbError {
114    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
115        write!(f, "{}: {}", self.status.code(), self.status.message())?;
116        if let Some(detail) = &self.detail {
117            if !detail.backend.is_empty() {
118                write!(f, " [backend={}", detail.backend)?;
119                if !detail.operation.is_empty() {
120                    write!(f, " op={}", detail.operation)?;
121                }
122                write!(f, "]")?;
123            }
124            if !detail.capability_required.is_empty() {
125                write!(f, " (requires {})", detail.capability_required)?;
126            }
127        }
128        Ok(())
129    }
130}
131
132impl std::error::Error for UdbError {}
133
134impl From<Status> for UdbError {
135    fn from(status: Status) -> Self {
136        Self::from_status(status)
137    }
138}
139
140/// Per-call deadline and retry policy.
141///
142/// Retries are bounded and only ever applied where the caller has said the
143/// operation is safe to repeat — see [`CallPolicy::idempotent`].
144#[derive(Debug, Clone, Copy)]
145pub struct CallPolicy {
146    pub deadline: Option<Duration>,
147    pub max_attempts: u32,
148    pub base_backoff: Duration,
149    /// The longest broker-requested delay (`ErrorDetail.retry_after_ms`) the
150    /// client waits for automatically; a longer one is returned to the caller
151    /// at once (the [`UdbError`] keeps `retry_after`). Defaults to 10 s.
152    pub max_retry_after: Duration,
153    /// Whether the operation may be safely repeated. Defaults to `false`: a
154    /// blanket retry on a mutation is how one payment becomes two.
155    pub idempotent: bool,
156}
157
158impl Default for CallPolicy {
159    fn default() -> Self {
160        Self {
161            deadline: Some(Duration::from_secs(30)),
162            max_attempts: 3,
163            base_backoff: Duration::from_millis(100),
164            max_retry_after: Duration::from_secs(10),
165            idempotent: false,
166        }
167    }
168}
169
170impl CallPolicy {
171    /// A read: safe to repeat.
172    pub fn idempotent() -> Self {
173        Self {
174            idempotent: true,
175            ..Default::default()
176        }
177    }
178
179    /// The policy the CONTRACT implies for one RPC path.
180    ///
181    /// This replaced a hand-written per-method judgement, which was wrong in both
182    /// directions: it refused to retry `Upsert`, `Update` and `Delete` even
183    /// though the broker declares them replayable, costing availability on a
184    /// transient failure the broker was happy to see again.
185    ///
186    /// Deciding from the descriptor instead of a method name is the point of the
187    /// `operation_kind` annotation. An unknown path yields the conservative
188    /// default (no retries).
189    pub fn from_contract(path: &str) -> Self {
190        if crate::generated_rpcs::is_retry_safe(path) {
191            Self::idempotent()
192        } else {
193            Self::default()
194        }
195    }
196
197    /// No retries at all.
198    pub fn once() -> Self {
199        Self {
200            max_attempts: 1,
201            ..Default::default()
202        }
203    }
204
205    pub fn with_deadline(mut self, deadline: Duration) -> Self {
206        self.deadline = Some(deadline);
207        self
208    }
209
210    pub fn with_max_attempts(mut self, attempts: u32) -> Self {
211        self.max_attempts = attempts.max(1);
212        self
213    }
214
215    /// The longest broker-requested retry delay this policy waits for; a longer
216    /// one is returned to the caller at once.
217    pub fn with_max_retry_after(mut self, cap: Duration) -> Self {
218        self.max_retry_after = cap;
219        self
220    }
221
222    /// Backoff before `attempt` (1-based): the larger of this policy's own
223    /// backoff and the broker's `retry_after`. A `retry_after` above
224    /// `max_retry_after` never gets here: [`Self::should_retry`] refuses it.
225    ///
226    /// The broker's delay is a floor, not a replacement: a short server hint
227    /// must not shrink an already-grown backoff.
228    pub(crate) fn backoff_for(&self, attempt: u32, err: &UdbError) -> Duration {
229        // Exponential, capped. No jitter here: callers that need decorrelated
230        // retries across a fleet should drive their own loop, and inventing
231        // randomness inside a client makes failures harder to reproduce.
232        let exp = self
233            .base_backoff
234            .saturating_mul(1u32 << attempt.min(6).saturating_sub(1))
235            .min(Duration::from_secs(5));
236        match err.retry_after() {
237            Some(after) => exp.max(after),
238            None => exp,
239        }
240    }
241
242    /// Whether another attempt is permitted. A broker-requested delay longer
243    /// than `max_retry_after` is not waited for: the error goes back to the
244    /// caller at once, its `retry_after` intact, so the caller decides.
245    pub(crate) fn should_retry(&self, attempt: u32, err: &UdbError) -> bool {
246        if err
247            .retry_after()
248            .is_some_and(|after| after > self.max_retry_after)
249        {
250            return false;
251        }
252        self.idempotent && attempt < self.max_attempts && err.is_retryable()
253    }
254}
255
256#[cfg(test)]
257mod tests {
258    use super::*;
259
260    fn status_without_detail(code: Code) -> UdbError {
261        UdbError::from_status(Status::new(code, "boom"))
262    }
263
264    #[test]
265    fn missing_trailer_falls_back_to_safe_codes() {
266        assert!(status_without_detail(Code::Unavailable).is_retryable());
267        assert!(status_without_detail(Code::ResourceExhausted).is_retryable());
268        assert!(
269            !status_without_detail(Code::InvalidArgument).is_retryable(),
270            "a rejected argument will be rejected again"
271        );
272        assert!(
273            !status_without_detail(Code::Internal).is_retryable(),
274            "INTERNAL may have applied a mutation; never assume repeatable"
275        );
276    }
277
278    #[test]
279    fn malformed_trailer_is_treated_as_absent() {
280        let mut status = Status::new(Code::Internal, "boom");
281        status.metadata_mut().insert_bin(
282            ERROR_DETAIL_TRAILER,
283            tonic::metadata::MetadataValue::from_bytes(b"not-a-protobuf-at-all"),
284        );
285        let err = UdbError::from_status(status);
286        // Decoding failure must not mask the original status.
287        assert_eq!(err.code(), Code::Internal);
288        assert_eq!(err.message(), "boom");
289    }
290
291    #[test]
292    fn broker_detail_overrides_the_code_guess() {
293        let detail = ErrorDetail {
294            retryable: true,
295            retry_after_ms: 250,
296            capability_required: "postgres_backend".into(),
297            correlation_id: "corr-9".into(),
298            ..Default::default()
299        };
300        let mut status = Status::new(Code::Internal, "boom");
301        let mut buf = Vec::new();
302        prost::Message::encode(&detail, &mut buf).expect("encode");
303        status.metadata_mut().insert_bin(
304            ERROR_DETAIL_TRAILER,
305            tonic::metadata::MetadataValue::from_bytes(&buf),
306        );
307
308        let err = UdbError::from_status(status);
309        assert!(
310            err.is_retryable(),
311            "INTERNAL, but the broker said retryable"
312        );
313        assert_eq!(err.retry_after(), Some(Duration::from_millis(250)));
314        assert_eq!(err.capability_required(), Some("postgres_backend"));
315        assert_eq!(err.correlation_id(), Some("corr-9"));
316    }
317
318    fn refusal(reason: &str) -> UdbError {
319        let detail = ErrorDetail {
320            reason: reason.into(),
321            ..Default::default()
322        };
323        let mut status = Status::new(Code::Unauthenticated, "refused");
324        let mut buf = Vec::new();
325        prost::Message::encode(&detail, &mut buf).expect("encode");
326        status.metadata_mut().insert_bin(
327            ERROR_DETAIL_TRAILER,
328            tonic::metadata::MetadataValue::from_bytes(&buf),
329        );
330        UdbError::from_status(status)
331    }
332
333    #[test]
334    fn expired_and_revoked_bearers_are_told_apart() {
335        let expired = refusal("UDB_TOKEN_EXPIRED");
336        let revoked = refusal("UDB_TOKEN_REVOKED");
337        assert_eq!(expired.code(), Code::Unauthenticated);
338        assert_eq!(expired.reason(), Some("UDB_TOKEN_EXPIRED"));
339        assert!(expired.is_token_expired() && !expired.is_token_revoked());
340        assert!(revoked.is_token_revoked() && !revoked.is_token_expired());
341        assert!(!refusal("UDB_AUTHENTICATION_FAILED").is_token_expired());
342        let text_only = UdbError::from_status(Status::unauthenticated("UDB_TOKEN_EXPIRED"));
343        assert!(
344            !text_only.is_token_expired(),
345            "the message text is never matched"
346        );
347        assert_eq!(text_only.reason(), None);
348    }
349
350    #[test]
351    fn mutations_are_not_retried_by_default() {
352        let err = status_without_detail(Code::Unavailable);
353        assert!(
354            !CallPolicy::default().should_retry(1, &err),
355            "default policy must not repeat a possible mutation"
356        );
357        assert!(CallPolicy::idempotent().should_retry(1, &err));
358    }
359
360    #[test]
361    fn retry_stops_at_max_attempts() {
362        let err = status_without_detail(Code::Unavailable);
363        let policy = CallPolicy::idempotent().with_max_attempts(2);
364        assert!(policy.should_retry(1, &err));
365        assert!(!policy.should_retry(2, &err), "attempt 2 of 2 is the last");
366    }
367
368    #[test]
369    fn backoff_prefers_the_brokers_retry_after() {
370        let detail = ErrorDetail {
371            retryable: true,
372            retry_after_ms: 1234,
373            ..Default::default()
374        };
375        let err = UdbError {
376            status: Status::new(Code::Unavailable, "boom"),
377            detail: Some(detail),
378        };
379        assert_eq!(
380            CallPolicy::idempotent().backoff_for(3, &err),
381            Duration::from_millis(1234)
382        );
383    }
384
385    #[test]
386    fn backoff_never_shrinks_below_its_own_step_for_a_short_retry_after() {
387        let err = UdbError {
388            status: Status::new(Code::ResourceExhausted, "slow down"),
389            detail: Some(ErrorDetail {
390                retryable: true,
391                retry_after_ms: 1200,
392                ..Default::default()
393            }),
394        };
395        // Attempt 6: own backoff 100ms * 32 = 3.2s, above the broker's 1.2s.
396        assert_eq!(
397            CallPolicy::idempotent().backoff_for(6, &err),
398            Duration::from_millis(3200)
399        );
400        // Attempt 1: own backoff 100ms, so the broker's 1.2s wins.
401        assert_eq!(
402            CallPolicy::idempotent().backoff_for(1, &err),
403            Duration::from_millis(1200)
404        );
405    }
406
407    #[test]
408    fn retry_after_above_the_cap_is_returned_not_retried() {
409        let throttled = |ms: i64| UdbError {
410            status: Status::new(Code::ResourceExhausted, "slow down"),
411            detail: Some(ErrorDetail {
412                retryable: true,
413                retry_after_ms: ms,
414                ..Default::default()
415            }),
416        };
417        let err = throttled(5000);
418        let policy = CallPolicy::idempotent().with_max_retry_after(Duration::from_millis(1500));
419        assert!(!policy.should_retry(1, &err), "5s is above the 1.5s cap");
420        assert_eq!(err.code(), Code::ResourceExhausted);
421        assert_eq!(err.retry_after(), Some(Duration::from_millis(5000)));
422
423        assert_eq!(
424            CallPolicy::default().max_retry_after,
425            Duration::from_secs(10),
426            "default cap"
427        );
428        assert!(!CallPolicy::idempotent().should_retry(1, &throttled(60_000)));
429        // A delay at the default cap is still waited for.
430        let at_cap = throttled(10_000);
431        assert!(CallPolicy::idempotent().should_retry(1, &at_cap));
432        assert_eq!(
433            CallPolicy::idempotent().backoff_for(1, &at_cap),
434            Duration::from_secs(10)
435        );
436    }
437
438    #[test]
439    fn backoff_grows_and_is_capped() {
440        let err = status_without_detail(Code::Unavailable);
441        let p = CallPolicy::idempotent();
442        assert_eq!(p.backoff_for(1, &err), Duration::from_millis(100));
443        assert_eq!(p.backoff_for(2, &err), Duration::from_millis(200));
444        assert_eq!(p.backoff_for(3, &err), Duration::from_millis(400));
445        assert!(p.backoff_for(20, &err) <= Duration::from_secs(5));
446    }
447}