mariadb-cli 0.1.1

One-shot agent-native command line client for MariaDB servers
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
//! Trying again, but only when trying again can possibly help.
//!
//! Two rules make this safe to apply automatically.
//!
//! The first is that only a failure marked as repeatable is ever repeated. A
//! refused credential repeated three times is three refusals, three log lines
//! on the server and one account closer to being locked out; the answer was
//! final the first time and the mark on the failure says so.
//!
//! The second is that the wait between attempts grows. A server that is still
//! starting, or a lock that is still held, is not helped by three attempts in
//! the same millisecond; it is helped by being left alone for a moment.
//!
//! # Why the wait is not the same wait everywhere
//!
//! This product does work across several destinations inside one invocation. A
//! server they share going away takes all of them down at the same instant, and
//! a wait computed only from the attempt number brings all of them back at the
//! same instant too, in waves. The wait that recovers one caller is the wait
//! that keeps a recovering server down when every caller uses it.
//!
//! So each wait is spread, and the spread is what makes two callers that failed
//! together come back apart. Half the wait is kept as a floor, because the
//! point of waiting at all is to leave the server alone for a moment.
//!
//! # Why attempts alone are not a bound
//!
//! Attempts bound how many times, and the ceiling bounds how long each one is.
//! Neither bounds the whole, and a caller waiting on a policy cares about the
//! whole: five attempts under a one-second ceiling is a five-second stall that
//! nothing in the policy declared. The deadline is that missing bound, and it
//! is checked against real elapsed time rather than against the sum of the
//! waits, so the work between attempts counts too.

use crate::error::{Failure, Result};
use std::future::Future;
use std::time::{Duration, Instant};
use tokio_util::sync::CancellationToken;

/// How many times an operation is repeated, and how long the waits are.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Backoff {
    /// Extra attempts beyond the first one.
    pub extra_attempts: u32,
    /// The wait before the first repeat.
    pub base: Duration,
    /// The longest any single wait may become.
    pub ceiling: Duration,
    /// The longest the whole series of attempts may take.
    ///
    /// Zero means no bound on the whole, which is what a policy that never
    /// repeats needs and what a caller that owns its own deadline may ask for.
    pub deadline: Duration,
}

impl Backoff {
    /// Builds a policy from the configured attempts, first wait and bounds.
    ///
    /// Both bounds are taken here rather than one being optional, because a
    /// ceiling without a deadline bounds each wait and not the wait. A policy
    /// with many attempts under a low ceiling still turns a short operation
    /// into one that appears to hang, and the caller was never told how long.
    #[must_use]
    pub const fn new(
        extra_attempts: u32,
        base: Duration,
        ceiling: Duration,
        deadline: Duration,
    ) -> Self {
        Self {
            extra_attempts,
            base,
            ceiling,
            deadline,
        }
    }

    /// A policy that never repeats anything.
    #[must_use]
    pub const fn none() -> Self {
        Self::new(
            0,
            Duration::from_millis(0),
            Duration::from_millis(0),
            Duration::from_millis(0),
        )
    }

    /// The wait before the given repeat, counting the first repeat as one.
    ///
    /// The wait doubles each time and then stops at the ceiling, so the growth
    /// is bounded rather than open-ended.
    ///
    /// # Examples
    ///
    /// The doubling is visible in the first three repeats, and the ceiling
    /// catches the growth rather than letting it run away:
    ///
    /// ```
    /// use mariadb_cli::runtime::Backoff;
    /// use std::time::Duration;
    ///
    /// let policy = Backoff::new(
    ///     5,
    ///     Duration::from_millis(100),
    ///     Duration::from_millis(400),
    ///     Duration::from_secs(30),
    /// );
    ///
    /// assert_eq!(policy.wait_before(1), Duration::from_millis(100));
    /// assert_eq!(policy.wait_before(2), Duration::from_millis(200));
    /// assert_eq!(policy.wait_before(3), Duration::from_millis(400));
    /// assert_eq!(policy.wait_before(9), Duration::from_millis(400));
    /// ```
    ///
    /// There is no wait before the first attempt, which is not a repeat:
    ///
    /// ```
    /// use mariadb_cli::runtime::Backoff;
    /// use std::time::Duration;
    ///
    /// let policy = Backoff::new(
    ///     3,
    ///     Duration::from_millis(50),
    ///     Duration::from_secs(1),
    ///     Duration::from_secs(30),
    /// );
    /// assert_eq!(policy.wait_before(0), Duration::from_millis(0));
    /// ```
    #[must_use]
    pub fn wait_before(&self, repeat: u32) -> Duration {
        if repeat == 0 {
            return Duration::from_millis(0);
        }
        let factor = 1_u32.checked_shl(repeat - 1).unwrap_or(u32::MAX);
        self.base
            .checked_mul(factor)
            .unwrap_or(self.ceiling)
            .min(self.ceiling)
    }

    /// One nominal wait, spread so two callers do not come back together.
    ///
    /// Half the nominal wait is kept and the other half is drawn, so the
    /// result lands somewhere in the upper half of what the growth asked for.
    /// Drawing the whole wait would let a repeat land immediately, which is the
    /// one thing waiting exists to prevent; keeping the whole wait is what
    /// makes every caller come back at the same instant.
    ///
    /// # Examples
    ///
    /// The spread never leaves the half it was promised:
    ///
    /// ```
    /// use mariadb_cli::runtime::Backoff;
    /// use std::time::Duration;
    ///
    /// let policy = Backoff::new(
    ///     3,
    ///     Duration::from_millis(200),
    ///     Duration::from_millis(800),
    ///     Duration::from_secs(30),
    /// );
    /// let nominal = policy.wait_before(2);
    /// let spread = policy.spread(nominal);
    ///
    /// assert!(spread >= nominal / 2);
    /// assert!(spread <= nominal);
    /// ```
    #[must_use]
    pub fn spread(&self, nominal: Duration) -> Duration {
        let kept = nominal / 2;
        // A wait too small to halve has nothing to spread, and turning it into
        // no wait at all would remove the pause rather than move it. Waits at
        // this scale do not exist in practice; the arm is here so the floor
        // holds at every input rather than at the ones that were thought of.
        if kept.is_zero() {
            return nominal;
        }
        // Saturating rather than plain, though halving can never overshoot the
        // whole. Stating that in the operator costs nothing and means a reader
        // does not have to reconstruct the argument to be sure.
        let width = u64::try_from(nominal.saturating_sub(kept).as_nanos()).unwrap_or(u64::MAX);
        kept + Duration::from_nanos(draw() % width.saturating_add(1))
    }

    /// Whether waiting this long would spend more time than the policy allows.
    ///
    /// A zero deadline is no bound at all, which is what a policy that never
    /// repeats carries and what a caller owning its own deadline asks for.
    #[must_use]
    pub fn exceeds_deadline(&self, spent: Duration, pause: Duration) -> bool {
        !self.deadline.is_zero() && spent.saturating_add(pause) > self.deadline
    }
}

/// A number that differs between processes and between draws.
///
/// Taken from the standard library's own randomness rather than from a crate,
/// because the only property needed is that two callers failing together do not
/// agree, and a dependency bought for that would be a dependency to audit, to
/// license and to keep current.
///
/// This is not a source anything may rely on for secrecy, and nothing does: it
/// is read once per wait, to decide a number of nanoseconds.
fn draw() -> u64 {
    use std::hash::{BuildHasher, Hasher};
    std::collections::hash_map::RandomState::new()
        .build_hasher()
        .finish()
}

/// Runs an operation, repeating it while the failure says repeating may help.
///
/// The operation is handed the attempt number, counting from one, so a caller
/// can report which attempt it is on without keeping the count itself.
///
/// # Errors
///
/// Returns the failure of the last attempt. A failure not marked as repeatable
/// is returned at once, on the first attempt, without any wait. The same
/// failure is returned when the attempts run out and when the deadline would be
/// passed, because in all three the answer the caller gets is the one the work
/// actually produced. Returns an interrupted failure when the invocation is
/// told to stop during a wait, so a signal is never left waiting for the policy
/// to run out.
pub async fn retry<T, F, Fut>(
    policy: &Backoff,
    token: &CancellationToken,
    mut operation: F,
) -> Result<T>
where
    F: FnMut(u32) -> Fut,
    Fut: Future<Output = Result<T>>,
{
    // Measured from before the first attempt, so the deadline bounds the whole
    // series and not the waits inside it. An attempt that hangs for most of the
    // budget has spent the budget, whatever the waits would have added.
    let started = Instant::now();
    let mut repeat = 0;
    loop {
        match operation(repeat + 1).await {
            Ok(value) => return Ok(value),
            Err(failure) => {
                // A final answer and a policy that has run out are different
                // things to a reader: the first says asking again is pointless,
                // the second says asking again was allowed and is spent. Only
                // the first earns the line that says so.
                if !failure.is_retryable() {
                    crate::i18n::report::attempt_permanent();
                    return Err(failure);
                }
                if repeat >= policy.extra_attempts {
                    return Err(failure);
                }
                let attempt = repeat + 1;
                repeat += 1;
                let pause = policy.spread(policy.wait_before(repeat));

                // Named rather than passed over in silence, because attempts
                // were left and a reader looking at the count would otherwise
                // have no way to learn why they were not used.
                let spent = started.elapsed();
                if policy.exceeds_deadline(spent, pause) {
                    crate::i18n::report::attempt_budget_spent(
                        spent.as_millis(),
                        policy.deadline.as_millis(),
                    );
                    return Err(failure);
                }

                crate::i18n::report::attempt_retrying(
                    attempt,
                    policy.extra_attempts + 1,
                    pause.as_millis(),
                );
                wait(pause, token).await?;
            }
        }
    }
}

/// Waits, unless the invocation is told to stop first.
///
/// # Errors
///
/// Returns an interrupted failure when the stop signal arrives during the wait.
async fn wait(duration: Duration, token: &CancellationToken) -> Result<()> {
    tokio::select! {
        () = token.cancelled() => Err(Failure::interrupted(
            "interrupted.waiting_to_retry",
            "the invocation was stopped while waiting to try again",
        )),
        () = tokio::time::sleep(duration) => Ok(()),
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::Cell;

    fn policy() -> Backoff {
        Backoff::new(
            3,
            Duration::from_millis(1),
            Duration::from_millis(8),
            Duration::from_secs(30),
        )
    }

    #[test]
    fn the_wait_doubles_and_then_stops_at_the_ceiling() {
        let policy = policy();
        assert_eq!(policy.wait_before(0), Duration::from_millis(0));
        assert_eq!(policy.wait_before(1), Duration::from_millis(1));
        assert_eq!(policy.wait_before(2), Duration::from_millis(2));
        assert_eq!(policy.wait_before(3), Duration::from_millis(4));
        assert_eq!(policy.wait_before(4), Duration::from_millis(8));
        assert_eq!(policy.wait_before(50), Duration::from_millis(8));
    }

    /// Two callers that failed together do not come back together.
    ///
    /// This is the whole reason the spread exists. Several destinations in one
    /// invocation lean on the same server, so that server going away takes them
    /// all down at the same instant; a wait computed only from the attempt
    /// number then brings them all back at the same instant, in waves, and the
    /// recovering server meets the same crowd that was there before.
    ///
    /// Asserted over many draws rather than two, because two equal draws are a
    /// coincidence and a hundred equal draws are a policy that does not spread.
    #[test]
    fn two_waits_of_the_same_length_do_not_land_on_the_same_instant() {
        let policy = Backoff::new(
            5,
            Duration::from_millis(200),
            Duration::from_millis(6_400),
            Duration::from_secs(30),
        );
        let nominal = policy.wait_before(3);

        let drawn: std::collections::BTreeSet<Duration> =
            (0..100).map(|_| policy.spread(nominal)).collect();

        assert!(
            drawn.len() > 50,
            "a hundred waits produced {} distinct lengths, which is a herd",
            drawn.len()
        );
    }

    /// The spread stays inside the half it was promised, at every step.
    ///
    /// A draw over the whole wait would let a repeat land at once, which is the
    /// one thing waiting exists to prevent, and the growth would stop meaning
    /// anything.
    #[test]
    fn a_spread_wait_never_leaves_the_upper_half_of_the_nominal_one() {
        let policy = Backoff::new(
            8,
            Duration::from_millis(200),
            Duration::from_millis(6_400),
            Duration::from_secs(30),
        );

        for repeat in 1..=8 {
            let nominal = policy.wait_before(repeat);
            for _ in 0..50 {
                let spread = policy.spread(nominal);
                assert!(spread >= nominal / 2, "repeat {repeat}: {spread:?}");
                assert!(spread <= nominal, "repeat {repeat}: {spread:?}");
            }
        }
    }

    /// A wait too short to divide is left alone rather than turned into none.
    #[test]
    fn a_wait_with_nothing_to_spread_is_kept_whole() {
        let policy = policy();
        assert_eq!(policy.spread(Duration::ZERO), Duration::ZERO);
        assert_eq!(
            policy.spread(Duration::from_nanos(1)),
            Duration::from_nanos(1)
        );
    }

    /// The whole series is bounded, which neither the count nor the ceiling
    /// bounds on its own.
    #[test]
    fn the_deadline_bounds_the_series_and_not_each_wait() {
        let policy = Backoff::new(
            9,
            Duration::from_secs(1),
            Duration::from_secs(1),
            Duration::from_secs(4),
        );

        assert!(
            !policy.exceeds_deadline(Duration::from_secs(2), Duration::from_secs(1)),
            "three seconds of a four second budget still fits"
        );
        assert!(
            policy.exceeds_deadline(Duration::from_secs(4), Duration::from_secs(1)),
            "the budget is spent and six attempts are still left"
        );
    }

    /// A policy carrying no deadline is bounded by its attempts alone, which
    /// is what a caller owning its own deadline asks for.
    #[test]
    fn a_policy_with_no_deadline_never_stops_for_one() {
        let unbounded = Backoff::new(
            3,
            Duration::from_millis(1),
            Duration::from_millis(8),
            Duration::ZERO,
        );
        assert!(!unbounded.exceeds_deadline(Duration::from_secs(3_600), Duration::from_secs(1)));
    }

    /// The deadline ends the series while attempts are still left, and the
    /// caller still receives the failure the work produced.
    #[tokio::test]
    async fn a_spent_budget_ends_the_series_before_the_attempts_do() {
        let attempts = Cell::new(0);
        let tight = Backoff::new(
            20,
            Duration::from_millis(40),
            Duration::from_millis(80),
            Duration::from_millis(1),
        );

        let failure = retry(&tight, &CancellationToken::new(), |_| {
            attempts.set(attempts.get() + 1);
            async { Err::<u8, Failure>(Failure::network("network.unreachable", "lost")) }
        })
        .await
        .expect_err("must fail");

        assert_eq!(
            attempts.get(),
            1,
            "the budget was spent before the second attempt, and twenty were left"
        );
        assert_eq!(
            failure.code(),
            "network.unreachable",
            "the caller gets what the work produced, not a report about the policy"
        );
    }

    #[tokio::test]
    async fn a_clean_call_is_never_repeated() {
        let attempts = Cell::new(0);
        let value = retry(&policy(), &CancellationToken::new(), |_| {
            attempts.set(attempts.get() + 1);
            async { Ok::<u8, Failure>(7) }
        })
        .await
        .expect("clean");
        assert_eq!(value, 7);
        assert_eq!(attempts.get(), 1);
    }

    #[tokio::test]
    async fn a_refused_credential_is_never_repeated() {
        let attempts = Cell::new(0);
        let failure = retry(&policy(), &CancellationToken::new(), |_| {
            attempts.set(attempts.get() + 1);
            async {
                Err::<u8, Failure>(Failure::authentication(
                    "auth.refused",
                    "the credential was refused",
                ))
            }
        })
        .await
        .expect_err("must fail");

        assert_eq!(attempts.get(), 1, "a final answer is never asked twice");
        assert_eq!(failure.exit_number().code(), 6);
    }

    #[tokio::test]
    async fn a_repeatable_failure_is_tried_again_until_it_clears() {
        let attempts = Cell::new(0);
        let value = retry(&policy(), &CancellationToken::new(), |attempt| {
            attempts.set(attempts.get() + 1);
            async move {
                if attempt < 3 {
                    return Err(Failure::conflict("conflict.deadlock", "deadlock found"));
                }
                Ok(attempt)
            }
        })
        .await
        .expect("cleared on the third attempt");

        assert_eq!(value, 3);
        assert_eq!(attempts.get(), 3);
    }

    #[tokio::test]
    async fn the_last_failure_survives_when_the_attempts_run_out() {
        let attempts = Cell::new(0);
        let failure = retry(&policy(), &CancellationToken::new(), |_| {
            attempts.set(attempts.get() + 1);
            async {
                Err::<u8, Failure>(Failure::network(
                    "network.unreachable",
                    "the server could not be reached",
                ))
            }
        })
        .await
        .expect_err("must fail");

        assert_eq!(attempts.get(), 4, "the first attempt plus three repeats");
        assert_eq!(failure.code(), "network.unreachable");
    }

    #[tokio::test]
    async fn a_policy_that_never_repeats_asks_exactly_once() {
        let attempts = Cell::new(0);
        let _ = retry(&Backoff::none(), &CancellationToken::new(), |_| {
            attempts.set(attempts.get() + 1);
            async { Err::<u8, Failure>(Failure::network("network.unreachable", "lost")) }
        })
        .await;
        assert_eq!(attempts.get(), 1);
    }

    #[tokio::test]
    async fn a_stopped_invocation_never_finishes_its_wait() {
        let token = CancellationToken::new();
        let slow = Backoff::new(
            5,
            Duration::from_secs(60),
            Duration::from_secs(60),
            Duration::from_secs(600),
        );
        token.cancel();

        let failure = retry(&slow, &token, |_| async {
            Err::<u8, Failure>(Failure::network("network.unreachable", "lost"))
        })
        .await
        .expect_err("must stop");

        assert_eq!(failure.category(), crate::error::Category::Interrupted);
    }
}