upcloud_api/

net.rs

//! **The transport boundary: what is retried and what never is.**
//!
//! Moved here from `private-gunnar-ops/xtask/src/remote/net.rs` (lane T14,
//! 2026-09-21) so the ONE wire implementation of [`crate::UpCloudApi`] carries
//! the same policy every client in that crate already had — and so that crate's
//! other HTTP clients (Cloudflare, the health probe) re-use this copy instead of
//! keeping a second one.
//!
//! MEASURED 2026-09-20, the live bring-up: `GET /1.3/price` on the account
//! answered "error sending request for url" ONCE, and a credential probe turned
//! that one dropped connection into "provider unavailable". The token was fine.
//! The link hiccuped.
//!
//! # The boundary, and it is not negotiable in the permissive direction
//!
//! * **TRANSPORT** — the question never reached the far end, or its answer never
//!   came back whole: a refused connection, a reset, a timeout, a name that did
//!   not resolve, a TLS handshake that fell over, an HTTP/1.1 framing error
//!   mid-flight. Nobody composed that; it is the wire. RETRYABLE, with a bounded
//!   budget and a backoff, and **every retry is printed** so a flaky link can
//!   never be mistaken for a healthy one.
//! * **PROTOCOL** — the far end understood the question and composed an answer:
//!   a status code, a refusal body, a URI this crate built wrong. That is DATA.
//!   It is never retried and never softened. A 401 is a revoked credential, a
//!   403 is a scope, a 412 is UpCloud saying out_of_stock, a 404 is an absence —
//!   and retrying any of them turns a clear refusal into a hang.
//!
//! Every client in this crate is built with `http_status_as_error(false)`, so a
//! status code never even arrives here as an `Err`: it comes back as an ordinary
//! response and is judged by the code that asked. That is not an accident, it is
//! the boundary made structural — the only thing this module can see is the wire.

use std::time::Duration;

/// Which side of the boundary an error fell on.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum Side {
    /// The wire. Bounded retry.
    Transport,
    /// A composed answer, or our own mistake. Final.
    Protocol,
}

/// Classify a `ureq` error. The match is on the VARIANT, not on the text of the
/// message, because a message is a sentence someone can reword and a variant is
/// a decision the library made.
pub fn side(e: &ureq::Error) -> Side {
    match e {
        // The wire.
        ureq::Error::Io(_)
        | ureq::Error::Timeout(_)
        | ureq::Error::HostNotFound
        | ureq::Error::ConnectionFailed
        | ureq::Error::Tls(_)
        | ureq::Error::Protocol(_) => Side::Transport,
        // An answer, or a request this crate built wrong. Both are final: the
        // first because it is data, the second because trying it again will
        // build the same broken request.
        _ => Side::Protocol,
    }
}

/// How long the wire is given to stop hiccuping.
#[derive(Debug, Clone, Copy)]
pub struct Budget {
    /// Total attempts, the first one included. 1 = no retry at all.
    pub attempts: u32,
    /// The wait after the first failure; doubled each time, capped at `cap`.
    pub first: Duration,
    pub cap: Duration,
    /// Retry only failures that prove the request NEVER LEFT THIS BOX.
    ///
    /// For a GET or a DELETE this is off: they are idempotent, and asking twice
    /// costs nothing. For a POST that CREATES something it is on, because a
    /// connection reset after the bytes went out is ambiguous — the server may
    /// have made the thing and lost the answer — and retrying that is how one
    /// order becomes two servers on the invoice. A refused connection and a
    /// name that did not resolve are not ambiguous: nothing was sent.
    pub unsent_only: bool,
}

impl Budget {
    /// What an API read gets: four attempts over ~7 s of waiting. Long enough
    /// for a dropped connection and a re-dial, short enough that a cloud that
    /// is genuinely down is named within the minute rather than leaned on.
    pub const fn api() -> Budget {
        Budget { attempts: 4, first: Duration::from_secs(1), cap: Duration::from_secs(4), unsent_only: false }
    }

    /// What a WRITE gets: the same budget, but only for failures that prove the
    /// request never left this box. See `unsent_only`.
    pub const fn write() -> Budget {
        Budget { unsent_only: true, ..Budget::api() }
    }

    /// One attempt: the boundary is still declared, and nothing is retried.
    #[allow(dead_code)]
    pub const fn once() -> Budget {
        Budget { attempts: 1, first: Duration::from_secs(0), cap: Duration::from_secs(0), unsent_only: false }
    }
}

/// Did this failure happen BEFORE a single byte of the request went out?
///
/// A connection that was refused, a name that did not resolve, a connector that
/// gave up: nothing reached the far end, so asking again cannot duplicate
/// anything. A reset or a timeout mid-flight is NOT in this set — the request
/// may well have arrived and been acted on.
pub fn never_sent(e: &ureq::Error) -> bool {
    match e {
        ureq::Error::HostNotFound | ureq::Error::ConnectionFailed => true,
        ureq::Error::Io(io) => matches!(
            io.kind(),
            std::io::ErrorKind::ConnectionRefused
                | std::io::ErrorKind::NetworkUnreachable
                | std::io::ErrorKind::HostUnreachable
                | std::io::ErrorKind::AddrNotAvailable
        ),
        _ => false,
    }
}

/// Run `op`, retrying ONLY what `side` calls `Transport`, saying every retry out
/// loud. `what` names the call in the line that is printed and in the error.
pub fn call<T>(
    what: &str,
    budget: Budget,
    op: impl FnMut() -> Result<T, ureq::Error>,
) -> Result<T, ureq::Error> {
    call_with(what, budget, &mut |l| eprintln!("{l}"), &mut std::thread::sleep, op)
}

/// `call`, with the clock and the voice injected so the budget can be tested
/// without a test that sleeps.
pub fn call_with<T>(
    what: &str,
    budget: Budget,
    say: &mut dyn FnMut(&str),
    sleep: &mut dyn FnMut(Duration),
    mut op: impl FnMut() -> Result<T, ureq::Error>,
) -> Result<T, ureq::Error> {
    let mut wait = budget.first;
    for attempt in 1..=budget.attempts.max(1) {
        let e = match op() {
            Ok(v) => {
                if attempt > 1 {
                    say(&format!("   ok     {what}: answered on attempt {attempt} of {}", budget.attempts));
                }
                return Ok(v);
            }
            Err(e) => e,
        };
        if side(&e) == Side::Protocol {
            // A composed answer. It is the point of the call, not an obstacle.
            return Err(e);
        }
        if budget.unsent_only && !never_sent(&e) {
            say(&format!(
                "   RED    {what}: the wire failed after the request went out ({e}) — NOT retried, \
                 because a write that may have landed must not be sent twice"
            ));
            return Err(e);
        }
        if attempt == budget.attempts.max(1) {
            say(&format!(
                "   RED    {what}: the wire failed all {} attempts — {e}",
                budget.attempts.max(1)
            ));
            return Err(e);
        }
        say(&format!(
            "   …      {what}: transport failure on attempt {attempt} of {} ({e}) — retrying in {} s",
            budget.attempts,
            wait.as_secs_f32()
        ));
        sleep(wait);
        wait = (wait * 2).min(budget.cap);
    }
    unreachable!("the loop returns on the last attempt")
}

#[cfg(test)]
mod tests {
    use super::*;
    use std::cell::RefCell;

    fn io(kind: std::io::ErrorKind) -> ureq::Error {
        ureq::Error::Io(std::io::Error::from(kind))
    }

    #[test]
    fn the_wire_is_transport_and_an_answer_is_protocol() {
        assert_eq!(side(&io(std::io::ErrorKind::ConnectionRefused)), Side::Transport);
        assert_eq!(side(&io(std::io::ErrorKind::ConnectionReset)), Side::Transport);
        assert_eq!(side(&ureq::Error::HostNotFound), Side::Transport);
        assert_eq!(side(&ureq::Error::ConnectionFailed), Side::Transport);
        assert_eq!(side(&ureq::Error::Tls("handshake")), Side::Transport);
        // Composed by the far end, or by us. Never retried.
        assert_eq!(side(&ureq::Error::StatusCode(401)), Side::Protocol);
        assert_eq!(side(&ureq::Error::StatusCode(403)), Side::Protocol);
        assert_eq!(side(&ureq::Error::StatusCode(412)), Side::Protocol);
        assert_eq!(side(&ureq::Error::BadUri("nonsense".into())), Side::Protocol);
    }

    /// TODAY'S BUG: one dropped connection, and the second attempt would have
    /// answered. Without the retry this call is a permanent refusal.
    #[test]
    fn one_hiccup_then_an_answer_is_an_answer_and_it_is_said_out_loud() {
        let n = RefCell::new(0);
        let said = RefCell::new(Vec::new());
        let mut slept = Vec::new();
        let got = call_with(
            "GET /1.3/price",
            Budget::api(),
            &mut |l| said.borrow_mut().push(l.to_string()),
            &mut |d| slept.push(d),
            || {
                *n.borrow_mut() += 1;
                if *n.borrow() == 1 { Err(io(std::io::ErrorKind::ConnectionReset)) } else { Ok(200u16) }
            },
        )
        .expect("the second attempt answered");
        assert_eq!(got, 200);
        assert_eq!(*n.borrow(), 2);
        assert_eq!(slept, vec![Duration::from_secs(1)], "one backoff, and it is the first");
        let said = said.borrow();
        assert!(said.iter().any(|l| l.contains("transport failure on attempt 1")), "{said:?}");
        assert!(said.iter().any(|l| l.contains("answered on attempt 2")), "a retry is never silent: {said:?}");
    }

    /// The other direction: a real answer is NOT retried, ever. A revoked
    /// credential must fail on the first call, not after four.
    #[test]
    fn a_composed_refusal_is_tried_exactly_once() {
        for code in [401u16, 403, 404, 412] {
            let n = RefCell::new(0);
            let err = call_with(
                "POST /1.3/server",
                Budget::api(),
                &mut |_| {},
                &mut |_| panic!("a protocol answer must never sleep"),
                || {
                    *n.borrow_mut() += 1;
                    Err::<(), _>(ureq::Error::StatusCode(code))
                },
            )
            .unwrap_err();
            assert!(matches!(err, ureq::Error::StatusCode(c) if c == code));
            assert_eq!(*n.borrow(), 1, "{code} was retried");
        }
    }

    #[test]
    fn a_wire_that_never_comes_back_is_named_red_after_the_whole_budget() {
        let n = RefCell::new(0);
        let said = RefCell::new(Vec::new());
        let mut slept = Vec::new();
        let err = call_with(
            "GET /zones",
            Budget::api(),
            &mut |l| said.borrow_mut().push(l.to_string()),
            &mut |d| slept.push(d),
            || {
                *n.borrow_mut() += 1;
                Err::<(), _>(ureq::Error::ConnectionFailed)
            },
        )
        .unwrap_err();
        assert!(matches!(err, ureq::Error::ConnectionFailed));
        assert_eq!(*n.borrow(), 4);
        // Doubling, capped: 1, 2, 4 — and no sleep after the last attempt.
        assert_eq!(slept, vec![Duration::from_secs(1), Duration::from_secs(2), Duration::from_secs(4)]);
        assert!(said.borrow().iter().any(|l| l.contains("failed all 4 attempts")), "{:?}", said.borrow());
    }

    /// A write is retried only when nothing can possibly have landed.
    #[test]
    fn a_write_is_never_sent_twice_after_the_bytes_went_out() {
        // Refused: nothing left the box, so ask again.
        let n = RefCell::new(0);
        let _ = call_with(
            "POST /1.3/server",
            Budget::write(),
            &mut |_| {},
            &mut |_| {},
            || {
                *n.borrow_mut() += 1;
                Err::<(), _>(io(std::io::ErrorKind::ConnectionRefused))
            },
        );
        assert_eq!(*n.borrow(), 4, "a refused connection created nothing");

        // Reset mid-flight: the server may have made the thing. Once, and once only.
        let n = RefCell::new(0);
        let _ = call_with(
            "POST /1.3/server",
            Budget::write(),
            &mut |_| {},
            &mut |_| panic!("an ambiguous write must not sleep and try again"),
            || {
                *n.borrow_mut() += 1;
                Err::<(), _>(io(std::io::ErrorKind::ConnectionReset))
            },
        );
        assert_eq!(*n.borrow(), 1);

        // …and a GET of the same shape is still retried, because it is idempotent.
        let n = RefCell::new(0);
        let _ = call_with(
            "GET /1.3/server",
            Budget::api(),
            &mut |_| {},
            &mut |_| {},
            || {
                *n.borrow_mut() += 1;
                Err::<(), _>(io(std::io::ErrorKind::ConnectionReset))
            },
        );
        assert_eq!(*n.borrow(), 4);
    }

    #[test]
    fn a_budget_of_one_retries_nothing() {
        let n = RefCell::new(0);
        let _ = call_with(
            "GET /account",
            Budget::once(),
            &mut |_| {},
            &mut |_| panic!("no sleep with no retry"),
            || {
                *n.borrow_mut() += 1;
                Err::<(), _>(ureq::Error::ConnectionFailed)
            },
        );
        assert_eq!(*n.borrow(), 1);
    }
}