fakecloud-core 0.52.0

//! Image pulls for the runtimes that launch containers: ECS (and Batch
//! through it) and Lambda `PackageType=Image` functions pull on every launch
//! with [`pull_image`]; EC2 instances and CodeBuild builds, whose images
//! stand in for an AMI or a curated build image, pull only when the image is
//! missing with [`ensure_image`].
//!
//! A bare `docker pull` always contacts the registry, even when the image is
//! already in the local cache, so a momentary registry failure fails the
//! launch. The common one is rate limiting: anonymous pulls from
//! `public.ecr.aws` are capped per source IP, and a burst of task launches --
//! or several processes sharing one NAT address -- gets
//! `429 Too Many Requests` back.
//!
//! A transient failure (throttling, a registry 5xx, a network timeout) is
//! retried with backoff, and falls back to the image already cached locally
//! instead of failing the launch. A refusal is final: an image that no longer
//! exists or a pull the registry denies fails at once with the registry's own
//! error, even when a stale copy is cached. Otherwise an image deleted from
//! ECR, or one a repository policy denies, would keep launching from the
//! local copy -- which neither Fargate nor Lambda, having no per-host image
//! cache, ever does.

use std::path::Path;
use std::time::Duration;

use tokio::process::Command;

/// Pull attempts made while the registry keeps failing transiently, before
/// the last error is returned.
const MAX_PULL_ATTEMPTS: u32 = 5;

/// Delay before the first retry; doubles on each further retry.
const BASE_RETRY_DELAY: Duration = Duration::from_secs(1);

/// Budget for one `pull`. Generous, since a multi-gigabyte image on a slow
/// link legitimately takes minutes, but finite: the CLI has no deadline of
/// its own, so a registry or CDN connection that stalls mid-download would
/// otherwise hold the launch (and a task in `PENDING`) forever. A pull that
/// outruns it is killed; see [`PULL_TOTAL_TIMEOUT`] for the retry.
const PULL_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15 * 60);

/// Budget for every attempt of one pull together, backoff included. Twice
/// [`PULL_ATTEMPT_TIMEOUT`], so a pull that stalled for a whole attempt gets
/// exactly one more: a stall is almost always one dead connection (one CDN
/// edge, one NAT mapping), a fresh `pull` opens new ones, and the engine
/// keeps the layers it already finished, so the retry rarely starts from
/// zero. A third full-length attempt would mostly just double the wait on a
/// registry that is really down.
const PULL_TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60);

/// Budget for `image inspect`, a local lookup that only a wedged daemon
/// makes slow; treated as "not cached" when it outruns it.
const INSPECT_TIMEOUT: Duration = Duration::from_secs(60);

/// Budget for the cache check after a pull timed out. A healthy daemon
/// answers `image inspect` in milliseconds, and if the timeout was the
/// daemon's own (wedged, not the registry stalling) another full
/// [`INSPECT_TIMEOUT`] would only add a minute to a launch that is failing.
const INSPECT_AFTER_STALL_TIMEOUT: Duration = Duration::from_secs(5);

/// Retry and timeout knobs of a pull; tests shrink them.
#[derive(Debug, Clone, Copy)]
struct PullTiming {
    retry_delay: Duration,
    attempt_timeout: Duration,
    total: Duration,
}

const PULL_TIMING: PullTiming = PullTiming {
    retry_delay: BASE_RETRY_DELAY,
    attempt_timeout: PULL_ATTEMPT_TIMEOUT,
    total: PULL_TOTAL_TIMEOUT,
};

/// How the registry a pulled reference names is reached.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum RegistryTransport {
    /// TLS, as every upstream registry (ECR Public, Docker Hub, ...) serves.
    Https,
    /// Plain HTTP: fakecloud's own OCI registry, which an AWS ECR URI is
    /// rewritten to. Docker treats a loopback registry as insecure on its
    /// own, but Podman insists on TLS for every registry unless told
    /// otherwise, so its pull fails with "server gave HTTP response to HTTPS
    /// client" without `--tls-verify=false`.
    PlainHttp,
}

impl RegistryTransport {
    /// The transport for a pull whose reference was (`true`) or was not
    /// rewritten to fakecloud's registry by
    /// [`crate::ecr_uri::translate_to_local_at`].
    pub fn for_local_rewrite(rewritten: bool) -> Self {
        if rewritten {
            Self::PlainHttp
        } else {
            Self::Https
        }
    }
}

/// How an image became available for a launch.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum PulledImage {
    /// The registry served the image.
    Pulled,
    /// The pull failed transiently but the image was already cached locally,
    /// so the cached copy is used. Carries the pull's error for logging.
    Cached { pull_error: String },
    /// [`ensure_image`] found the image already cached and did not pull.
    Present,
}

/// Pull `reference` with the container `cli`. A transient registry failure
/// falls back to a locally cached copy, or is retried with backoff when
/// nothing is cached; any other failure is returned at once. `docker_config`
/// is exported as `DOCKER_CONFIG` for the pull so registry credentials
/// resolve. `transport` says whether the registry is fakecloud's own
/// plain-HTTP one.
///
/// Returns the pull's stderr as the error.
pub async fn pull_image(
    cli: &str,
    docker_config: Option<&Path>,
    reference: &str,
    transport: RegistryTransport,
) -> Result<PulledImage, String> {
    pull_image_with(cli, docker_config, reference, transport, PULL_TIMING).await
}

/// Make `reference` available locally, pulling it only when it is not
/// already cached -- what `docker run` does with its implicit pull, but with
/// [`pull_image`]'s retry when the registry fails transiently. `docker run`
/// gives up on the first `429 Too Many Requests`, which on a host with an
/// empty cache fails every launch that races the first pull of an image.
pub async fn ensure_image(
    cli: &str,
    docker_config: Option<&Path>,
    reference: &str,
) -> Result<PulledImage, String> {
    ensure_image_with(cli, docker_config, reference, PULL_TIMING).await
}

/// [`ensure_image`] for a caller with a budget of its own: the cache check
/// and every pull attempt together end within `budget` (each attempt also
/// still within the usual per-attempt limit). Every CLI call is killed when it
/// outruns its share, so nothing is left running past the budget.
pub async fn ensure_image_within(
    cli: &str,
    docker_config: Option<&Path>,
    reference: &str,
    budget: Duration,
) -> Result<PulledImage, String> {
    let timing = PullTiming {
        attempt_timeout: PULL_ATTEMPT_TIMEOUT.min(budget),
        total: budget,
        ..PULL_TIMING
    };
    ensure_image_with(cli, docker_config, reference, timing).await
}

async fn ensure_image_with(
    cli: &str,
    docker_config: Option<&Path>,
    reference: &str,
    timing: PullTiming,
) -> Result<PulledImage, String> {
    let start = tokio::time::Instant::now();
    if image_cached(
        cli,
        docker_config,
        reference,
        INSPECT_TIMEOUT.min(timing.total),
    )
    .await
    {
        return Ok(PulledImage::Present);
    }
    let timing = PullTiming {
        total: timing.total.saturating_sub(start.elapsed()),
        ..timing
    };
    pull_image_with(
        cli,
        docker_config,
        reference,
        RegistryTransport::Https,
        timing,
    )
    .await
}

async fn pull_image_with(
    cli: &str,
    docker_config: Option<&Path>,
    reference: &str,
    transport: RegistryTransport,
    timing: PullTiming,
) -> Result<PulledImage, String> {
    // Only a plain-HTTP pull branches on the engine, so only it pays for
    // (the first, then cached) engine probe.
    let podman = transport == RegistryTransport::PlainHttp
        && crate::container_net::is_podman_async(cli).await;
    let deadline = tokio::time::Instant::now() + timing.total;
    let remaining = || deadline.saturating_duration_since(tokio::time::Instant::now());
    let mut delay = timing.retry_delay;
    let mut attempt = 1;
    let mut stalled_once = false;
    loop {
        let budget = timing.attempt_timeout.min(remaining());
        let mut cmd = Command::new(cli);
        if let Some(p) = docker_config {
            cmd.env("DOCKER_CONFIG", p);
        }
        cmd.args(pull_args(podman, reference, transport));
        let out = match crate::container_net::output_within(&mut cmd, budget).await {
            Ok(out) => out,
            Err(crate::container_net::CliRunError::TimedOut(_)) => {
                let pull_error = format!(
                    "{cli} pull {reference} did not finish within {}s and was killed",
                    budget.as_secs_f64()
                );
                if image_cached(cli, docker_config, reference, INSPECT_AFTER_STALL_TIMEOUT).await {
                    tracing::warn!(
                        image = %reference,
                        error = %pull_error,
                        "image pull timed out; using the locally cached image"
                    );
                    return Ok(PulledImage::Cached { pull_error });
                }
                // One more attempt, on fresh connections, if the overall
                // budget still has room for it.
                if stalled_once || remaining().is_zero() {
                    return Err(pull_error);
                }
                stalled_once = true;
                tracing::warn!(
                    image = %reference,
                    error = %pull_error,
                    "image pull timed out; retrying once"
                );
                continue;
            }
            Err(e) => return Err(format!("{cli} pull: {e}")),
        };
        if out.status.success() {
            return Ok(PulledImage::Pulled);
        }
        let pull_error = String::from_utf8_lossy(&out.stderr).trim().to_string();
        if !is_transient(&pull_error, reference) {
            return Err(pull_error);
        }
        let check = INSPECT_TIMEOUT
            .min(remaining())
            .max(INSPECT_AFTER_STALL_TIMEOUT);
        if image_cached(cli, docker_config, reference, check).await {
            tracing::warn!(
                image = %reference,
                error = %pull_error,
                "image pull failed transiently; using the locally cached image"
            );
            return Ok(PulledImage::Cached { pull_error });
        }
        if attempt >= MAX_PULL_ATTEMPTS || remaining() <= delay {
            return Err(pull_error);
        }
        tracing::info!(
            image = %reference,
            attempt,
            retry_in_ms = delay.as_millis() as u64,
            "image pull failed transiently; retrying"
        );
        tokio::time::sleep(delay).await;
        delay *= 2;
        attempt += 1;
    }
}

/// The `pull` arguments, `podman` saying whether the CLI drives Podman (by
/// engine, not name: the `podman-docker` shim is a `docker` that is Podman).
/// Only Podman needs telling that a plain-HTTP registry is one: Docker accepts
/// it for a loopback registry, and its `pull` has no `--tls-verify` flag at all.
fn pull_args(podman: bool, reference: &str, transport: RegistryTransport) -> Vec<String> {
    let mut args = vec!["pull".to_string()];
    if transport == RegistryTransport::PlainHttp && podman {
        args.push("--tls-verify=false".to_string());
    }
    args.push(reference.to_string());
    args
}

/// Whether `reference` resolves to an image in the local cache. Runs with
/// the same `DOCKER_CONFIG` as the pull: the config also selects the Docker
/// context, so without it the check could consult a different daemon than
/// the one that pulls and later runs the image.
async fn image_cached(
    cli: &str,
    docker_config: Option<&Path>,
    reference: &str,
    timeout: Duration,
) -> bool {
    let mut cmd = Command::new(cli);
    if let Some(p) = docker_config {
        cmd.env("DOCKER_CONFIG", p);
    }
    cmd.args(["image", "inspect", reference]);
    crate::container_net::output_within(&mut cmd, timeout)
        .await
        .is_ok_and(|out| out.status.success())
}

/// Whether a pull error is one a later attempt could succeed past: the
/// registry throttling the caller (the `429 Too Many Requests` status line,
/// the `toomanyrequests` error code, ECR Public's `Rate exceeded`), any 5xx
/// from the registry, or the network timing out or dropping the connection.
///
/// A refusal (not found, access denied, unauthorized) is never transient, and
/// wins over transient wording in the same message. Both are matched only
/// after the image's own name is removed from the message: the name is chosen
/// by the user and quoted in the error, so a repository spelled like either
/// kind of marker (`toomanyrequests/app`, `acme/access-denied-page`) must not
/// flip the classification.
fn is_transient(stderr: &str, reference: &str) -> bool {
    const REFUSED: [&str; 6] = [
        "manifest unknown",
        "not found",
        "denied",
        "unauthorized",
        "forbidden",
        "does not exist",
    ];
    const TRANSIENT: [&str; 8] = [
        "toomanyrequests",
        "too many requests",
        "rate exceeded",
        "i/o timeout",
        "tls handshake timeout",
        "connection reset by peer",
        "context deadline exceeded",
        "request canceled while waiting for connection",
    ];
    let message = without_image_name(&stderr.to_ascii_lowercase(), reference);
    if REFUSED.iter().any(|m| message.contains(m)) {
        return false;
    }
    TRANSIENT.iter().any(|m| message.contains(m)) || has_server_error_status(&message)
}

/// `message` (lowercase) with each way an error can quote the image blanked
/// out: the reference as given, every trailing path of its repository
/// (`public.ecr.aws/acme/app`, `acme/app`, `app`), and its registry host --
/// registries put the host and repository path in URLs, and Podman expands short names to
/// `docker.io/library/<name>`. Only whole names are removed, bounded by
/// characters a name cannot contain, so a short repository like `d` never
/// cuts letters out of the surrounding words.
fn without_image_name(message: &str, reference: &str) -> String {
    let reference = reference.to_ascii_lowercase();
    let untagged = reference.split('@').next().unwrap_or(&reference);
    // A `:` after the last `/` starts the tag; one before it is a registry port.
    let repository = match (untagged.rfind(':'), untagged.rfind('/')) {
        (Some(colon), Some(slash)) if colon < slash => untagged,
        (Some(colon), _) => &untagged[..colon],
        (None, _) => untagged,
    };
    let mut names = vec![reference.as_str(), repository];
    names.extend(
        repository
            .match_indices('/')
            .map(|(i, _)| &repository[i + 1..]),
    );
    // The registry host, which URLs in the error quote on its own.
    if let Some((host, _)) = repository.split_once('/') {
        if host.contains(['.', ':']) || host == "localhost" {
            names.push(host);
        }
    }
    names.sort_by_key(|n| std::cmp::Reverse(n.len()));

    let mut out = message.to_string();
    for name in names.into_iter().filter(|n| !n.is_empty()) {
        out = remove_whole(&out, name);
    }
    out
}

/// `haystack` with every occurrence of `name` that is not part of a longer
/// name (flanked by a letter, digit, `.`, `_` or `-`) replaced by a space.
fn remove_whole(haystack: &str, name: &str) -> String {
    let is_name_char = |c: char| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-');
    let mut out = String::with_capacity(haystack.len());
    let mut rest = haystack;
    while let Some(i) = rest.find(name) {
        let end = i + name.len();
        let before = rest[..i].chars().next_back();
        let after = rest[end..].chars().next();
        if before.is_some_and(is_name_char) || after.is_some_and(is_name_char) {
            out.push_str(&rest[..end]);
        } else {
            out.push_str(&rest[..i]);
            out.push(' ');
        }
        rest = &rest[end..];
    }
    out.push_str(rest);
    out
}

/// Whether the message carries a 5xx HTTP status. Docker and Podman quote the
/// status as `: 503 Service Unavailable`, `status: 500`, or
/// `status code 502`; a three-digit number in that position from 500 to 599
/// counts, whatever reason phrase follows.
fn has_server_error_status(message: &str) -> bool {
    let bytes = message.as_bytes();
    ["status code ", "status: ", "status ", ": "]
        .iter()
        .flat_map(|prefix| message.match_indices(prefix).map(|(i, p)| i + p.len()))
        .any(|start| {
            let code = &bytes[start..bytes.len().min(start + 3)];
            code.len() == 3
                && code[0] == b'5'
                && code.iter().all(u8::is_ascii_digit)
                && !bytes
                    .get(start + 3)
                    .is_some_and(|c| c.is_ascii_alphanumeric())
        })
}

#[cfg(all(test, unix))]
mod tests {
    use super::*;

    /// A stand-in container CLI. `pull` fails with `pull_stderr` for the first
    /// `pull_failures` calls and succeeds after; `image inspect` succeeds only
    /// when `cached`. Every invocation is appended to `calls.log`.
    struct FakeCli {
        dir: tempfile::TempDir,
        name: String,
    }

    impl FakeCli {
        fn new(pull_failures: u32, pull_stderr: &str, cached: bool) -> Self {
            Self::named("cli", pull_failures, pull_stderr, cached)
        }

        /// A fake installed under `name`, which is what decides whether
        /// fakecloud drives it as Podman.
        fn named(name: &str, pull_failures: u32, pull_stderr: &str, cached: bool) -> Self {
            let version = if name.contains("podman") {
                "podman version 5.2.0"
            } else {
                "Docker version 27.3.1, build ce12230"
            };
            Self::with_version(name, version, pull_failures, pull_stderr, cached)
        }

        /// A fake whose `--version` prints `version` -- the engine probe's
        /// answer. That call is not logged, so `calls()` shows only the work.
        fn with_version(
            name: &str,
            version: &str,
            pull_failures: u32,
            pull_stderr: &str,
            cached: bool,
        ) -> Self {
            let dir = tempfile::tempdir().unwrap();
            let script = format!(
                r#"#!/bin/sh
d="{dir}"
if [ "$1" = --version ]; then
  echo '{version}'
  exit 0
fi
echo "$*" >> "$d/calls.log"
case "$1" in
  pull)
    n=$(cat "$d/pulls" 2>/dev/null || echo 0)
    n=$((n + 1))
    echo "$n" > "$d/pulls"
    if [ "$n" -le {pull_failures} ]; then
      echo '{pull_stderr}' >&2
      exit 1
    fi
    exit 0 ;;
  image)
    [ "{cached}" = "true" ] && exit 0
    echo 'Error: No such image' >&2
    exit 1 ;;
esac
exit 2
"#,
                dir = dir.path().display(),
            );
            let path = dir.path().join(name);
            crate::test_support::write_executable(&path, &script);
            let _ = std::fs::remove_file(dir.path().join("calls.log"));
            Self {
                dir,
                name: name.to_string(),
            }
        }

        fn cli(&self) -> String {
            self.dir.path().join(&self.name).display().to_string()
        }

        fn calls(&self) -> Vec<String> {
            std::fs::read_to_string(self.dir.path().join("calls.log"))
                .unwrap_or_default()
                .lines()
                .map(String::from)
                .collect()
        }

        async fn pull(&self) -> Result<PulledImage, String> {
            self.pull_ref("alpine:3.20").await
        }

        async fn pull_ref(&self, reference: &str) -> Result<PulledImage, String> {
            self.pull_via(reference, RegistryTransport::Https).await
        }

        async fn pull_via(
            &self,
            reference: &str,
            transport: RegistryTransport,
        ) -> Result<PulledImage, String> {
            pull_image_with(&self.cli(), None, reference, transport, TEST_TIMING).await
        }

        async fn ensure(&self) -> Result<PulledImage, String> {
            ensure_image_with(&self.cli(), None, "alpine:3.20", TEST_TIMING).await
        }
    }

    const TEST_TIMING: PullTiming = PullTiming {
        retry_delay: Duration::from_millis(1),
        attempt_timeout: Duration::from_secs(30),
        total: Duration::from_secs(120),
    };

    fn is_transient_for_test(stderr: &str) -> bool {
        is_transient(stderr, "alpine:3.20")
    }

    const THROTTLED: &str = "Error response from daemon: unexpected status from HEAD request to https://public.ecr.aws/v2/docker/library/alpine/manifests/3.20: 429 Too Many Requests";

    /// A fake CLI whose `pull` never finishes and whose `image inspect`
    /// succeeds only when `cached`; calls go to `calls.log`.
    fn stalling_cli(cached: bool) -> (tempfile::TempDir, String) {
        crate::test_support::fake_cli(
            "cli",
            &format!(
                "d=$(dirname \"$0\")\n[ \"$1\" = warmup ] && exit 0\necho \"$*\" >> \"$d/calls.log\"\n\
                 case \"$1\" in\n  pull) exec sleep 1000 ;;\n  image) [ {cached} = true ] ;;\nesac\n"
            ),
        )
    }

    fn calls_in(dir: &tempfile::TempDir) -> Vec<String> {
        std::fs::read_to_string(dir.path().join("calls.log"))
            .unwrap_or_default()
            .lines()
            .map(String::from)
            .collect()
    }

    /// A pull whose download stalls is killed at its budget instead of holding
    /// the launch forever, and falls back to a cached copy when there is one.
    #[tokio::test]
    async fn a_stalled_pull_falls_back_to_the_cache() {
        let (dir, cli) = stalling_cli(true);
        let timing = PullTiming {
            retry_delay: Duration::from_millis(1),
            attempt_timeout: Duration::from_millis(300),
            total: Duration::from_secs(20),
        };
        let got =
            pull_image_with(&cli, None, "alpine:3.20", RegistryTransport::Https, timing).await;
        match got {
            Ok(PulledImage::Cached { pull_error }) => {
                assert!(
                    pull_error.contains("did not finish within 0.3s"),
                    "{pull_error}"
                )
            }
            other => panic!("unexpected {other:?}"),
        }
        assert_eq!(
            calls_in(&dir),
            ["pull alpine:3.20", "image inspect alpine:3.20"]
        );
    }

    /// With nothing cached, a stalled pull gets exactly one more attempt on a
    /// fresh connection, and then fails.
    #[tokio::test]
    async fn a_stalled_pull_is_retried_once() {
        let (dir, cli) = stalling_cli(false);
        let timing = PullTiming {
            retry_delay: Duration::from_millis(1),
            attempt_timeout: Duration::from_millis(300),
            total: Duration::from_secs(20),
        };
        let start = std::time::Instant::now();
        let err = pull_image_with(&cli, None, "alpine:3.20", RegistryTransport::Https, timing)
            .await
            .expect_err("nothing cached");
        assert!(start.elapsed() < Duration::from_secs(10));
        assert!(err.contains("did not finish within 0.3s"), "{err}");
        assert_eq!(
            calls_in(&dir),
            [
                "pull alpine:3.20",
                "image inspect alpine:3.20",
                "pull alpine:3.20",
                "image inspect alpine:3.20"
            ]
        );
    }

    /// The overall budget caps the retry: it only gets what is left.
    #[tokio::test]
    async fn the_retry_after_a_stall_only_gets_the_remaining_budget() {
        let (dir, cli) = stalling_cli(false);
        let start = std::time::Instant::now();
        let err = ensure_image_within(&cli, None, "alpine:3.20", Duration::from_millis(600))
            .await
            .expect_err("nothing cached");
        assert!(start.elapsed() < Duration::from_secs(10));
        assert!(err.contains("did not finish within"), "{err}");
        // inspect, a 0.6s pull, then the budget is spent.
        assert_eq!(
            calls_in(&dir),
            [
                "image inspect alpine:3.20",
                "pull alpine:3.20",
                "image inspect alpine:3.20"
            ]
        );
    }

    #[tokio::test]
    async fn ensure_image_uses_a_cached_image_without_contacting_the_registry() {
        let cli = FakeCli::new(u32::MAX, THROTTLED, true);
        assert_eq!(cli.ensure().await, Ok(PulledImage::Present));
        assert_eq!(cli.calls(), ["image inspect alpine:3.20"]);
    }

    #[tokio::test]
    async fn ensure_image_retries_a_throttled_first_pull() {
        let cli = FakeCli::new(2, THROTTLED, false);
        assert_eq!(cli.ensure().await, Ok(PulledImage::Pulled));
        let pulls = cli.calls().iter().filter(|c| c.starts_with("pull")).count();
        assert_eq!(pulls, 3);
    }

    #[tokio::test]
    async fn ensure_image_fails_on_a_refused_pull() {
        let missing =
            "Error response from daemon: manifest for alpine:3.20 not found: manifest unknown";
        let cli = FakeCli::new(u32::MAX, missing, false);
        assert_eq!(cli.ensure().await, Err(missing.to_string()));
    }

    #[tokio::test]
    async fn podman_pulls_fakecloud_registry_over_plain_http() {
        // Podman defaults to TLS even for a loopback registry, and fakecloud's
        // serves plain HTTP (issue #2585).
        let cli = FakeCli::named("podman", 0, "", false);
        let got = cli
            .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp)
            .await;
        assert_eq!(got, Ok(PulledImage::Pulled));
        assert_eq!(
            cli.calls(),
            ["pull --tls-verify=false 127.0.0.1:4566/test:healthy"]
        );
    }

    #[tokio::test]
    async fn podman_keeps_tls_for_upstream_registries() {
        let cli = FakeCli::named("podman", 0, "", false);
        let got = cli
            .pull_via(
                "public.ecr.aws/docker/library/alpine:3.20",
                RegistryTransport::Https,
            )
            .await;
        assert_eq!(got, Ok(PulledImage::Pulled));
        assert_eq!(
            cli.calls(),
            ["pull public.ecr.aws/docker/library/alpine:3.20"]
        );
    }

    #[tokio::test]
    async fn podman_docker_shim_pulls_fakecloud_registry_over_plain_http() {
        // The podman-docker package installs a `docker` that execs podman
        // (issue #2599): it is Podman, and needs the flag, whatever its name.
        let cli = FakeCli::with_version("docker", "podman version 5.2.0", 0, "", false);
        let got = cli
            .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp)
            .await;
        assert_eq!(got, Ok(PulledImage::Pulled));
        assert_eq!(
            cli.calls(),
            ["pull --tls-verify=false 127.0.0.1:4566/test:healthy"]
        );
    }

    #[tokio::test]
    async fn docker_pulls_fakecloud_registry_without_a_tls_flag() {
        // `docker pull` has no --tls-verify; the daemon already treats a
        // loopback registry as insecure.
        let cli = FakeCli::named("docker", 0, "", false);
        let got = cli
            .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp)
            .await;
        assert_eq!(got, Ok(PulledImage::Pulled));
        assert_eq!(cli.calls(), ["pull 127.0.0.1:4566/test:healthy"]);
    }

    #[test]
    fn transport_follows_the_local_rewrite() {
        assert_eq!(
            RegistryTransport::for_local_rewrite(true),
            RegistryTransport::PlainHttp
        );
        assert_eq!(
            RegistryTransport::for_local_rewrite(false),
            RegistryTransport::Https
        );
    }

    #[tokio::test]
    async fn a_successful_pull_needs_no_cache_check() {
        let cli = FakeCli::new(0, "", false);
        assert_eq!(cli.pull().await, Ok(PulledImage::Pulled));
        assert_eq!(cli.calls(), ["pull alpine:3.20"]);
    }

    #[tokio::test]
    async fn a_throttled_pull_uses_the_cached_image() {
        let cli = FakeCli::new(u32::MAX, THROTTLED, true);
        let got = cli.pull().await;
        assert_eq!(
            got,
            Ok(PulledImage::Cached {
                pull_error: THROTTLED.to_string()
            })
        );
        assert_eq!(
            cli.calls(),
            ["pull alpine:3.20", "image inspect alpine:3.20"],
            "a cached image is used at once, without retrying the pull"
        );
    }

    #[tokio::test]
    async fn a_rate_limited_pull_with_nothing_cached_is_retried() {
        let cli = FakeCli::new(2, THROTTLED, false);
        assert_eq!(cli.pull().await, Ok(PulledImage::Pulled));
        let pulls = cli.calls().iter().filter(|c| c.starts_with("pull")).count();
        assert_eq!(pulls, 3);
    }

    #[tokio::test]
    async fn retries_stop_after_the_attempt_cap() {
        let cli = FakeCli::new(u32::MAX, THROTTLED, false);
        assert_eq!(cli.pull().await, Err(THROTTLED.to_string()));
        let pulls = cli.calls().iter().filter(|c| c.starts_with("pull")).count();
        assert_eq!(pulls, MAX_PULL_ATTEMPTS as usize);
    }

    #[tokio::test]
    async fn a_missing_image_fails_without_retrying() {
        let missing =
            "Error response from daemon: manifest for alpine:nope not found: manifest unknown";
        let cli = FakeCli::new(u32::MAX, missing, false);
        assert_eq!(cli.pull().await, Err(missing.to_string()));
        assert_eq!(
            cli.calls(),
            ["pull alpine:3.20"],
            "a refused pull is neither retried nor checked against the cache"
        );
    }

    #[tokio::test]
    async fn a_refused_pull_fails_even_with_a_stale_cached_copy() {
        // The image was deleted from the registry, or a policy now denies the
        // pull. A copy cached by an earlier launch must not be used.
        for refused in [
            "Error response from daemon: manifest for alpine:3.20 not found: manifest unknown",
            "Error response from daemon: pull access denied for alpine, repository does not exist or may require authorization: denied",
        ] {
            let cli = FakeCli::new(u32::MAX, refused, true);
            assert_eq!(cli.pull().await, Err(refused.to_string()));
            assert_eq!(cli.calls(), ["pull alpine:3.20"]);
        }
    }

    #[tokio::test]
    async fn a_registry_server_error_uses_the_cached_image() {
        let unavailable =
            "Error response from daemon: received unexpected HTTP status: 503 Service Unavailable";
        let cli = FakeCli::new(u32::MAX, unavailable, true);
        assert_eq!(
            cli.pull().await,
            Ok(PulledImage::Cached {
                pull_error: unavailable.to_string()
            })
        );
    }

    #[tokio::test]
    async fn a_refused_pull_of_a_repository_named_like_a_marker_is_still_refused() {
        // The message quotes the image name. A repository spelled like a
        // throttling code must not make a refused pull look transient.
        let refused = "Error response from daemon: manifest for toomanyrequests:latest not found: manifest unknown: manifest unknown";
        let cli = FakeCli::new(u32::MAX, refused, true);
        assert_eq!(
            cli.pull_ref("toomanyrequests:latest").await,
            Err(refused.to_string())
        );
        assert_eq!(cli.calls(), ["pull toomanyrequests:latest"]);
    }

    #[tokio::test]
    async fn a_throttled_pull_of_a_repository_named_like_a_refusal_uses_the_cache() {
        // Only removing the image name from the message keeps `denied` in the
        // repository from reading as a refusal; without it this pull would
        // fail instead of falling back to the cached copy.
        let reference = "public.ecr.aws/acme/access-denied-page:1";
        let throttled = "Error response from daemon: unexpected status from HEAD request to https://public.ecr.aws/v2/acme/access-denied-page/manifests/1: 429 Too Many Requests";
        let cli = FakeCli::new(u32::MAX, throttled, true);
        assert_eq!(
            cli.pull_ref(reference).await,
            Ok(PulledImage::Cached {
                pull_error: throttled.to_string()
            })
        );
        assert_eq!(
            cli.calls(),
            [
                format!("pull {reference}"),
                format!("image inspect {reference}")
            ]
        );
    }

    #[test]
    fn transient_detection_separates_retryable_from_refused() {
        assert!(is_transient_for_test(THROTTLED));
        assert!(is_transient_for_test(
            "toomanyrequests: You have reached your pull rate limit."
        ));
        assert!(is_transient_for_test("Error: Rate exceeded"));
        assert!(is_transient_for_test(
            "received unexpected HTTP status: 502 Bad Gateway"
        ));
        assert!(is_transient_for_test(
            "Get \"https://public.ecr.aws/v2/\": net/http: TLS handshake timeout"
        ));
        assert!(is_transient_for_test(
            "read tcp 10.0.0.2:4431->1.2.3.4:443: read: connection reset by peer"
        ));
        assert!(!is_transient_for_test("manifest unknown"));
        assert!(!is_transient_for_test("pull access denied for foo"));
        assert!(!is_transient_for_test(
            "unauthorized: authentication required"
        ));
        assert!(!is_transient_for_test(
            "pull access denied for toomanyrequests, repository does not exist or may require authorization"
        ));
    }

    #[test]
    fn any_5xx_status_is_transient_whatever_its_reason_phrase() {
        for msg in [
            "received unexpected HTTP status: 500 Internal Server Error",
            "unexpected status from GET request to https://r.example/v2/: 507 Insufficient Storage",
            "unexpected status code 520",
            "error pulling image: status: 599",
            "unexpected status from HEAD request to https://r.example/v2/a/manifests/1: 503",
        ] {
            assert!(is_transient_for_test(msg), "{msg}");
        }
        for msg in [
            // A registry port or a 4xx is not a server error.
            "Get \"http://127.0.0.1:5000/v2/\": dial tcp 127.0.0.1:5000: connect: connection refused",
            "unexpected status code 400 Bad Request",
            "status: 5001",
        ] {
            assert!(!is_transient_for_test(msg), "{msg}");
        }
    }

    #[test]
    fn a_throttled_pull_of_a_repository_named_like_a_refusal_is_still_transient() {
        let reference = "public.ecr.aws/acme/access-denied-page:1";
        for msg in [
            "Error response from daemon: unexpected status from HEAD request to https://public.ecr.aws/v2/acme/access-denied-page/manifests/1: 429 Too Many Requests",
            "Error response from daemon: toomanyrequests: Rate exceeded for public.ecr.aws/acme/access-denied-page:1",
        ] {
            assert!(is_transient(msg, reference), "{msg}");
        }
        // Its genuine refusals are still refusals.
        assert!(!is_transient(
            "Error response from daemon: manifest for public.ecr.aws/acme/access-denied-page:1 not found: manifest unknown",
            reference
        ));
    }

    #[test]
    fn a_registry_host_named_like_a_refusal_does_not_hide_a_throttle() {
        assert!(is_transient(
            "Error response from daemon: unexpected status from HEAD request to https://denied.example/v2/app/manifests/1: 429 Too Many Requests",
            "denied.example/app:1"
        ));
    }

    #[test]
    fn only_whole_names_are_removed() {
        // A one-letter repository must not cut the `d` out of `denied`.
        assert!(!is_transient(
            "Error response from daemon: pull access denied for d, repository does not exist",
            "d"
        ));
        assert_eq!(
            without_image_name(
                "pull access denied for docker.io/library/alpine",
                "alpine:3.20"
            ),
            "pull access denied for docker.io/library/ "
        );
        assert_eq!(
            without_image_name(
                "get https://127.0.0.1:5000/v2/team/app/manifests/v1",
                "127.0.0.1:5000/team/app:v1"
            ),
            "get https:// /v2/ /manifests/v1"
        );
    }
}