fakecloud_core/
container_image.rs

1//! Image pulls for the runtimes that launch containers: ECS (and Batch
2//! through it) and Lambda `PackageType=Image` functions pull on every launch
3//! with [`pull_image`]; EC2 instances and CodeBuild builds, whose images
4//! stand in for an AMI or a curated build image, pull only when the image is
5//! missing with [`ensure_image`].
6//!
7//! A bare `docker pull` always contacts the registry, even when the image is
8//! already in the local cache, so a momentary registry failure fails the
9//! launch. The common one is rate limiting: anonymous pulls from
10//! `public.ecr.aws` are capped per source IP, and a burst of task launches --
11//! or several processes sharing one NAT address -- gets
12//! `429 Too Many Requests` back.
13//!
14//! A transient failure (throttling, a registry 5xx, a network timeout) is
15//! retried with backoff, and falls back to the image already cached locally
16//! instead of failing the launch. A refusal is final: an image that no longer
17//! exists or a pull the registry denies fails at once with the registry's own
18//! error, even when a stale copy is cached. Otherwise an image deleted from
19//! ECR, or one a repository policy denies, would keep launching from the
20//! local copy -- which neither Fargate nor Lambda, having no per-host image
21//! cache, ever does.
22
23use std::path::Path;
24use std::time::Duration;
25
26use tokio::process::Command;
27
28/// Pull attempts made while the registry keeps failing transiently, before
29/// the last error is returned.
30const MAX_PULL_ATTEMPTS: u32 = 5;
31
32/// Delay before the first retry; doubles on each further retry.
33const BASE_RETRY_DELAY: Duration = Duration::from_secs(1);
34
35/// Budget for one `pull`. Generous, since a multi-gigabyte image on a slow
36/// link legitimately takes minutes, but finite: the CLI has no deadline of
37/// its own, so a registry or CDN connection that stalls mid-download would
38/// otherwise hold the launch (and a task in `PENDING`) forever. A pull that
39/// outruns it is killed; see [`PULL_TOTAL_TIMEOUT`] for the retry.
40const PULL_ATTEMPT_TIMEOUT: Duration = Duration::from_secs(15 * 60);
41
42/// Budget for every attempt of one pull together, backoff included. Twice
43/// [`PULL_ATTEMPT_TIMEOUT`], so a pull that stalled for a whole attempt gets
44/// exactly one more: a stall is almost always one dead connection (one CDN
45/// edge, one NAT mapping), a fresh `pull` opens new ones, and the engine
46/// keeps the layers it already finished, so the retry rarely starts from
47/// zero. A third full-length attempt would mostly just double the wait on a
48/// registry that is really down.
49const PULL_TOTAL_TIMEOUT: Duration = Duration::from_secs(30 * 60);
50
51/// Budget for `image inspect`, a local lookup that only a wedged daemon
52/// makes slow; treated as "not cached" when it outruns it.
53const INSPECT_TIMEOUT: Duration = Duration::from_secs(60);
54
55/// Budget for the cache check after a pull timed out. A healthy daemon
56/// answers `image inspect` in milliseconds, and if the timeout was the
57/// daemon's own (wedged, not the registry stalling) another full
58/// [`INSPECT_TIMEOUT`] would only add a minute to a launch that is failing.
59const INSPECT_AFTER_STALL_TIMEOUT: Duration = Duration::from_secs(5);
60
61/// Retry and timeout knobs of a pull; tests shrink them.
62#[derive(Debug, Clone, Copy)]
63struct PullTiming {
64    retry_delay: Duration,
65    attempt_timeout: Duration,
66    total: Duration,
67}
68
69const PULL_TIMING: PullTiming = PullTiming {
70    retry_delay: BASE_RETRY_DELAY,
71    attempt_timeout: PULL_ATTEMPT_TIMEOUT,
72    total: PULL_TOTAL_TIMEOUT,
73};
74
75/// How the registry a pulled reference names is reached.
76#[derive(Debug, Clone, Copy, PartialEq, Eq)]
77pub enum RegistryTransport {
78    /// TLS, as every upstream registry (ECR Public, Docker Hub, ...) serves.
79    Https,
80    /// Plain HTTP: fakecloud's own OCI registry, which an AWS ECR URI is
81    /// rewritten to. Docker treats a loopback registry as insecure on its
82    /// own, but Podman insists on TLS for every registry unless told
83    /// otherwise, so its pull fails with "server gave HTTP response to HTTPS
84    /// client" without `--tls-verify=false`.
85    PlainHttp,
86}
87
88impl RegistryTransport {
89    /// The transport for a pull whose reference was (`true`) or was not
90    /// rewritten to fakecloud's registry by
91    /// [`crate::ecr_uri::translate_to_local_at`].
92    pub fn for_local_rewrite(rewritten: bool) -> Self {
93        if rewritten {
94            Self::PlainHttp
95        } else {
96            Self::Https
97        }
98    }
99}
100
101/// How an image became available for a launch.
102#[derive(Debug, Clone, PartialEq, Eq)]
103pub enum PulledImage {
104    /// The registry served the image.
105    Pulled,
106    /// The pull failed transiently but the image was already cached locally,
107    /// so the cached copy is used. Carries the pull's error for logging.
108    Cached { pull_error: String },
109    /// [`ensure_image`] found the image already cached and did not pull.
110    Present,
111}
112
113/// Pull `reference` with the container `cli`. A transient registry failure
114/// falls back to a locally cached copy, or is retried with backoff when
115/// nothing is cached; any other failure is returned at once. `docker_config`
116/// is exported as `DOCKER_CONFIG` for the pull so registry credentials
117/// resolve. `transport` says whether the registry is fakecloud's own
118/// plain-HTTP one.
119///
120/// Returns the pull's stderr as the error.
121pub async fn pull_image(
122    cli: &str,
123    docker_config: Option<&Path>,
124    reference: &str,
125    transport: RegistryTransport,
126) -> Result<PulledImage, String> {
127    pull_image_with(cli, docker_config, reference, transport, PULL_TIMING).await
128}
129
130/// Make `reference` available locally, pulling it only when it is not
131/// already cached -- what `docker run` does with its implicit pull, but with
132/// [`pull_image`]'s retry when the registry fails transiently. `docker run`
133/// gives up on the first `429 Too Many Requests`, which on a host with an
134/// empty cache fails every launch that races the first pull of an image.
135pub async fn ensure_image(
136    cli: &str,
137    docker_config: Option<&Path>,
138    reference: &str,
139) -> Result<PulledImage, String> {
140    ensure_image_with(cli, docker_config, reference, PULL_TIMING).await
141}
142
143/// [`ensure_image`] for a caller with a budget of its own: the cache check
144/// and every pull attempt together end within `budget` (each attempt also
145/// still within the usual per-attempt limit). Every CLI call is killed when it
146/// outruns its share, so nothing is left running past the budget.
147pub async fn ensure_image_within(
148    cli: &str,
149    docker_config: Option<&Path>,
150    reference: &str,
151    budget: Duration,
152) -> Result<PulledImage, String> {
153    let timing = PullTiming {
154        attempt_timeout: PULL_ATTEMPT_TIMEOUT.min(budget),
155        total: budget,
156        ..PULL_TIMING
157    };
158    ensure_image_with(cli, docker_config, reference, timing).await
159}
160
161async fn ensure_image_with(
162    cli: &str,
163    docker_config: Option<&Path>,
164    reference: &str,
165    timing: PullTiming,
166) -> Result<PulledImage, String> {
167    let start = tokio::time::Instant::now();
168    if image_cached(
169        cli,
170        docker_config,
171        reference,
172        INSPECT_TIMEOUT.min(timing.total),
173    )
174    .await
175    {
176        return Ok(PulledImage::Present);
177    }
178    let timing = PullTiming {
179        total: timing.total.saturating_sub(start.elapsed()),
180        ..timing
181    };
182    pull_image_with(
183        cli,
184        docker_config,
185        reference,
186        RegistryTransport::Https,
187        timing,
188    )
189    .await
190}
191
192async fn pull_image_with(
193    cli: &str,
194    docker_config: Option<&Path>,
195    reference: &str,
196    transport: RegistryTransport,
197    timing: PullTiming,
198) -> Result<PulledImage, String> {
199    // Only a plain-HTTP pull branches on the engine, so only it pays for
200    // (the first, then cached) engine probe.
201    let podman = transport == RegistryTransport::PlainHttp
202        && crate::container_net::is_podman_async(cli).await;
203    let deadline = tokio::time::Instant::now() + timing.total;
204    let remaining = || deadline.saturating_duration_since(tokio::time::Instant::now());
205    let mut delay = timing.retry_delay;
206    let mut attempt = 1;
207    let mut stalled_once = false;
208    loop {
209        let budget = timing.attempt_timeout.min(remaining());
210        let mut cmd = Command::new(cli);
211        if let Some(p) = docker_config {
212            cmd.env("DOCKER_CONFIG", p);
213        }
214        cmd.args(pull_args(podman, reference, transport));
215        let out = match crate::container_net::output_within(&mut cmd, budget).await {
216            Ok(out) => out,
217            Err(crate::container_net::CliRunError::TimedOut(_)) => {
218                let pull_error = format!(
219                    "{cli} pull {reference} did not finish within {}s and was killed",
220                    budget.as_secs_f64()
221                );
222                if image_cached(cli, docker_config, reference, INSPECT_AFTER_STALL_TIMEOUT).await {
223                    tracing::warn!(
224                        image = %reference,
225                        error = %pull_error,
226                        "image pull timed out; using the locally cached image"
227                    );
228                    return Ok(PulledImage::Cached { pull_error });
229                }
230                // One more attempt, on fresh connections, if the overall
231                // budget still has room for it.
232                if stalled_once || remaining().is_zero() {
233                    return Err(pull_error);
234                }
235                stalled_once = true;
236                tracing::warn!(
237                    image = %reference,
238                    error = %pull_error,
239                    "image pull timed out; retrying once"
240                );
241                continue;
242            }
243            Err(e) => return Err(format!("{cli} pull: {e}")),
244        };
245        if out.status.success() {
246            return Ok(PulledImage::Pulled);
247        }
248        let pull_error = String::from_utf8_lossy(&out.stderr).trim().to_string();
249        if !is_transient(&pull_error, reference) {
250            return Err(pull_error);
251        }
252        let check = INSPECT_TIMEOUT
253            .min(remaining())
254            .max(INSPECT_AFTER_STALL_TIMEOUT);
255        if image_cached(cli, docker_config, reference, check).await {
256            tracing::warn!(
257                image = %reference,
258                error = %pull_error,
259                "image pull failed transiently; using the locally cached image"
260            );
261            return Ok(PulledImage::Cached { pull_error });
262        }
263        if attempt >= MAX_PULL_ATTEMPTS || remaining() <= delay {
264            return Err(pull_error);
265        }
266        tracing::info!(
267            image = %reference,
268            attempt,
269            retry_in_ms = delay.as_millis() as u64,
270            "image pull failed transiently; retrying"
271        );
272        tokio::time::sleep(delay).await;
273        delay *= 2;
274        attempt += 1;
275    }
276}
277
278/// The `pull` arguments, `podman` saying whether the CLI drives Podman (by
279/// engine, not name: the `podman-docker` shim is a `docker` that is Podman).
280/// Only Podman needs telling that a plain-HTTP registry is one: Docker accepts
281/// it for a loopback registry, and its `pull` has no `--tls-verify` flag at all.
282fn pull_args(podman: bool, reference: &str, transport: RegistryTransport) -> Vec<String> {
283    let mut args = vec!["pull".to_string()];
284    if transport == RegistryTransport::PlainHttp && podman {
285        args.push("--tls-verify=false".to_string());
286    }
287    args.push(reference.to_string());
288    args
289}
290
291/// Whether `reference` resolves to an image in the local cache. Runs with
292/// the same `DOCKER_CONFIG` as the pull: the config also selects the Docker
293/// context, so without it the check could consult a different daemon than
294/// the one that pulls and later runs the image.
295async fn image_cached(
296    cli: &str,
297    docker_config: Option<&Path>,
298    reference: &str,
299    timeout: Duration,
300) -> bool {
301    let mut cmd = Command::new(cli);
302    if let Some(p) = docker_config {
303        cmd.env("DOCKER_CONFIG", p);
304    }
305    cmd.args(["image", "inspect", reference]);
306    crate::container_net::output_within(&mut cmd, timeout)
307        .await
308        .is_ok_and(|out| out.status.success())
309}
310
311/// Whether a pull error is one a later attempt could succeed past: the
312/// registry throttling the caller (the `429 Too Many Requests` status line,
313/// the `toomanyrequests` error code, ECR Public's `Rate exceeded`), any 5xx
314/// from the registry, or the network timing out or dropping the connection.
315///
316/// A refusal (not found, access denied, unauthorized) is never transient, and
317/// wins over transient wording in the same message. Both are matched only
318/// after the image's own name is removed from the message: the name is chosen
319/// by the user and quoted in the error, so a repository spelled like either
320/// kind of marker (`toomanyrequests/app`, `acme/access-denied-page`) must not
321/// flip the classification.
322fn is_transient(stderr: &str, reference: &str) -> bool {
323    const REFUSED: [&str; 6] = [
324        "manifest unknown",
325        "not found",
326        "denied",
327        "unauthorized",
328        "forbidden",
329        "does not exist",
330    ];
331    const TRANSIENT: [&str; 8] = [
332        "toomanyrequests",
333        "too many requests",
334        "rate exceeded",
335        "i/o timeout",
336        "tls handshake timeout",
337        "connection reset by peer",
338        "context deadline exceeded",
339        "request canceled while waiting for connection",
340    ];
341    let message = without_image_name(&stderr.to_ascii_lowercase(), reference);
342    if REFUSED.iter().any(|m| message.contains(m)) {
343        return false;
344    }
345    TRANSIENT.iter().any(|m| message.contains(m)) || has_server_error_status(&message)
346}
347
348/// `message` (lowercase) with each way an error can quote the image blanked
349/// out: the reference as given, every trailing path of its repository
350/// (`public.ecr.aws/acme/app`, `acme/app`, `app`), and its registry host --
351/// registries put the host and repository path in URLs, and Podman expands short names to
352/// `docker.io/library/<name>`. Only whole names are removed, bounded by
353/// characters a name cannot contain, so a short repository like `d` never
354/// cuts letters out of the surrounding words.
355fn without_image_name(message: &str, reference: &str) -> String {
356    let reference = reference.to_ascii_lowercase();
357    let untagged = reference.split('@').next().unwrap_or(&reference);
358    // A `:` after the last `/` starts the tag; one before it is a registry port.
359    let repository = match (untagged.rfind(':'), untagged.rfind('/')) {
360        (Some(colon), Some(slash)) if colon < slash => untagged,
361        (Some(colon), _) => &untagged[..colon],
362        (None, _) => untagged,
363    };
364    let mut names = vec![reference.as_str(), repository];
365    names.extend(
366        repository
367            .match_indices('/')
368            .map(|(i, _)| &repository[i + 1..]),
369    );
370    // The registry host, which URLs in the error quote on its own.
371    if let Some((host, _)) = repository.split_once('/') {
372        if host.contains(['.', ':']) || host == "localhost" {
373            names.push(host);
374        }
375    }
376    names.sort_by_key(|n| std::cmp::Reverse(n.len()));
377
378    let mut out = message.to_string();
379    for name in names.into_iter().filter(|n| !n.is_empty()) {
380        out = remove_whole(&out, name);
381    }
382    out
383}
384
385/// `haystack` with every occurrence of `name` that is not part of a longer
386/// name (flanked by a letter, digit, `.`, `_` or `-`) replaced by a space.
387fn remove_whole(haystack: &str, name: &str) -> String {
388    let is_name_char = |c: char| c.is_ascii_alphanumeric() || matches!(c, '.' | '_' | '-');
389    let mut out = String::with_capacity(haystack.len());
390    let mut rest = haystack;
391    while let Some(i) = rest.find(name) {
392        let end = i + name.len();
393        let before = rest[..i].chars().next_back();
394        let after = rest[end..].chars().next();
395        if before.is_some_and(is_name_char) || after.is_some_and(is_name_char) {
396            out.push_str(&rest[..end]);
397        } else {
398            out.push_str(&rest[..i]);
399            out.push(' ');
400        }
401        rest = &rest[end..];
402    }
403    out.push_str(rest);
404    out
405}
406
407/// Whether the message carries a 5xx HTTP status. Docker and Podman quote the
408/// status as `: 503 Service Unavailable`, `status: 500`, or
409/// `status code 502`; a three-digit number in that position from 500 to 599
410/// counts, whatever reason phrase follows.
411fn has_server_error_status(message: &str) -> bool {
412    let bytes = message.as_bytes();
413    ["status code ", "status: ", "status ", ": "]
414        .iter()
415        .flat_map(|prefix| message.match_indices(prefix).map(|(i, p)| i + p.len()))
416        .any(|start| {
417            let code = &bytes[start..bytes.len().min(start + 3)];
418            code.len() == 3
419                && code[0] == b'5'
420                && code.iter().all(u8::is_ascii_digit)
421                && !bytes
422                    .get(start + 3)
423                    .is_some_and(|c| c.is_ascii_alphanumeric())
424        })
425}
426
427#[cfg(all(test, unix))]
428mod tests {
429    use super::*;
430
431    /// A stand-in container CLI. `pull` fails with `pull_stderr` for the first
432    /// `pull_failures` calls and succeeds after; `image inspect` succeeds only
433    /// when `cached`. Every invocation is appended to `calls.log`.
434    struct FakeCli {
435        dir: tempfile::TempDir,
436        name: String,
437    }
438
439    impl FakeCli {
440        fn new(pull_failures: u32, pull_stderr: &str, cached: bool) -> Self {
441            Self::named("cli", pull_failures, pull_stderr, cached)
442        }
443
444        /// A fake installed under `name`, which is what decides whether
445        /// fakecloud drives it as Podman.
446        fn named(name: &str, pull_failures: u32, pull_stderr: &str, cached: bool) -> Self {
447            let version = if name.contains("podman") {
448                "podman version 5.2.0"
449            } else {
450                "Docker version 27.3.1, build ce12230"
451            };
452            Self::with_version(name, version, pull_failures, pull_stderr, cached)
453        }
454
455        /// A fake whose `--version` prints `version` -- the engine probe's
456        /// answer. That call is not logged, so `calls()` shows only the work.
457        fn with_version(
458            name: &str,
459            version: &str,
460            pull_failures: u32,
461            pull_stderr: &str,
462            cached: bool,
463        ) -> Self {
464            let dir = tempfile::tempdir().unwrap();
465            let script = format!(
466                r#"#!/bin/sh
467d="{dir}"
468if [ "$1" = --version ]; then
469  echo '{version}'
470  exit 0
471fi
472echo "$*" >> "$d/calls.log"
473case "$1" in
474  pull)
475    n=$(cat "$d/pulls" 2>/dev/null || echo 0)
476    n=$((n + 1))
477    echo "$n" > "$d/pulls"
478    if [ "$n" -le {pull_failures} ]; then
479      echo '{pull_stderr}' >&2
480      exit 1
481    fi
482    exit 0 ;;
483  image)
484    [ "{cached}" = "true" ] && exit 0
485    echo 'Error: No such image' >&2
486    exit 1 ;;
487esac
488exit 2
489"#,
490                dir = dir.path().display(),
491            );
492            let path = dir.path().join(name);
493            crate::test_support::write_executable(&path, &script);
494            let _ = std::fs::remove_file(dir.path().join("calls.log"));
495            Self {
496                dir,
497                name: name.to_string(),
498            }
499        }
500
501        fn cli(&self) -> String {
502            self.dir.path().join(&self.name).display().to_string()
503        }
504
505        fn calls(&self) -> Vec<String> {
506            std::fs::read_to_string(self.dir.path().join("calls.log"))
507                .unwrap_or_default()
508                .lines()
509                .map(String::from)
510                .collect()
511        }
512
513        async fn pull(&self) -> Result<PulledImage, String> {
514            self.pull_ref("alpine:3.20").await
515        }
516
517        async fn pull_ref(&self, reference: &str) -> Result<PulledImage, String> {
518            self.pull_via(reference, RegistryTransport::Https).await
519        }
520
521        async fn pull_via(
522            &self,
523            reference: &str,
524            transport: RegistryTransport,
525        ) -> Result<PulledImage, String> {
526            pull_image_with(&self.cli(), None, reference, transport, TEST_TIMING).await
527        }
528
529        async fn ensure(&self) -> Result<PulledImage, String> {
530            ensure_image_with(&self.cli(), None, "alpine:3.20", TEST_TIMING).await
531        }
532    }
533
534    const TEST_TIMING: PullTiming = PullTiming {
535        retry_delay: Duration::from_millis(1),
536        attempt_timeout: Duration::from_secs(30),
537        total: Duration::from_secs(120),
538    };
539
540    fn is_transient_for_test(stderr: &str) -> bool {
541        is_transient(stderr, "alpine:3.20")
542    }
543
544    const THROTTLED: &str = "Error response from daemon: unexpected status from HEAD request to https://public.ecr.aws/v2/docker/library/alpine/manifests/3.20: 429 Too Many Requests";
545
546    /// A fake CLI whose `pull` never finishes and whose `image inspect`
547    /// succeeds only when `cached`; calls go to `calls.log`.
548    fn stalling_cli(cached: bool) -> (tempfile::TempDir, String) {
549        crate::test_support::fake_cli(
550            "cli",
551            &format!(
552                "d=$(dirname \"$0\")\n[ \"$1\" = warmup ] && exit 0\necho \"$*\" >> \"$d/calls.log\"\n\
553                 case \"$1\" in\n  pull) exec sleep 1000 ;;\n  image) [ {cached} = true ] ;;\nesac\n"
554            ),
555        )
556    }
557
558    fn calls_in(dir: &tempfile::TempDir) -> Vec<String> {
559        std::fs::read_to_string(dir.path().join("calls.log"))
560            .unwrap_or_default()
561            .lines()
562            .map(String::from)
563            .collect()
564    }
565
566    /// A pull whose download stalls is killed at its budget instead of holding
567    /// the launch forever, and falls back to a cached copy when there is one.
568    #[tokio::test]
569    async fn a_stalled_pull_falls_back_to_the_cache() {
570        let (dir, cli) = stalling_cli(true);
571        let timing = PullTiming {
572            retry_delay: Duration::from_millis(1),
573            attempt_timeout: Duration::from_millis(300),
574            total: Duration::from_secs(20),
575        };
576        let got =
577            pull_image_with(&cli, None, "alpine:3.20", RegistryTransport::Https, timing).await;
578        match got {
579            Ok(PulledImage::Cached { pull_error }) => {
580                assert!(
581                    pull_error.contains("did not finish within 0.3s"),
582                    "{pull_error}"
583                )
584            }
585            other => panic!("unexpected {other:?}"),
586        }
587        assert_eq!(
588            calls_in(&dir),
589            ["pull alpine:3.20", "image inspect alpine:3.20"]
590        );
591    }
592
593    /// With nothing cached, a stalled pull gets exactly one more attempt on a
594    /// fresh connection, and then fails.
595    #[tokio::test]
596    async fn a_stalled_pull_is_retried_once() {
597        let (dir, cli) = stalling_cli(false);
598        let timing = PullTiming {
599            retry_delay: Duration::from_millis(1),
600            attempt_timeout: Duration::from_millis(300),
601            total: Duration::from_secs(20),
602        };
603        let start = std::time::Instant::now();
604        let err = pull_image_with(&cli, None, "alpine:3.20", RegistryTransport::Https, timing)
605            .await
606            .expect_err("nothing cached");
607        assert!(start.elapsed() < Duration::from_secs(10));
608        assert!(err.contains("did not finish within 0.3s"), "{err}");
609        assert_eq!(
610            calls_in(&dir),
611            [
612                "pull alpine:3.20",
613                "image inspect alpine:3.20",
614                "pull alpine:3.20",
615                "image inspect alpine:3.20"
616            ]
617        );
618    }
619
620    /// The overall budget caps the retry: it only gets what is left.
621    #[tokio::test]
622    async fn the_retry_after_a_stall_only_gets_the_remaining_budget() {
623        let (dir, cli) = stalling_cli(false);
624        let start = std::time::Instant::now();
625        let err = ensure_image_within(&cli, None, "alpine:3.20", Duration::from_millis(600))
626            .await
627            .expect_err("nothing cached");
628        assert!(start.elapsed() < Duration::from_secs(10));
629        assert!(err.contains("did not finish within"), "{err}");
630        // inspect, a 0.6s pull, then the budget is spent.
631        assert_eq!(
632            calls_in(&dir),
633            [
634                "image inspect alpine:3.20",
635                "pull alpine:3.20",
636                "image inspect alpine:3.20"
637            ]
638        );
639    }
640
641    #[tokio::test]
642    async fn ensure_image_uses_a_cached_image_without_contacting_the_registry() {
643        let cli = FakeCli::new(u32::MAX, THROTTLED, true);
644        assert_eq!(cli.ensure().await, Ok(PulledImage::Present));
645        assert_eq!(cli.calls(), ["image inspect alpine:3.20"]);
646    }
647
648    #[tokio::test]
649    async fn ensure_image_retries_a_throttled_first_pull() {
650        let cli = FakeCli::new(2, THROTTLED, false);
651        assert_eq!(cli.ensure().await, Ok(PulledImage::Pulled));
652        let pulls = cli.calls().iter().filter(|c| c.starts_with("pull")).count();
653        assert_eq!(pulls, 3);
654    }
655
656    #[tokio::test]
657    async fn ensure_image_fails_on_a_refused_pull() {
658        let missing =
659            "Error response from daemon: manifest for alpine:3.20 not found: manifest unknown";
660        let cli = FakeCli::new(u32::MAX, missing, false);
661        assert_eq!(cli.ensure().await, Err(missing.to_string()));
662    }
663
664    #[tokio::test]
665    async fn podman_pulls_fakecloud_registry_over_plain_http() {
666        // Podman defaults to TLS even for a loopback registry, and fakecloud's
667        // serves plain HTTP (issue #2585).
668        let cli = FakeCli::named("podman", 0, "", false);
669        let got = cli
670            .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp)
671            .await;
672        assert_eq!(got, Ok(PulledImage::Pulled));
673        assert_eq!(
674            cli.calls(),
675            ["pull --tls-verify=false 127.0.0.1:4566/test:healthy"]
676        );
677    }
678
679    #[tokio::test]
680    async fn podman_keeps_tls_for_upstream_registries() {
681        let cli = FakeCli::named("podman", 0, "", false);
682        let got = cli
683            .pull_via(
684                "public.ecr.aws/docker/library/alpine:3.20",
685                RegistryTransport::Https,
686            )
687            .await;
688        assert_eq!(got, Ok(PulledImage::Pulled));
689        assert_eq!(
690            cli.calls(),
691            ["pull public.ecr.aws/docker/library/alpine:3.20"]
692        );
693    }
694
695    #[tokio::test]
696    async fn podman_docker_shim_pulls_fakecloud_registry_over_plain_http() {
697        // The podman-docker package installs a `docker` that execs podman
698        // (issue #2599): it is Podman, and needs the flag, whatever its name.
699        let cli = FakeCli::with_version("docker", "podman version 5.2.0", 0, "", false);
700        let got = cli
701            .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp)
702            .await;
703        assert_eq!(got, Ok(PulledImage::Pulled));
704        assert_eq!(
705            cli.calls(),
706            ["pull --tls-verify=false 127.0.0.1:4566/test:healthy"]
707        );
708    }
709
710    #[tokio::test]
711    async fn docker_pulls_fakecloud_registry_without_a_tls_flag() {
712        // `docker pull` has no --tls-verify; the daemon already treats a
713        // loopback registry as insecure.
714        let cli = FakeCli::named("docker", 0, "", false);
715        let got = cli
716            .pull_via("127.0.0.1:4566/test:healthy", RegistryTransport::PlainHttp)
717            .await;
718        assert_eq!(got, Ok(PulledImage::Pulled));
719        assert_eq!(cli.calls(), ["pull 127.0.0.1:4566/test:healthy"]);
720    }
721
722    #[test]
723    fn transport_follows_the_local_rewrite() {
724        assert_eq!(
725            RegistryTransport::for_local_rewrite(true),
726            RegistryTransport::PlainHttp
727        );
728        assert_eq!(
729            RegistryTransport::for_local_rewrite(false),
730            RegistryTransport::Https
731        );
732    }
733
734    #[tokio::test]
735    async fn a_successful_pull_needs_no_cache_check() {
736        let cli = FakeCli::new(0, "", false);
737        assert_eq!(cli.pull().await, Ok(PulledImage::Pulled));
738        assert_eq!(cli.calls(), ["pull alpine:3.20"]);
739    }
740
741    #[tokio::test]
742    async fn a_throttled_pull_uses_the_cached_image() {
743        let cli = FakeCli::new(u32::MAX, THROTTLED, true);
744        let got = cli.pull().await;
745        assert_eq!(
746            got,
747            Ok(PulledImage::Cached {
748                pull_error: THROTTLED.to_string()
749            })
750        );
751        assert_eq!(
752            cli.calls(),
753            ["pull alpine:3.20", "image inspect alpine:3.20"],
754            "a cached image is used at once, without retrying the pull"
755        );
756    }
757
758    #[tokio::test]
759    async fn a_rate_limited_pull_with_nothing_cached_is_retried() {
760        let cli = FakeCli::new(2, THROTTLED, false);
761        assert_eq!(cli.pull().await, Ok(PulledImage::Pulled));
762        let pulls = cli.calls().iter().filter(|c| c.starts_with("pull")).count();
763        assert_eq!(pulls, 3);
764    }
765
766    #[tokio::test]
767    async fn retries_stop_after_the_attempt_cap() {
768        let cli = FakeCli::new(u32::MAX, THROTTLED, false);
769        assert_eq!(cli.pull().await, Err(THROTTLED.to_string()));
770        let pulls = cli.calls().iter().filter(|c| c.starts_with("pull")).count();
771        assert_eq!(pulls, MAX_PULL_ATTEMPTS as usize);
772    }
773
774    #[tokio::test]
775    async fn a_missing_image_fails_without_retrying() {
776        let missing =
777            "Error response from daemon: manifest for alpine:nope not found: manifest unknown";
778        let cli = FakeCli::new(u32::MAX, missing, false);
779        assert_eq!(cli.pull().await, Err(missing.to_string()));
780        assert_eq!(
781            cli.calls(),
782            ["pull alpine:3.20"],
783            "a refused pull is neither retried nor checked against the cache"
784        );
785    }
786
787    #[tokio::test]
788    async fn a_refused_pull_fails_even_with_a_stale_cached_copy() {
789        // The image was deleted from the registry, or a policy now denies the
790        // pull. A copy cached by an earlier launch must not be used.
791        for refused in [
792            "Error response from daemon: manifest for alpine:3.20 not found: manifest unknown",
793            "Error response from daemon: pull access denied for alpine, repository does not exist or may require authorization: denied",
794        ] {
795            let cli = FakeCli::new(u32::MAX, refused, true);
796            assert_eq!(cli.pull().await, Err(refused.to_string()));
797            assert_eq!(cli.calls(), ["pull alpine:3.20"]);
798        }
799    }
800
801    #[tokio::test]
802    async fn a_registry_server_error_uses_the_cached_image() {
803        let unavailable =
804            "Error response from daemon: received unexpected HTTP status: 503 Service Unavailable";
805        let cli = FakeCli::new(u32::MAX, unavailable, true);
806        assert_eq!(
807            cli.pull().await,
808            Ok(PulledImage::Cached {
809                pull_error: unavailable.to_string()
810            })
811        );
812    }
813
814    #[tokio::test]
815    async fn a_refused_pull_of_a_repository_named_like_a_marker_is_still_refused() {
816        // The message quotes the image name. A repository spelled like a
817        // throttling code must not make a refused pull look transient.
818        let refused = "Error response from daemon: manifest for toomanyrequests:latest not found: manifest unknown: manifest unknown";
819        let cli = FakeCli::new(u32::MAX, refused, true);
820        assert_eq!(
821            cli.pull_ref("toomanyrequests:latest").await,
822            Err(refused.to_string())
823        );
824        assert_eq!(cli.calls(), ["pull toomanyrequests:latest"]);
825    }
826
827    #[tokio::test]
828    async fn a_throttled_pull_of_a_repository_named_like_a_refusal_uses_the_cache() {
829        // Only removing the image name from the message keeps `denied` in the
830        // repository from reading as a refusal; without it this pull would
831        // fail instead of falling back to the cached copy.
832        let reference = "public.ecr.aws/acme/access-denied-page:1";
833        let throttled = "Error response from daemon: unexpected status from HEAD request to https://public.ecr.aws/v2/acme/access-denied-page/manifests/1: 429 Too Many Requests";
834        let cli = FakeCli::new(u32::MAX, throttled, true);
835        assert_eq!(
836            cli.pull_ref(reference).await,
837            Ok(PulledImage::Cached {
838                pull_error: throttled.to_string()
839            })
840        );
841        assert_eq!(
842            cli.calls(),
843            [
844                format!("pull {reference}"),
845                format!("image inspect {reference}")
846            ]
847        );
848    }
849
850    #[test]
851    fn transient_detection_separates_retryable_from_refused() {
852        assert!(is_transient_for_test(THROTTLED));
853        assert!(is_transient_for_test(
854            "toomanyrequests: You have reached your pull rate limit."
855        ));
856        assert!(is_transient_for_test("Error: Rate exceeded"));
857        assert!(is_transient_for_test(
858            "received unexpected HTTP status: 502 Bad Gateway"
859        ));
860        assert!(is_transient_for_test(
861            "Get \"https://public.ecr.aws/v2/\": net/http: TLS handshake timeout"
862        ));
863        assert!(is_transient_for_test(
864            "read tcp 10.0.0.2:4431->1.2.3.4:443: read: connection reset by peer"
865        ));
866        assert!(!is_transient_for_test("manifest unknown"));
867        assert!(!is_transient_for_test("pull access denied for foo"));
868        assert!(!is_transient_for_test(
869            "unauthorized: authentication required"
870        ));
871        assert!(!is_transient_for_test(
872            "pull access denied for toomanyrequests, repository does not exist or may require authorization"
873        ));
874    }
875
876    #[test]
877    fn any_5xx_status_is_transient_whatever_its_reason_phrase() {
878        for msg in [
879            "received unexpected HTTP status: 500 Internal Server Error",
880            "unexpected status from GET request to https://r.example/v2/: 507 Insufficient Storage",
881            "unexpected status code 520",
882            "error pulling image: status: 599",
883            "unexpected status from HEAD request to https://r.example/v2/a/manifests/1: 503",
884        ] {
885            assert!(is_transient_for_test(msg), "{msg}");
886        }
887        for msg in [
888            // A registry port or a 4xx is not a server error.
889            "Get \"http://127.0.0.1:5000/v2/\": dial tcp 127.0.0.1:5000: connect: connection refused",
890            "unexpected status code 400 Bad Request",
891            "status: 5001",
892        ] {
893            assert!(!is_transient_for_test(msg), "{msg}");
894        }
895    }
896
897    #[test]
898    fn a_throttled_pull_of_a_repository_named_like_a_refusal_is_still_transient() {
899        let reference = "public.ecr.aws/acme/access-denied-page:1";
900        for msg in [
901            "Error response from daemon: unexpected status from HEAD request to https://public.ecr.aws/v2/acme/access-denied-page/manifests/1: 429 Too Many Requests",
902            "Error response from daemon: toomanyrequests: Rate exceeded for public.ecr.aws/acme/access-denied-page:1",
903        ] {
904            assert!(is_transient(msg, reference), "{msg}");
905        }
906        // Its genuine refusals are still refusals.
907        assert!(!is_transient(
908            "Error response from daemon: manifest for public.ecr.aws/acme/access-denied-page:1 not found: manifest unknown",
909            reference
910        ));
911    }
912
913    #[test]
914    fn a_registry_host_named_like_a_refusal_does_not_hide_a_throttle() {
915        assert!(is_transient(
916            "Error response from daemon: unexpected status from HEAD request to https://denied.example/v2/app/manifests/1: 429 Too Many Requests",
917            "denied.example/app:1"
918        ));
919    }
920
921    #[test]
922    fn only_whole_names_are_removed() {
923        // A one-letter repository must not cut the `d` out of `denied`.
924        assert!(!is_transient(
925            "Error response from daemon: pull access denied for d, repository does not exist",
926            "d"
927        ));
928        assert_eq!(
929            without_image_name(
930                "pull access denied for docker.io/library/alpine",
931                "alpine:3.20"
932            ),
933            "pull access denied for docker.io/library/ "
934        );
935        assert_eq!(
936            without_image_name(
937                "get https://127.0.0.1:5000/v2/team/app/manifests/v1",
938                "127.0.0.1:5000/team/app:v1"
939            ),
940            "get https:// /v2/ /manifests/v1"
941        );
942    }
943}
fakecloud_core/container_image.rs

fakecloud_core/
container_image.rs
