microvms_app/control/
sweep.rs

1// SPDX-License-Identifier: Apache-2.0
2//! Prefix-scoped cleanup. Discovery finishes before mutation so deletion cannot move a
3//! pagination cursor past another resource. Keep entries protect both an image and its VMs.
4
5use std::collections::{BTreeMap, HashSet};
6use std::time::Duration;
7
8use serde::Serialize;
9
10use super::transport::{Call, paths};
11use super::{ControlPlane, ops};
12use crate::error::{Error, ErrorKind};
13use crate::sandbox::{DEFAULT_LIFECYCLE_TIMEOUT, LIFECYCLE_POLL_INTERVAL};
14
15/// Options for an explicitly scoped sweep. The default only discovers resources.
16#[derive(Clone, Debug)]
17pub struct SweepOptions {
18    /// Exact image names or ARNs whose images and VMs must survive.
19    pub keep: Vec<String>,
20    /// Discover and report candidates without terminating VMs or deleting images.
21    pub dry_run: bool,
22}
23
24impl Default for SweepOptions {
25    fn default() -> Self {
26        Self {
27            keep: Vec::new(),
28            dry_run: true,
29        }
30    }
31}
32
33/// A failed resource operation. Other candidates are still processed.
34#[derive(Clone, Debug, Serialize)]
35#[serde(rename_all = "camelCase")]
36pub struct SweepFailure {
37    pub resource: String,
38    pub message: String,
39}
40
41/// An inventory and confirmed cleanup outcomes. Dry runs populate only the inventory.
42#[derive(Clone, Debug, Default, Serialize)]
43#[serde(rename_all = "camelCase")]
44pub struct SweepReport {
45    pub dry_run: bool,
46    /// ARNs whose image name starts with the requested prefix, including kept images.
47    pub images_matched: Vec<String>,
48    pub images_kept: Vec<String>,
49    /// Live VM IDs belonging to matched images outside the keep list.
50    pub microvms_matched: Vec<String>,
51    /// VM IDs observed in TERMINATED after the termination request.
52    pub microvms_terminated: Vec<String>,
53    /// Image ARNs observed absent or DELETED after a delete request.
54    pub images_deleted: Vec<String>,
55    /// Image ARNs still held by a live VM, or rejected by the service on deletion.
56    pub images_in_use: Vec<String>,
57    pub failures: Vec<SweepFailure>,
58}
59
60// The service caps all of these listings at 50. Spacing discovery, polling and version
61// pages avoids a sweep producing a burst of ListMicrovms calls against a busy account.
62const PAGE_SIZE: &str = "maxResults=50";
63const LIST_SPACING: Duration = Duration::from_secs(1);
64
65impl ControlPlane {
66    pub(super) async fn sweep_page<T: serde::de::DeserializeOwned>(
67        &self,
68        operation: &'static str,
69        path: String,
70        last: &mut Option<Duration>,
71    ) -> Result<T, Error> {
72        if let Some(previous) = *last {
73            let elapsed = self.clock().elapsed().saturating_sub(previous);
74            self.clock()
75                .sleep(LIST_SPACING.saturating_sub(elapsed))
76                .await;
77        }
78        let separator = if path.contains('?') { '&' } else { '?' };
79        let call = Call::get(operation, format!("{path}{separator}{PAGE_SIZE}"));
80        *last = Some(self.clock().elapsed());
81        self.send(call).await?.json(operation)
82    }
83
84    async fn sweep_images(
85        &self,
86        prefix: &str,
87        last: &mut Option<Duration>,
88    ) -> Result<Vec<ops::MicrovmImageSummaryWire>, Error> {
89        let mut items = BTreeMap::new();
90        let mut token = None;
91        let mut seen = HashSet::new();
92        loop {
93            let page: ops::ListImagesResponseWire = self
94                .sweep_page(
95                    "ListMicrovmImages",
96                    paths::microvm_images_list(Some(prefix), token.as_deref()),
97                    last,
98                )
99                .await?;
100            for image in page.items {
101                // nameFilter is substring matching. This check is the destructive scope boundary.
102                if image.name.starts_with(prefix) && image.state != "DELETED" {
103                    items.insert(image.image_arn.clone(), image);
104                }
105            }
106            match page.next_token {
107                Some(next) if seen.insert(next.clone()) => token = Some(next),
108                Some(_) => {
109                    return Err(Error::new(
110                        ErrorKind::Platform,
111                        "image listing repeated its pagination token",
112                    ));
113                }
114                None => return Ok(items.into_values().collect()),
115            }
116        }
117    }
118
119    async fn sweep_vms(
120        &self,
121        last: &mut Option<Duration>,
122    ) -> Result<Vec<ops::MicrovmSummaryWire>, Error> {
123        let mut items = BTreeMap::new();
124        let mut token = None;
125        let mut seen = HashSet::new();
126        loop {
127            let page: ops::ListMicrovmSummariesResponseWire = self
128                .sweep_page("ListMicrovms", paths::microvms_list(token.as_deref()), last)
129                .await?;
130            for vm in page.items {
131                items.insert(vm.microvm_id.clone(), vm);
132            }
133            match page.next_token {
134                Some(next) if seen.insert(next.clone()) => token = Some(next),
135                Some(_) => {
136                    return Err(Error::new(
137                        ErrorKind::Platform,
138                        "VM listing repeated its pagination token",
139                    ));
140                }
141                None => return Ok(items.into_values().collect()),
142            }
143        }
144    }
145
146    /// Discover or clean up images whose names start with `prefix`, and their live VMs.
147    ///
148    /// A nonempty image-name prefix is required. Keep entries are exact names or ARNs;
149    /// their VMs are protected too. Destructive cleanup can stop every VM using a
150    /// selected, non-kept image, including VMs owned by other runners. It cannot isolate
151    /// a failed runner's VMs when runners share an image. Keep shared images or use a
152    /// separate image-name prefix for each job.
153    ///
154    /// Cleanup waits for termination before deletion, then confirms image disappearance.
155    /// Concurrent launches may keep an image in use; they
156    /// are reported rather than repeatedly terminated. This is not an account-wide lock.
157    pub async fn sweep(&self, prefix: &str, options: &SweepOptions) -> Result<SweepReport, Error> {
158        super::require_valid_image_name(prefix)?;
159        let mut last = None;
160        let images = self.sweep_images(prefix, &mut last).await?;
161        let mut report = SweepReport {
162            dry_run: options.dry_run,
163            ..SweepReport::default()
164        };
165        let mut candidates = HashSet::new();
166        for image in &images {
167            report.images_matched.push(image.image_arn.clone());
168            if options.keep.contains(&image.name) || options.keep.contains(&image.image_arn) {
169                report.images_kept.push(image.image_arn.clone());
170            } else {
171                candidates.insert(image.image_arn.clone());
172            }
173        }
174        let vms = self.sweep_vms(&mut last).await?;
175        report.microvms_matched = vms
176            .iter()
177            .filter(|vm| candidates.contains(&vm.image_arn) && vm.state != "TERMINATED")
178            .map(|vm| vm.microvm_id.clone())
179            .collect();
180        if options.dry_run {
181            return Ok(report);
182        }
183        for id in &report.microvms_matched {
184            if let Err(error) = self.terminate(id).await {
185                report.failures.push(SweepFailure {
186                    resource: id.clone(),
187                    message: error.to_string(),
188                });
189            }
190        }
191        let started = self.clock().elapsed();
192        let mut live;
193        loop {
194            live = match self.sweep_vms(&mut last).await {
195                Ok(live) => live,
196                Err(error) => {
197                    report.failures.push(SweepFailure {
198                        resource: "ListMicrovms".into(),
199                        message: error.to_string(),
200                    });
201                    // Discovery has become uncertain after mutation. Preserve the report and
202                    // do not delete an image without observing its selected VMs terminate.
203                    report.images_in_use = candidates.into_iter().collect();
204                    report.images_in_use.sort();
205                    return Ok(report);
206                }
207            };
208            // Confirmed terminations may disappear from later lists. Retain that evidence,
209            // but never count an absent VM whose termination was not observed.
210            report.microvms_terminated = report
211                .microvms_matched
212                .iter()
213                .filter(|id| {
214                    report.microvms_terminated.contains(id)
215                        || live
216                            .iter()
217                            .any(|vm| &vm.microvm_id == *id && vm.state == "TERMINATED")
218                })
219                .cloned()
220                .collect();
221            let pending = report.microvms_terminated.len() != report.microvms_matched.len();
222            if !pending
223                || self.clock().elapsed().saturating_sub(started) >= DEFAULT_LIFECYCLE_TIMEOUT
224            {
225                break;
226            }
227            self.clock().sleep(LIFECYCLE_POLL_INTERVAL).await;
228        }
229        let mut deleting = HashSet::new();
230        for image in images
231            .iter()
232            .filter(|image| candidates.contains(&image.image_arn))
233        {
234            let unconfirmed = vms.iter().any(|vm| {
235                vm.image_arn == image.image_arn
236                    && report.microvms_matched.contains(&vm.microvm_id)
237                    && !report.microvms_terminated.contains(&vm.microvm_id)
238            });
239            let held = unconfirmed
240                || live
241                    .iter()
242                    .any(|vm| vm.image_arn == image.image_arn && vm.state != "TERMINATED");
243            if held {
244                report.images_in_use.push(image.image_arn.clone());
245            } else if let Err(error) = self
246                .try_delete_image_paced(&image.image_arn, Some(&mut last))
247                .await
248            {
249                report.images_in_use.push(image.image_arn.clone());
250                report.failures.push(SweepFailure {
251                    resource: image.image_arn.clone(),
252                    message: error.to_string(),
253                });
254            } else {
255                deleting.insert(image.image_arn.clone());
256            }
257        }
258        let started = self.clock().elapsed();
259        while !deleting.is_empty() {
260            let remaining = match self.sweep_images(prefix, &mut last).await {
261                Ok(remaining) => remaining,
262                Err(error) => {
263                    report.failures.push(SweepFailure {
264                        resource: "ListMicrovmImages".into(),
265                        message: error.to_string(),
266                    });
267                    return Ok(report);
268                }
269            };
270            let absent: Vec<_> = deleting
271                .iter()
272                .filter(|arn| !remaining.iter().any(|image| &image.image_arn == *arn))
273                .cloned()
274                .collect();
275            for arn in absent {
276                deleting.remove(&arn);
277                report.images_deleted.push(arn);
278            }
279            if deleting.is_empty() {
280                break;
281            }
282            if self.clock().elapsed().saturating_sub(started) >= DEFAULT_LIFECYCLE_TIMEOUT {
283                for arn in deleting {
284                    report.failures.push(SweepFailure {
285                        resource: arn,
286                        message: "image deletion was not confirmed before the cleanup deadline"
287                            .into(),
288                    });
289                }
290                break;
291            }
292            self.clock().sleep(LIFECYCLE_POLL_INTERVAL).await;
293        }
294        report.images_deleted.sort();
295        Ok(report)
296    }
297}
298
299#[cfg(test)]
300mod tests {
301    use super::*;
302    use crate::region::Region;
303    use crate::testing::{Answer, FakeControlPlane, TestClock, control_plane, list_versions_page};
304    use std::sync::Arc;
305
306    fn fixture() -> (Arc<FakeControlPlane>, Arc<TestClock>, ControlPlane) {
307        let fake = Arc::new(FakeControlPlane::new());
308        let clock = Arc::new(TestClock::new());
309        let plane = control_plane(fake.clone(), Region::US_EAST_1, clock.clone());
310        (fake, clock, plane)
311    }
312
313    const IMAGES: &str = r#"{"items":[{"imageArn":"arn:old","name":"job-old","state":"CREATED"},{"imageArn":"arn:keep","name":"job-keep","state":"CREATED"},{"imageArn":"arn:foreign","name":"other-job-old","state":"CREATED"}]}"#;
314    const VMS: &str = r#"{"items":[{"microvmId":"mvm-old","imageArn":"arn:old","imageVersion":"1","state":"RUNNING","startedAt":1234.5},{"microvmId":"mvm-keep","imageArn":"arn:keep","imageVersion":"1","state":"RUNNING"},{"microvmId":"mvm-foreign","imageArn":"arn:foreign","imageVersion":"1","state":"RUNNING"},{"microvmId":"mvm-done","imageArn":"arn:old","imageVersion":"1","state":"TERMINATED"}]}"#;
315    const EMPTY: &str = r#"{"items":[]}"#;
316
317    fn polls_through_deadline(fake: &FakeControlPlane, body: &str) {
318        for _ in 0..=DEFAULT_LIFECYCLE_TIMEOUT.as_secs() / LIFECYCLE_POLL_INTERVAL.as_secs() {
319            fake.answer("ListMicrovms", Answer::ok(body));
320        }
321        // A broken loop condition must fail the report assertions, not repeat a ready
322        // future forever and prevent the test runner from observing the failure.
323        fake.answer(
324            "ListMicrovms",
325            Answer::failure(403, "polling exceeded the deadline"),
326        );
327    }
328
329    #[tokio::test]
330    async fn sweep_honors_the_control_plane_retry_attempt_budget() {
331        let (fake, clock, plane) = fixture();
332        fake.answer("ListMicrovmImages", Answer::failure(429, "throttled"));
333        fake.answer("ListMicrovmImages", Answer::ok(EMPTY));
334        let plane = plane.with_retry(
335            super::super::transport::RetryPolicy::new(1, Duration::from_secs(30)).unwrap(),
336        );
337        let error = plane
338            .sweep("job-", &SweepOptions::default())
339            .await
340            .unwrap_err();
341        assert_eq!(fake.call_count("ListMicrovmImages"), 1);
342        assert!(error.throttled());
343        assert_eq!(error.attempt_count(), 1);
344        assert_eq!(clock.now(), Duration::ZERO);
345    }
346
347    #[tokio::test]
348    async fn sweep_decode_failures_retain_completed_request_retry_details() {
349        let (fake, clock, plane) = fixture();
350        fake.answer("ListMicrovmImages", Answer::throttled_as_400());
351        fake.answer("ListMicrovmImages", Answer::ok(r#"{"items":["#));
352        let plane = plane.with_retry(
353            super::super::transport::RetryPolicy::new(2, Duration::from_secs(30)).unwrap(),
354        );
355
356        let error = plane
357            .sweep("job-", &SweepOptions::default())
358            .await
359            .unwrap_err();
360
361        assert_eq!(error.kind(), ErrorKind::Platform);
362        assert!(!error.throttled());
363        assert_eq!(error.attempt_count(), 2);
364        assert_eq!(error.backoff(), clock.now());
365        assert!(error.backoff() >= Duration::from_millis(200));
366        assert!(error.backoff() < Duration::from_millis(400));
367        assert_eq!(fake.call_count("ListMicrovmImages"), 2);
368        assert_eq!(fake.call_count("ListMicrovms"), 0);
369    }
370
371    #[tokio::test]
372    async fn dry_run_protects_keep_names_and_arns_and_filters_substrings() {
373        for keep in ["job-keep", "arn:keep"] {
374            let (fake, clock, plane) = fixture();
375            fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
376            fake.answer("ListMicrovms", Answer::ok(VMS));
377            let report = plane
378                .sweep(
379                    "job-",
380                    &SweepOptions {
381                        keep: vec![keep.into()],
382                        ..SweepOptions::default()
383                    },
384                )
385                .await
386                .unwrap();
387            assert!(report.dry_run);
388            assert_eq!(report.images_matched, ["arn:keep", "arn:old"]);
389            assert_eq!(report.images_kept, ["arn:keep"]);
390            assert_eq!(report.microvms_matched, ["mvm-old"]);
391            assert!(report.microvms_terminated.is_empty() && report.images_deleted.is_empty());
392            assert!(
393                fake.calls()
394                    .iter()
395                    .all(|call| call.method == super::super::transport::Method::Get)
396            );
397            assert!(
398                fake.calls()
399                    .iter()
400                    .all(|call| call.path.contains("maxResults=50"))
401            );
402            assert!(fake.calls()[0].path.contains("nameFilter=job-"));
403            assert_eq!(clock.now(), Duration::from_secs(1));
404        }
405    }
406
407    #[tokio::test]
408    async fn sweep_reads_every_page_and_preserves_started_at() {
409        let (fake, clock, plane) = fixture();
410        fake.answer(
411            "ListMicrovmImages",
412            Answer::ok(r#"{"items":[],"nextToken":"i+=/"}"#),
413        );
414        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
415        fake.answer(
416            "ListMicrovms",
417            Answer::ok(r#"{"items":[],"nextToken":"v+=/"}"#),
418        );
419        fake.answer("ListMicrovms", Answer::ok(VMS));
420        let report = plane.sweep("job-", &SweepOptions::default()).await.unwrap();
421        assert_eq!(report.microvms_matched, ["mvm-keep", "mvm-old"]);
422        assert_eq!(clock.now(), Duration::from_secs(3));
423        assert!(fake.calls()[1].path.contains("nextToken=i%2B%3D%2F"));
424        assert!(fake.calls()[3].path.contains("nextToken=v%2B%3D%2F"));
425        let items = plane
426            .list_microvm_summaries(&Default::default())
427            .await
428            .unwrap();
429        assert_eq!(items[0].started_at, Some(1234.5));
430        assert_eq!(items[1].started_at, None);
431    }
432
433    #[tokio::test]
434    async fn cleanup_confirms_termination_before_deletion_and_image_absence() {
435        let (fake, _, plane) = fixture();
436        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
437        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
438        fake.answer(
439            "ListMicrovmImages",
440            Answer::ok(
441                r#"{"items":[{"imageArn":"arn:keep","name":"job-keep","state":"CREATED"}]}"#,
442            ),
443        );
444        fake.answer(
445            "ListMicrovmImages",
446            Answer::failure(403, "deletion was already confirmed"),
447        );
448        fake.answer("ListMicrovms", Answer::ok(VMS));
449        fake.answer("ListMicrovms", Answer::ok(r#"{"items":[{"microvmId":"mvm-old","imageArn":"arn:old","imageVersion":"1","state":"TERMINATING"}]}"#));
450        fake.answer("ListMicrovms", Answer::ok(r#"{"items":[{"microvmId":"mvm-old","imageArn":"arn:old","imageVersion":"1","state":"TERMINATED"},{"microvmId":"mvm-foreign","imageArn":"arn:foreign","imageVersion":"1","state":"RUNNING"}]}"#));
451        fake.answer(
452            "ListMicrovms",
453            Answer::failure(403, "termination was already confirmed"),
454        );
455        fake.answer("TerminateMicrovm", Answer::ok("{}"));
456        fake.answer(
457            "ListMicrovmImageVersions",
458            Answer::ok(list_versions_page(&["1"], Some("versions-next"))),
459        );
460        fake.answer(
461            "ListMicrovmImageVersions",
462            Answer::ok(list_versions_page(&["2"], None)),
463        );
464        fake.answer("DeleteMicrovmImageVersion", Answer::ok("{}"));
465        fake.answer(
466            "DeleteMicrovmImage",
467            Answer::ok(r#"{"imageIdentifier":"arn:old","state":"DELETING"}"#),
468        );
469        let report = plane
470            .sweep(
471                "job-",
472                &SweepOptions {
473                    keep: vec!["job-keep".into()],
474                    dry_run: false,
475                },
476            )
477            .await
478            .unwrap();
479        assert_eq!(report.microvms_terminated, ["mvm-old"]);
480        assert_eq!(report.images_deleted, ["arn:old"]);
481        assert!(report.images_in_use.is_empty() && report.failures.is_empty());
482        assert_eq!(fake.call_count("ListMicrovmImages"), 3);
483        assert_eq!(fake.call_count("ListMicrovmImageVersions"), 2);
484        assert_eq!(fake.call_count("DeleteMicrovmImageVersion"), 1);
485        assert!(fake.calls().iter().any(|call| {
486            call.operation == "ListMicrovmImageVersions"
487                && call.path.contains("nextToken=versions-next")
488        }));
489        let calls = fake.calls();
490        let deleted = calls
491            .iter()
492            .position(|call| call.operation == "DeleteMicrovmImage")
493            .unwrap();
494        assert_eq!(
495            calls[..deleted]
496                .iter()
497                .filter(|call| call.operation == "ListMicrovms")
498                .count(),
499            3
500        );
501        assert_eq!(
502            calls
503                .iter()
504                .filter(|call| call.operation == "TerminateMicrovm")
505                .count(),
506            1
507        );
508        assert!(
509            calls
510                .iter()
511                .filter(|call| call.method == super::super::transport::Method::Delete)
512                .all(|call| !call.path.contains("keep") && !call.path.contains("foreign"))
513        );
514    }
515
516    #[tokio::test]
517    async fn confirmed_terminations_survive_later_listing_omissions() {
518        let (fake, _, plane) = fixture();
519        fake.answer("ListMicrovmImages", Answer::ok(r#"{"items":[{"imageArn":"arn:a","name":"job-a","state":"CREATED"},{"imageArn":"arn:b","name":"job-b","state":"CREATED"}]}"#));
520        fake.answer("ListMicrovmImages", Answer::ok(EMPTY));
521        fake.answer("ListMicrovms", Answer::ok(r#"{"items":[{"microvmId":"mvm-a","imageArn":"arn:a","imageVersion":"1","state":"RUNNING"},{"microvmId":"mvm-b","imageArn":"arn:b","imageVersion":"1","state":"RUNNING"}]}"#));
522        fake.answer("ListMicrovms", Answer::ok(r#"{"items":[{"microvmId":"mvm-a","imageArn":"arn:a","imageVersion":"1","state":"TERMINATED"},{"microvmId":"mvm-b","imageArn":"arn:b","imageVersion":"1","state":"TERMINATING"}]}"#));
523        fake.answer("ListMicrovms", Answer::ok(r#"{"items":[{"microvmId":"mvm-b","imageArn":"arn:b","imageVersion":"1","state":"TERMINATED"}]}"#));
524        fake.answer(
525            "ListMicrovms",
526            Answer::failure(403, "both terminations were already confirmed"),
527        );
528        fake.answer("TerminateMicrovm", Answer::ok("{}"));
529        fake.answer("ListMicrovmImageVersions", Answer::ok(EMPTY));
530        fake.answer(
531            "DeleteMicrovmImage",
532            Answer::ok(r#"{"imageIdentifier":"arn:a","state":"DELETING"}"#),
533        );
534        fake.answer(
535            "DeleteMicrovmImage",
536            Answer::ok(r#"{"imageIdentifier":"arn:b","state":"DELETING"}"#),
537        );
538
539        let report = plane
540            .sweep(
541                "job-",
542                &SweepOptions {
543                    dry_run: false,
544                    ..SweepOptions::default()
545                },
546            )
547            .await
548            .unwrap();
549
550        assert_eq!(report.microvms_matched, ["mvm-a", "mvm-b"]);
551        assert_eq!(report.microvms_terminated, ["mvm-a", "mvm-b"]);
552        assert_eq!(report.images_deleted, ["arn:a", "arn:b"]);
553        assert!(report.images_in_use.is_empty());
554        assert!(report.failures.is_empty());
555        assert_eq!(fake.call_count("ListMicrovms"), 3);
556        assert_eq!(fake.call_count("TerminateMicrovm"), 2);
557        assert_eq!(fake.call_count("DeleteMicrovmImage"), 2);
558    }
559
560    #[tokio::test]
561    async fn cleanup_reports_held_images_without_deleting_them() {
562        let (fake, clock, plane) = fixture();
563        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
564        fake.answer("ListMicrovms", Answer::ok(VMS));
565        polls_through_deadline(&fake, VMS);
566        fake.answer("TerminateMicrovm", Answer::failure(403, "denied"));
567        let report = plane
568            .sweep(
569                "job-",
570                &SweepOptions {
571                    keep: vec!["job-keep".into()],
572                    dry_run: false,
573                },
574            )
575            .await
576            .unwrap();
577        assert_eq!(report.images_in_use, ["arn:old"]);
578        assert_eq!(report.failures.len(), 1);
579        assert_eq!(report.failures[0].resource, "mvm-old");
580        assert!(clock.now() >= DEFAULT_LIFECYCLE_TIMEOUT);
581        assert!(report.microvms_terminated.is_empty() && report.images_deleted.is_empty());
582        assert!(
583            fake.calls()
584                .iter()
585                .all(|call| call.operation != "DeleteMicrovmImage")
586        );
587    }
588
589    #[tokio::test]
590    async fn a_missing_vm_never_becomes_permission_to_delete_its_image() {
591        let (fake, _, plane) = fixture();
592        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
593        fake.answer("ListMicrovms", Answer::ok(VMS));
594        polls_through_deadline(&fake, EMPTY);
595        fake.answer("TerminateMicrovm", Answer::ok("{}"));
596        let report = plane
597            .sweep(
598                "job-",
599                &SweepOptions {
600                    keep: vec!["job-keep".into()],
601                    dry_run: false,
602                },
603            )
604            .await
605            .unwrap();
606        assert!(report.microvms_terminated.is_empty());
607        assert_eq!(report.images_in_use, ["arn:old"]);
608        assert!(report.failures.is_empty());
609        assert!(
610            fake.calls()
611                .iter()
612                .all(|call| call.operation != "ListMicrovmImageVersions")
613        );
614    }
615
616    #[tokio::test]
617    async fn polling_errors_preserve_the_inventory_and_prior_outcomes() {
618        let (fake, _, plane) = fixture();
619        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
620        fake.answer("ListMicrovms", Answer::ok(VMS));
621        fake.answer("ListMicrovms", Answer::failure(403, "denied"));
622        fake.answer("TerminateMicrovm", Answer::ok("{}"));
623        let options = SweepOptions {
624            keep: vec!["job-keep".into()],
625            dry_run: false,
626        };
627        let report = plane.sweep("job-", &options).await.unwrap();
628        assert_eq!(report.microvms_matched, ["mvm-old"]);
629        assert_eq!(report.images_in_use, ["arn:old"]);
630        assert_eq!(report.failures[0].resource, "ListMicrovms");
631
632        let (fake, _, plane) = fixture();
633        fake.answer("ListMicrovmImages", Answer::ok(IMAGES));
634        fake.answer("ListMicrovmImages", Answer::failure(403, "denied"));
635        fake.answer("ListMicrovms", Answer::ok(EMPTY));
636        fake.answer("ListMicrovms", Answer::ok(EMPTY));
637        fake.answer(
638            "ListMicrovms",
639            Answer::failure(403, "termination was already confirmed"),
640        );
641        fake.answer("ListMicrovmImageVersions", Answer::ok(EMPTY));
642        fake.answer(
643            "DeleteMicrovmImage",
644            Answer::ok(r#"{"imageIdentifier":"arn:old","state":"DELETING"}"#),
645        );
646        let report = plane.sweep("job-", &options).await.unwrap();
647        assert_eq!(report.images_matched, ["arn:keep", "arn:old"]);
648        assert_eq!(report.failures[0].resource, "ListMicrovmImages");
649        assert!(report.images_deleted.is_empty());
650        assert!(
651            fake.calls()
652                .iter()
653                .any(|call| call.operation == "DeleteMicrovmImage")
654        );
655    }
656
657    #[tokio::test]
658    async fn sweep_refuses_empty_scope_and_repeated_pagination_before_mutation() {
659        let (fake, _, plane) = fixture();
660        assert!(plane.sweep("", &SweepOptions::default()).await.is_err());
661        assert!(fake.calls().is_empty());
662        fake.answer(
663            "ListMicrovmImages",
664            Answer::ok(r#"{"items":[],"nextToken":"again"}"#),
665        );
666        assert!(plane.sweep("job-", &SweepOptions::default()).await.is_err());
667        assert_eq!(fake.calls().len(), 2);
668    }
669}
microvms_app/control/sweep.rs

microvms_app/control/
sweep.rs
