Skip to main content

s4_server/
repair.rs

1//! v0.9 #106: standalone sidecar repair / verify / sweep tooling.
2//!
3//! The S4 server writes a `<key>.s4index` sidecar after every framed PUT so
4//! Range GETs can do a partial fetch instead of streaming the whole body.
5//! Three failure modes leave the sidecar diverged from the live object and
6//! degrade Range GET to the full-read fallback:
7//!
8//! 1. The sidecar PUT failed after the main object committed (network blip,
9//!    backend throttle).
10//! 2. An operator overwrote the object directly through the backend, leaving
11//!    the sidecar stale (ETag / size mismatch with the new body).
12//! 3. The v0.8.15 H-g multipart-Complete-on-Versioning-Enabled bug emitted
13//!    sidecars bound to the parent key while the body landed under the
14//!    versioning shadow path (`<key>.__s4ver__/<id>`). Those orphans never
15//!    re-pair and lifecycle doesn't reap them.
16//!
17//! [`verify_sidecar`] reports the current state without writing,
18//! [`repair_sidecar`] rebuilds a single sidecar by re-scanning the main
19//! body, and [`sweep_orphan_sidecars`] walks every `*.s4index` in a bucket
20//! and reports / deletes the ones whose paired key is missing or stale.
21//!
22//! All three operate directly against an `aws_sdk_s3::Client` (the operator
23//! points it at the backend, not the S4 gateway, because the gateway hides
24//! `.s4index` from list output by design).
25
26use aws_sdk_s3::Client;
27use s4_codec::index::{
28    SIDECAR_SUFFIX, build_index_from_body, decode_index, encode_index, sidecar_key,
29};
30use thiserror::Error;
31
32/// Default cap on bytes loaded into RAM for sidecar rebuild. Matches the
33/// `--max-body-bytes` default (#178, 5 GiB) — repair needs the full body in
34/// memory because `build_index_from_body` is a single-pass scan. Operators
35/// with larger objects pass `--max-body-bytes` to raise this explicitly so a
36/// runaway `repair-sidecar` on a 50 GiB object surfaces a clear error
37/// instead of swapping the host.
38pub const DEFAULT_REPAIR_BODY_BYTES_CAP: u64 = 5 * 1024 * 1024 * 1024;
39
40/// v0.9 #106-audit-R5 P2-R5 (Codex): hard cap on `<key>.s4index` body
41/// bytes read by `verify-sidecar` / `sweep-orphan-sidecars`. The codec
42/// spec bounds a legitimate sidecar at `MAX_FRAMES (16M) * ENTRY_BYTES
43/// (32) + header (≤ 74 B)` ≈ 512 MiB. Any sidecar object larger than
44/// this cap is either an attacker payload aimed at OOM-ing the
45/// operator's repair process or a confused legacy reserved-name user
46/// data file — neither is something we want to load into RAM before
47/// `decode_index` can reject it. 600 MiB leaves a safety margin over
48/// the 512 MiB legitimate ceiling. Operators with anomalously large
49/// LEGITIMATE sidecars (multi-million-frame objects) should raise the
50/// cap explicitly; until then 600 MiB is the safe-by-default value.
51pub const MAX_SIDECAR_BODY_BYTES: u64 = 600 * 1024 * 1024;
52
53/// v0.10 #A1 (Codex round 1): worst-case S4E6 envelope overhead
54/// added on top of a plaintext body. Used by
55/// [`repair_sidecar_with_keyring`] to relax the operator-supplied
56/// `body_bytes_cap` (which is conceptually a plaintext cap matching
57/// the gateway's `--max-body-bytes`) when the SSE-S4 keyring path
58/// is active — without the relaxation, an object whose plaintext
59/// fits the cap can still be rejected as `BodyTooLarge` purely on
60/// the encryption overhead the gateway added at PUT time.
61///
62/// Derivation: an S4E6 envelope is a fixed 24-byte header
63/// ([`crate::sse::S4E6_HEADER_BYTES`]) + a 16-byte AES-GCM tag per
64/// chunk ([`crate::sse::S4E5_PER_CHUNK_OVERHEAD`]), with the
65/// 24-bit chunk-index field capping `chunk_count` at
66/// [`crate::sse::S4E6_MAX_CHUNK_COUNT`] (= `2^24 - 1`). So the
67/// worst-case overhead is
68/// `24 + 16 × (2^24 - 1)` ≈ 256 MiB. Every realistic chunk_size
69/// (1 KiB or larger) drops actual overhead by orders of magnitude;
70/// the worst-case ceiling here is the safe-by-default headroom.
71pub const SSE_S4_REPAIR_MAX_OVERHEAD_BYTES: u64 = (crate::sse::S4E6_HEADER_BYTES as u64)
72    + (crate::sse::S4E6_MAX_CHUNK_COUNT as u64) * (crate::sse::S4E5_PER_CHUNK_OVERHEAD as u64);
73
74/// v0.10 #A1 (Codex round 4): hard ceiling on the final-chunk slack
75/// the SSE-S4 repair path is willing to grant on top of the operator's
76/// `body_bytes_cap` when calling `decrypt_chunked_buffered`. The on-
77/// disk S4E6 header carries an attacker-controlled `chunk_size` field
78/// (only weakly bounded by the body-length consistency check inside
79/// `parse_chunked_header`); trusting that value verbatim would let a
80/// tampered header declare `chunk_size = u32::MAX, chunk_count = 1`
81/// and force `Vec::with_capacity(u32::MAX)` ≈ 4 GiB before the
82/// operator-cap check fires (the helper's cap check is against the
83/// raised `body_bytes_cap + slack` we pass it).
84///
85/// The effective slack is `min(hdr.chunk_size, SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES)`.
86/// 16 MiB comfortably covers the server-default `--sse-chunk-size = 1
87/// MiB` (by 16×) and any practical operator-chosen chunk size; in the
88/// rare case the operator used `--sse-chunk-size > 16 MiB` at PUT
89/// time AND the actual plaintext is within `chunk_size` bytes of the
90/// repair cap, they can simply pass `--max-body-bytes` raised by
91/// `chunk_size` to compensate.
92pub const SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES: u64 = 16 * 1024 * 1024;
93
94#[derive(Debug, Error)]
95pub enum RepairError {
96    #[error("S3 backend error on {op} {bucket}/{key}: {cause}")]
97    Backend {
98        op: &'static str,
99        bucket: String,
100        key: String,
101        // Named `cause` (not `source`) so thiserror doesn't auto-treat it
102        // as a `#[source]` chain field — the upstream SDK error is already
103        // stringified into `cause`.
104        cause: String,
105    },
106    #[error("frame scan failed on {bucket}/{key}: {cause}")]
107    FrameScan {
108        bucket: String,
109        key: String,
110        cause: String,
111    },
112    #[error("object body {size} bytes exceeds repair cap {cap}; pass --max-body-bytes to raise")]
113    BodyTooLarge { size: u64, cap: u64 },
114    /// HEAD on `{bucket}/{key}` returned no `Content-Length` header. The
115    /// body-size cap that prevents OOM on a runaway repair relies on this
116    /// being available, so the tool fails closed rather than treating a
117    /// missing length as zero (which would silently bypass the cap).
118    #[error(
119        "HEAD {bucket}/{key} returned no Content-Length; cannot enforce body cap, refusing to proceed"
120    )]
121    MissingContentLength { bucket: String, key: String },
122    /// `If-Match` race detector: the object was overwritten between the
123    /// initial HEAD (whose ETag we stamped into the sidecar) and the GET.
124    /// Returned by `repair_sidecar` so the operator can re-run instead of
125    /// writing a sidecar that's immediately stale.
126    #[error(
127        "object {bucket}/{key} was overwritten during repair (HEAD ETag {head_etag} != GET response); re-run repair-sidecar"
128    )]
129    OverwrittenDuringRepair {
130        bucket: String,
131        key: String,
132        head_etag: String,
133    },
134    /// v0.9 #106-audit-R5 P2-R5 (Codex): the `<key>.s4index` body
135    /// the backend reports exceeds [`MAX_SIDECAR_BODY_BYTES`], which
136    /// exceeds the codec spec's max legitimate sidecar (~512 MiB).
137    /// Surfaced before the GET to avoid loading a multi-GiB corrupt
138    /// or attacker-supplied `.s4index` blob into the operator's
139    /// repair process (DoS hardening). Operators with anomalously
140    /// large legitimate sidecars (multi-million-frame objects) can
141    /// raise the cap by changing the constant — but the practical
142    /// answer is "treat the underlying object as not-sidecared
143    /// (the GET path already falls back to a full read in that
144    /// case)" rather than chasing larger sidecars.
145    #[error(
146        "sidecar object {bucket}/{key} is {size} bytes (> {cap}-byte cap); refusing to load — \
147         most likely a legacy reserved-name user object or attacker payload aimed at OOM"
148    )]
149    SidecarTooLarge {
150        bucket: String,
151        key: String,
152        size: u64,
153        cap: u64,
154    },
155    /// v0.9 #106-audit-R3 P2-R3: the object body has no S4F2 frame
156    /// magic — it's a passthrough / raw-bytes object the server
157    /// intentionally never sidecared (service.rs::put_object only
158    /// builds a sidecar when `is_framed && !will_encrypt`). Writing
159    /// an empty `<key>.s4index` would silently break Range GET:
160    /// `FrameIndex::lookup_range` over zero entries returns `None`,
161    /// the GET path falls into the "invalid range" branch instead of
162    /// the correct passthrough-range fallback that exists for
163    /// sidecar-less objects. Surface as a typed error so the
164    /// operator knows the object isn't a candidate for sidecar
165    /// repair (and `verify-sidecar` will already classify it as
166    /// `MissingHarmless` with frame_count=0).
167    #[error(
168        "object {bucket}/{key} body has no S4F2 frame magic — it's a passthrough or \
169         raw-bytes object that the server intentionally never sidecared; \
170         sidecar repair would silently break Range GET. No action required."
171    )]
172    NotFramed { bucket: String, key: String },
173    /// v0.9 #106-audit-R2 P2-INT-1 (introduced) / v0.10 #A1 (refined):
174    /// the object body the backend returned is an SSE-S4 encrypted
175    /// envelope (`S4E1`/`S4E2`/`S4E3`/`S4E4`/`S4E5`/`S4E6`) and the
176    /// repair tool either was not given a matching keyring or the
177    /// envelope is not the chunked S4E6 variant the repair tool can
178    /// rebuild from.
179    ///
180    /// `repair_sidecar` runs against the BACKEND (not the gateway), so
181    /// the body it sees is ciphertext — feeding that to the frame
182    /// scanner would surface as a confusing `FrameScan` because the
183    /// S4F2 frame magic is hidden inside the encrypted payload. With a
184    /// keyring + the chunked S4E6 envelope, the new
185    /// [`repair_sidecar_with_keyring`] path decrypts in-process and
186    /// stamps a v3 sidecar carrying the SSE binding (key_id / salt /
187    /// chunk_size / chunk_count / plaintext_len / header_bytes); the
188    /// non-S4E6 envelopes (S4E1/E2/E3/E4/E5) are intentionally out of
189    /// scope (buffered AEAD frames have no per-chunk geometry, and
190    /// SSE-C / SSE-KMS need different key-material plumbing) — they
191    /// surface here so the operator routes those repairs through a
192    /// server-mode rebuild path or re-PUT.
193    #[error(
194        "object {bucket}/{key} body is an SSE-S4 encrypted envelope ({message}); \
195         encrypted-sidecar repair requires the matching SSE-S4 keyring (pass \
196         `--sse-s4-key` / `--sse-s4-key-rotated`) AND the chunked S4E6 envelope; \
197         non-S4E6 envelopes (S4E1/E2/E3/E4/E5) need a server-mode rebuild path \
198         or re-PUT the object to regenerate the sidecar"
199    )]
200    EncryptedSidecarUnsupported {
201        bucket: String,
202        key: String,
203        message: String,
204    },
205    /// v0.10 #A1: a keyring WAS supplied for an SSE-S4 chunked (S4E6)
206    /// object, but parsing the envelope header or decrypting one of
207    /// the AES-GCM chunks failed. The most common cause is a key
208    /// mismatch: the operator's `--sse-s4-key` is not the slot the
209    /// object was encrypted under at PUT time. Other causes are
210    /// envelope truncation / tampering (chunk auth-tag verify fails).
211    /// Surfaced as a distinct variant from `EncryptedSidecarUnsupported`
212    /// so the CLI can give operator-actionable guidance ("check your
213    /// `--sse-s4-key-rotated` list against the PUT-time slot") instead
214    /// of the unsupported-envelope hint.
215    #[error(
216        "SSE-S4 decrypt of {bucket}/{key} failed during sidecar repair: {cause}; \
217         check that `--sse-s4-key` (and any `--sse-s4-key-rotated`) covers the \
218         keyring slot the object was encrypted under at PUT time"
219    )]
220    SseDecryptFailed {
221        bucket: String,
222        key: String,
223        cause: String,
224    },
225}
226
227/// Status reported by [`verify_sidecar`]. Discriminates the outcomes a
228/// CI / cron job needs to branch on. The three `Missing*` variants
229/// resolve the P2-C ambiguity Codex caught: small single-frame objects
230/// intentionally have no sidecar (server only writes when
231/// `entries.len() > 1`), so a blanket `Missing` = exit-1 would false-
232/// alert on healthy objects.
233#[derive(Debug, Clone, PartialEq, Eq)]
234pub enum SidecarStatus {
235    /// Sidecar present, parses cleanly, and its v2 etag + size binding
236    /// matches the live HEAD.
237    Ok { frame_count: u64, sidecar_size: u64 },
238    /// No `<key>.s4index` AND the main body scans as a single frame
239    /// (server skips sidecar emission for `entries.len() <= 1` by
240    /// design). Healthy state — Range GET falls back to a full body
241    /// read, but a single-frame object's "full read" *is* its only
242    /// frame, so there's no fast-path to lose. Exit 0.
243    MissingHarmless { frame_count: u64 },
244    /// No `<key>.s4index` AND the main body has 2+ frames. Range GET
245    /// fast-path is lost; `repair-sidecar` will restore it. Exit 1.
246    MissingDivergent { frame_count: u64 },
247    /// No `<key>.s4index` AND the main object body exceeds the deep-
248    /// scan cap, so we can't tell whether it's a healthy single-frame
249    /// or a real divergence. Operator should raise `--max-body-bytes`
250    /// or run `repair-sidecar` to settle it. Exit 0 (ambiguous, not a
251    /// confirmed divergence — better to under-alert than spam).
252    MissingUnknown { size: u64, cap: u64 },
253    /// Sidecar present but its `source_etag` doesn't match the live HEAD —
254    /// the main object was overwritten or the sidecar is from a different
255    /// commit point.
256    StaleEtag {
257        sidecar_etag: String,
258        live_etag: String,
259    },
260    /// Sidecar present and ETag matches, but the recorded body size differs
261    /// (some backends, e.g. lifecycle moves, change bytes without bumping
262    /// ETag). Treated as stale.
263    StaleSize { sidecar_size: u64, live_size: u64 },
264    /// Pre-v0.8.4 sidecar (no source_etag / source_compressed_size). Still
265    /// usable read-only, but a repair will upgrade it to v2.
266    LegacyV1 { frame_count: u64 },
267    /// Sidecar bytes failed to decode. The body is corrupt or someone PUT
268    /// non-S4IX data at the `.s4index` key. A `repair-sidecar` overwrites
269    /// it cleanly.
270    DecodeError { message: String },
271}
272
273#[derive(Debug, Clone)]
274pub struct VerifyReport {
275    pub bucket: String,
276    pub key: String,
277    pub status: SidecarStatus,
278}
279
280impl VerifyReport {
281    /// True when the sidecar is in a state operators don't need to
282    /// action. Used by the CLI to decide exit code (true → 0, false → 1).
283    /// `MissingHarmless` is clean (single-frame objects have no sidecar
284    /// by design); `MissingUnknown` is also reported clean so the CLI
285    /// doesn't false-alert on objects too large to deep-scan — operator
286    /// can still see the hint in stdout and raise `--max-body-bytes`.
287    pub fn is_clean(&self) -> bool {
288        matches!(
289            self.status,
290            SidecarStatus::Ok { .. }
291                | SidecarStatus::LegacyV1 { .. }
292                | SidecarStatus::MissingHarmless { .. }
293                | SidecarStatus::MissingUnknown { .. }
294        )
295    }
296}
297
298#[derive(Debug, Clone)]
299pub struct RepairReport {
300    pub bucket: String,
301    pub key: String,
302    pub frame_count: u64,
303    pub sidecar_bytes_written: u64,
304    pub source_etag: Option<String>,
305    pub source_compressed_size: u64,
306    /// True when a sidecar already existed (we overwrote it). False when we
307    /// wrote one for the first time.
308    pub rebuilt_from_existing: bool,
309    /// v0.10 #A1: `Some(..)` when the source body was an SSE-S4 chunked
310    /// (S4E6) envelope and the repair tool decrypted it in-process with
311    /// the operator-supplied keyring; the v3 `sse_v3` binding was
312    /// stamped onto the sidecar so Range GETs take the encryption-aware
313    /// partial-fetch fast-path. `None` for plaintext-body repairs (the
314    /// existing v0.9 path) — the sidecar is the v2 layout.
315    pub sse_v3_binding: Option<RepairSseBinding>,
316}
317
318/// v0.10 #A1: distilled view of the SSE-S4 chunked binding stamped by
319/// [`repair_sidecar_with_keyring`]. Mirrors the codec's
320/// `SseChunkBinding` fields the CLI surfaces in its OK line; kept as a
321/// repair-module-local type so `s4-server` callers don't have to import
322/// `s4-codec` just to format the report.
323#[derive(Debug, Clone, Copy, PartialEq, Eq)]
324pub struct RepairSseBinding {
325    pub enc_chunk_size: u32,
326    pub enc_chunk_count: u32,
327    pub enc_key_id: u16,
328    pub enc_plaintext_len: u64,
329    pub enc_header_bytes: u32,
330}
331
332#[derive(Debug, Clone, PartialEq, Eq)]
333pub enum OrphanReason {
334    /// The paired logical key has no HEAD — sidecar is dangling.
335    PairedMissing,
336    /// Paired key exists but the sidecar's recorded ETag is stale.
337    PairedEtagMismatch {
338        sidecar_etag: String,
339        live_etag: String,
340    },
341    /// Paired key exists, ETag matches, but size differs.
342    PairedSizeMismatch { sidecar_size: u64, live_size: u64 },
343    /// The sidecar bytes failed to decode — either corruption or a non-
344    /// sidecar object that happened to land at a `.s4index` key.
345    SidecarUndecodable { message: String },
346}
347
348#[derive(Debug, Clone)]
349pub struct OrphanReport {
350    pub sidecar_key: String,
351    pub paired_key: String,
352    pub reason: OrphanReason,
353}
354
355#[derive(Debug, Clone)]
356pub struct SweepReport {
357    pub bucket: String,
358    pub sidecars_scanned: u64,
359    pub orphans: Vec<OrphanReport>,
360    /// Count actually deleted when `delete = true` was passed. Always 0 in
361    /// dry-run mode.
362    pub deleted: u64,
363}
364
365/// Verify a single `<bucket>/<key>` sidecar without writing.
366///
367/// When the sidecar is absent, this fetches the main body (capped at
368/// `deep_scan_body_cap`) to scan its frame count — single-frame objects
369/// intentionally have no sidecar (server skips emission when
370/// `entries.len() <= 1`), so the absent-sidecar verdict is
371/// `MissingHarmless` for those rather than a false-alert `Missing`.
372/// Pass [`DEFAULT_REPAIR_BODY_BYTES_CAP`] (5 GiB) for the standard CLI
373/// behaviour.
374pub async fn verify_sidecar(
375    client: &Client,
376    bucket: &str,
377    key: &str,
378    deep_scan_body_cap: u64,
379) -> Result<VerifyReport, RepairError> {
380    let HeadInfo {
381        raw_etag: live_raw_etag,
382        normalized_etag: live_etag,
383        size: live_size,
384    } = head_main(client, bucket, key).await?;
385    let sidecar_k = sidecar_key(key);
386    // v0.9 #106-audit-R5 P2-R5 (Codex): bounded sidecar fetch.
387    // A multi-GiB corrupt or legacy reserved-name user `.s4index`
388    // object would OOM the operator's repair process if we did the
389    // naive unbounded GET. Cap on HEAD-reported size.
390    let bytes = match get_sidecar_bytes_capped(client, bucket, &sidecar_k).await {
391        Ok(Some(b)) => b,
392        Ok(None) => {
393            // P2-C (Codex R3): disambiguate Missing via a body scan
394            // before deciding whether this is a healthy single-frame
395            // object or a real divergence.
396            return Ok(VerifyReport {
397                bucket: bucket.into(),
398                key: key.into(),
399                status: classify_missing_sidecar(
400                    client,
401                    bucket,
402                    key,
403                    live_raw_etag.as_deref(),
404                    live_size,
405                    deep_scan_body_cap,
406                )
407                .await?,
408            });
409        }
410        Err(SidecarFetchOutcome::TooLarge { size, cap }) => {
411            return Err(RepairError::SidecarTooLarge {
412                bucket: bucket.into(),
413                key: sidecar_k,
414                size,
415                cap,
416            });
417        }
418        Err(SidecarFetchOutcome::Other(msg)) => {
419            return Err(RepairError::Backend {
420                op: "GET",
421                bucket: bucket.into(),
422                key: sidecar_k,
423                cause: msg,
424            });
425        }
426    };
427    let sidecar_size = bytes.len() as u64;
428    let idx = match decode_index(bytes) {
429        Ok(i) => i,
430        Err(e) => {
431            return Ok(VerifyReport {
432                bucket: bucket.into(),
433                key: key.into(),
434                status: SidecarStatus::DecodeError {
435                    message: e.to_string(),
436                },
437            });
438        }
439    };
440    let frame_count = idx.entries.len() as u64;
441    // P2-D (Codex R4): both sides of the etag comparison are now
442    // `Option<&str>` so an ETag-less backend `None == None` round-trips
443    // as Ok rather than tripping the stale path.
444    //
445    // P3-A (Codex R5): the size-only binding case `(None, Some(z))` is
446    // a fully valid v2 sidecar (just no ETag because the backend
447    // doesn't emit one). Treat any present-size binding as Ok rather
448    // than falling through to `LegacyV1`, which would falsely tell
449    // the operator that `repair-sidecar` could "upgrade" a sidecar
450    // that already IS the v2 it can produce on that backend.
451    // `LegacyV1` is only the true pre-v0.8.4 case where neither
452    // binding field is present.
453    let status = match (idx.source_etag.as_deref(), idx.source_compressed_size) {
454        (Some(side_etag), _) if Some(side_etag) != live_etag.as_deref() => {
455            SidecarStatus::StaleEtag {
456                sidecar_etag: side_etag.into(),
457                live_etag: live_etag.unwrap_or_default(),
458            }
459        }
460        (_, Some(side_size)) if side_size != live_size => SidecarStatus::StaleSize {
461            sidecar_size: side_size,
462            live_size,
463        },
464        // Any present size binding → Ok (covers full v2 AND the
465        // size-only-binding case from ETag-less repair, P3-A).
466        (_, Some(_)) => SidecarStatus::Ok {
467            frame_count,
468            sidecar_size,
469        },
470        // No size binding at all → genuinely legacy v1. Covers both
471        // (None, None) and the anomalous (Some, None) shape (which
472        // encode_index never emits, but match exhaustiveness needs
473        // coverage).
474        (_, None) => SidecarStatus::LegacyV1 { frame_count },
475    };
476    Ok(VerifyReport {
477        bucket: bucket.into(),
478        key: key.into(),
479        status,
480    })
481}
482
483/// Rebuild `<bucket>/<key>.s4index` from the main object body. Overwrites
484/// any existing sidecar (including stale or corrupt ones). Returns an error
485/// when the main body exceeds `body_bytes_cap`.
486///
487/// Back-compat shim around [`repair_sidecar_with_keyring`] for callers that
488/// don't carry an SSE-S4 keyring (= every v0.9 caller). Encrypted bodies
489/// surface as [`RepairError::EncryptedSidecarUnsupported`] same as before.
490pub async fn repair_sidecar(
491    client: &Client,
492    bucket: &str,
493    key: &str,
494    body_bytes_cap: u64,
495) -> Result<RepairReport, RepairError> {
496    repair_sidecar_with_keyring(client, bucket, key, body_bytes_cap, None).await
497}
498
499/// v0.10 #A1: same as [`repair_sidecar`] but optionally accepts an
500/// SSE-S4 keyring. When the on-disk body is an SSE-S4 chunked (S4E6)
501/// envelope AND a keyring is supplied, the repair path:
502///
503/// 1. parses the S4E6 fixed header (`parse_s4e6_header`) to recover
504///    the per-PUT salt / key_id / chunk_size / chunk_count,
505/// 2. decrypts the body in-process via
506///    [`crate::sse::decrypt_chunked_buffered`] (full plaintext into
507///    RAM — repair already had the encrypted body in RAM),
508/// 3. runs the frame scan on the recovered plaintext, and
509/// 4. stamps a v3 sidecar carrying the `sse_v3` chunked binding so
510///    subsequent Range GETs take the encryption-aware partial-fetch
511///    fast-path the v0.9 PUT path already wires.
512///
513/// Non-S4E6 envelopes (S4E1/E2/E3/E4/E5) and missing-keyring cases
514/// fall through to [`RepairError::EncryptedSidecarUnsupported`] — see
515/// the variant doc for the reasoning. Decrypt failures (key mismatch,
516/// chunk-tag verify) surface as the new
517/// [`RepairError::SseDecryptFailed`].
518pub async fn repair_sidecar_with_keyring(
519    client: &Client,
520    bucket: &str,
521    key: &str,
522    body_bytes_cap: u64,
523    sse_keyring: Option<&crate::sse::SharedSseKeyring>,
524) -> Result<RepairReport, RepairError> {
525    let HeadInfo {
526        raw_etag: head_raw_etag,
527        normalized_etag: head_normalized_etag,
528        size: live_size,
529    } = head_main(client, bucket, key).await?;
530    // v0.10 #A1 (Codex R1+R2): `body_bytes_cap` is conceptually a
531    // plaintext-size cap (matches the gateway's `--max-body-bytes`,
532    // which bounds PRE-encrypt PUT bodies).
533    //
534    // - Plaintext bodies: `live_size` IS the body size; enforce the
535    //   cap strictly (`live_size > body_bytes_cap` → `BodyTooLarge`).
536    // - SSE-S4 chunked (S4E6) bodies: the on-disk ciphertext carries
537    //   up to `S4E6_HEADER_BYTES + S4E6_MAX_CHUNK_COUNT × TAG_LEN` ≈
538    //   256 MiB of envelope overhead on top of the plaintext, so
539    //   blindly enforcing the cap against `live_size` would reject
540    //   an object whose plaintext fits the cap purely on encryption
541    //   overhead. Relax the live-size check by the worst-case
542    //   envelope overhead — but ONLY after a 4-byte magic peek
543    //   confirms the body actually IS S4E6 (Codex R2 caught the
544    //   earlier "keyring supplied ⇒ relax cap" shortcut, which
545    //   silently bypassed the operator's RAM cap on plaintext
546    //   objects that happened to be repaired with a keyring set
547    //   for another bucket / key).
548    //
549    // The post-decrypt plaintext is still capped at `body_bytes_cap`
550    // by `decrypt_chunked_buffered` (`max_body_bytes` arg), so the
551    // operator's RAM budget stays bounded by `body_bytes_cap` plus
552    // the envelope overhead even after the relaxation.
553    if live_size > body_bytes_cap {
554        let should_relax = if sse_keyring.is_some()
555            && live_size <= body_bytes_cap.saturating_add(SSE_S4_REPAIR_MAX_OVERHEAD_BYTES)
556        {
557            // The body MIGHT be S4E6 within the overhead headroom —
558            // peek the first 4 bytes to confirm before relaxing. A
559            // single bounded Range GET keeps the cost ≤ 4 bytes of
560            // network + 1 round-trip; far cheaper than the
561            // unconditional relaxation that v0.10 #A1 first shipped.
562            match peek_body_magic(client, bucket, key, head_raw_etag.as_deref()).await {
563                Ok(Some(magic)) => &magic == b"S4E6",
564                Ok(None) => false,
565                Err(e) => {
566                    return Err(RepairError::Backend {
567                        op: "GET",
568                        bucket: bucket.into(),
569                        key: key.into(),
570                        cause: format!("4-byte magic peek for SSE-overhead-cap relaxation: {e}"),
571                    });
572                }
573            }
574        } else {
575            false
576        };
577        if !should_relax {
578            // Report the operator-supplied cap (not the relaxed one)
579            // so the actionable guidance stays "pass --max-body-bytes
580            // to raise".
581            return Err(RepairError::BodyTooLarge {
582                size: live_size,
583                cap: body_bytes_cap,
584            });
585        }
586    }
587    // v0.9 #106 TOCTOU guard: pin the GET to the HEAD's ETag via If-Match.
588    // Without this, an overwrite between HEAD and GET would yield a body
589    // whose actual ETag is E2 while we stamp `source_etag = E1`, producing
590    // a sidecar that fails its own version-binding check on the very next
591    // Range GET (operator sees "repair succeeded" then nothing changed).
592    // Backend returns 412 PreconditionFailed if the object changed.
593    //
594    // P1-B (Codex review R1): pass the RAW etag (quoted entity-tag) per
595    // RFC 7232, not the normalized form. Strict S3-compatible backends
596    // reject `If-Match: abc-2` (missing quotes) with 400/412 and the
597    // repair never succeeds. Tolerant backends accept either. The
598    // sidecar's stored `source_etag` still uses the normalized form to
599    // match the server's PUT-path stamping convention.
600    //
601    // P2-D (Codex R4): when the backend doesn't return an ETag at all,
602    // skip `If-Match` entirely. Same posture the server takes in that
603    // case (it stamps `source_etag = None`); the race window stays open
604    // for those backends, but they don't have an ETag we could pin
605    // against anyway.
606    let get_builder = client.get_object().bucket(bucket).key(key);
607    let get_builder = match &head_raw_etag {
608        Some(t) => get_builder.if_match(t.clone()),
609        None => get_builder,
610    };
611    let body = match get_builder.send().await {
612        Ok(resp) => resp
613            .body
614            .collect()
615            .await
616            .map(|agg| agg.into_bytes())
617            .map_err(|e| RepairError::Backend {
618                op: "GET",
619                bucket: bucket.into(),
620                key: key.into(),
621                cause: format!("read body: {e}"),
622            })?,
623        Err(e) => {
624            // PreconditionFailed (412) → object was overwritten between
625            // HEAD and GET. Surface as a typed error so the operator can
626            // re-run instead of writing a stale sidecar.
627            let s = format!("{e}");
628            if s.contains("PreconditionFailed") || s.contains("412") {
629                return Err(RepairError::OverwrittenDuringRepair {
630                    bucket: bucket.into(),
631                    key: key.into(),
632                    head_etag: head_normalized_etag.clone().unwrap_or_default(),
633                });
634            }
635            if is_get_not_found(&e) {
636                return Err(RepairError::Backend {
637                    op: "GET",
638                    bucket: bucket.into(),
639                    key: key.into(),
640                    cause: "object not found (NoSuchKey)".into(),
641                });
642            }
643            return Err(RepairError::Backend {
644                op: "GET",
645                bucket: bucket.into(),
646                key: key.into(),
647                cause: s,
648            });
649        }
650    };
651    // Defense in depth: even with If-Match, double-check the bytes we got
652    // are the size HEAD promised. Backends with quirky range / cache
653    // behaviour have surprised us before — see codec memo on partial
654    // serves that succeeded with the wrong length.
655    if (body.len() as u64) != live_size {
656        return Err(RepairError::Backend {
657            op: "GET",
658            bucket: bucket.into(),
659            key: key.into(),
660            cause: format!(
661                "got {} bytes but HEAD said {}; backend served wrong content length",
662                body.len(),
663                live_size
664            ),
665        });
666    }
667    // v0.9 #106-audit-R2 P2-INT-1 (initial) / v0.10 #A1 (keyring
668    // plumbing): detect SSE-S4 encrypted envelopes BEFORE handing the
669    // body to the frame scanner. The backend serves the on-disk
670    // ciphertext (S4E1..S4E6 magic prefix); `build_index_from_body`
671    // would scan for `S4F2` frame magic inside that ciphertext and
672    // surface an opaque `FrameScan` error.
673    //
674    // - `S4E6` + keyring supplied → decrypt in-process, frame-scan the
675    //   plaintext, stamp a v3 sidecar with the chunked binding so
676    //   Range GETs take the encryption-aware fast-path.
677    // - `S4E6` + no keyring → `EncryptedSidecarUnsupported` directing
678    //   the operator to pass `--sse-s4-key`.
679    // - Any non-S4E6 envelope (S4E1/E2/E3/E4/E5) → unsupported,
680    //   `EncryptedSidecarUnsupported`; buffered AEAD frames have no
681    //   per-chunk geometry and SSE-C / SSE-KMS need different
682    //   key-material plumbing (both deferred to v0.11+).
683    let sse_repair: Option<(bytes::Bytes, s4_codec::index::SseChunkBinding)> =
684        match detect_sse_magic(&body) {
685            Some("S4E6") => match sse_keyring {
686                Some(keyring) => {
687                    let (plaintext, binding) =
688                        decrypt_s4e6_for_repair(&body, keyring, body_bytes_cap, bucket, key)?;
689                    Some((plaintext, binding))
690                }
691                None => {
692                    return Err(RepairError::EncryptedSidecarUnsupported {
693                        bucket: bucket.into(),
694                        key: key.into(),
695                        message: "body magic S4E6 indicates SSE-S4 envelope; \
696                              pass `--sse-s4-key` to decrypt and rebuild the v3 sidecar"
697                            .into(),
698                    });
699                }
700            },
701            Some(magic) => {
702                return Err(RepairError::EncryptedSidecarUnsupported {
703                    bucket: bucket.into(),
704                    key: key.into(),
705                    message: format!(
706                        "body magic {magic} indicates SSE-S4 envelope (only chunked S4E6 is \
707                     repair-supported; buffered / SSE-C / SSE-KMS envelopes need a \
708                     server-mode rebuild path)"
709                    ),
710                });
711            }
712            None => None,
713        };
714    // Pick the body the frame scanner walks. For non-encrypted bodies
715    // this is the backend bytes we just GET'd; for S4E6 + keyring it
716    // is the decrypted plaintext (= the post-compression, pre-encrypt
717    // body the v0.9 PUT path stamps the sidecar against).
718    let scan_body: &bytes::Bytes = match &sse_repair {
719        Some((plaintext, _)) => plaintext,
720        None => &body,
721    };
722    let sidecar_k = sidecar_key(key);
723    let rebuilt_from_existing = client
724        .head_object()
725        .bucket(bucket)
726        .key(&sidecar_k)
727        .send()
728        .await
729        .is_ok();
730    let mut idx = build_index_from_body(scan_body).map_err(|e| RepairError::FrameScan {
731        bucket: bucket.into(),
732        key: key.into(),
733        cause: e.to_string(),
734    })?;
735    // v0.9 #106-audit-R3 P2-R3 (Codex): `build_index_from_body`
736    // on a non-S4F2 body (passthrough / raw bytes) returns Ok with
737    // an empty entries vec rather than an error. Writing that as a
738    // sidecar would silently break Range GET — `lookup_range` over
739    // zero entries returns None, and the GET path then takes the
740    // "no plan" branch instead of the passthrough-range fallback
741    // that exists for sidecar-less objects. Reject cleanly so the
742    // operator knows the object isn't a sidecar-repair candidate.
743    if idx.entries.is_empty() {
744        return Err(RepairError::NotFramed {
745            bucket: bucket.into(),
746            key: key.into(),
747        });
748    }
749    // Stamp the NORMALIZED form so server-side
750    // `sidecar_version_binding_ok` (which compares against the s3s
751    // `ETag::value()` stripped form) sees a match. The raw form was
752    // only needed for the wire-level `If-Match` header above.
753    //
754    // P2-D (Codex R4): pass through `None` when the backend doesn't
755    // return an ETag — the server's binding check treats `None` as
756    // the legacy/back-compat best-effort path. Stamping `Some("")`
757    // would force the check into the mismatch branch and the sidecar
758    // would be immediately rejected as stale.
759    idx.source_etag = head_normalized_etag.clone();
760    idx.source_compressed_size = Some(body.len() as u64);
761    // v0.10 #A1: stamp the SSE-S4 chunked binding so the GET path takes
762    // the encryption-aware partial-fetch fast-path. Mirrors the v0.9
763    // PUT path's binding construction (service.rs `sse_binding`); the
764    // salt is not secret (it lives in the encrypted body's plaintext
765    // header anyway), so duplicating it in the sidecar saves the GET
766    // path an extra HEAD/GET round-trip per Range request.
767    if let Some((_plaintext, binding)) = sse_repair.as_ref() {
768        idx.sse_v3 = Some(*binding);
769    }
770    let encoded = encode_index(&idx);
771    let encoded_len = encoded.len() as u64;
772    let frame_count = idx.entries.len() as u64;
773    client
774        .put_object()
775        .bucket(bucket)
776        .key(&sidecar_k)
777        .body(aws_sdk_s3::primitives::ByteStream::from(encoded.to_vec()))
778        .content_type("application/x-s4-index")
779        .send()
780        .await
781        .map_err(|e| RepairError::Backend {
782            op: "PUT",
783            bucket: bucket.into(),
784            key: sidecar_k.clone(),
785            cause: format!("{e}"),
786        })?;
787    // v0.9 #106 P2-B (Codex review round 2): `If-Match` on the GET
788    // only proves the body hadn't changed at GET time. The main object
789    // can still be overwritten during the (a) build_index_from_body
790    // scan and (b) sidecar PUT window — leaving a freshly-written
791    // sidecar stamped with the OLD ETag against the NEW body. The
792    // server-side `sidecar_version_binding_ok` would then trip on
793    // every Range GET and we'd silently report "repair succeeded".
794    //
795    // Final HEAD: if the main object's ETag changed since we read it,
796    // the sidecar we just wrote is already stale. Delete it (so the
797    // operator's next Range GET falls back to the safe full-read path,
798    // not the bad fast-path) and surface `OverwrittenDuringRepair`
799    // so the operator re-runs the repair under quieter conditions.
800    let post = head_main(client, bucket, key).await?;
801    if post.normalized_etag != head_normalized_etag || post.size != live_size {
802        // Best-effort cleanup; ignore the delete's outcome because the
803        // primary error is the race, not the cleanup itself.
804        let _ = client
805            .delete_object()
806            .bucket(bucket)
807            .key(&sidecar_k)
808            .send()
809            .await;
810        return Err(RepairError::OverwrittenDuringRepair {
811            bucket: bucket.into(),
812            key: key.into(),
813            head_etag: head_normalized_etag.unwrap_or_default(),
814        });
815    }
816    let sse_v3_binding = sse_repair.as_ref().map(|(_, b)| RepairSseBinding {
817        enc_chunk_size: b.enc_chunk_size,
818        enc_chunk_count: b.enc_chunk_count,
819        enc_key_id: b.enc_key_id,
820        enc_plaintext_len: b.enc_plaintext_len,
821        enc_header_bytes: b.enc_header_bytes,
822    });
823    Ok(RepairReport {
824        bucket: bucket.into(),
825        key: key.into(),
826        frame_count,
827        sidecar_bytes_written: encoded_len,
828        source_etag: idx.source_etag,
829        source_compressed_size: live_size,
830        rebuilt_from_existing,
831        sse_v3_binding,
832    })
833}
834
835/// v0.10 #A1 (Codex R2): peek the first 4 bytes of the main object
836/// via a single Range GET (`Range: bytes=0-3`) so the body-cap
837/// relaxation in [`repair_sidecar_with_keyring`] can confirm the
838/// body is an S4E6 envelope BEFORE allowing the worst-case
839/// envelope-overhead headroom on top of `body_bytes_cap`. Returns:
840///
841/// - `Ok(Some([4 bytes]))` when the GET succeeded and the body had
842///   at least 4 bytes,
843/// - `Ok(None)` when the GET succeeded but the body was shorter
844///   than 4 bytes (= no magic to test against; caller treats as
845///   not-S4E6 and fails closed),
846/// - `Err(_)` on any backend error — caller surfaces as
847///   `RepairError::Backend` so the operator sees the underlying
848///   cause (network, permissions, etc.).
849///
850/// Pinned to the HEAD's `If-Match` ETag (when available) so a
851/// concurrent swap between the HEAD and this peek can't be used to
852/// bypass the cap (small object swapped in to pass the magic check,
853/// then the full GET delivers the original oversized ciphertext).
854async fn peek_body_magic(
855    client: &Client,
856    bucket: &str,
857    key: &str,
858    if_match_raw: Option<&str>,
859) -> Result<Option<[u8; 4]>, String> {
860    let get_builder = client
861        .get_object()
862        .bucket(bucket)
863        .key(key)
864        .range("bytes=0-3");
865    let get_builder = match if_match_raw {
866        Some(t) => get_builder.if_match(t.to_owned()),
867        None => get_builder,
868    };
869    let resp = get_builder.send().await.map_err(|e| format!("{e}"))?;
870    let bytes = resp
871        .body
872        .collect()
873        .await
874        .map(|agg| agg.into_bytes())
875        .map_err(|e| format!("read peek body: {e}"))?;
876    if bytes.len() < 4 {
877        return Ok(None);
878    }
879    let mut magic = [0u8; 4];
880    magic.copy_from_slice(&bytes[..4]);
881    Ok(Some(magic))
882}
883
884/// v0.10 #A1: decrypt an S4E6-magic body in-process so
885/// [`repair_sidecar_with_keyring`] can frame-scan the recovered
886/// plaintext and stamp a v3 sidecar binding. Returns the plaintext
887/// bytes paired with the [`s4_codec::index::SseChunkBinding`] reflecting
888/// the on-disk envelope geometry — mirrors the v0.9 PUT-path binding
889/// construction in `service.rs::put_object` so a sidecar rebuilt here
890/// is byte-equivalent (modulo ETag bind) to one written at PUT time.
891///
892/// `body_bytes_cap` is reused as the plaintext-size cap fed to
893/// [`crate::sse::decrypt_chunked_buffered`]; on 64-bit it saturates at
894/// `usize::MAX` so a 5 GiB cap survives the conversion intact. The
895/// outer `repair_sidecar_with_keyring` already enforced the same cap
896/// against the on-disk ciphertext (`live_size > body_bytes_cap` →
897/// `BodyTooLarge`), so by the time we land here the plaintext can only
898/// be marginally smaller than the cap — passing the same value through
899/// keeps the failure mode consistent.
900fn decrypt_s4e6_for_repair(
901    body: &[u8],
902    keyring: &crate::sse::SharedSseKeyring,
903    body_bytes_cap: u64,
904    bucket: &str,
905    key: &str,
906) -> Result<(bytes::Bytes, s4_codec::index::SseChunkBinding), RepairError> {
907    let hdr = crate::sse::parse_s4e6_header(body).map_err(|e| RepairError::SseDecryptFailed {
908        bucket: bucket.into(),
909        key: key.into(),
910        cause: format!("parse S4E6 header: {e}"),
911    })?;
912    // v0.10 #A1 (Codex R3 + R4): `decrypt_chunked_buffered` enforces
913    // its `max_body_bytes` against the DECLARED upper bound
914    // `chunk_size × chunk_count` — not the actual plaintext length.
915    // Because the final chunk may be partial (encoder uses
916    // `ceil(plaintext / chunk_size)`), declared can exceed actual
917    // by up to `chunk_size - 1` bytes. Passing `body_bytes_cap`
918    // verbatim therefore wrongly rejects an object whose ACTUAL
919    // plaintext fits the cap but whose final-chunk slack pushes
920    // declared above it. Bump the decrypt cap by one `chunk_size`
921    // worth of slack, then re-verify the actual plaintext length
922    // against the operator-supplied cap after decrypt.
923    //
924    // Codex R4: the on-disk `hdr.chunk_size` is attacker-controlled
925    // (a tampered S4E6 header could declare `chunk_size = u32::MAX`
926    // and `chunk_count = 1`, then trick the buffered decrypt's
927    // `Vec::with_capacity(chunk_size × chunk_count)` into a ~4 GiB
928    // allocation under a much smaller operator cap). Cap the slack
929    // at the trusted [`SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES`]
930    // ceiling so the worst-case RAM increase from this slack is
931    // bounded regardless of what the on-disk header claims.
932    let chunk_slack = (hdr.chunk_size as u64).min(SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES);
933    let cap_with_slack = body_bytes_cap.saturating_add(chunk_slack);
934    let cap_usize: usize = cap_with_slack.min(usize::MAX as u64) as usize;
935    let plaintext = crate::sse::decrypt_chunked_buffered(body, keyring.as_ref(), cap_usize)
936        .map_err(|e| RepairError::SseDecryptFailed {
937            bucket: bucket.into(),
938            key: key.into(),
939            cause: format!("decrypt S4E6 chunks: {e}"),
940        })?;
941    // Operator cap applies to actual plaintext bytes, not declared
942    // upper bound. Surfaces as the same `BodyTooLarge` variant the
943    // outer ciphertext-cap path uses so the CLI's error formatting
944    // stays consistent.
945    if (plaintext.len() as u64) > body_bytes_cap {
946        return Err(RepairError::BodyTooLarge {
947            size: plaintext.len() as u64,
948            cap: body_bytes_cap,
949        });
950    }
951    let binding = s4_codec::index::SseChunkBinding {
952        enc_chunk_size: hdr.chunk_size,
953        enc_chunk_count: hdr.chunk_count,
954        enc_key_id: hdr.key_id,
955        enc_salt: *hdr.salt,
956        enc_plaintext_len: plaintext.len() as u64,
957        // Carried explicitly so a future S4E7-style header bump can't
958        // silently break v3 sidecar decode (mirrors the v0.9 PUT-path
959        // stamp at `service.rs::put_object` `enc_header_bytes`).
960        enc_header_bytes: crate::sse::S4E6_HEADER_BYTES as u32,
961    };
962    Ok((plaintext, binding))
963}
964
965/// Knob controlling which orphan categories `sweep_orphan_sidecars` is
966/// allowed to delete. `SidecarUndecodable` is kept out of the default
967/// `--delete` because v0.8.17-era operators on the
968/// `--allow-legacy-reserved-key-reads` migration hatch can have
969/// legitimate user-PUT objects whose key happens to end in `.s4index` —
970/// those would fail to decode and `--delete` would nuke real user data.
971/// Escalation to `DeletePolicy::IncludeUndecodable` is an explicit
972/// operator opt-in (`--delete-undecodable` on the CLI).
973#[derive(Debug, Clone, Copy, PartialEq, Eq)]
974pub enum DeletePolicy {
975    /// Pure dry-run: classify only, never write to the backend.
976    DryRun,
977    /// Delete `PairedMissing` / `PairedEtagMismatch` / `PairedSizeMismatch`
978    /// orphans. Leave `SidecarUndecodable` in the report — operator must
979    /// inspect those and rerun with `IncludeUndecodable` if they truly
980    /// are corrupt sidecars (and not legacy reserved-name user data).
981    PairBoundOnly,
982    /// All four categories. Use only after confirming there's no legacy
983    /// `--allow-legacy-reserved-key-reads` user data in this bucket.
984    IncludeUndecodable,
985}
986
987impl DeletePolicy {
988    fn allows(&self, reason: &OrphanReason) -> bool {
989        match (self, reason) {
990            (DeletePolicy::DryRun, _) => false,
991            (DeletePolicy::PairBoundOnly, OrphanReason::SidecarUndecodable { .. }) => false,
992            (DeletePolicy::PairBoundOnly, _) => true,
993            (DeletePolicy::IncludeUndecodable, _) => true,
994        }
995    }
996}
997
998/// List every `*.s4index` in `bucket` and report (and optionally delete) the
999/// orphans — sidecars whose paired key is missing or whose recorded
1000/// ETag / size disagree with the live HEAD.
1001///
1002/// See [`DeletePolicy`] for the three deletion levels. Always run
1003/// [`DeletePolicy::DryRun`] first to inspect the orphan list.
1004pub async fn sweep_orphan_sidecars(
1005    client: &Client,
1006    bucket: &str,
1007    policy: DeletePolicy,
1008) -> Result<SweepReport, RepairError> {
1009    let mut sidecars_scanned: u64 = 0;
1010    let mut orphans: Vec<OrphanReport> = Vec::new();
1011    let mut continuation: Option<String> = None;
1012    loop {
1013        let mut req = client.list_objects_v2().bucket(bucket);
1014        if let Some(c) = continuation.as_ref() {
1015            req = req.continuation_token(c);
1016        }
1017        let resp = req.send().await.map_err(|e| RepairError::Backend {
1018            op: "ListObjectsV2",
1019            bucket: bucket.into(),
1020            key: String::new(),
1021            cause: format!("{e}"),
1022        })?;
1023        for obj in resp.contents() {
1024            let Some(k) = obj.key() else { continue };
1025            if !k.ends_with(SIDECAR_SUFFIX) {
1026                continue;
1027            }
1028            sidecars_scanned += 1;
1029            let paired = &k[..k.len() - SIDECAR_SUFFIX.len()];
1030            classify_one(client, bucket, k, paired, &mut orphans).await?;
1031        }
1032        if resp.is_truncated().unwrap_or(false) {
1033            continuation = resp.next_continuation_token().map(str::to_owned);
1034            if continuation.is_none() {
1035                // Defensive: a truncated response with no continuation token
1036                // is a backend bug; bail rather than infinite-loop.
1037                break;
1038            }
1039        } else {
1040            break;
1041        }
1042    }
1043    let mut deleted = 0u64;
1044    for orph in &orphans {
1045        if !policy.allows(&orph.reason) {
1046            continue;
1047        }
1048        client
1049            .delete_object()
1050            .bucket(bucket)
1051            .key(&orph.sidecar_key)
1052            .send()
1053            .await
1054            .map_err(|e| RepairError::Backend {
1055                op: "DELETE",
1056                bucket: bucket.into(),
1057                key: orph.sidecar_key.clone(),
1058                cause: format!("{e}"),
1059            })?;
1060        deleted += 1;
1061    }
1062    Ok(SweepReport {
1063        bucket: bucket.into(),
1064        sidecars_scanned,
1065        orphans,
1066        deleted,
1067    })
1068}
1069
1070/// P2-C (Codex R3): the server skips sidecar emission for objects whose
1071/// frame count is ≤ 1 (small single-PUTs / single-chunk multiparts), so
1072/// a missing sidecar can be EITHER an intentional skip OR a real
1073/// divergence. Disambiguate by fetching the body (capped) and counting
1074/// frames. Returns [`SidecarStatus::MissingUnknown`] when the body
1075/// exceeds the cap, so verify-sidecar doesn't false-alert on
1076/// large-but-can't-confirm objects.
1077async fn classify_missing_sidecar(
1078    client: &Client,
1079    bucket: &str,
1080    key: &str,
1081    live_raw_etag: Option<&str>,
1082    live_size: u64,
1083    cap: u64,
1084) -> Result<SidecarStatus, RepairError> {
1085    if live_size > cap {
1086        return Ok(SidecarStatus::MissingUnknown {
1087            size: live_size,
1088            cap,
1089        });
1090    }
1091    // Pin the GET to the HEAD's ETag (RFC 7232 quoted form). If a race
1092    // overwrites the object between HEAD and GET we'd otherwise scan a
1093    // different body than the one HEAD reported on — surface as a
1094    // typed error so the operator re-runs.
1095    //
1096    // P2-D: backends without an ETag have nothing to pin against;
1097    // skip If-Match (matches the server-side `None`-tolerance path).
1098    let get_builder = client.get_object().bucket(bucket).key(key);
1099    let get_builder = match live_raw_etag {
1100        Some(t) => get_builder.if_match(t.to_owned()),
1101        None => get_builder,
1102    };
1103    let body = match get_builder.send().await {
1104        Ok(resp) => resp
1105            .body
1106            .collect()
1107            .await
1108            .map(|agg| agg.into_bytes())
1109            .map_err(|e| RepairError::Backend {
1110                op: "GET",
1111                bucket: bucket.into(),
1112                key: key.into(),
1113                cause: format!("read body: {e}"),
1114            })?,
1115        Err(e) => {
1116            let s = format!("{e}");
1117            if s.contains("PreconditionFailed") || s.contains("412") {
1118                return Err(RepairError::OverwrittenDuringRepair {
1119                    bucket: bucket.into(),
1120                    key: key.into(),
1121                    head_etag: live_raw_etag.map(normalize_etag).unwrap_or_default(),
1122                });
1123            }
1124            if is_get_not_found(&e) {
1125                return Err(RepairError::Backend {
1126                    op: "GET",
1127                    bucket: bucket.into(),
1128                    key: key.into(),
1129                    cause: "object not found (NoSuchKey)".into(),
1130                });
1131            }
1132            return Err(RepairError::Backend {
1133                op: "GET",
1134                bucket: bucket.into(),
1135                key: key.into(),
1136                cause: s,
1137            });
1138        }
1139    };
1140    // v0.9 #106-audit self-review (post-R2): mirror the encrypted-body
1141    // guard from `repair_sidecar` here. Without it, running
1142    // `verify-sidecar` against an SSE-S4 chunked object (whose sidecar
1143    // is missing — e.g. PUT happened pre-v0.9 before v3 sidecars
1144    // shipped) would surface as a confusing FrameScan error instead of
1145    // the friendly EncryptedSidecarUnsupported the repair tool already
1146    // returns. Same root cause as P2-INT-1; same surface error.
1147    if let Some(magic) = detect_sse_magic(&body) {
1148        return Err(RepairError::EncryptedSidecarUnsupported {
1149            bucket: bucket.into(),
1150            key: key.into(),
1151            message: format!("body magic {magic} indicates SSE-S4 envelope"),
1152        });
1153    }
1154    // v0.9 #106-audit-R4 P2-R4 (Codex): a passthrough / raw-bytes
1155    // body (no S4F2 magic) trips `build_index_from_body` with a
1156    // `BadMagic` `FrameError`. From the verify-sidecar perspective
1157    // that's the same outcome as a single-frame body: server never
1158    // sidecared it, Range GET takes the full-read path, no operator
1159    // action needed. Surface `MissingHarmless { frame_count: 0 }`
1160    // (clean, exit 0) instead of a FrameScan repair error (exit 1)
1161    // so CI / cron jobs don't false-alert on healthy passthrough
1162    // objects. Twin of R3 P2-R3 on the repair-side.
1163    let idx = match build_index_from_body(&body) {
1164        Ok(i) => i,
1165        Err(crate::codec::multipart::FrameError::BadMagic { .. }) => {
1166            return Ok(SidecarStatus::MissingHarmless { frame_count: 0 });
1167        }
1168        Err(e) => {
1169            return Err(RepairError::FrameScan {
1170                bucket: bucket.into(),
1171                key: key.into(),
1172                cause: e.to_string(),
1173            });
1174        }
1175    };
1176    let frame_count = idx.entries.len() as u64;
1177    if frame_count <= 1 {
1178        Ok(SidecarStatus::MissingHarmless { frame_count })
1179    } else {
1180        Ok(SidecarStatus::MissingDivergent { frame_count })
1181    }
1182}
1183
1184async fn classify_one(
1185    client: &Client,
1186    bucket: &str,
1187    sidecar_k: &str,
1188    paired: &str,
1189    out: &mut Vec<OrphanReport>,
1190) -> Result<(), RepairError> {
1191    // v0.9 #106 review P1-A (Codex): MUST decode the listed object first.
1192    // Branching on "HEAD paired-key" before reading the candidate would
1193    // mis-classify a legitimate `--allow-legacy-reserved-key-reads`
1194    // user object (whose key happens to end in `.s4index` and whose
1195    // paired stripped key may not exist) as `PairedMissing` — and
1196    // `DeletePolicy::PairBoundOnly` would silently delete user data.
1197    // The rule is: bytes that don't parse as S4IX magic = user data,
1198    // never an orphan-eligible-for-default-delete.
1199    // v0.9 #106-audit-R5 P2-R5 (Codex): bounded sidecar fetch.
1200    // sweep walks every `*.s4index` in the bucket — a single
1201    // multi-GiB attacker-supplied or legacy-user `.s4index` object
1202    // would OOM the sweep process with the naive unbounded GET.
1203    // TooLarge surfaces as a `SidecarUndecodable` orphan with a
1204    // size-explaining message rather than aborting the whole sweep
1205    // (one bad sidecar shouldn't stop the rest from being inspected).
1206    let bytes = match get_sidecar_bytes_capped(client, bucket, sidecar_k).await {
1207        Ok(Some(b)) => b,
1208        // ListObjectsV2 saw it; if GET says NotFound now, treat as a
1209        // sidecar that vanished mid-sweep — skip rather than report.
1210        Ok(None) => return Ok(()),
1211        Err(SidecarFetchOutcome::TooLarge { size, cap }) => {
1212            out.push(OrphanReport {
1213                sidecar_key: sidecar_k.into(),
1214                paired_key: paired.into(),
1215                reason: OrphanReason::SidecarUndecodable {
1216                    message: format!(
1217                        "sidecar size {size} > cap {cap}; refused to load (likely legacy user data or attack payload)"
1218                    ),
1219                },
1220            });
1221            return Ok(());
1222        }
1223        Err(SidecarFetchOutcome::Other(msg)) => {
1224            return Err(RepairError::Backend {
1225                op: "GET",
1226                bucket: bucket.into(),
1227                key: sidecar_k.into(),
1228                cause: msg,
1229            });
1230        }
1231    };
1232    let idx = match decode_index(bytes) {
1233        Ok(i) => i,
1234        Err(e) => {
1235            // Not a real S4IX sidecar — flag it under the safer
1236            // category. `DeletePolicy::PairBoundOnly` does NOT remove
1237            // these; the operator must escalate to
1238            // `IncludeUndecodable` after confirming it isn't legacy
1239            // user data.
1240            out.push(OrphanReport {
1241                sidecar_key: sidecar_k.into(),
1242                paired_key: paired.into(),
1243                reason: OrphanReason::SidecarUndecodable {
1244                    message: e.to_string(),
1245                },
1246            });
1247            return Ok(());
1248        }
1249    };
1250    // Bytes decoded as S4IX — now we can safely check the paired key
1251    // status. A missing paired key combined with a decodable sidecar
1252    // IS a real orphan (the v0.8.15 H-g case, for example).
1253    let head_res = client.head_object().bucket(bucket).key(paired).send().await;
1254    let (live_etag_norm, live_size) = match head_res {
1255        Ok(h) => {
1256            // P2-D: `None` means the backend didn't return an ETag.
1257            // Preserve the absence rather than coercing to `""` —
1258            // comparing `Some("xyz")` from the sidecar against
1259            // `Some("")` would always trip stale, falsely orphaning
1260            // every paired-OK sidecar on an ETag-less backend.
1261            let etag: Option<String> = h.e_tag().map(normalize_etag);
1262            let size = h.content_length().unwrap_or(0).max(0) as u64;
1263            (etag, size)
1264        }
1265        Err(e) => {
1266            if is_head_not_found(&e) {
1267                out.push(OrphanReport {
1268                    sidecar_key: sidecar_k.into(),
1269                    paired_key: paired.into(),
1270                    reason: OrphanReason::PairedMissing,
1271                });
1272                return Ok(());
1273            }
1274            return Err(RepairError::Backend {
1275                op: "HEAD",
1276                bucket: bucket.into(),
1277                key: paired.into(),
1278                cause: format!("{e}"),
1279            });
1280        }
1281    };
1282    // ETag mismatch only fires when BOTH sides have an ETag. If the
1283    // sidecar carries Some("x") and the live HEAD has None, that's
1284    // not a definitive divergence — could be a backend that recently
1285    // dropped ETag support. Skip the mismatch flag for the None side
1286    // (matches the server's `sidecar_version_binding_ok` `None`-
1287    // tolerance posture).
1288    if let (Some(side_etag), Some(live_e)) = (idx.source_etag.as_deref(), live_etag_norm.as_deref())
1289        && side_etag != live_e
1290    {
1291        out.push(OrphanReport {
1292            sidecar_key: sidecar_k.into(),
1293            paired_key: paired.into(),
1294            reason: OrphanReason::PairedEtagMismatch {
1295                sidecar_etag: side_etag.into(),
1296                live_etag: live_e.into(),
1297            },
1298        });
1299        return Ok(());
1300    }
1301    if let Some(side_size) = idx.source_compressed_size
1302        && side_size != live_size
1303    {
1304        out.push(OrphanReport {
1305            sidecar_key: sidecar_k.into(),
1306            paired_key: paired.into(),
1307            reason: OrphanReason::PairedSizeMismatch {
1308                sidecar_size: side_size,
1309                live_size,
1310            },
1311        });
1312    }
1313    // Legacy v1 sidecars (no binding fields) are intentionally
1314    // tolerated here — read-only Range GETs still work and the
1315    // operator gets warned by `verify-sidecar` separately.
1316    Ok(())
1317}
1318
1319/// HEAD response distilled to the fields the repair tools care about.
1320///
1321/// Both etag fields are `Option<String>` so the absent-ETag case
1322/// round-trips cleanly through to the sidecar (P2-D, Codex R4). When
1323/// `raw_etag = None`, the backend didn't return one — we MUST stamp
1324/// `FrameIndex::source_etag = None` to match the server PUT path's
1325/// `resp.e_tag.as_ref().map(...)` shape, otherwise
1326/// `sidecar_version_binding_ok` would compare `Some("")` against a
1327/// missing live ETag and always trip "stale".
1328///
1329/// - `raw_etag`: wire form (typically `"..."`) — pass to `If-Match`
1330///   headers, which per RFC 7232 want the full entity-tag. `None`
1331///   means skip `If-Match` entirely (best-effort, same posture the
1332///   server takes for ETag-less backends).
1333/// - `normalized_etag`: stripped form for comparing against
1334///   `FrameIndex::source_etag` (the s3s `ETag::value()` accessor
1335///   used by the server PUT path strips quotes).
1336struct HeadInfo {
1337    raw_etag: Option<String>,
1338    normalized_etag: Option<String>,
1339    size: u64,
1340}
1341
1342async fn head_main(client: &Client, bucket: &str, key: &str) -> Result<HeadInfo, RepairError> {
1343    let head = client
1344        .head_object()
1345        .bucket(bucket)
1346        .key(key)
1347        .send()
1348        .await
1349        .map_err(|e| RepairError::Backend {
1350            op: "HEAD",
1351            bucket: bucket.into(),
1352            key: key.into(),
1353            cause: format!("{e}"),
1354        })?;
1355    let raw_etag = head.e_tag().map(str::to_owned);
1356    let normalized_etag = raw_etag.as_deref().map(normalize_etag);
1357    // `content_length` is `Option<i64>` on the SDK type — `None` means the
1358    // backend didn't return a Content-Length header. We fail closed rather
1359    // than treating that as zero (which would silently bypass the
1360    // `body_bytes_cap` in `repair_sidecar` and let an unbounded GET
1361    // exhaust RAM). AWS S3 / MinIO / Garage / Ceph RGW all return
1362    // Content-Length on HEAD, so this only trips on exotic / broken
1363    // backends — which the operator should know about.
1364    let size = match head.content_length() {
1365        Some(n) if n >= 0 => n as u64,
1366        Some(_) | None => {
1367            return Err(RepairError::MissingContentLength {
1368                bucket: bucket.into(),
1369                key: key.into(),
1370            });
1371        }
1372    };
1373    Ok(HeadInfo {
1374        raw_etag,
1375        normalized_etag,
1376        size,
1377    })
1378}
1379
1380/// Strip the surrounding `"..."` quotes from an RFC 7232 entity-tag so
1381/// the on-wire form (aws-sdk-s3 returns raw `"..."`) matches the form
1382/// the S4 gateway stamps into `FrameIndex::source_etag` (the s3s
1383/// `ETag::value()` accessor that drives the PUT path strips quotes).
1384///
1385/// Without this normalization, a freshly-written sidecar would falsely
1386/// flag as `StaleEtag` because the strings differ only by the wrapping
1387/// quotes. Both the PUT side (server) and the repair side (this CLI)
1388/// must agree on the canonical form — the de-facto canonical is "no
1389/// surrounding quotes", since that's what the server already writes
1390/// into every v2 sidecar in the wild.
1391fn normalize_etag(s: &str) -> String {
1392    s.trim_matches('"').to_owned()
1393}
1394
1395/// v0.9 #106-audit-R2 P2-INT-1: detect SSE-S4 encrypted envelopes by
1396/// magic prefix. Returns `Some(name)` when the first four bytes match
1397/// one of the SSE frame magics (`S4E1`..`S4E6`); returns `None` for any
1398/// other body, including S4 framed plaintext (`S4F2`) and raw
1399/// compressed / passthrough bodies.
1400///
1401/// Intentionally duplicated here as a 4-byte prefix compare instead of
1402/// reusing `sse::peek_magic` because `peek_magic` length-gates on the
1403/// full S4E1/S4E2 header size (36 bytes) and would return `None` for a
1404/// very short S4E6 stub the way an empty-key edge-case might land —
1405/// the gate is for cryptographic frame validity, not for the
1406/// "is encrypted at all" question this helper answers. The exact magic
1407/// bytes are stable wire-format constants (see `sse::SSE_MAGIC_V{1..6}`)
1408/// and are echoed here so the repair module has no circular dep on the
1409/// SSE module's full surface.
1410fn detect_sse_magic(body: &[u8]) -> Option<&'static str> {
1411    if body.len() < 4 {
1412        return None;
1413    }
1414    match &body[..4] {
1415        b"S4E1" => Some("S4E1"),
1416        b"S4E2" => Some("S4E2"),
1417        b"S4E3" => Some("S4E3"),
1418        b"S4E4" => Some("S4E4"),
1419        b"S4E5" => Some("S4E5"),
1420        b"S4E6" => Some("S4E6"),
1421        _ => None,
1422    }
1423}
1424
1425/// v0.9 #106-audit-R5 P2-R5 (Codex): bounded sidecar fetch.
1426/// HEADs the sidecar key first to learn its size; refuses to GET
1427/// (and thus refuses to allocate) if the size exceeds
1428/// [`MAX_SIDECAR_BODY_BYTES`]. Used by both `verify_sidecar` and
1429/// `classify_one` (sweep) so a multi-GiB corrupt or legacy user
1430/// `.s4index` object can't OOM the operator's repair process.
1431///
1432/// Returns:
1433///   - `Ok(Some(bytes))` when the sidecar exists and fits in the cap
1434///   - `Ok(None)` when the sidecar HEAD returns NotFound (caller
1435///     classifies as `Missing*`)
1436///   - `Err(SidecarFetchOutcome::Other)` when HEAD returns
1437///     Content-Length missing or any other backend error
1438///   - `Err(SidecarFetchOutcome::TooLarge { .. })` when size > cap
1439async fn get_sidecar_bytes_capped(
1440    client: &Client,
1441    bucket: &str,
1442    key: &str,
1443) -> Result<Option<bytes::Bytes>, SidecarFetchOutcome> {
1444    let head = match client.head_object().bucket(bucket).key(key).send().await {
1445        Ok(h) => h,
1446        Err(e) => {
1447            return if is_head_not_found(&e) {
1448                Ok(None)
1449            } else {
1450                Err(SidecarFetchOutcome::Other(format!("HEAD: {e}")))
1451            };
1452        }
1453    };
1454    let size = match head.content_length() {
1455        Some(n) if n >= 0 => n as u64,
1456        Some(_) | None => {
1457            return Err(SidecarFetchOutcome::Other(
1458                "sidecar HEAD returned no Content-Length; refusing to GET unbounded".into(),
1459            ));
1460        }
1461    };
1462    if size > MAX_SIDECAR_BODY_BYTES {
1463        return Err(SidecarFetchOutcome::TooLarge {
1464            size,
1465            cap: MAX_SIDECAR_BODY_BYTES,
1466        });
1467    }
1468    // v0.9 #106-audit-R6 P2-R6 (Codex): pin the GET to the HEAD's
1469    // ETag so a sidecar swap between HEAD and GET can't bypass
1470    // the cap. Without this, an attacker who races
1471    // HEAD(small) → swap(massive) → GET could still OOM the
1472    // process because `collect()` reads whatever the GET response
1473    // delivers, ignoring the HEAD-reported size. With If-Match
1474    // pinned, the swap surfaces as 412 PreconditionFailed → we
1475    // refuse the body without allocating it.
1476    //
1477    // Backends that don't return ETags fall back to a post-GET
1478    // length check below (still a window where collect() runs to
1479    // completion, but the typed `TooLarge` exit replaces what
1480    // would otherwise be a silent OOM-pass).
1481    let raw_etag = head.e_tag().map(str::to_owned);
1482    let get_builder = client.get_object().bucket(bucket).key(key);
1483    let get_builder = match raw_etag {
1484        Some(ref t) => get_builder.if_match(t.clone()),
1485        None => get_builder,
1486    };
1487    match get_builder.send().await {
1488        Ok(resp) => {
1489            let agg = resp
1490                .body
1491                .collect()
1492                .await
1493                .map_err(|e| SidecarFetchOutcome::Other(format!("read body: {e}")))?;
1494            let bytes = agg.into_bytes();
1495            // Defense-in-depth: ETag-less backends bypass
1496            // If-Match; If-Match-non-honouring backends also exist.
1497            // Check the actual body length AFTER collect to catch
1498            // a race-during-collect that exceeded the cap.
1499            if (bytes.len() as u64) > MAX_SIDECAR_BODY_BYTES {
1500                return Err(SidecarFetchOutcome::TooLarge {
1501                    size: bytes.len() as u64,
1502                    cap: MAX_SIDECAR_BODY_BYTES,
1503                });
1504            }
1505            Ok(Some(bytes))
1506        }
1507        Err(e) => {
1508            let s = format!("{e}");
1509            if is_get_not_found(&e) {
1510                // Race: existed at HEAD, gone by GET. Treat as missing.
1511                Ok(None)
1512            } else if s.contains("PreconditionFailed") || s.contains("412") {
1513                // Race: sidecar replaced between HEAD and GET. The
1514                // new sidecar's size is whatever the swap-in is;
1515                // we refuse to load it without re-HEAD'ing under
1516                // operator supervision.
1517                Err(SidecarFetchOutcome::Other(format!(
1518                    "sidecar at {bucket}/{key} was replaced between HEAD and GET (412 \
1519                     PreconditionFailed); re-run when the sidecar is stable"
1520                )))
1521            } else {
1522                Err(SidecarFetchOutcome::Other(format!("GET: {s}")))
1523            }
1524        }
1525    }
1526}
1527
1528enum SidecarFetchOutcome {
1529    Other(String),
1530    TooLarge { size: u64, cap: u64 },
1531}
1532
1533fn is_head_not_found(
1534    e: &aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::head_object::HeadObjectError>,
1535) -> bool {
1536    matches!(
1537        e,
1538        aws_sdk_s3::error::SdkError::ServiceError(svc)
1539            if matches!(
1540                svc.err(),
1541                aws_sdk_s3::operation::head_object::HeadObjectError::NotFound(_)
1542            )
1543    )
1544}
1545
1546fn is_get_not_found(
1547    e: &aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::get_object::GetObjectError>,
1548) -> bool {
1549    matches!(
1550        e,
1551        aws_sdk_s3::error::SdkError::ServiceError(svc)
1552            if matches!(
1553                svc.err(),
1554                aws_sdk_s3::operation::get_object::GetObjectError::NoSuchKey(_)
1555            )
1556    )
1557}
1558
1559/// Parse a `bucket/key` CLI argument. Splits on the **first** `/` only so
1560/// keys with slashes (e.g. `prefix/sub/file.bin`) round-trip cleanly.
1561pub fn parse_bucket_key(arg: &str) -> Result<(&str, &str), String> {
1562    match arg.split_once('/') {
1563        Some((b, k)) if !b.is_empty() && !k.is_empty() => Ok((b, k)),
1564        Some(_) => Err(format!(
1565            "expected `bucket/key`, got {arg:?} — bucket and key must both be non-empty"
1566        )),
1567        None => Err(format!("expected `bucket/key`, got {arg:?} — missing `/`")),
1568    }
1569}
1570
1571#[cfg(test)]
1572mod tests {
1573    use super::*;
1574
1575    #[test]
1576    fn parse_bucket_key_simple() {
1577        assert_eq!(
1578            parse_bucket_key("mybucket/foo.txt"),
1579            Ok(("mybucket", "foo.txt"))
1580        );
1581    }
1582
1583    #[test]
1584    fn parse_bucket_key_with_slashes_in_key() {
1585        assert_eq!(parse_bucket_key("b/a/b/c"), Ok(("b", "a/b/c")));
1586    }
1587
1588    #[test]
1589    fn parse_bucket_key_missing_slash() {
1590        assert!(parse_bucket_key("nokey").is_err());
1591    }
1592
1593    #[test]
1594    fn parse_bucket_key_empty_key() {
1595        assert!(parse_bucket_key("bucket/").is_err());
1596    }
1597
1598    #[test]
1599    fn parse_bucket_key_empty_bucket() {
1600        assert!(parse_bucket_key("/key").is_err());
1601    }
1602
1603    #[test]
1604    fn verify_report_is_clean_truth_table() {
1605        let mk = |status| VerifyReport {
1606            bucket: "b".into(),
1607            key: "k".into(),
1608            status,
1609        };
1610        assert!(
1611            mk(SidecarStatus::Ok {
1612                frame_count: 1,
1613                sidecar_size: 100,
1614            })
1615            .is_clean()
1616        );
1617        assert!(mk(SidecarStatus::LegacyV1 { frame_count: 3 }).is_clean());
1618        // P2-C (Codex R3): single-frame objects intentionally have no
1619        // sidecar — clean state, not divergence.
1620        assert!(mk(SidecarStatus::MissingHarmless { frame_count: 1 }).is_clean());
1621        // Ambiguous (body too large to deep-scan) — report cleanly so
1622        // CI doesn't false-alert; operator sees the hint in stdout.
1623        assert!(
1624            mk(SidecarStatus::MissingUnknown {
1625                size: 10 * 1024 * 1024 * 1024,
1626                cap: 5 * 1024 * 1024 * 1024,
1627            })
1628            .is_clean()
1629        );
1630        // Multi-frame + missing sidecar = real divergence.
1631        assert!(!mk(SidecarStatus::MissingDivergent { frame_count: 5 }).is_clean());
1632        assert!(
1633            !mk(SidecarStatus::StaleEtag {
1634                sidecar_etag: "a".into(),
1635                live_etag: "b".into(),
1636            })
1637            .is_clean()
1638        );
1639        assert!(
1640            !mk(SidecarStatus::StaleSize {
1641                sidecar_size: 1,
1642                live_size: 2,
1643            })
1644            .is_clean()
1645        );
1646        assert!(
1647            !mk(SidecarStatus::DecodeError {
1648                message: "bad".into()
1649            })
1650            .is_clean()
1651        );
1652    }
1653
1654    #[test]
1655    fn delete_policy_allows_truth_table() {
1656        let missing = OrphanReason::PairedMissing;
1657        let etag = OrphanReason::PairedEtagMismatch {
1658            sidecar_etag: "a".into(),
1659            live_etag: "b".into(),
1660        };
1661        let size = OrphanReason::PairedSizeMismatch {
1662            sidecar_size: 1,
1663            live_size: 2,
1664        };
1665        let undecodable = OrphanReason::SidecarUndecodable {
1666            message: "bad bytes".into(),
1667        };
1668
1669        // DryRun: never deletes anything.
1670        assert!(!DeletePolicy::DryRun.allows(&missing));
1671        assert!(!DeletePolicy::DryRun.allows(&etag));
1672        assert!(!DeletePolicy::DryRun.allows(&size));
1673        assert!(!DeletePolicy::DryRun.allows(&undecodable));
1674
1675        // PairBoundOnly: deletes the three pair-bound categories,
1676        // skips Undecodable (HIGH-2 review fix: protects v0.8.17
1677        // legacy reserved-name user data).
1678        assert!(DeletePolicy::PairBoundOnly.allows(&missing));
1679        assert!(DeletePolicy::PairBoundOnly.allows(&etag));
1680        assert!(DeletePolicy::PairBoundOnly.allows(&size));
1681        assert!(!DeletePolicy::PairBoundOnly.allows(&undecodable));
1682
1683        // IncludeUndecodable: explicit operator opt-in deletes all.
1684        assert!(DeletePolicy::IncludeUndecodable.allows(&missing));
1685        assert!(DeletePolicy::IncludeUndecodable.allows(&etag));
1686        assert!(DeletePolicy::IncludeUndecodable.allows(&size));
1687        assert!(DeletePolicy::IncludeUndecodable.allows(&undecodable));
1688    }
1689
1690    /// P3-A (Codex R5): a v2 sidecar with size binding but no ETag
1691    /// (rebuilt on an ETag-less backend) classifies as `Ok`, NOT
1692    /// `LegacyV1`. The latter would tell operators to "repair to
1693    /// upgrade" a sidecar already at the highest binding level the
1694    /// backend supports. This test asserts the exact pattern the
1695    /// status match in `verify_sidecar` relies on.
1696    #[test]
1697    fn verify_status_classifies_etag_less_v2_as_ok_not_legacy() {
1698        // The actual match arms in `verify_sidecar`:
1699        //
1700        //   (Some(s), _) if Some(s) != live → StaleEtag
1701        //   (_, Some(z)) if z != live_size → StaleSize
1702        //   (_, Some(_))                   → Ok        // P3-A fix
1703        //   (None, None)                   → LegacyV1
1704        //
1705        // Mirror that decision tree inline so refactors to the real
1706        // function can't quietly regress without flipping this test.
1707        fn classify(side_etag: Option<&str>, side_size: Option<u64>) -> &'static str {
1708            const LIVE_ETAG: Option<&str> = Some("xyz");
1709            const LIVE_SIZE: u64 = 100;
1710            match (side_etag, side_size) {
1711                (Some(s), _) if Some(s) != LIVE_ETAG => "StaleEtag",
1712                (_, Some(z)) if z != LIVE_SIZE => "StaleSize",
1713                (_, Some(_)) => "Ok",
1714                (_, None) => "LegacyV1",
1715            }
1716        }
1717        // P3-A core case: ETag-less repair stamps (None, Some(size)).
1718        // Must classify as Ok, not LegacyV1.
1719        assert_eq!(classify(None, Some(100)), "Ok");
1720        // Full v2 binding with matching etag + size.
1721        assert_eq!(classify(Some("xyz"), Some(100)), "Ok");
1722        // True v1 legacy (neither field) still surfaces as LegacyV1.
1723        assert_eq!(classify(None, None), "LegacyV1");
1724        // Mismatches still detected.
1725        assert_eq!(classify(Some("abc"), Some(100)), "StaleEtag");
1726        assert_eq!(classify(Some("xyz"), Some(999)), "StaleSize");
1727    }
1728
1729    /// P2-D (Codex R4): on an ETag-less backend the server stamps
1730    /// `source_etag = None`; the verifier MUST treat that as the
1731    /// legacy / best-effort path (Ok / LegacyV1), not flag every
1732    /// such sidecar as stale. This unit test pins the discriminator
1733    /// the `verify_sidecar` status-match arm relies on (the
1734    /// `Option<&str>` equality).
1735    #[test]
1736    fn etag_option_equality_treats_none_none_as_match() {
1737        let side: Option<&str> = None;
1738        let live: Option<&str> = None;
1739        assert!(side == live, "None == None must hold for the no-ETag path");
1740
1741        let side: Option<&str> = Some("abc");
1742        let live: Option<&str> = Some("abc");
1743        assert!(side == live);
1744
1745        let side: Option<&str> = Some("");
1746        let live: Option<&str> = None;
1747        assert!(side != live, "Some(\"\") must NOT equal None — P2-D guard");
1748    }
1749
1750    #[test]
1751    fn normalize_etag_strips_surrounding_quotes() {
1752        // aws-sdk-s3 returns the wire form (with quotes); s3s `value()`
1753        // returns the stripped form. The sidecar's `source_etag` is
1754        // canonical-stripped, so both sides must agree.
1755        assert_eq!(normalize_etag("\"abc-1\""), "abc-1");
1756        // Multipart ETags are `<hex>-<n>` and still get quoted on wire.
1757        assert_eq!(
1758            normalize_etag("\"067e3167e8c481c2aea3650ebb273198-2\""),
1759            "067e3167e8c481c2aea3650ebb273198-2"
1760        );
1761        // Already-stripped form is a no-op (the helper is idempotent so
1762        // callers don't need to branch on the source).
1763        assert_eq!(normalize_etag("abc-1"), "abc-1");
1764        // Defensive: an empty etag stays empty (head responses with no
1765        // ETag header round-trip to the empty string in head_main).
1766        assert_eq!(normalize_etag(""), "");
1767    }
1768
1769    /// P2-R5 (Codex R5 audit): the bounded sidecar fetch helper
1770    /// must enforce [`MAX_SIDECAR_BODY_BYTES`] and surface a typed
1771    /// `SidecarTooLarge` error before allocating. Pin the wire
1772    /// shape of the variant so a future refactor can't silently
1773    /// drop the cap and re-introduce the OOM vector.
1774    #[test]
1775    fn sidecar_too_large_error_shape() {
1776        let err = RepairError::SidecarTooLarge {
1777            bucket: "b".into(),
1778            key: "k.s4index".into(),
1779            size: 2 * MAX_SIDECAR_BODY_BYTES,
1780            cap: MAX_SIDECAR_BODY_BYTES,
1781        };
1782        let rendered = format!("{err}");
1783        assert!(
1784            rendered.contains("b/k.s4index"),
1785            "Display must mention bucket/key — got {rendered:?}"
1786        );
1787        assert!(
1788            rendered.contains(&MAX_SIDECAR_BODY_BYTES.to_string()),
1789            "Display must mention the cap — got {rendered:?}"
1790        );
1791        assert!(
1792            rendered.contains("OOM") || rendered.contains("legacy") || rendered.contains("attack"),
1793            "Display must hint at the threat model — got {rendered:?}"
1794        );
1795        match err {
1796            RepairError::SidecarTooLarge {
1797                bucket,
1798                key,
1799                size,
1800                cap,
1801            } => {
1802                assert_eq!(bucket, "b");
1803                assert_eq!(key, "k.s4index");
1804                assert_eq!(size, 2 * MAX_SIDECAR_BODY_BYTES);
1805                assert_eq!(cap, MAX_SIDECAR_BODY_BYTES);
1806            }
1807            _ => unreachable!("SidecarTooLarge must match its own variant"),
1808        }
1809    }
1810
1811    /// P2-R5: the cap value is load-bearing — too small breaks
1812    /// legitimate sidecars, too large defeats the OOM guard. Pin
1813    /// it at the codec-spec-derived ceiling (16M frames × 32 B per
1814    /// entry + header ≈ 512 MiB, rounded up with safety margin to
1815    /// 600 MiB). Bump only with explicit operator justification.
1816    #[test]
1817    fn max_sidecar_body_bytes_cap_value_pinned() {
1818        assert_eq!(MAX_SIDECAR_BODY_BYTES, 600 * 1024 * 1024);
1819        // Sanity: cap must comfortably exceed the codec spec's
1820        // max legitimate sidecar geometry. Computed dynamically
1821        // from the codec constants so a bump to either side
1822        // surfaces here (clippy flags `assert!(const)` as
1823        // pointless, so we use `assert_eq!` against `false` for
1824        // the negative — if the cap ever DROPS below the spec
1825        // max, this fails loudly).
1826        let spec_max_legitimate: u64 = s4_codec::index::MAX_FRAMES
1827            * (s4_codec::index::ENTRY_BYTES as u64)
1828            + (s4_codec::index::HEADER_FIXED_V2 as u64)
1829            + (s4_codec::index::MAX_ETAG_BYTES as u64);
1830        assert!(
1831            MAX_SIDECAR_BODY_BYTES > spec_max_legitimate,
1832            "cap {MAX_SIDECAR_BODY_BYTES} must exceed spec-max {spec_max_legitimate}",
1833        );
1834    }
1835
1836    /// P2-R3 (Codex R3 audit): `repair-sidecar` on a passthrough /
1837    /// raw-bytes object would previously write an empty sidecar
1838    /// that silently breaks Range GET. Pin the typed error's wire
1839    /// shape so a future refactor can't quietly drop the
1840    /// `NotFramed` branch.
1841    #[test]
1842    fn not_framed_error_shape() {
1843        let err = RepairError::NotFramed {
1844            bucket: "b".into(),
1845            key: "k".into(),
1846        };
1847        let rendered = format!("{err}");
1848        assert!(rendered.contains("b/k"), "Display must mention bucket/key");
1849        assert!(
1850            rendered.contains("S4F2") || rendered.contains("passthrough"),
1851            "Display must hint at the framing reason"
1852        );
1853        // Pattern-match guard: any rename of bucket/key here is a
1854        // compile error both here AND at the repair_sidecar
1855        // construction site.
1856        match err {
1857            RepairError::NotFramed { bucket, key } => {
1858                assert_eq!(bucket, "b");
1859                assert_eq!(key, "k");
1860            }
1861            _ => unreachable!("NotFramed must match its own variant"),
1862        }
1863    }
1864
1865    /// CI-unblock (post-v0.9 audit): the MinIO E2E race test
1866    /// (`repair_sidecar_detects_post_get_overwrite_race`) is
1867    /// inherently timing-dependent and flakes on fast CI runners
1868    /// where the entire repair pipeline completes before the
1869    /// spawned overwrite lands. This deterministic guard pins
1870    /// the error type's wire shape (Display + field accessors)
1871    /// so the post-PUT divergence detector branch in
1872    /// `repair_sidecar` can't be silently refactored into a
1873    /// different error variant without flipping this assertion.
1874    #[test]
1875    fn overwritten_during_repair_error_shape() {
1876        let err = RepairError::OverwrittenDuringRepair {
1877            bucket: "b".into(),
1878            key: "k".into(),
1879            head_etag: "abc-1".into(),
1880        };
1881        let rendered = format!("{err}");
1882        assert!(
1883            rendered.contains("b/k"),
1884            "Display must mention bucket/key — got {rendered:?}"
1885        );
1886        assert!(
1887            rendered.contains("abc-1"),
1888            "Display must mention the pre-race ETag — got {rendered:?}"
1889        );
1890        assert!(
1891            rendered.contains("re-run") || rendered.contains("overwritten"),
1892            "Display must hint that the operator should re-run — got {rendered:?}"
1893        );
1894        // Pattern-match guard: any future destructure of this
1895        // variant elsewhere in the crate must keep these three
1896        // named fields. A rename here would surface as a compile
1897        // error here AND at the production call sites in
1898        // repair_sidecar / classify_missing_sidecar.
1899        match err {
1900            RepairError::OverwrittenDuringRepair {
1901                bucket,
1902                key,
1903                head_etag,
1904            } => {
1905                assert_eq!(bucket, "b");
1906                assert_eq!(key, "k");
1907                assert_eq!(head_etag, "abc-1");
1908            }
1909            _ => unreachable!("OverwrittenDuringRepair must match its own variant"),
1910        }
1911    }
1912
1913    #[test]
1914    fn default_repair_body_cap_matches_max_body_default() {
1915        // Tied to s4-server `--max-body-bytes` default (5 GiB, #178). If
1916        // the default changes there, update both in lockstep.
1917        assert_eq!(DEFAULT_REPAIR_BODY_BYTES_CAP, 5 * 1024 * 1024 * 1024);
1918    }
1919
1920    /// v0.9 #106-audit-R2 P2-INT-1: `detect_sse_magic` returns the
1921    /// correct frame label for every S4Ex prefix, and `None` for the
1922    /// plaintext frame magic (`S4F2`) and short / random inputs. The
1923    /// helper is the discriminator the `EncryptedSidecarUnsupported`
1924    /// branch in `repair_sidecar` relies on; pinning its outputs
1925    /// guards against a silent regression that would resurrect the
1926    /// confusing `FrameScan` failure on encrypted bodies.
1927    #[test]
1928    fn detect_sse_magic_covers_all_envelope_variants() {
1929        assert_eq!(detect_sse_magic(b"S4E1\0\0\0\0"), Some("S4E1"));
1930        assert_eq!(detect_sse_magic(b"S4E2\0\0\0\0"), Some("S4E2"));
1931        assert_eq!(detect_sse_magic(b"S4E3\0\0\0\0"), Some("S4E3"));
1932        assert_eq!(detect_sse_magic(b"S4E4\0\0\0\0"), Some("S4E4"));
1933        assert_eq!(detect_sse_magic(b"S4E5\0\0\0\0"), Some("S4E5"));
1934        assert_eq!(detect_sse_magic(b"S4E6\0\0\0\0"), Some("S4E6"));
1935        // S4F2 = plaintext framed body; must NOT match (or repair
1936        // would falsely reject every framed object as encrypted).
1937        assert_eq!(detect_sse_magic(b"S4F2\0\0\0\0"), None);
1938        // Random bytes, short inputs, and empty body all return None.
1939        assert_eq!(detect_sse_magic(b"NOPE\0"), None);
1940        assert_eq!(detect_sse_magic(b"S4"), None);
1941        assert_eq!(detect_sse_magic(b""), None);
1942    }
1943
1944    /// v0.10 #A1: `SseDecryptFailed` is a distinct typed variant
1945    /// (NOT lumped under `Backend` / `FrameScan`) so the CLI can give
1946    /// operator-actionable guidance pointing at `--sse-s4-key` /
1947    /// `--sse-s4-key-rotated`. Pin the Display + struct shape so a
1948    /// future refactor can't silently demote it to the generic
1949    /// `Backend` variant.
1950    #[test]
1951    fn sse_decrypt_failed_error_shape() {
1952        let err = RepairError::SseDecryptFailed {
1953            bucket: "b".into(),
1954            key: "k".into(),
1955            cause: "chunk 0 auth-tag verify failed".into(),
1956        };
1957        let rendered = format!("{err}");
1958        assert!(
1959            rendered.contains("b/k"),
1960            "Display must mention bucket/key — got {rendered:?}"
1961        );
1962        assert!(
1963            rendered.contains("SSE-S4 decrypt"),
1964            "Display must name the failure mode — got {rendered:?}"
1965        );
1966        assert!(
1967            rendered.contains("--sse-s4-key"),
1968            "Display must point at the operator-actionable flag — got {rendered:?}"
1969        );
1970        // Pattern guard: any rename of the three fields surfaces here
1971        // AND at the construction site in `decrypt_s4e6_for_repair`.
1972        match err {
1973            RepairError::SseDecryptFailed { bucket, key, cause } => {
1974                assert_eq!(bucket, "b");
1975                assert_eq!(key, "k");
1976                assert!(cause.contains("chunk 0"));
1977            }
1978            _ => unreachable!("SseDecryptFailed must match its own variant"),
1979        }
1980    }
1981
1982    /// v0.10 #A1 (Codex R4 fix): the final-chunk slack the SSE-S4
1983    /// repair path grants on top of `body_bytes_cap` MUST be bounded
1984    /// by [`SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES`] regardless of what
1985    /// the on-disk `chunk_size` declares. Pin the constant + the
1986    /// `min(hdr.chunk_size, ceiling)` arithmetic so a future
1987    /// refactor can't quietly resurrect the OOM vector by trusting
1988    /// the attacker-controlled header field verbatim.
1989    #[test]
1990    fn sse_s4_repair_max_chunk_slack_bounds_attacker_controlled_header() {
1991        // Pin the constant value at its documented 16 MiB ceiling.
1992        // 16 MiB comfortably covers the server-default 1 MiB chunk
1993        // size by 16× while staying well below any cap that would
1994        // double the operator's RAM budget.
1995        assert_eq!(SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES, 16 * 1024 * 1024);
1996
1997        // Inline the slack-computation arithmetic so the regression
1998        // guard fires here AND at the production call site in
1999        // `decrypt_s4e6_for_repair`. Tests three regimes:
2000        //   - small legitimate chunk (1 MiB) → slack == chunk_size
2001        //   - on-ceiling legitimate chunk (16 MiB) → slack == ceiling
2002        //   - attacker-controlled huge chunk (u32::MAX) → slack
2003        //     pinned at ceiling (OOM vector closed)
2004        fn compute_slack(hdr_chunk_size: u32) -> u64 {
2005            (hdr_chunk_size as u64).min(SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES)
2006        }
2007        assert_eq!(compute_slack(1024 * 1024), 1024 * 1024);
2008        assert_eq!(
2009            compute_slack(16 * 1024 * 1024),
2010            SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES
2011        );
2012        assert_eq!(compute_slack(u32::MAX), SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES);
2013
2014        // The slack ceiling MUST stay smaller than the worst-case
2015        // SSE envelope overhead (256 MiB) — they cover different
2016        // axes (chunk-size slack vs envelope tag/header overhead)
2017        // but a slack > envelope overhead would be nonsensical.
2018        // Bind the constants to runtime locals so clippy's
2019        // `assertions_on_constants` lint doesn't fire on the
2020        // pinned values.
2021        let slack_cap = SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES;
2022        let overhead_cap = SSE_S4_REPAIR_MAX_OVERHEAD_BYTES;
2023        assert!(slack_cap < overhead_cap);
2024    }
2025
2026    /// v0.10 #A1 (Codex R1 fix): `SSE_S4_REPAIR_MAX_OVERHEAD_BYTES`
2027    /// is load-bearing for the body-cap relaxation in
2028    /// `repair_sidecar_with_keyring` — too small reintroduces
2029    /// `BodyTooLarge` rejection on valid S4E6 objects whose
2030    /// plaintext fits the cap; too large bloats the GET's RAM
2031    /// budget unnecessarily. Pin it at the codec-spec ceiling
2032    /// (`S4E6_HEADER_BYTES + S4E6_MAX_CHUNK_COUNT × TAG_LEN`)
2033    /// so a bump to either side of the SSE constants surfaces
2034    /// here.
2035    #[test]
2036    fn sse_s4_repair_max_overhead_bytes_matches_codec_spec() {
2037        let expected = (crate::sse::S4E6_HEADER_BYTES as u64)
2038            + (crate::sse::S4E6_MAX_CHUNK_COUNT as u64)
2039                * (crate::sse::S4E5_PER_CHUNK_OVERHEAD as u64);
2040        assert_eq!(SSE_S4_REPAIR_MAX_OVERHEAD_BYTES, expected);
2041        // Sanity bounds — must comfortably exceed any realistic
2042        // S4E6 envelope overhead but stay well below the 5 GiB
2043        // single-PUT ceiling so the relaxation can't double the
2044        // operator's RAM budget. Build the comparison values from
2045        // the codec spec dynamically (instead of a literal const)
2046        // so clippy's `assertions_on_constants` lint doesn't fire
2047        // on the pinned constant.
2048        let min_reasonable: u64 = 1024 * 1024; // 1 MiB
2049        let max_reasonable: u64 = 1024 * 1024 * 1024; // 1 GiB
2050        assert!(
2051            SSE_S4_REPAIR_MAX_OVERHEAD_BYTES > min_reasonable,
2052            "overhead headroom {SSE_S4_REPAIR_MAX_OVERHEAD_BYTES} must accommodate the \
2053             worst-case S4E6 envelope (>= 1 MiB)",
2054        );
2055        assert!(
2056            SSE_S4_REPAIR_MAX_OVERHEAD_BYTES < max_reasonable,
2057            "overhead headroom {SSE_S4_REPAIR_MAX_OVERHEAD_BYTES} must stay below 1 GiB so \
2058             it doesn't double the operator's --max-body-bytes RAM budget",
2059        );
2060    }
2061
2062    /// v0.10 #A1: `RepairReport::sse_v3_binding` carries the chunked
2063    /// geometry the CLI surfaces in the OK line. Pin the field shape
2064    /// so a struct rename surfaces as a compile error here AND at the
2065    /// CLI's print site in `run_repair_sidecar`.
2066    #[test]
2067    fn repair_sse_binding_shape() {
2068        let b = RepairSseBinding {
2069            enc_chunk_size: 1_048_576,
2070            enc_chunk_count: 4,
2071            enc_key_id: 1,
2072            enc_plaintext_len: 4_000_000,
2073            enc_header_bytes: 24,
2074        };
2075        // Field accesses double as a compile-time guard that the names
2076        // don't drift away from the codec's `SseChunkBinding` (the
2077        // module re-export from `s4_codec::index`).
2078        assert_eq!(b.enc_chunk_size, 1_048_576);
2079        assert_eq!(b.enc_chunk_count, 4);
2080        assert_eq!(b.enc_key_id, 1);
2081        assert_eq!(b.enc_plaintext_len, 4_000_000);
2082        assert_eq!(b.enc_header_bytes, 24);
2083    }
2084
2085    /// v0.9 #106-audit-R2 P2-INT-1 (initial) / v0.10 #A1 (refined):
2086    /// pin the Display text + struct shape of the new variant so
2087    /// refactors can't silently drop the operator guidance
2088    /// (server-mode rebuild / re-PUT / `--sse-s4-key` plumbing) or
2089    /// rename the fields the CLI's error formatter reads. Mirrors
2090    /// the existing `overwritten_during_repair_error_shape` test
2091    /// pattern.
2092    #[test]
2093    fn repair_sidecar_rejects_encrypted_body_with_typed_error() {
2094        let err = RepairError::EncryptedSidecarUnsupported {
2095            bucket: "b".into(),
2096            key: "k".into(),
2097            message: "body magic S4E6 indicates SSE-S4 envelope".into(),
2098        };
2099        let rendered = format!("{err}");
2100        assert!(
2101            rendered.contains("b/k"),
2102            "Display must mention bucket/key — got {rendered:?}"
2103        );
2104        assert!(
2105            rendered.contains("S4E6"),
2106            "Display must echo the body magic for operator triage — got {rendered:?}"
2107        );
2108        assert!(
2109            rendered.contains("encrypted-sidecar repair"),
2110            "Display must name the failure mode — got {rendered:?}"
2111        );
2112        assert!(
2113            rendered.contains("re-PUT") || rendered.contains("server-mode"),
2114            "Display must hint at the recovery path — got {rendered:?}"
2115        );
2116        match err {
2117            RepairError::EncryptedSidecarUnsupported {
2118                bucket,
2119                key,
2120                message,
2121            } => {
2122                assert_eq!(bucket, "b");
2123                assert_eq!(key, "k");
2124                assert!(message.contains("S4E6"));
2125            }
2126            _ => unreachable!("EncryptedSidecarUnsupported must match its own variant"),
2127        }
2128    }
2129}