s4_server/repair.rs
1//! v0.9 #106: standalone sidecar repair / verify / sweep tooling.
2//!
3//! The S4 server writes a `<key>.s4index` sidecar after every framed PUT so
4//! Range GETs can do a partial fetch instead of streaming the whole body.
5//! Three failure modes leave the sidecar diverged from the live object and
6//! degrade Range GET to the full-read fallback:
7//!
8//! 1. The sidecar PUT failed after the main object committed (network blip,
9//! backend throttle).
10//! 2. An operator overwrote the object directly through the backend, leaving
11//! the sidecar stale (ETag / size mismatch with the new body).
12//! 3. The v0.8.15 H-g multipart-Complete-on-Versioning-Enabled bug emitted
13//! sidecars bound to the parent key while the body landed under the
14//! versioning shadow path (`<key>.__s4ver__/<id>`). Those orphans never
15//! re-pair and lifecycle doesn't reap them.
16//!
17//! [`verify_sidecar`] reports the current state without writing,
18//! [`repair_sidecar`] rebuilds a single sidecar by re-scanning the main
19//! body, and [`sweep_orphan_sidecars`] walks every `*.s4index` in a bucket
20//! and reports / deletes the ones whose paired key is missing or stale.
21//!
22//! All three operate directly against an `aws_sdk_s3::Client` (the operator
23//! points it at the backend, not the S4 gateway, because the gateway hides
24//! `.s4index` from list output by design).
25
26use aws_sdk_s3::Client;
27use s4_codec::index::{
28 SIDECAR_SUFFIX, build_index_from_body, decode_index, encode_index, sidecar_key,
29};
30use thiserror::Error;
31
32/// Default cap on bytes loaded into RAM for sidecar rebuild. Matches the
33/// `--max-body-bytes` default (#178, 5 GiB) — repair needs the full body in
34/// memory because `build_index_from_body` is a single-pass scan. Operators
35/// with larger objects pass `--max-body-bytes` to raise this explicitly so a
36/// runaway `repair-sidecar` on a 50 GiB object surfaces a clear error
37/// instead of swapping the host.
38pub const DEFAULT_REPAIR_BODY_BYTES_CAP: u64 = 5 * 1024 * 1024 * 1024;
39
40/// v0.9 #106-audit-R5 P2-R5 (Codex): hard cap on `<key>.s4index` body
41/// bytes read by `verify-sidecar` / `sweep-orphan-sidecars`. The codec
42/// spec bounds a legitimate sidecar at `MAX_FRAMES (16M) * ENTRY_BYTES
43/// (32) + header (≤ 74 B)` ≈ 512 MiB. Any sidecar object larger than
44/// this cap is either an attacker payload aimed at OOM-ing the
45/// operator's repair process or a confused legacy reserved-name user
46/// data file — neither is something we want to load into RAM before
47/// `decode_index` can reject it. 600 MiB leaves a safety margin over
48/// the 512 MiB legitimate ceiling. Operators with anomalously large
49/// LEGITIMATE sidecars (multi-million-frame objects) should raise the
50/// cap explicitly; until then 600 MiB is the safe-by-default value.
51pub const MAX_SIDECAR_BODY_BYTES: u64 = 600 * 1024 * 1024;
52
53/// v0.10 #A1 (Codex round 1): worst-case S4E6 envelope overhead
54/// added on top of a plaintext body. Used by
55/// [`repair_sidecar_with_keyring`] to relax the operator-supplied
56/// `body_bytes_cap` (which is conceptually a plaintext cap matching
57/// the gateway's `--max-body-bytes`) when the SSE-S4 keyring path
58/// is active — without the relaxation, an object whose plaintext
59/// fits the cap can still be rejected as `BodyTooLarge` purely on
60/// the encryption overhead the gateway added at PUT time.
61///
62/// Derivation: an S4E6 envelope is a fixed 24-byte header
63/// ([`crate::sse::S4E6_HEADER_BYTES`]) + a 16-byte AES-GCM tag per
64/// chunk ([`crate::sse::S4E5_PER_CHUNK_OVERHEAD`]), with the
65/// 24-bit chunk-index field capping `chunk_count` at
66/// [`crate::sse::S4E6_MAX_CHUNK_COUNT`] (= `2^24 - 1`). So the
67/// worst-case overhead is
68/// `24 + 16 × (2^24 - 1)` ≈ 256 MiB. Every realistic chunk_size
69/// (1 KiB or larger) drops actual overhead by orders of magnitude;
70/// the worst-case ceiling here is the safe-by-default headroom.
71pub const SSE_S4_REPAIR_MAX_OVERHEAD_BYTES: u64 = (crate::sse::S4E6_HEADER_BYTES as u64)
72 + (crate::sse::S4E6_MAX_CHUNK_COUNT as u64) * (crate::sse::S4E5_PER_CHUNK_OVERHEAD as u64);
73
74/// v0.10 #A1 (Codex round 4): hard ceiling on the final-chunk slack
75/// the SSE-S4 repair path is willing to grant on top of the operator's
76/// `body_bytes_cap` when calling `decrypt_chunked_buffered`. The on-
77/// disk S4E6 header carries an attacker-controlled `chunk_size` field
78/// (only weakly bounded by the body-length consistency check inside
79/// `parse_chunked_header`); trusting that value verbatim would let a
80/// tampered header declare `chunk_size = u32::MAX, chunk_count = 1`
81/// and force `Vec::with_capacity(u32::MAX)` ≈ 4 GiB before the
82/// operator-cap check fires (the helper's cap check is against the
83/// raised `body_bytes_cap + slack` we pass it).
84///
85/// The effective slack is `min(hdr.chunk_size, SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES)`.
86/// 16 MiB comfortably covers the server-default `--sse-chunk-size = 1
87/// MiB` (by 16×) and any practical operator-chosen chunk size; in the
88/// rare case the operator used `--sse-chunk-size > 16 MiB` at PUT
89/// time AND the actual plaintext is within `chunk_size` bytes of the
90/// repair cap, they can simply pass `--max-body-bytes` raised by
91/// `chunk_size` to compensate.
92pub const SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES: u64 = 16 * 1024 * 1024;
93
94#[derive(Debug, Error)]
95pub enum RepairError {
96 #[error("S3 backend error on {op} {bucket}/{key}: {cause}")]
97 Backend {
98 op: &'static str,
99 bucket: String,
100 key: String,
101 // Named `cause` (not `source`) so thiserror doesn't auto-treat it
102 // as a `#[source]` chain field — the upstream SDK error is already
103 // stringified into `cause`.
104 cause: String,
105 },
106 #[error("frame scan failed on {bucket}/{key}: {cause}")]
107 FrameScan {
108 bucket: String,
109 key: String,
110 cause: String,
111 },
112 #[error("object body {size} bytes exceeds repair cap {cap}; pass --max-body-bytes to raise")]
113 BodyTooLarge { size: u64, cap: u64 },
114 /// HEAD on `{bucket}/{key}` returned no `Content-Length` header. The
115 /// body-size cap that prevents OOM on a runaway repair relies on this
116 /// being available, so the tool fails closed rather than treating a
117 /// missing length as zero (which would silently bypass the cap).
118 #[error(
119 "HEAD {bucket}/{key} returned no Content-Length; cannot enforce body cap, refusing to proceed"
120 )]
121 MissingContentLength { bucket: String, key: String },
122 /// `If-Match` race detector: the object was overwritten between the
123 /// initial HEAD (whose ETag we stamped into the sidecar) and the GET.
124 /// Returned by `repair_sidecar` so the operator can re-run instead of
125 /// writing a sidecar that's immediately stale.
126 #[error(
127 "object {bucket}/{key} was overwritten during repair (HEAD ETag {head_etag} != GET response); re-run repair-sidecar"
128 )]
129 OverwrittenDuringRepair {
130 bucket: String,
131 key: String,
132 head_etag: String,
133 },
134 /// v0.9 #106-audit-R5 P2-R5 (Codex): the `<key>.s4index` body
135 /// the backend reports exceeds [`MAX_SIDECAR_BODY_BYTES`], which
136 /// exceeds the codec spec's max legitimate sidecar (~512 MiB).
137 /// Surfaced before the GET to avoid loading a multi-GiB corrupt
138 /// or attacker-supplied `.s4index` blob into the operator's
139 /// repair process (DoS hardening). Operators with anomalously
140 /// large legitimate sidecars (multi-million-frame objects) can
141 /// raise the cap by changing the constant — but the practical
142 /// answer is "treat the underlying object as not-sidecared
143 /// (the GET path already falls back to a full read in that
144 /// case)" rather than chasing larger sidecars.
145 #[error(
146 "sidecar object {bucket}/{key} is {size} bytes (> {cap}-byte cap); refusing to load — \
147 most likely a legacy reserved-name user object or attacker payload aimed at OOM"
148 )]
149 SidecarTooLarge {
150 bucket: String,
151 key: String,
152 size: u64,
153 cap: u64,
154 },
155 /// v0.9 #106-audit-R3 P2-R3: the object body has no S4F2 frame
156 /// magic — it's a passthrough / raw-bytes object the server
157 /// intentionally never sidecared (service.rs::put_object only
158 /// builds a sidecar when `is_framed && !will_encrypt`). Writing
159 /// an empty `<key>.s4index` would silently break Range GET:
160 /// `FrameIndex::lookup_range` over zero entries returns `None`,
161 /// the GET path falls into the "invalid range" branch instead of
162 /// the correct passthrough-range fallback that exists for
163 /// sidecar-less objects. Surface as a typed error so the
164 /// operator knows the object isn't a candidate for sidecar
165 /// repair (and `verify-sidecar` will already classify it as
166 /// `MissingHarmless` with frame_count=0).
167 #[error(
168 "object {bucket}/{key} body has no S4F2 frame magic — it's a passthrough or \
169 raw-bytes object that the server intentionally never sidecared; \
170 sidecar repair would silently break Range GET. No action required."
171 )]
172 NotFramed { bucket: String, key: String },
173 /// v0.9 #106-audit-R2 P2-INT-1 (introduced) / v0.10 #A1 (refined):
174 /// the object body the backend returned is an SSE-S4 encrypted
175 /// envelope (`S4E1`/`S4E2`/`S4E3`/`S4E4`/`S4E5`/`S4E6`) and the
176 /// repair tool either was not given a matching keyring or the
177 /// envelope is not the chunked S4E6 variant the repair tool can
178 /// rebuild from.
179 ///
180 /// `repair_sidecar` runs against the BACKEND (not the gateway), so
181 /// the body it sees is ciphertext — feeding that to the frame
182 /// scanner would surface as a confusing `FrameScan` because the
183 /// S4F2 frame magic is hidden inside the encrypted payload. With a
184 /// keyring + the chunked S4E6 envelope, the new
185 /// [`repair_sidecar_with_keyring`] path decrypts in-process and
186 /// stamps a v3 sidecar carrying the SSE binding (key_id / salt /
187 /// chunk_size / chunk_count / plaintext_len / header_bytes); the
188 /// non-S4E6 envelopes (S4E1/E2/E3/E4/E5) are intentionally out of
189 /// scope (buffered AEAD frames have no per-chunk geometry, and
190 /// SSE-C / SSE-KMS need different key-material plumbing) — they
191 /// surface here so the operator routes those repairs through a
192 /// server-mode rebuild path or re-PUT.
193 #[error(
194 "object {bucket}/{key} body is an SSE-S4 encrypted envelope ({message}); \
195 encrypted-sidecar repair requires the matching SSE-S4 keyring (pass \
196 `--sse-s4-key` / `--sse-s4-key-rotated`) AND the chunked S4E6 envelope; \
197 non-S4E6 envelopes (S4E1/E2/E3/E4/E5) need a server-mode rebuild path \
198 or re-PUT the object to regenerate the sidecar"
199 )]
200 EncryptedSidecarUnsupported {
201 bucket: String,
202 key: String,
203 message: String,
204 },
205 /// v0.10 #A1: a keyring WAS supplied for an SSE-S4 chunked (S4E6)
206 /// object, but parsing the envelope header or decrypting one of
207 /// the AES-GCM chunks failed. The most common cause is a key
208 /// mismatch: the operator's `--sse-s4-key` is not the slot the
209 /// object was encrypted under at PUT time. Other causes are
210 /// envelope truncation / tampering (chunk auth-tag verify fails).
211 /// Surfaced as a distinct variant from `EncryptedSidecarUnsupported`
212 /// so the CLI can give operator-actionable guidance ("check your
213 /// `--sse-s4-key-rotated` list against the PUT-time slot") instead
214 /// of the unsupported-envelope hint.
215 #[error(
216 "SSE-S4 decrypt of {bucket}/{key} failed during sidecar repair: {cause}; \
217 check that `--sse-s4-key` (and any `--sse-s4-key-rotated`) covers the \
218 keyring slot the object was encrypted under at PUT time"
219 )]
220 SseDecryptFailed {
221 bucket: String,
222 key: String,
223 cause: String,
224 },
225}
226
227/// Status reported by [`verify_sidecar`]. Discriminates the outcomes a
228/// CI / cron job needs to branch on. The three `Missing*` variants
229/// resolve the P2-C ambiguity Codex caught: small single-frame objects
230/// intentionally have no sidecar (server only writes when
231/// `entries.len() > 1`), so a blanket `Missing` = exit-1 would false-
232/// alert on healthy objects.
233#[derive(Debug, Clone, PartialEq, Eq)]
234pub enum SidecarStatus {
235 /// Sidecar present, parses cleanly, and its v2 etag + size binding
236 /// matches the live HEAD.
237 Ok { frame_count: u64, sidecar_size: u64 },
238 /// No `<key>.s4index` AND the main body scans as a single frame
239 /// (server skips sidecar emission for `entries.len() <= 1` by
240 /// design). Healthy state — Range GET falls back to a full body
241 /// read, but a single-frame object's "full read" *is* its only
242 /// frame, so there's no fast-path to lose. Exit 0.
243 MissingHarmless { frame_count: u64 },
244 /// No `<key>.s4index` AND the main body has 2+ frames. Range GET
245 /// fast-path is lost; `repair-sidecar` will restore it. Exit 1.
246 MissingDivergent { frame_count: u64 },
247 /// No `<key>.s4index` AND the main object body exceeds the deep-
248 /// scan cap, so we can't tell whether it's a healthy single-frame
249 /// or a real divergence. Operator should raise `--max-body-bytes`
250 /// or run `repair-sidecar` to settle it. Exit 0 (ambiguous, not a
251 /// confirmed divergence — better to under-alert than spam).
252 MissingUnknown { size: u64, cap: u64 },
253 /// Sidecar present but its `source_etag` doesn't match the live HEAD —
254 /// the main object was overwritten or the sidecar is from a different
255 /// commit point.
256 StaleEtag {
257 sidecar_etag: String,
258 live_etag: String,
259 },
260 /// Sidecar present and ETag matches, but the recorded body size differs
261 /// (some backends, e.g. lifecycle moves, change bytes without bumping
262 /// ETag). Treated as stale.
263 StaleSize { sidecar_size: u64, live_size: u64 },
264 /// Pre-v0.8.4 sidecar (no source_etag / source_compressed_size). Still
265 /// usable read-only, but a repair will upgrade it to v2.
266 LegacyV1 { frame_count: u64 },
267 /// Sidecar bytes failed to decode. The body is corrupt or someone PUT
268 /// non-S4IX data at the `.s4index` key. A `repair-sidecar` overwrites
269 /// it cleanly.
270 DecodeError { message: String },
271}
272
273#[derive(Debug, Clone)]
274pub struct VerifyReport {
275 pub bucket: String,
276 pub key: String,
277 pub status: SidecarStatus,
278}
279
280impl VerifyReport {
281 /// True when the sidecar is in a state operators don't need to
282 /// action. Used by the CLI to decide exit code (true → 0, false → 1).
283 /// `MissingHarmless` is clean (single-frame objects have no sidecar
284 /// by design); `MissingUnknown` is also reported clean so the CLI
285 /// doesn't false-alert on objects too large to deep-scan — operator
286 /// can still see the hint in stdout and raise `--max-body-bytes`.
287 pub fn is_clean(&self) -> bool {
288 matches!(
289 self.status,
290 SidecarStatus::Ok { .. }
291 | SidecarStatus::LegacyV1 { .. }
292 | SidecarStatus::MissingHarmless { .. }
293 | SidecarStatus::MissingUnknown { .. }
294 )
295 }
296}
297
298#[derive(Debug, Clone)]
299pub struct RepairReport {
300 pub bucket: String,
301 pub key: String,
302 pub frame_count: u64,
303 pub sidecar_bytes_written: u64,
304 pub source_etag: Option<String>,
305 pub source_compressed_size: u64,
306 /// True when a sidecar already existed (we overwrote it). False when we
307 /// wrote one for the first time.
308 pub rebuilt_from_existing: bool,
309 /// v0.10 #A1: `Some(..)` when the source body was an SSE-S4 chunked
310 /// (S4E6) envelope and the repair tool decrypted it in-process with
311 /// the operator-supplied keyring; the v3 `sse_v3` binding was
312 /// stamped onto the sidecar so Range GETs take the encryption-aware
313 /// partial-fetch fast-path. `None` for plaintext-body repairs (the
314 /// existing v0.9 path) — the sidecar is the v2 layout.
315 pub sse_v3_binding: Option<RepairSseBinding>,
316}
317
318/// v0.10 #A1: distilled view of the SSE-S4 chunked binding stamped by
319/// [`repair_sidecar_with_keyring`]. Mirrors the codec's
320/// `SseChunkBinding` fields the CLI surfaces in its OK line; kept as a
321/// repair-module-local type so `s4-server` callers don't have to import
322/// `s4-codec` just to format the report.
323#[derive(Debug, Clone, Copy, PartialEq, Eq)]
324pub struct RepairSseBinding {
325 pub enc_chunk_size: u32,
326 pub enc_chunk_count: u32,
327 pub enc_key_id: u16,
328 pub enc_plaintext_len: u64,
329 pub enc_header_bytes: u32,
330}
331
332#[derive(Debug, Clone, PartialEq, Eq)]
333pub enum OrphanReason {
334 /// The paired logical key has no HEAD — sidecar is dangling.
335 PairedMissing,
336 /// Paired key exists but the sidecar's recorded ETag is stale.
337 PairedEtagMismatch {
338 sidecar_etag: String,
339 live_etag: String,
340 },
341 /// Paired key exists, ETag matches, but size differs.
342 PairedSizeMismatch { sidecar_size: u64, live_size: u64 },
343 /// The sidecar bytes failed to decode — either corruption or a non-
344 /// sidecar object that happened to land at a `.s4index` key.
345 SidecarUndecodable { message: String },
346}
347
348#[derive(Debug, Clone)]
349pub struct OrphanReport {
350 pub sidecar_key: String,
351 pub paired_key: String,
352 pub reason: OrphanReason,
353}
354
355#[derive(Debug, Clone)]
356pub struct SweepReport {
357 pub bucket: String,
358 pub sidecars_scanned: u64,
359 pub orphans: Vec<OrphanReport>,
360 /// Count actually deleted when `delete = true` was passed. Always 0 in
361 /// dry-run mode.
362 pub deleted: u64,
363}
364
365/// Verify a single `<bucket>/<key>` sidecar without writing.
366///
367/// When the sidecar is absent, this fetches the main body (capped at
368/// `deep_scan_body_cap`) to scan its frame count — single-frame objects
369/// intentionally have no sidecar (server skips emission when
370/// `entries.len() <= 1`), so the absent-sidecar verdict is
371/// `MissingHarmless` for those rather than a false-alert `Missing`.
372/// Pass [`DEFAULT_REPAIR_BODY_BYTES_CAP`] (5 GiB) for the standard CLI
373/// behaviour.
374pub async fn verify_sidecar(
375 client: &Client,
376 bucket: &str,
377 key: &str,
378 deep_scan_body_cap: u64,
379) -> Result<VerifyReport, RepairError> {
380 let HeadInfo {
381 raw_etag: live_raw_etag,
382 normalized_etag: live_etag,
383 size: live_size,
384 } = head_main(client, bucket, key).await?;
385 let sidecar_k = sidecar_key(key);
386 // v0.9 #106-audit-R5 P2-R5 (Codex): bounded sidecar fetch.
387 // A multi-GiB corrupt or legacy reserved-name user `.s4index`
388 // object would OOM the operator's repair process if we did the
389 // naive unbounded GET. Cap on HEAD-reported size.
390 let bytes = match get_sidecar_bytes_capped(client, bucket, &sidecar_k).await {
391 Ok(Some(b)) => b,
392 Ok(None) => {
393 // P2-C (Codex R3): disambiguate Missing via a body scan
394 // before deciding whether this is a healthy single-frame
395 // object or a real divergence.
396 return Ok(VerifyReport {
397 bucket: bucket.into(),
398 key: key.into(),
399 status: classify_missing_sidecar(
400 client,
401 bucket,
402 key,
403 live_raw_etag.as_deref(),
404 live_size,
405 deep_scan_body_cap,
406 )
407 .await?,
408 });
409 }
410 Err(SidecarFetchOutcome::TooLarge { size, cap }) => {
411 return Err(RepairError::SidecarTooLarge {
412 bucket: bucket.into(),
413 key: sidecar_k,
414 size,
415 cap,
416 });
417 }
418 Err(SidecarFetchOutcome::Other(msg)) => {
419 return Err(RepairError::Backend {
420 op: "GET",
421 bucket: bucket.into(),
422 key: sidecar_k,
423 cause: msg,
424 });
425 }
426 };
427 let sidecar_size = bytes.len() as u64;
428 let idx = match decode_index(bytes) {
429 Ok(i) => i,
430 Err(e) => {
431 return Ok(VerifyReport {
432 bucket: bucket.into(),
433 key: key.into(),
434 status: SidecarStatus::DecodeError {
435 message: e.to_string(),
436 },
437 });
438 }
439 };
440 let frame_count = idx.entries.len() as u64;
441 // P2-D (Codex R4): both sides of the etag comparison are now
442 // `Option<&str>` so an ETag-less backend `None == None` round-trips
443 // as Ok rather than tripping the stale path.
444 //
445 // P3-A (Codex R5): the size-only binding case `(None, Some(z))` is
446 // a fully valid v2 sidecar (just no ETag because the backend
447 // doesn't emit one). Treat any present-size binding as Ok rather
448 // than falling through to `LegacyV1`, which would falsely tell
449 // the operator that `repair-sidecar` could "upgrade" a sidecar
450 // that already IS the v2 it can produce on that backend.
451 // `LegacyV1` is only the true pre-v0.8.4 case where neither
452 // binding field is present.
453 let status = match (idx.source_etag.as_deref(), idx.source_compressed_size) {
454 (Some(side_etag), _) if Some(side_etag) != live_etag.as_deref() => {
455 SidecarStatus::StaleEtag {
456 sidecar_etag: side_etag.into(),
457 live_etag: live_etag.unwrap_or_default(),
458 }
459 }
460 (_, Some(side_size)) if side_size != live_size => SidecarStatus::StaleSize {
461 sidecar_size: side_size,
462 live_size,
463 },
464 // Any present size binding → Ok (covers full v2 AND the
465 // size-only-binding case from ETag-less repair, P3-A).
466 (_, Some(_)) => SidecarStatus::Ok {
467 frame_count,
468 sidecar_size,
469 },
470 // No size binding at all → genuinely legacy v1. Covers both
471 // (None, None) and the anomalous (Some, None) shape (which
472 // encode_index never emits, but match exhaustiveness needs
473 // coverage).
474 (_, None) => SidecarStatus::LegacyV1 { frame_count },
475 };
476 Ok(VerifyReport {
477 bucket: bucket.into(),
478 key: key.into(),
479 status,
480 })
481}
482
483/// Rebuild `<bucket>/<key>.s4index` from the main object body. Overwrites
484/// any existing sidecar (including stale or corrupt ones). Returns an error
485/// when the main body exceeds `body_bytes_cap`.
486///
487/// Back-compat shim around [`repair_sidecar_with_keyring`] for callers that
488/// don't carry an SSE-S4 keyring (= every v0.9 caller). Encrypted bodies
489/// surface as [`RepairError::EncryptedSidecarUnsupported`] same as before.
490pub async fn repair_sidecar(
491 client: &Client,
492 bucket: &str,
493 key: &str,
494 body_bytes_cap: u64,
495) -> Result<RepairReport, RepairError> {
496 repair_sidecar_with_keyring(client, bucket, key, body_bytes_cap, None).await
497}
498
499/// v0.10 #A1: same as [`repair_sidecar`] but optionally accepts an
500/// SSE-S4 keyring. When the on-disk body is an SSE-S4 chunked (S4E6)
501/// envelope AND a keyring is supplied, the repair path:
502///
503/// 1. parses the S4E6 fixed header (`parse_s4e6_header`) to recover
504/// the per-PUT salt / key_id / chunk_size / chunk_count,
505/// 2. decrypts the body in-process via
506/// [`crate::sse::decrypt_chunked_buffered`] (full plaintext into
507/// RAM — repair already had the encrypted body in RAM),
508/// 3. runs the frame scan on the recovered plaintext, and
509/// 4. stamps a v3 sidecar carrying the `sse_v3` chunked binding so
510/// subsequent Range GETs take the encryption-aware partial-fetch
511/// fast-path the v0.9 PUT path already wires.
512///
513/// Non-S4E6 envelopes (S4E1/E2/E3/E4/E5) and missing-keyring cases
514/// fall through to [`RepairError::EncryptedSidecarUnsupported`] — see
515/// the variant doc for the reasoning. Decrypt failures (key mismatch,
516/// chunk-tag verify) surface as the new
517/// [`RepairError::SseDecryptFailed`].
518pub async fn repair_sidecar_with_keyring(
519 client: &Client,
520 bucket: &str,
521 key: &str,
522 body_bytes_cap: u64,
523 sse_keyring: Option<&crate::sse::SharedSseKeyring>,
524) -> Result<RepairReport, RepairError> {
525 let HeadInfo {
526 raw_etag: head_raw_etag,
527 normalized_etag: head_normalized_etag,
528 size: live_size,
529 } = head_main(client, bucket, key).await?;
530 // v0.10 #A1 (Codex R1+R2): `body_bytes_cap` is conceptually a
531 // plaintext-size cap (matches the gateway's `--max-body-bytes`,
532 // which bounds PRE-encrypt PUT bodies).
533 //
534 // - Plaintext bodies: `live_size` IS the body size; enforce the
535 // cap strictly (`live_size > body_bytes_cap` → `BodyTooLarge`).
536 // - SSE-S4 chunked (S4E6) bodies: the on-disk ciphertext carries
537 // up to `S4E6_HEADER_BYTES + S4E6_MAX_CHUNK_COUNT × TAG_LEN` ≈
538 // 256 MiB of envelope overhead on top of the plaintext, so
539 // blindly enforcing the cap against `live_size` would reject
540 // an object whose plaintext fits the cap purely on encryption
541 // overhead. Relax the live-size check by the worst-case
542 // envelope overhead — but ONLY after a 4-byte magic peek
543 // confirms the body actually IS S4E6 (Codex R2 caught the
544 // earlier "keyring supplied ⇒ relax cap" shortcut, which
545 // silently bypassed the operator's RAM cap on plaintext
546 // objects that happened to be repaired with a keyring set
547 // for another bucket / key).
548 //
549 // The post-decrypt plaintext is still capped at `body_bytes_cap`
550 // by `decrypt_chunked_buffered` (`max_body_bytes` arg), so the
551 // operator's RAM budget stays bounded by `body_bytes_cap` plus
552 // the envelope overhead even after the relaxation.
553 if live_size > body_bytes_cap {
554 let should_relax = if sse_keyring.is_some()
555 && live_size <= body_bytes_cap.saturating_add(SSE_S4_REPAIR_MAX_OVERHEAD_BYTES)
556 {
557 // The body MIGHT be S4E6 within the overhead headroom —
558 // peek the first 4 bytes to confirm before relaxing. A
559 // single bounded Range GET keeps the cost ≤ 4 bytes of
560 // network + 1 round-trip; far cheaper than the
561 // unconditional relaxation that v0.10 #A1 first shipped.
562 match peek_body_magic(client, bucket, key, head_raw_etag.as_deref()).await {
563 Ok(Some(magic)) => &magic == b"S4E6",
564 Ok(None) => false,
565 Err(e) => {
566 return Err(RepairError::Backend {
567 op: "GET",
568 bucket: bucket.into(),
569 key: key.into(),
570 cause: format!("4-byte magic peek for SSE-overhead-cap relaxation: {e}"),
571 });
572 }
573 }
574 } else {
575 false
576 };
577 if !should_relax {
578 // Report the operator-supplied cap (not the relaxed one)
579 // so the actionable guidance stays "pass --max-body-bytes
580 // to raise".
581 return Err(RepairError::BodyTooLarge {
582 size: live_size,
583 cap: body_bytes_cap,
584 });
585 }
586 }
587 // v0.9 #106 TOCTOU guard: pin the GET to the HEAD's ETag via If-Match.
588 // Without this, an overwrite between HEAD and GET would yield a body
589 // whose actual ETag is E2 while we stamp `source_etag = E1`, producing
590 // a sidecar that fails its own version-binding check on the very next
591 // Range GET (operator sees "repair succeeded" then nothing changed).
592 // Backend returns 412 PreconditionFailed if the object changed.
593 //
594 // P1-B (Codex review R1): pass the RAW etag (quoted entity-tag) per
595 // RFC 7232, not the normalized form. Strict S3-compatible backends
596 // reject `If-Match: abc-2` (missing quotes) with 400/412 and the
597 // repair never succeeds. Tolerant backends accept either. The
598 // sidecar's stored `source_etag` still uses the normalized form to
599 // match the server's PUT-path stamping convention.
600 //
601 // P2-D (Codex R4): when the backend doesn't return an ETag at all,
602 // skip `If-Match` entirely. Same posture the server takes in that
603 // case (it stamps `source_etag = None`); the race window stays open
604 // for those backends, but they don't have an ETag we could pin
605 // against anyway.
606 let get_builder = client.get_object().bucket(bucket).key(key);
607 let get_builder = match &head_raw_etag {
608 Some(t) => get_builder.if_match(t.clone()),
609 None => get_builder,
610 };
611 let body = match get_builder.send().await {
612 Ok(resp) => resp
613 .body
614 .collect()
615 .await
616 .map(|agg| agg.into_bytes())
617 .map_err(|e| RepairError::Backend {
618 op: "GET",
619 bucket: bucket.into(),
620 key: key.into(),
621 cause: format!("read body: {e}"),
622 })?,
623 Err(e) => {
624 // PreconditionFailed (412) → object was overwritten between
625 // HEAD and GET. Surface as a typed error so the operator can
626 // re-run instead of writing a stale sidecar.
627 let s = format!("{e}");
628 if s.contains("PreconditionFailed") || s.contains("412") {
629 return Err(RepairError::OverwrittenDuringRepair {
630 bucket: bucket.into(),
631 key: key.into(),
632 head_etag: head_normalized_etag.clone().unwrap_or_default(),
633 });
634 }
635 if is_get_not_found(&e) {
636 return Err(RepairError::Backend {
637 op: "GET",
638 bucket: bucket.into(),
639 key: key.into(),
640 cause: "object not found (NoSuchKey)".into(),
641 });
642 }
643 return Err(RepairError::Backend {
644 op: "GET",
645 bucket: bucket.into(),
646 key: key.into(),
647 cause: s,
648 });
649 }
650 };
651 // Defense in depth: even with If-Match, double-check the bytes we got
652 // are the size HEAD promised. Backends with quirky range / cache
653 // behaviour have surprised us before — see codec memo on partial
654 // serves that succeeded with the wrong length.
655 if (body.len() as u64) != live_size {
656 return Err(RepairError::Backend {
657 op: "GET",
658 bucket: bucket.into(),
659 key: key.into(),
660 cause: format!(
661 "got {} bytes but HEAD said {}; backend served wrong content length",
662 body.len(),
663 live_size
664 ),
665 });
666 }
667 // v0.9 #106-audit-R2 P2-INT-1 (initial) / v0.10 #A1 (keyring
668 // plumbing): detect SSE-S4 encrypted envelopes BEFORE handing the
669 // body to the frame scanner. The backend serves the on-disk
670 // ciphertext (S4E1..S4E6 magic prefix); `build_index_from_body`
671 // would scan for `S4F2` frame magic inside that ciphertext and
672 // surface an opaque `FrameScan` error.
673 //
674 // - `S4E6` + keyring supplied → decrypt in-process, frame-scan the
675 // plaintext, stamp a v3 sidecar with the chunked binding so
676 // Range GETs take the encryption-aware fast-path.
677 // - `S4E6` + no keyring → `EncryptedSidecarUnsupported` directing
678 // the operator to pass `--sse-s4-key`.
679 // - Any non-S4E6 envelope (S4E1/E2/E3/E4/E5) → unsupported,
680 // `EncryptedSidecarUnsupported`; buffered AEAD frames have no
681 // per-chunk geometry and SSE-C / SSE-KMS need different
682 // key-material plumbing (both deferred to v0.11+).
683 let sse_repair: Option<(bytes::Bytes, s4_codec::index::SseChunkBinding)> =
684 match detect_sse_magic(&body) {
685 Some("S4E6") => match sse_keyring {
686 Some(keyring) => {
687 let (plaintext, binding) =
688 decrypt_s4e6_for_repair(&body, keyring, body_bytes_cap, bucket, key)?;
689 Some((plaintext, binding))
690 }
691 None => {
692 return Err(RepairError::EncryptedSidecarUnsupported {
693 bucket: bucket.into(),
694 key: key.into(),
695 message: "body magic S4E6 indicates SSE-S4 envelope; \
696 pass `--sse-s4-key` to decrypt and rebuild the v3 sidecar"
697 .into(),
698 });
699 }
700 },
701 Some(magic) => {
702 return Err(RepairError::EncryptedSidecarUnsupported {
703 bucket: bucket.into(),
704 key: key.into(),
705 message: format!(
706 "body magic {magic} indicates SSE-S4 envelope (only chunked S4E6 is \
707 repair-supported; buffered / SSE-C / SSE-KMS envelopes need a \
708 server-mode rebuild path)"
709 ),
710 });
711 }
712 None => None,
713 };
714 // Pick the body the frame scanner walks. For non-encrypted bodies
715 // this is the backend bytes we just GET'd; for S4E6 + keyring it
716 // is the decrypted plaintext (= the post-compression, pre-encrypt
717 // body the v0.9 PUT path stamps the sidecar against).
718 let scan_body: &bytes::Bytes = match &sse_repair {
719 Some((plaintext, _)) => plaintext,
720 None => &body,
721 };
722 let sidecar_k = sidecar_key(key);
723 let rebuilt_from_existing = client
724 .head_object()
725 .bucket(bucket)
726 .key(&sidecar_k)
727 .send()
728 .await
729 .is_ok();
730 let mut idx = build_index_from_body(scan_body).map_err(|e| RepairError::FrameScan {
731 bucket: bucket.into(),
732 key: key.into(),
733 cause: e.to_string(),
734 })?;
735 // v0.9 #106-audit-R3 P2-R3 (Codex): `build_index_from_body`
736 // on a non-S4F2 body (passthrough / raw bytes) returns Ok with
737 // an empty entries vec rather than an error. Writing that as a
738 // sidecar would silently break Range GET — `lookup_range` over
739 // zero entries returns None, and the GET path then takes the
740 // "no plan" branch instead of the passthrough-range fallback
741 // that exists for sidecar-less objects. Reject cleanly so the
742 // operator knows the object isn't a sidecar-repair candidate.
743 if idx.entries.is_empty() {
744 return Err(RepairError::NotFramed {
745 bucket: bucket.into(),
746 key: key.into(),
747 });
748 }
749 // Stamp the NORMALIZED form so server-side
750 // `sidecar_version_binding_ok` (which compares against the s3s
751 // `ETag::value()` stripped form) sees a match. The raw form was
752 // only needed for the wire-level `If-Match` header above.
753 //
754 // P2-D (Codex R4): pass through `None` when the backend doesn't
755 // return an ETag — the server's binding check treats `None` as
756 // the legacy/back-compat best-effort path. Stamping `Some("")`
757 // would force the check into the mismatch branch and the sidecar
758 // would be immediately rejected as stale.
759 idx.source_etag = head_normalized_etag.clone();
760 idx.source_compressed_size = Some(body.len() as u64);
761 // v0.10 #A1: stamp the SSE-S4 chunked binding so the GET path takes
762 // the encryption-aware partial-fetch fast-path. Mirrors the v0.9
763 // PUT path's binding construction (service.rs `sse_binding`); the
764 // salt is not secret (it lives in the encrypted body's plaintext
765 // header anyway), so duplicating it in the sidecar saves the GET
766 // path an extra HEAD/GET round-trip per Range request.
767 if let Some((_plaintext, binding)) = sse_repair.as_ref() {
768 idx.sse_v3 = Some(*binding);
769 }
770 let encoded = encode_index(&idx);
771 let encoded_len = encoded.len() as u64;
772 let frame_count = idx.entries.len() as u64;
773 client
774 .put_object()
775 .bucket(bucket)
776 .key(&sidecar_k)
777 .body(aws_sdk_s3::primitives::ByteStream::from(encoded.to_vec()))
778 .content_type("application/x-s4-index")
779 .send()
780 .await
781 .map_err(|e| RepairError::Backend {
782 op: "PUT",
783 bucket: bucket.into(),
784 key: sidecar_k.clone(),
785 cause: format!("{e}"),
786 })?;
787 // v0.9 #106 P2-B (Codex review round 2): `If-Match` on the GET
788 // only proves the body hadn't changed at GET time. The main object
789 // can still be overwritten during the (a) build_index_from_body
790 // scan and (b) sidecar PUT window — leaving a freshly-written
791 // sidecar stamped with the OLD ETag against the NEW body. The
792 // server-side `sidecar_version_binding_ok` would then trip on
793 // every Range GET and we'd silently report "repair succeeded".
794 //
795 // Final HEAD: if the main object's ETag changed since we read it,
796 // the sidecar we just wrote is already stale. Delete it (so the
797 // operator's next Range GET falls back to the safe full-read path,
798 // not the bad fast-path) and surface `OverwrittenDuringRepair`
799 // so the operator re-runs the repair under quieter conditions.
800 let post = head_main(client, bucket, key).await?;
801 if post.normalized_etag != head_normalized_etag || post.size != live_size {
802 // Best-effort cleanup; ignore the delete's outcome because the
803 // primary error is the race, not the cleanup itself.
804 let _ = client
805 .delete_object()
806 .bucket(bucket)
807 .key(&sidecar_k)
808 .send()
809 .await;
810 return Err(RepairError::OverwrittenDuringRepair {
811 bucket: bucket.into(),
812 key: key.into(),
813 head_etag: head_normalized_etag.unwrap_or_default(),
814 });
815 }
816 let sse_v3_binding = sse_repair.as_ref().map(|(_, b)| RepairSseBinding {
817 enc_chunk_size: b.enc_chunk_size,
818 enc_chunk_count: b.enc_chunk_count,
819 enc_key_id: b.enc_key_id,
820 enc_plaintext_len: b.enc_plaintext_len,
821 enc_header_bytes: b.enc_header_bytes,
822 });
823 Ok(RepairReport {
824 bucket: bucket.into(),
825 key: key.into(),
826 frame_count,
827 sidecar_bytes_written: encoded_len,
828 source_etag: idx.source_etag,
829 source_compressed_size: live_size,
830 rebuilt_from_existing,
831 sse_v3_binding,
832 })
833}
834
835/// v0.10 #A1 (Codex R2): peek the first 4 bytes of the main object
836/// via a single Range GET (`Range: bytes=0-3`) so the body-cap
837/// relaxation in [`repair_sidecar_with_keyring`] can confirm the
838/// body is an S4E6 envelope BEFORE allowing the worst-case
839/// envelope-overhead headroom on top of `body_bytes_cap`. Returns:
840///
841/// - `Ok(Some([4 bytes]))` when the GET succeeded and the body had
842/// at least 4 bytes,
843/// - `Ok(None)` when the GET succeeded but the body was shorter
844/// than 4 bytes (= no magic to test against; caller treats as
845/// not-S4E6 and fails closed),
846/// - `Err(_)` on any backend error — caller surfaces as
847/// `RepairError::Backend` so the operator sees the underlying
848/// cause (network, permissions, etc.).
849///
850/// Pinned to the HEAD's `If-Match` ETag (when available) so a
851/// concurrent swap between the HEAD and this peek can't be used to
852/// bypass the cap (small object swapped in to pass the magic check,
853/// then the full GET delivers the original oversized ciphertext).
854async fn peek_body_magic(
855 client: &Client,
856 bucket: &str,
857 key: &str,
858 if_match_raw: Option<&str>,
859) -> Result<Option<[u8; 4]>, String> {
860 let get_builder = client
861 .get_object()
862 .bucket(bucket)
863 .key(key)
864 .range("bytes=0-3");
865 let get_builder = match if_match_raw {
866 Some(t) => get_builder.if_match(t.to_owned()),
867 None => get_builder,
868 };
869 let resp = get_builder.send().await.map_err(|e| format!("{e}"))?;
870 let bytes = resp
871 .body
872 .collect()
873 .await
874 .map(|agg| agg.into_bytes())
875 .map_err(|e| format!("read peek body: {e}"))?;
876 if bytes.len() < 4 {
877 return Ok(None);
878 }
879 let mut magic = [0u8; 4];
880 magic.copy_from_slice(&bytes[..4]);
881 Ok(Some(magic))
882}
883
884/// v0.10 #A1: decrypt an S4E6-magic body in-process so
885/// [`repair_sidecar_with_keyring`] can frame-scan the recovered
886/// plaintext and stamp a v3 sidecar binding. Returns the plaintext
887/// bytes paired with the [`s4_codec::index::SseChunkBinding`] reflecting
888/// the on-disk envelope geometry — mirrors the v0.9 PUT-path binding
889/// construction in `service.rs::put_object` so a sidecar rebuilt here
890/// is byte-equivalent (modulo ETag bind) to one written at PUT time.
891///
892/// `body_bytes_cap` is reused as the plaintext-size cap fed to
893/// [`crate::sse::decrypt_chunked_buffered`]; on 64-bit it saturates at
894/// `usize::MAX` so a 5 GiB cap survives the conversion intact. The
895/// outer `repair_sidecar_with_keyring` already enforced the same cap
896/// against the on-disk ciphertext (`live_size > body_bytes_cap` →
897/// `BodyTooLarge`), so by the time we land here the plaintext can only
898/// be marginally smaller than the cap — passing the same value through
899/// keeps the failure mode consistent.
900fn decrypt_s4e6_for_repair(
901 body: &[u8],
902 keyring: &crate::sse::SharedSseKeyring,
903 body_bytes_cap: u64,
904 bucket: &str,
905 key: &str,
906) -> Result<(bytes::Bytes, s4_codec::index::SseChunkBinding), RepairError> {
907 let hdr = crate::sse::parse_s4e6_header(body).map_err(|e| RepairError::SseDecryptFailed {
908 bucket: bucket.into(),
909 key: key.into(),
910 cause: format!("parse S4E6 header: {e}"),
911 })?;
912 // v0.10 #A1 (Codex R3 + R4): `decrypt_chunked_buffered` enforces
913 // its `max_body_bytes` against the DECLARED upper bound
914 // `chunk_size × chunk_count` — not the actual plaintext length.
915 // Because the final chunk may be partial (encoder uses
916 // `ceil(plaintext / chunk_size)`), declared can exceed actual
917 // by up to `chunk_size - 1` bytes. Passing `body_bytes_cap`
918 // verbatim therefore wrongly rejects an object whose ACTUAL
919 // plaintext fits the cap but whose final-chunk slack pushes
920 // declared above it. Bump the decrypt cap by one `chunk_size`
921 // worth of slack, then re-verify the actual plaintext length
922 // against the operator-supplied cap after decrypt.
923 //
924 // Codex R4: the on-disk `hdr.chunk_size` is attacker-controlled
925 // (a tampered S4E6 header could declare `chunk_size = u32::MAX`
926 // and `chunk_count = 1`, then trick the buffered decrypt's
927 // `Vec::with_capacity(chunk_size × chunk_count)` into a ~4 GiB
928 // allocation under a much smaller operator cap). Cap the slack
929 // at the trusted [`SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES`]
930 // ceiling so the worst-case RAM increase from this slack is
931 // bounded regardless of what the on-disk header claims.
932 let chunk_slack = (hdr.chunk_size as u64).min(SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES);
933 let cap_with_slack = body_bytes_cap.saturating_add(chunk_slack);
934 let cap_usize: usize = cap_with_slack.min(usize::MAX as u64) as usize;
935 let plaintext = crate::sse::decrypt_chunked_buffered(body, keyring.as_ref(), cap_usize)
936 .map_err(|e| RepairError::SseDecryptFailed {
937 bucket: bucket.into(),
938 key: key.into(),
939 cause: format!("decrypt S4E6 chunks: {e}"),
940 })?;
941 // Operator cap applies to actual plaintext bytes, not declared
942 // upper bound. Surfaces as the same `BodyTooLarge` variant the
943 // outer ciphertext-cap path uses so the CLI's error formatting
944 // stays consistent.
945 if (plaintext.len() as u64) > body_bytes_cap {
946 return Err(RepairError::BodyTooLarge {
947 size: plaintext.len() as u64,
948 cap: body_bytes_cap,
949 });
950 }
951 let binding = s4_codec::index::SseChunkBinding {
952 enc_chunk_size: hdr.chunk_size,
953 enc_chunk_count: hdr.chunk_count,
954 enc_key_id: hdr.key_id,
955 enc_salt: *hdr.salt,
956 enc_plaintext_len: plaintext.len() as u64,
957 // Carried explicitly so a future S4E7-style header bump can't
958 // silently break v3 sidecar decode (mirrors the v0.9 PUT-path
959 // stamp at `service.rs::put_object` `enc_header_bytes`).
960 enc_header_bytes: crate::sse::S4E6_HEADER_BYTES as u32,
961 };
962 Ok((plaintext, binding))
963}
964
965/// Knob controlling which orphan categories `sweep_orphan_sidecars` is
966/// allowed to delete. `SidecarUndecodable` is kept out of the default
967/// `--delete` because v0.8.17-era operators on the
968/// `--allow-legacy-reserved-key-reads` migration hatch can have
969/// legitimate user-PUT objects whose key happens to end in `.s4index` —
970/// those would fail to decode and `--delete` would nuke real user data.
971/// Escalation to `DeletePolicy::IncludeUndecodable` is an explicit
972/// operator opt-in (`--delete-undecodable` on the CLI).
973#[derive(Debug, Clone, Copy, PartialEq, Eq)]
974pub enum DeletePolicy {
975 /// Pure dry-run: classify only, never write to the backend.
976 DryRun,
977 /// Delete `PairedMissing` / `PairedEtagMismatch` / `PairedSizeMismatch`
978 /// orphans. Leave `SidecarUndecodable` in the report — operator must
979 /// inspect those and rerun with `IncludeUndecodable` if they truly
980 /// are corrupt sidecars (and not legacy reserved-name user data).
981 PairBoundOnly,
982 /// All four categories. Use only after confirming there's no legacy
983 /// `--allow-legacy-reserved-key-reads` user data in this bucket.
984 IncludeUndecodable,
985}
986
987impl DeletePolicy {
988 fn allows(&self, reason: &OrphanReason) -> bool {
989 match (self, reason) {
990 (DeletePolicy::DryRun, _) => false,
991 (DeletePolicy::PairBoundOnly, OrphanReason::SidecarUndecodable { .. }) => false,
992 (DeletePolicy::PairBoundOnly, _) => true,
993 (DeletePolicy::IncludeUndecodable, _) => true,
994 }
995 }
996}
997
998/// List every `*.s4index` in `bucket` and report (and optionally delete) the
999/// orphans — sidecars whose paired key is missing or whose recorded
1000/// ETag / size disagree with the live HEAD.
1001///
1002/// See [`DeletePolicy`] for the three deletion levels. Always run
1003/// [`DeletePolicy::DryRun`] first to inspect the orphan list.
1004pub async fn sweep_orphan_sidecars(
1005 client: &Client,
1006 bucket: &str,
1007 policy: DeletePolicy,
1008) -> Result<SweepReport, RepairError> {
1009 let mut sidecars_scanned: u64 = 0;
1010 let mut orphans: Vec<OrphanReport> = Vec::new();
1011 let mut continuation: Option<String> = None;
1012 loop {
1013 let mut req = client.list_objects_v2().bucket(bucket);
1014 if let Some(c) = continuation.as_ref() {
1015 req = req.continuation_token(c);
1016 }
1017 let resp = req.send().await.map_err(|e| RepairError::Backend {
1018 op: "ListObjectsV2",
1019 bucket: bucket.into(),
1020 key: String::new(),
1021 cause: format!("{e}"),
1022 })?;
1023 for obj in resp.contents() {
1024 let Some(k) = obj.key() else { continue };
1025 if !k.ends_with(SIDECAR_SUFFIX) {
1026 continue;
1027 }
1028 sidecars_scanned += 1;
1029 let paired = &k[..k.len() - SIDECAR_SUFFIX.len()];
1030 classify_one(client, bucket, k, paired, &mut orphans).await?;
1031 }
1032 if resp.is_truncated().unwrap_or(false) {
1033 continuation = resp.next_continuation_token().map(str::to_owned);
1034 if continuation.is_none() {
1035 // Defensive: a truncated response with no continuation token
1036 // is a backend bug; bail rather than infinite-loop.
1037 break;
1038 }
1039 } else {
1040 break;
1041 }
1042 }
1043 let mut deleted = 0u64;
1044 for orph in &orphans {
1045 if !policy.allows(&orph.reason) {
1046 continue;
1047 }
1048 client
1049 .delete_object()
1050 .bucket(bucket)
1051 .key(&orph.sidecar_key)
1052 .send()
1053 .await
1054 .map_err(|e| RepairError::Backend {
1055 op: "DELETE",
1056 bucket: bucket.into(),
1057 key: orph.sidecar_key.clone(),
1058 cause: format!("{e}"),
1059 })?;
1060 deleted += 1;
1061 }
1062 Ok(SweepReport {
1063 bucket: bucket.into(),
1064 sidecars_scanned,
1065 orphans,
1066 deleted,
1067 })
1068}
1069
1070/// P2-C (Codex R3): the server skips sidecar emission for objects whose
1071/// frame count is ≤ 1 (small single-PUTs / single-chunk multiparts), so
1072/// a missing sidecar can be EITHER an intentional skip OR a real
1073/// divergence. Disambiguate by fetching the body (capped) and counting
1074/// frames. Returns [`SidecarStatus::MissingUnknown`] when the body
1075/// exceeds the cap, so verify-sidecar doesn't false-alert on
1076/// large-but-can't-confirm objects.
1077async fn classify_missing_sidecar(
1078 client: &Client,
1079 bucket: &str,
1080 key: &str,
1081 live_raw_etag: Option<&str>,
1082 live_size: u64,
1083 cap: u64,
1084) -> Result<SidecarStatus, RepairError> {
1085 if live_size > cap {
1086 return Ok(SidecarStatus::MissingUnknown {
1087 size: live_size,
1088 cap,
1089 });
1090 }
1091 // Pin the GET to the HEAD's ETag (RFC 7232 quoted form). If a race
1092 // overwrites the object between HEAD and GET we'd otherwise scan a
1093 // different body than the one HEAD reported on — surface as a
1094 // typed error so the operator re-runs.
1095 //
1096 // P2-D: backends without an ETag have nothing to pin against;
1097 // skip If-Match (matches the server-side `None`-tolerance path).
1098 let get_builder = client.get_object().bucket(bucket).key(key);
1099 let get_builder = match live_raw_etag {
1100 Some(t) => get_builder.if_match(t.to_owned()),
1101 None => get_builder,
1102 };
1103 let body = match get_builder.send().await {
1104 Ok(resp) => resp
1105 .body
1106 .collect()
1107 .await
1108 .map(|agg| agg.into_bytes())
1109 .map_err(|e| RepairError::Backend {
1110 op: "GET",
1111 bucket: bucket.into(),
1112 key: key.into(),
1113 cause: format!("read body: {e}"),
1114 })?,
1115 Err(e) => {
1116 let s = format!("{e}");
1117 if s.contains("PreconditionFailed") || s.contains("412") {
1118 return Err(RepairError::OverwrittenDuringRepair {
1119 bucket: bucket.into(),
1120 key: key.into(),
1121 head_etag: live_raw_etag.map(normalize_etag).unwrap_or_default(),
1122 });
1123 }
1124 if is_get_not_found(&e) {
1125 return Err(RepairError::Backend {
1126 op: "GET",
1127 bucket: bucket.into(),
1128 key: key.into(),
1129 cause: "object not found (NoSuchKey)".into(),
1130 });
1131 }
1132 return Err(RepairError::Backend {
1133 op: "GET",
1134 bucket: bucket.into(),
1135 key: key.into(),
1136 cause: s,
1137 });
1138 }
1139 };
1140 // v0.9 #106-audit self-review (post-R2): mirror the encrypted-body
1141 // guard from `repair_sidecar` here. Without it, running
1142 // `verify-sidecar` against an SSE-S4 chunked object (whose sidecar
1143 // is missing — e.g. PUT happened pre-v0.9 before v3 sidecars
1144 // shipped) would surface as a confusing FrameScan error instead of
1145 // the friendly EncryptedSidecarUnsupported the repair tool already
1146 // returns. Same root cause as P2-INT-1; same surface error.
1147 if let Some(magic) = detect_sse_magic(&body) {
1148 return Err(RepairError::EncryptedSidecarUnsupported {
1149 bucket: bucket.into(),
1150 key: key.into(),
1151 message: format!("body magic {magic} indicates SSE-S4 envelope"),
1152 });
1153 }
1154 // v0.9 #106-audit-R4 P2-R4 (Codex): a passthrough / raw-bytes
1155 // body (no S4F2 magic) trips `build_index_from_body` with a
1156 // `BadMagic` `FrameError`. From the verify-sidecar perspective
1157 // that's the same outcome as a single-frame body: server never
1158 // sidecared it, Range GET takes the full-read path, no operator
1159 // action needed. Surface `MissingHarmless { frame_count: 0 }`
1160 // (clean, exit 0) instead of a FrameScan repair error (exit 1)
1161 // so CI / cron jobs don't false-alert on healthy passthrough
1162 // objects. Twin of R3 P2-R3 on the repair-side.
1163 let idx = match build_index_from_body(&body) {
1164 Ok(i) => i,
1165 Err(crate::codec::multipart::FrameError::BadMagic { .. }) => {
1166 return Ok(SidecarStatus::MissingHarmless { frame_count: 0 });
1167 }
1168 Err(e) => {
1169 return Err(RepairError::FrameScan {
1170 bucket: bucket.into(),
1171 key: key.into(),
1172 cause: e.to_string(),
1173 });
1174 }
1175 };
1176 let frame_count = idx.entries.len() as u64;
1177 if frame_count <= 1 {
1178 Ok(SidecarStatus::MissingHarmless { frame_count })
1179 } else {
1180 Ok(SidecarStatus::MissingDivergent { frame_count })
1181 }
1182}
1183
1184async fn classify_one(
1185 client: &Client,
1186 bucket: &str,
1187 sidecar_k: &str,
1188 paired: &str,
1189 out: &mut Vec<OrphanReport>,
1190) -> Result<(), RepairError> {
1191 // v0.9 #106 review P1-A (Codex): MUST decode the listed object first.
1192 // Branching on "HEAD paired-key" before reading the candidate would
1193 // mis-classify a legitimate `--allow-legacy-reserved-key-reads`
1194 // user object (whose key happens to end in `.s4index` and whose
1195 // paired stripped key may not exist) as `PairedMissing` — and
1196 // `DeletePolicy::PairBoundOnly` would silently delete user data.
1197 // The rule is: bytes that don't parse as S4IX magic = user data,
1198 // never an orphan-eligible-for-default-delete.
1199 // v0.9 #106-audit-R5 P2-R5 (Codex): bounded sidecar fetch.
1200 // sweep walks every `*.s4index` in the bucket — a single
1201 // multi-GiB attacker-supplied or legacy-user `.s4index` object
1202 // would OOM the sweep process with the naive unbounded GET.
1203 // TooLarge surfaces as a `SidecarUndecodable` orphan with a
1204 // size-explaining message rather than aborting the whole sweep
1205 // (one bad sidecar shouldn't stop the rest from being inspected).
1206 let bytes = match get_sidecar_bytes_capped(client, bucket, sidecar_k).await {
1207 Ok(Some(b)) => b,
1208 // ListObjectsV2 saw it; if GET says NotFound now, treat as a
1209 // sidecar that vanished mid-sweep — skip rather than report.
1210 Ok(None) => return Ok(()),
1211 Err(SidecarFetchOutcome::TooLarge { size, cap }) => {
1212 out.push(OrphanReport {
1213 sidecar_key: sidecar_k.into(),
1214 paired_key: paired.into(),
1215 reason: OrphanReason::SidecarUndecodable {
1216 message: format!(
1217 "sidecar size {size} > cap {cap}; refused to load (likely legacy user data or attack payload)"
1218 ),
1219 },
1220 });
1221 return Ok(());
1222 }
1223 Err(SidecarFetchOutcome::Other(msg)) => {
1224 return Err(RepairError::Backend {
1225 op: "GET",
1226 bucket: bucket.into(),
1227 key: sidecar_k.into(),
1228 cause: msg,
1229 });
1230 }
1231 };
1232 let idx = match decode_index(bytes) {
1233 Ok(i) => i,
1234 Err(e) => {
1235 // Not a real S4IX sidecar — flag it under the safer
1236 // category. `DeletePolicy::PairBoundOnly` does NOT remove
1237 // these; the operator must escalate to
1238 // `IncludeUndecodable` after confirming it isn't legacy
1239 // user data.
1240 out.push(OrphanReport {
1241 sidecar_key: sidecar_k.into(),
1242 paired_key: paired.into(),
1243 reason: OrphanReason::SidecarUndecodable {
1244 message: e.to_string(),
1245 },
1246 });
1247 return Ok(());
1248 }
1249 };
1250 // Bytes decoded as S4IX — now we can safely check the paired key
1251 // status. A missing paired key combined with a decodable sidecar
1252 // IS a real orphan (the v0.8.15 H-g case, for example).
1253 let head_res = client.head_object().bucket(bucket).key(paired).send().await;
1254 let (live_etag_norm, live_size) = match head_res {
1255 Ok(h) => {
1256 // P2-D: `None` means the backend didn't return an ETag.
1257 // Preserve the absence rather than coercing to `""` —
1258 // comparing `Some("xyz")` from the sidecar against
1259 // `Some("")` would always trip stale, falsely orphaning
1260 // every paired-OK sidecar on an ETag-less backend.
1261 let etag: Option<String> = h.e_tag().map(normalize_etag);
1262 let size = h.content_length().unwrap_or(0).max(0) as u64;
1263 (etag, size)
1264 }
1265 Err(e) => {
1266 if is_head_not_found(&e) {
1267 out.push(OrphanReport {
1268 sidecar_key: sidecar_k.into(),
1269 paired_key: paired.into(),
1270 reason: OrphanReason::PairedMissing,
1271 });
1272 return Ok(());
1273 }
1274 return Err(RepairError::Backend {
1275 op: "HEAD",
1276 bucket: bucket.into(),
1277 key: paired.into(),
1278 cause: format!("{e}"),
1279 });
1280 }
1281 };
1282 // ETag mismatch only fires when BOTH sides have an ETag. If the
1283 // sidecar carries Some("x") and the live HEAD has None, that's
1284 // not a definitive divergence — could be a backend that recently
1285 // dropped ETag support. Skip the mismatch flag for the None side
1286 // (matches the server's `sidecar_version_binding_ok` `None`-
1287 // tolerance posture).
1288 if let (Some(side_etag), Some(live_e)) = (idx.source_etag.as_deref(), live_etag_norm.as_deref())
1289 && side_etag != live_e
1290 {
1291 out.push(OrphanReport {
1292 sidecar_key: sidecar_k.into(),
1293 paired_key: paired.into(),
1294 reason: OrphanReason::PairedEtagMismatch {
1295 sidecar_etag: side_etag.into(),
1296 live_etag: live_e.into(),
1297 },
1298 });
1299 return Ok(());
1300 }
1301 if let Some(side_size) = idx.source_compressed_size
1302 && side_size != live_size
1303 {
1304 out.push(OrphanReport {
1305 sidecar_key: sidecar_k.into(),
1306 paired_key: paired.into(),
1307 reason: OrphanReason::PairedSizeMismatch {
1308 sidecar_size: side_size,
1309 live_size,
1310 },
1311 });
1312 }
1313 // Legacy v1 sidecars (no binding fields) are intentionally
1314 // tolerated here — read-only Range GETs still work and the
1315 // operator gets warned by `verify-sidecar` separately.
1316 Ok(())
1317}
1318
1319/// HEAD response distilled to the fields the repair tools care about.
1320///
1321/// Both etag fields are `Option<String>` so the absent-ETag case
1322/// round-trips cleanly through to the sidecar (P2-D, Codex R4). When
1323/// `raw_etag = None`, the backend didn't return one — we MUST stamp
1324/// `FrameIndex::source_etag = None` to match the server PUT path's
1325/// `resp.e_tag.as_ref().map(...)` shape, otherwise
1326/// `sidecar_version_binding_ok` would compare `Some("")` against a
1327/// missing live ETag and always trip "stale".
1328///
1329/// - `raw_etag`: wire form (typically `"..."`) — pass to `If-Match`
1330/// headers, which per RFC 7232 want the full entity-tag. `None`
1331/// means skip `If-Match` entirely (best-effort, same posture the
1332/// server takes for ETag-less backends).
1333/// - `normalized_etag`: stripped form for comparing against
1334/// `FrameIndex::source_etag` (the s3s `ETag::value()` accessor
1335/// used by the server PUT path strips quotes).
1336struct HeadInfo {
1337 raw_etag: Option<String>,
1338 normalized_etag: Option<String>,
1339 size: u64,
1340}
1341
1342async fn head_main(client: &Client, bucket: &str, key: &str) -> Result<HeadInfo, RepairError> {
1343 let head = client
1344 .head_object()
1345 .bucket(bucket)
1346 .key(key)
1347 .send()
1348 .await
1349 .map_err(|e| RepairError::Backend {
1350 op: "HEAD",
1351 bucket: bucket.into(),
1352 key: key.into(),
1353 cause: format!("{e}"),
1354 })?;
1355 let raw_etag = head.e_tag().map(str::to_owned);
1356 let normalized_etag = raw_etag.as_deref().map(normalize_etag);
1357 // `content_length` is `Option<i64>` on the SDK type — `None` means the
1358 // backend didn't return a Content-Length header. We fail closed rather
1359 // than treating that as zero (which would silently bypass the
1360 // `body_bytes_cap` in `repair_sidecar` and let an unbounded GET
1361 // exhaust RAM). AWS S3 / MinIO / Garage / Ceph RGW all return
1362 // Content-Length on HEAD, so this only trips on exotic / broken
1363 // backends — which the operator should know about.
1364 let size = match head.content_length() {
1365 Some(n) if n >= 0 => n as u64,
1366 Some(_) | None => {
1367 return Err(RepairError::MissingContentLength {
1368 bucket: bucket.into(),
1369 key: key.into(),
1370 });
1371 }
1372 };
1373 Ok(HeadInfo {
1374 raw_etag,
1375 normalized_etag,
1376 size,
1377 })
1378}
1379
1380/// Strip the surrounding `"..."` quotes from an RFC 7232 entity-tag so
1381/// the on-wire form (aws-sdk-s3 returns raw `"..."`) matches the form
1382/// the S4 gateway stamps into `FrameIndex::source_etag` (the s3s
1383/// `ETag::value()` accessor that drives the PUT path strips quotes).
1384///
1385/// Without this normalization, a freshly-written sidecar would falsely
1386/// flag as `StaleEtag` because the strings differ only by the wrapping
1387/// quotes. Both the PUT side (server) and the repair side (this CLI)
1388/// must agree on the canonical form — the de-facto canonical is "no
1389/// surrounding quotes", since that's what the server already writes
1390/// into every v2 sidecar in the wild.
1391fn normalize_etag(s: &str) -> String {
1392 s.trim_matches('"').to_owned()
1393}
1394
1395/// v0.9 #106-audit-R2 P2-INT-1: detect SSE-S4 encrypted envelopes by
1396/// magic prefix. Returns `Some(name)` when the first four bytes match
1397/// one of the SSE frame magics (`S4E1`..`S4E6`); returns `None` for any
1398/// other body, including S4 framed plaintext (`S4F2`) and raw
1399/// compressed / passthrough bodies.
1400///
1401/// Intentionally duplicated here as a 4-byte prefix compare instead of
1402/// reusing `sse::peek_magic` because `peek_magic` length-gates on the
1403/// full S4E1/S4E2 header size (36 bytes) and would return `None` for a
1404/// very short S4E6 stub the way an empty-key edge-case might land —
1405/// the gate is for cryptographic frame validity, not for the
1406/// "is encrypted at all" question this helper answers. The exact magic
1407/// bytes are stable wire-format constants (see `sse::SSE_MAGIC_V{1..6}`)
1408/// and are echoed here so the repair module has no circular dep on the
1409/// SSE module's full surface.
1410fn detect_sse_magic(body: &[u8]) -> Option<&'static str> {
1411 if body.len() < 4 {
1412 return None;
1413 }
1414 match &body[..4] {
1415 b"S4E1" => Some("S4E1"),
1416 b"S4E2" => Some("S4E2"),
1417 b"S4E3" => Some("S4E3"),
1418 b"S4E4" => Some("S4E4"),
1419 b"S4E5" => Some("S4E5"),
1420 b"S4E6" => Some("S4E6"),
1421 _ => None,
1422 }
1423}
1424
1425/// v0.9 #106-audit-R5 P2-R5 (Codex): bounded sidecar fetch.
1426/// HEADs the sidecar key first to learn its size; refuses to GET
1427/// (and thus refuses to allocate) if the size exceeds
1428/// [`MAX_SIDECAR_BODY_BYTES`]. Used by both `verify_sidecar` and
1429/// `classify_one` (sweep) so a multi-GiB corrupt or legacy user
1430/// `.s4index` object can't OOM the operator's repair process.
1431///
1432/// Returns:
1433/// - `Ok(Some(bytes))` when the sidecar exists and fits in the cap
1434/// - `Ok(None)` when the sidecar HEAD returns NotFound (caller
1435/// classifies as `Missing*`)
1436/// - `Err(SidecarFetchOutcome::Other)` when HEAD returns
1437/// Content-Length missing or any other backend error
1438/// - `Err(SidecarFetchOutcome::TooLarge { .. })` when size > cap
1439async fn get_sidecar_bytes_capped(
1440 client: &Client,
1441 bucket: &str,
1442 key: &str,
1443) -> Result<Option<bytes::Bytes>, SidecarFetchOutcome> {
1444 let head = match client.head_object().bucket(bucket).key(key).send().await {
1445 Ok(h) => h,
1446 Err(e) => {
1447 return if is_head_not_found(&e) {
1448 Ok(None)
1449 } else {
1450 Err(SidecarFetchOutcome::Other(format!("HEAD: {e}")))
1451 };
1452 }
1453 };
1454 let size = match head.content_length() {
1455 Some(n) if n >= 0 => n as u64,
1456 Some(_) | None => {
1457 return Err(SidecarFetchOutcome::Other(
1458 "sidecar HEAD returned no Content-Length; refusing to GET unbounded".into(),
1459 ));
1460 }
1461 };
1462 if size > MAX_SIDECAR_BODY_BYTES {
1463 return Err(SidecarFetchOutcome::TooLarge {
1464 size,
1465 cap: MAX_SIDECAR_BODY_BYTES,
1466 });
1467 }
1468 // v0.9 #106-audit-R6 P2-R6 (Codex): pin the GET to the HEAD's
1469 // ETag so a sidecar swap between HEAD and GET can't bypass
1470 // the cap. Without this, an attacker who races
1471 // HEAD(small) → swap(massive) → GET could still OOM the
1472 // process because `collect()` reads whatever the GET response
1473 // delivers, ignoring the HEAD-reported size. With If-Match
1474 // pinned, the swap surfaces as 412 PreconditionFailed → we
1475 // refuse the body without allocating it.
1476 //
1477 // Backends that don't return ETags fall back to a post-GET
1478 // length check below (still a window where collect() runs to
1479 // completion, but the typed `TooLarge` exit replaces what
1480 // would otherwise be a silent OOM-pass).
1481 let raw_etag = head.e_tag().map(str::to_owned);
1482 let get_builder = client.get_object().bucket(bucket).key(key);
1483 let get_builder = match raw_etag {
1484 Some(ref t) => get_builder.if_match(t.clone()),
1485 None => get_builder,
1486 };
1487 match get_builder.send().await {
1488 Ok(resp) => {
1489 let agg = resp
1490 .body
1491 .collect()
1492 .await
1493 .map_err(|e| SidecarFetchOutcome::Other(format!("read body: {e}")))?;
1494 let bytes = agg.into_bytes();
1495 // Defense-in-depth: ETag-less backends bypass
1496 // If-Match; If-Match-non-honouring backends also exist.
1497 // Check the actual body length AFTER collect to catch
1498 // a race-during-collect that exceeded the cap.
1499 if (bytes.len() as u64) > MAX_SIDECAR_BODY_BYTES {
1500 return Err(SidecarFetchOutcome::TooLarge {
1501 size: bytes.len() as u64,
1502 cap: MAX_SIDECAR_BODY_BYTES,
1503 });
1504 }
1505 Ok(Some(bytes))
1506 }
1507 Err(e) => {
1508 let s = format!("{e}");
1509 if is_get_not_found(&e) {
1510 // Race: existed at HEAD, gone by GET. Treat as missing.
1511 Ok(None)
1512 } else if s.contains("PreconditionFailed") || s.contains("412") {
1513 // Race: sidecar replaced between HEAD and GET. The
1514 // new sidecar's size is whatever the swap-in is;
1515 // we refuse to load it without re-HEAD'ing under
1516 // operator supervision.
1517 Err(SidecarFetchOutcome::Other(format!(
1518 "sidecar at {bucket}/{key} was replaced between HEAD and GET (412 \
1519 PreconditionFailed); re-run when the sidecar is stable"
1520 )))
1521 } else {
1522 Err(SidecarFetchOutcome::Other(format!("GET: {s}")))
1523 }
1524 }
1525 }
1526}
1527
1528enum SidecarFetchOutcome {
1529 Other(String),
1530 TooLarge { size: u64, cap: u64 },
1531}
1532
1533fn is_head_not_found(
1534 e: &aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::head_object::HeadObjectError>,
1535) -> bool {
1536 matches!(
1537 e,
1538 aws_sdk_s3::error::SdkError::ServiceError(svc)
1539 if matches!(
1540 svc.err(),
1541 aws_sdk_s3::operation::head_object::HeadObjectError::NotFound(_)
1542 )
1543 )
1544}
1545
1546fn is_get_not_found(
1547 e: &aws_sdk_s3::error::SdkError<aws_sdk_s3::operation::get_object::GetObjectError>,
1548) -> bool {
1549 matches!(
1550 e,
1551 aws_sdk_s3::error::SdkError::ServiceError(svc)
1552 if matches!(
1553 svc.err(),
1554 aws_sdk_s3::operation::get_object::GetObjectError::NoSuchKey(_)
1555 )
1556 )
1557}
1558
1559/// Parse a `bucket/key` CLI argument. Splits on the **first** `/` only so
1560/// keys with slashes (e.g. `prefix/sub/file.bin`) round-trip cleanly.
1561pub fn parse_bucket_key(arg: &str) -> Result<(&str, &str), String> {
1562 match arg.split_once('/') {
1563 Some((b, k)) if !b.is_empty() && !k.is_empty() => Ok((b, k)),
1564 Some(_) => Err(format!(
1565 "expected `bucket/key`, got {arg:?} — bucket and key must both be non-empty"
1566 )),
1567 None => Err(format!("expected `bucket/key`, got {arg:?} — missing `/`")),
1568 }
1569}
1570
1571#[cfg(test)]
1572mod tests {
1573 use super::*;
1574
1575 #[test]
1576 fn parse_bucket_key_simple() {
1577 assert_eq!(
1578 parse_bucket_key("mybucket/foo.txt"),
1579 Ok(("mybucket", "foo.txt"))
1580 );
1581 }
1582
1583 #[test]
1584 fn parse_bucket_key_with_slashes_in_key() {
1585 assert_eq!(parse_bucket_key("b/a/b/c"), Ok(("b", "a/b/c")));
1586 }
1587
1588 #[test]
1589 fn parse_bucket_key_missing_slash() {
1590 assert!(parse_bucket_key("nokey").is_err());
1591 }
1592
1593 #[test]
1594 fn parse_bucket_key_empty_key() {
1595 assert!(parse_bucket_key("bucket/").is_err());
1596 }
1597
1598 #[test]
1599 fn parse_bucket_key_empty_bucket() {
1600 assert!(parse_bucket_key("/key").is_err());
1601 }
1602
1603 #[test]
1604 fn verify_report_is_clean_truth_table() {
1605 let mk = |status| VerifyReport {
1606 bucket: "b".into(),
1607 key: "k".into(),
1608 status,
1609 };
1610 assert!(
1611 mk(SidecarStatus::Ok {
1612 frame_count: 1,
1613 sidecar_size: 100,
1614 })
1615 .is_clean()
1616 );
1617 assert!(mk(SidecarStatus::LegacyV1 { frame_count: 3 }).is_clean());
1618 // P2-C (Codex R3): single-frame objects intentionally have no
1619 // sidecar — clean state, not divergence.
1620 assert!(mk(SidecarStatus::MissingHarmless { frame_count: 1 }).is_clean());
1621 // Ambiguous (body too large to deep-scan) — report cleanly so
1622 // CI doesn't false-alert; operator sees the hint in stdout.
1623 assert!(
1624 mk(SidecarStatus::MissingUnknown {
1625 size: 10 * 1024 * 1024 * 1024,
1626 cap: 5 * 1024 * 1024 * 1024,
1627 })
1628 .is_clean()
1629 );
1630 // Multi-frame + missing sidecar = real divergence.
1631 assert!(!mk(SidecarStatus::MissingDivergent { frame_count: 5 }).is_clean());
1632 assert!(
1633 !mk(SidecarStatus::StaleEtag {
1634 sidecar_etag: "a".into(),
1635 live_etag: "b".into(),
1636 })
1637 .is_clean()
1638 );
1639 assert!(
1640 !mk(SidecarStatus::StaleSize {
1641 sidecar_size: 1,
1642 live_size: 2,
1643 })
1644 .is_clean()
1645 );
1646 assert!(
1647 !mk(SidecarStatus::DecodeError {
1648 message: "bad".into()
1649 })
1650 .is_clean()
1651 );
1652 }
1653
1654 #[test]
1655 fn delete_policy_allows_truth_table() {
1656 let missing = OrphanReason::PairedMissing;
1657 let etag = OrphanReason::PairedEtagMismatch {
1658 sidecar_etag: "a".into(),
1659 live_etag: "b".into(),
1660 };
1661 let size = OrphanReason::PairedSizeMismatch {
1662 sidecar_size: 1,
1663 live_size: 2,
1664 };
1665 let undecodable = OrphanReason::SidecarUndecodable {
1666 message: "bad bytes".into(),
1667 };
1668
1669 // DryRun: never deletes anything.
1670 assert!(!DeletePolicy::DryRun.allows(&missing));
1671 assert!(!DeletePolicy::DryRun.allows(&etag));
1672 assert!(!DeletePolicy::DryRun.allows(&size));
1673 assert!(!DeletePolicy::DryRun.allows(&undecodable));
1674
1675 // PairBoundOnly: deletes the three pair-bound categories,
1676 // skips Undecodable (HIGH-2 review fix: protects v0.8.17
1677 // legacy reserved-name user data).
1678 assert!(DeletePolicy::PairBoundOnly.allows(&missing));
1679 assert!(DeletePolicy::PairBoundOnly.allows(&etag));
1680 assert!(DeletePolicy::PairBoundOnly.allows(&size));
1681 assert!(!DeletePolicy::PairBoundOnly.allows(&undecodable));
1682
1683 // IncludeUndecodable: explicit operator opt-in deletes all.
1684 assert!(DeletePolicy::IncludeUndecodable.allows(&missing));
1685 assert!(DeletePolicy::IncludeUndecodable.allows(&etag));
1686 assert!(DeletePolicy::IncludeUndecodable.allows(&size));
1687 assert!(DeletePolicy::IncludeUndecodable.allows(&undecodable));
1688 }
1689
1690 /// P3-A (Codex R5): a v2 sidecar with size binding but no ETag
1691 /// (rebuilt on an ETag-less backend) classifies as `Ok`, NOT
1692 /// `LegacyV1`. The latter would tell operators to "repair to
1693 /// upgrade" a sidecar already at the highest binding level the
1694 /// backend supports. This test asserts the exact pattern the
1695 /// status match in `verify_sidecar` relies on.
1696 #[test]
1697 fn verify_status_classifies_etag_less_v2_as_ok_not_legacy() {
1698 // The actual match arms in `verify_sidecar`:
1699 //
1700 // (Some(s), _) if Some(s) != live → StaleEtag
1701 // (_, Some(z)) if z != live_size → StaleSize
1702 // (_, Some(_)) → Ok // P3-A fix
1703 // (None, None) → LegacyV1
1704 //
1705 // Mirror that decision tree inline so refactors to the real
1706 // function can't quietly regress without flipping this test.
1707 fn classify(side_etag: Option<&str>, side_size: Option<u64>) -> &'static str {
1708 const LIVE_ETAG: Option<&str> = Some("xyz");
1709 const LIVE_SIZE: u64 = 100;
1710 match (side_etag, side_size) {
1711 (Some(s), _) if Some(s) != LIVE_ETAG => "StaleEtag",
1712 (_, Some(z)) if z != LIVE_SIZE => "StaleSize",
1713 (_, Some(_)) => "Ok",
1714 (_, None) => "LegacyV1",
1715 }
1716 }
1717 // P3-A core case: ETag-less repair stamps (None, Some(size)).
1718 // Must classify as Ok, not LegacyV1.
1719 assert_eq!(classify(None, Some(100)), "Ok");
1720 // Full v2 binding with matching etag + size.
1721 assert_eq!(classify(Some("xyz"), Some(100)), "Ok");
1722 // True v1 legacy (neither field) still surfaces as LegacyV1.
1723 assert_eq!(classify(None, None), "LegacyV1");
1724 // Mismatches still detected.
1725 assert_eq!(classify(Some("abc"), Some(100)), "StaleEtag");
1726 assert_eq!(classify(Some("xyz"), Some(999)), "StaleSize");
1727 }
1728
1729 /// P2-D (Codex R4): on an ETag-less backend the server stamps
1730 /// `source_etag = None`; the verifier MUST treat that as the
1731 /// legacy / best-effort path (Ok / LegacyV1), not flag every
1732 /// such sidecar as stale. This unit test pins the discriminator
1733 /// the `verify_sidecar` status-match arm relies on (the
1734 /// `Option<&str>` equality).
1735 #[test]
1736 fn etag_option_equality_treats_none_none_as_match() {
1737 let side: Option<&str> = None;
1738 let live: Option<&str> = None;
1739 assert!(side == live, "None == None must hold for the no-ETag path");
1740
1741 let side: Option<&str> = Some("abc");
1742 let live: Option<&str> = Some("abc");
1743 assert!(side == live);
1744
1745 let side: Option<&str> = Some("");
1746 let live: Option<&str> = None;
1747 assert!(side != live, "Some(\"\") must NOT equal None — P2-D guard");
1748 }
1749
1750 #[test]
1751 fn normalize_etag_strips_surrounding_quotes() {
1752 // aws-sdk-s3 returns the wire form (with quotes); s3s `value()`
1753 // returns the stripped form. The sidecar's `source_etag` is
1754 // canonical-stripped, so both sides must agree.
1755 assert_eq!(normalize_etag("\"abc-1\""), "abc-1");
1756 // Multipart ETags are `<hex>-<n>` and still get quoted on wire.
1757 assert_eq!(
1758 normalize_etag("\"067e3167e8c481c2aea3650ebb273198-2\""),
1759 "067e3167e8c481c2aea3650ebb273198-2"
1760 );
1761 // Already-stripped form is a no-op (the helper is idempotent so
1762 // callers don't need to branch on the source).
1763 assert_eq!(normalize_etag("abc-1"), "abc-1");
1764 // Defensive: an empty etag stays empty (head responses with no
1765 // ETag header round-trip to the empty string in head_main).
1766 assert_eq!(normalize_etag(""), "");
1767 }
1768
1769 /// P2-R5 (Codex R5 audit): the bounded sidecar fetch helper
1770 /// must enforce [`MAX_SIDECAR_BODY_BYTES`] and surface a typed
1771 /// `SidecarTooLarge` error before allocating. Pin the wire
1772 /// shape of the variant so a future refactor can't silently
1773 /// drop the cap and re-introduce the OOM vector.
1774 #[test]
1775 fn sidecar_too_large_error_shape() {
1776 let err = RepairError::SidecarTooLarge {
1777 bucket: "b".into(),
1778 key: "k.s4index".into(),
1779 size: 2 * MAX_SIDECAR_BODY_BYTES,
1780 cap: MAX_SIDECAR_BODY_BYTES,
1781 };
1782 let rendered = format!("{err}");
1783 assert!(
1784 rendered.contains("b/k.s4index"),
1785 "Display must mention bucket/key — got {rendered:?}"
1786 );
1787 assert!(
1788 rendered.contains(&MAX_SIDECAR_BODY_BYTES.to_string()),
1789 "Display must mention the cap — got {rendered:?}"
1790 );
1791 assert!(
1792 rendered.contains("OOM") || rendered.contains("legacy") || rendered.contains("attack"),
1793 "Display must hint at the threat model — got {rendered:?}"
1794 );
1795 match err {
1796 RepairError::SidecarTooLarge {
1797 bucket,
1798 key,
1799 size,
1800 cap,
1801 } => {
1802 assert_eq!(bucket, "b");
1803 assert_eq!(key, "k.s4index");
1804 assert_eq!(size, 2 * MAX_SIDECAR_BODY_BYTES);
1805 assert_eq!(cap, MAX_SIDECAR_BODY_BYTES);
1806 }
1807 _ => unreachable!("SidecarTooLarge must match its own variant"),
1808 }
1809 }
1810
1811 /// P2-R5: the cap value is load-bearing — too small breaks
1812 /// legitimate sidecars, too large defeats the OOM guard. Pin
1813 /// it at the codec-spec-derived ceiling (16M frames × 32 B per
1814 /// entry + header ≈ 512 MiB, rounded up with safety margin to
1815 /// 600 MiB). Bump only with explicit operator justification.
1816 #[test]
1817 fn max_sidecar_body_bytes_cap_value_pinned() {
1818 assert_eq!(MAX_SIDECAR_BODY_BYTES, 600 * 1024 * 1024);
1819 // Sanity: cap must comfortably exceed the codec spec's
1820 // max legitimate sidecar geometry. Computed dynamically
1821 // from the codec constants so a bump to either side
1822 // surfaces here (clippy flags `assert!(const)` as
1823 // pointless, so we use `assert_eq!` against `false` for
1824 // the negative — if the cap ever DROPS below the spec
1825 // max, this fails loudly).
1826 let spec_max_legitimate: u64 = s4_codec::index::MAX_FRAMES
1827 * (s4_codec::index::ENTRY_BYTES as u64)
1828 + (s4_codec::index::HEADER_FIXED_V2 as u64)
1829 + (s4_codec::index::MAX_ETAG_BYTES as u64);
1830 assert!(
1831 MAX_SIDECAR_BODY_BYTES > spec_max_legitimate,
1832 "cap {MAX_SIDECAR_BODY_BYTES} must exceed spec-max {spec_max_legitimate}",
1833 );
1834 }
1835
1836 /// P2-R3 (Codex R3 audit): `repair-sidecar` on a passthrough /
1837 /// raw-bytes object would previously write an empty sidecar
1838 /// that silently breaks Range GET. Pin the typed error's wire
1839 /// shape so a future refactor can't quietly drop the
1840 /// `NotFramed` branch.
1841 #[test]
1842 fn not_framed_error_shape() {
1843 let err = RepairError::NotFramed {
1844 bucket: "b".into(),
1845 key: "k".into(),
1846 };
1847 let rendered = format!("{err}");
1848 assert!(rendered.contains("b/k"), "Display must mention bucket/key");
1849 assert!(
1850 rendered.contains("S4F2") || rendered.contains("passthrough"),
1851 "Display must hint at the framing reason"
1852 );
1853 // Pattern-match guard: any rename of bucket/key here is a
1854 // compile error both here AND at the repair_sidecar
1855 // construction site.
1856 match err {
1857 RepairError::NotFramed { bucket, key } => {
1858 assert_eq!(bucket, "b");
1859 assert_eq!(key, "k");
1860 }
1861 _ => unreachable!("NotFramed must match its own variant"),
1862 }
1863 }
1864
1865 /// CI-unblock (post-v0.9 audit): the MinIO E2E race test
1866 /// (`repair_sidecar_detects_post_get_overwrite_race`) is
1867 /// inherently timing-dependent and flakes on fast CI runners
1868 /// where the entire repair pipeline completes before the
1869 /// spawned overwrite lands. This deterministic guard pins
1870 /// the error type's wire shape (Display + field accessors)
1871 /// so the post-PUT divergence detector branch in
1872 /// `repair_sidecar` can't be silently refactored into a
1873 /// different error variant without flipping this assertion.
1874 #[test]
1875 fn overwritten_during_repair_error_shape() {
1876 let err = RepairError::OverwrittenDuringRepair {
1877 bucket: "b".into(),
1878 key: "k".into(),
1879 head_etag: "abc-1".into(),
1880 };
1881 let rendered = format!("{err}");
1882 assert!(
1883 rendered.contains("b/k"),
1884 "Display must mention bucket/key — got {rendered:?}"
1885 );
1886 assert!(
1887 rendered.contains("abc-1"),
1888 "Display must mention the pre-race ETag — got {rendered:?}"
1889 );
1890 assert!(
1891 rendered.contains("re-run") || rendered.contains("overwritten"),
1892 "Display must hint that the operator should re-run — got {rendered:?}"
1893 );
1894 // Pattern-match guard: any future destructure of this
1895 // variant elsewhere in the crate must keep these three
1896 // named fields. A rename here would surface as a compile
1897 // error here AND at the production call sites in
1898 // repair_sidecar / classify_missing_sidecar.
1899 match err {
1900 RepairError::OverwrittenDuringRepair {
1901 bucket,
1902 key,
1903 head_etag,
1904 } => {
1905 assert_eq!(bucket, "b");
1906 assert_eq!(key, "k");
1907 assert_eq!(head_etag, "abc-1");
1908 }
1909 _ => unreachable!("OverwrittenDuringRepair must match its own variant"),
1910 }
1911 }
1912
1913 #[test]
1914 fn default_repair_body_cap_matches_max_body_default() {
1915 // Tied to s4-server `--max-body-bytes` default (5 GiB, #178). If
1916 // the default changes there, update both in lockstep.
1917 assert_eq!(DEFAULT_REPAIR_BODY_BYTES_CAP, 5 * 1024 * 1024 * 1024);
1918 }
1919
1920 /// v0.9 #106-audit-R2 P2-INT-1: `detect_sse_magic` returns the
1921 /// correct frame label for every S4Ex prefix, and `None` for the
1922 /// plaintext frame magic (`S4F2`) and short / random inputs. The
1923 /// helper is the discriminator the `EncryptedSidecarUnsupported`
1924 /// branch in `repair_sidecar` relies on; pinning its outputs
1925 /// guards against a silent regression that would resurrect the
1926 /// confusing `FrameScan` failure on encrypted bodies.
1927 #[test]
1928 fn detect_sse_magic_covers_all_envelope_variants() {
1929 assert_eq!(detect_sse_magic(b"S4E1\0\0\0\0"), Some("S4E1"));
1930 assert_eq!(detect_sse_magic(b"S4E2\0\0\0\0"), Some("S4E2"));
1931 assert_eq!(detect_sse_magic(b"S4E3\0\0\0\0"), Some("S4E3"));
1932 assert_eq!(detect_sse_magic(b"S4E4\0\0\0\0"), Some("S4E4"));
1933 assert_eq!(detect_sse_magic(b"S4E5\0\0\0\0"), Some("S4E5"));
1934 assert_eq!(detect_sse_magic(b"S4E6\0\0\0\0"), Some("S4E6"));
1935 // S4F2 = plaintext framed body; must NOT match (or repair
1936 // would falsely reject every framed object as encrypted).
1937 assert_eq!(detect_sse_magic(b"S4F2\0\0\0\0"), None);
1938 // Random bytes, short inputs, and empty body all return None.
1939 assert_eq!(detect_sse_magic(b"NOPE\0"), None);
1940 assert_eq!(detect_sse_magic(b"S4"), None);
1941 assert_eq!(detect_sse_magic(b""), None);
1942 }
1943
1944 /// v0.10 #A1: `SseDecryptFailed` is a distinct typed variant
1945 /// (NOT lumped under `Backend` / `FrameScan`) so the CLI can give
1946 /// operator-actionable guidance pointing at `--sse-s4-key` /
1947 /// `--sse-s4-key-rotated`. Pin the Display + struct shape so a
1948 /// future refactor can't silently demote it to the generic
1949 /// `Backend` variant.
1950 #[test]
1951 fn sse_decrypt_failed_error_shape() {
1952 let err = RepairError::SseDecryptFailed {
1953 bucket: "b".into(),
1954 key: "k".into(),
1955 cause: "chunk 0 auth-tag verify failed".into(),
1956 };
1957 let rendered = format!("{err}");
1958 assert!(
1959 rendered.contains("b/k"),
1960 "Display must mention bucket/key — got {rendered:?}"
1961 );
1962 assert!(
1963 rendered.contains("SSE-S4 decrypt"),
1964 "Display must name the failure mode — got {rendered:?}"
1965 );
1966 assert!(
1967 rendered.contains("--sse-s4-key"),
1968 "Display must point at the operator-actionable flag — got {rendered:?}"
1969 );
1970 // Pattern guard: any rename of the three fields surfaces here
1971 // AND at the construction site in `decrypt_s4e6_for_repair`.
1972 match err {
1973 RepairError::SseDecryptFailed { bucket, key, cause } => {
1974 assert_eq!(bucket, "b");
1975 assert_eq!(key, "k");
1976 assert!(cause.contains("chunk 0"));
1977 }
1978 _ => unreachable!("SseDecryptFailed must match its own variant"),
1979 }
1980 }
1981
1982 /// v0.10 #A1 (Codex R4 fix): the final-chunk slack the SSE-S4
1983 /// repair path grants on top of `body_bytes_cap` MUST be bounded
1984 /// by [`SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES`] regardless of what
1985 /// the on-disk `chunk_size` declares. Pin the constant + the
1986 /// `min(hdr.chunk_size, ceiling)` arithmetic so a future
1987 /// refactor can't quietly resurrect the OOM vector by trusting
1988 /// the attacker-controlled header field verbatim.
1989 #[test]
1990 fn sse_s4_repair_max_chunk_slack_bounds_attacker_controlled_header() {
1991 // Pin the constant value at its documented 16 MiB ceiling.
1992 // 16 MiB comfortably covers the server-default 1 MiB chunk
1993 // size by 16× while staying well below any cap that would
1994 // double the operator's RAM budget.
1995 assert_eq!(SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES, 16 * 1024 * 1024);
1996
1997 // Inline the slack-computation arithmetic so the regression
1998 // guard fires here AND at the production call site in
1999 // `decrypt_s4e6_for_repair`. Tests three regimes:
2000 // - small legitimate chunk (1 MiB) → slack == chunk_size
2001 // - on-ceiling legitimate chunk (16 MiB) → slack == ceiling
2002 // - attacker-controlled huge chunk (u32::MAX) → slack
2003 // pinned at ceiling (OOM vector closed)
2004 fn compute_slack(hdr_chunk_size: u32) -> u64 {
2005 (hdr_chunk_size as u64).min(SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES)
2006 }
2007 assert_eq!(compute_slack(1024 * 1024), 1024 * 1024);
2008 assert_eq!(
2009 compute_slack(16 * 1024 * 1024),
2010 SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES
2011 );
2012 assert_eq!(compute_slack(u32::MAX), SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES);
2013
2014 // The slack ceiling MUST stay smaller than the worst-case
2015 // SSE envelope overhead (256 MiB) — they cover different
2016 // axes (chunk-size slack vs envelope tag/header overhead)
2017 // but a slack > envelope overhead would be nonsensical.
2018 // Bind the constants to runtime locals so clippy's
2019 // `assertions_on_constants` lint doesn't fire on the
2020 // pinned values.
2021 let slack_cap = SSE_S4_REPAIR_MAX_CHUNK_SLACK_BYTES;
2022 let overhead_cap = SSE_S4_REPAIR_MAX_OVERHEAD_BYTES;
2023 assert!(slack_cap < overhead_cap);
2024 }
2025
2026 /// v0.10 #A1 (Codex R1 fix): `SSE_S4_REPAIR_MAX_OVERHEAD_BYTES`
2027 /// is load-bearing for the body-cap relaxation in
2028 /// `repair_sidecar_with_keyring` — too small reintroduces
2029 /// `BodyTooLarge` rejection on valid S4E6 objects whose
2030 /// plaintext fits the cap; too large bloats the GET's RAM
2031 /// budget unnecessarily. Pin it at the codec-spec ceiling
2032 /// (`S4E6_HEADER_BYTES + S4E6_MAX_CHUNK_COUNT × TAG_LEN`)
2033 /// so a bump to either side of the SSE constants surfaces
2034 /// here.
2035 #[test]
2036 fn sse_s4_repair_max_overhead_bytes_matches_codec_spec() {
2037 let expected = (crate::sse::S4E6_HEADER_BYTES as u64)
2038 + (crate::sse::S4E6_MAX_CHUNK_COUNT as u64)
2039 * (crate::sse::S4E5_PER_CHUNK_OVERHEAD as u64);
2040 assert_eq!(SSE_S4_REPAIR_MAX_OVERHEAD_BYTES, expected);
2041 // Sanity bounds — must comfortably exceed any realistic
2042 // S4E6 envelope overhead but stay well below the 5 GiB
2043 // single-PUT ceiling so the relaxation can't double the
2044 // operator's RAM budget. Build the comparison values from
2045 // the codec spec dynamically (instead of a literal const)
2046 // so clippy's `assertions_on_constants` lint doesn't fire
2047 // on the pinned constant.
2048 let min_reasonable: u64 = 1024 * 1024; // 1 MiB
2049 let max_reasonable: u64 = 1024 * 1024 * 1024; // 1 GiB
2050 assert!(
2051 SSE_S4_REPAIR_MAX_OVERHEAD_BYTES > min_reasonable,
2052 "overhead headroom {SSE_S4_REPAIR_MAX_OVERHEAD_BYTES} must accommodate the \
2053 worst-case S4E6 envelope (>= 1 MiB)",
2054 );
2055 assert!(
2056 SSE_S4_REPAIR_MAX_OVERHEAD_BYTES < max_reasonable,
2057 "overhead headroom {SSE_S4_REPAIR_MAX_OVERHEAD_BYTES} must stay below 1 GiB so \
2058 it doesn't double the operator's --max-body-bytes RAM budget",
2059 );
2060 }
2061
2062 /// v0.10 #A1: `RepairReport::sse_v3_binding` carries the chunked
2063 /// geometry the CLI surfaces in the OK line. Pin the field shape
2064 /// so a struct rename surfaces as a compile error here AND at the
2065 /// CLI's print site in `run_repair_sidecar`.
2066 #[test]
2067 fn repair_sse_binding_shape() {
2068 let b = RepairSseBinding {
2069 enc_chunk_size: 1_048_576,
2070 enc_chunk_count: 4,
2071 enc_key_id: 1,
2072 enc_plaintext_len: 4_000_000,
2073 enc_header_bytes: 24,
2074 };
2075 // Field accesses double as a compile-time guard that the names
2076 // don't drift away from the codec's `SseChunkBinding` (the
2077 // module re-export from `s4_codec::index`).
2078 assert_eq!(b.enc_chunk_size, 1_048_576);
2079 assert_eq!(b.enc_chunk_count, 4);
2080 assert_eq!(b.enc_key_id, 1);
2081 assert_eq!(b.enc_plaintext_len, 4_000_000);
2082 assert_eq!(b.enc_header_bytes, 24);
2083 }
2084
2085 /// v0.9 #106-audit-R2 P2-INT-1 (initial) / v0.10 #A1 (refined):
2086 /// pin the Display text + struct shape of the new variant so
2087 /// refactors can't silently drop the operator guidance
2088 /// (server-mode rebuild / re-PUT / `--sse-s4-key` plumbing) or
2089 /// rename the fields the CLI's error formatter reads. Mirrors
2090 /// the existing `overwritten_during_repair_error_shape` test
2091 /// pattern.
2092 #[test]
2093 fn repair_sidecar_rejects_encrypted_body_with_typed_error() {
2094 let err = RepairError::EncryptedSidecarUnsupported {
2095 bucket: "b".into(),
2096 key: "k".into(),
2097 message: "body magic S4E6 indicates SSE-S4 envelope".into(),
2098 };
2099 let rendered = format!("{err}");
2100 assert!(
2101 rendered.contains("b/k"),
2102 "Display must mention bucket/key — got {rendered:?}"
2103 );
2104 assert!(
2105 rendered.contains("S4E6"),
2106 "Display must echo the body magic for operator triage — got {rendered:?}"
2107 );
2108 assert!(
2109 rendered.contains("encrypted-sidecar repair"),
2110 "Display must name the failure mode — got {rendered:?}"
2111 );
2112 assert!(
2113 rendered.contains("re-PUT") || rendered.contains("server-mode"),
2114 "Display must hint at the recovery path — got {rendered:?}"
2115 );
2116 match err {
2117 RepairError::EncryptedSidecarUnsupported {
2118 bucket,
2119 key,
2120 message,
2121 } => {
2122 assert_eq!(bucket, "b");
2123 assert_eq!(key, "k");
2124 assert!(message.contains("S4E6"));
2125 }
2126 _ => unreachable!("EncryptedSidecarUnsupported must match its own variant"),
2127 }
2128 }
2129}