flint-store 0.2.0

The flint object-store layer: the ObjectStore trait, conditional writes, generation stamps, CRC-64/NVME, an in-memory double and the S3 backend
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
//! The version-surface conformance probe.
//!
//! It lives HERE, in the store crate, so that every caller asks the one
//! question the same way — two copies of "is this bucket conformant?"
//! would eventually answer differently. Lean no longer depends on the
//! version surface (gated mode was removed 2026-09-13); the probe stays
//! as a tool for any store user that does.
//!
//! What it refuses, and why refusal rather than degradation: a
//! project-scoped proxy that strips `x-amz-version-id` makes every
//! staged entry carry `None`, and citation then falls back to etag
//! semantics on a key whose CURRENT version is uncited — precisely the
//! torn view gated mode exists to prevent. Silent degradation into the
//! hazard is the one outcome that must be impossible.

use bytes::Bytes;

use crate::{crc64_nvme, GenerationStamps, ObjectStore, PutCondition, StoreError};

/// The prefix a probe uses when it could not write its OWN object.
///
/// This is not a verdict about the store's conditional writes: it is a
/// credential, a bucket policy, or the network. A caller that refuses on
/// it has refused an ERROR while reporting an ANSWER — which is how a
/// read-only credential comes to be described as "this store does not
/// enforce conditional writes". Callers that act on a verdict must tell
/// the two apart; `lean/syncer/src/conformance.rs` does.
pub const PROBE_UNREACHABLE: &str = "cannot write the probe object";

/// Run the probe against `key`. `Ok(())` means the whole surface is
/// present; `Err` names the exact step that failed, for a condition
/// message an operator can act on.
///
/// Callers must use DISTINCT keys per principal: the syncer and the
/// operator probe on independent clocks, and a shared key would let two
/// conformant probes fail each other's `If-None-Match` write.
pub async fn probe_version_surface(store: &dyn ObjectStore, key: &str) -> Result<(), String> {
    let stamps = GenerationStamps {
        generation: 0,
        epoch: 0,
        flush_uuid: "version-probe".into(),
        boundary_source: None,
        posix: None,
    };
    let refuse = |m: &str| m.to_string();

    // The probe's OWN crash window, closed. It writes If-None-Match and
    // cleans up at the end; a crash — or one failed cleanup DELETE —
    // leaves the object behind and every later probe 412s on its first
    // write. On a gated workspace that would be a permanent startup
    // wedge produced by a transient error, so a leftover is swept.
    let b1 = Bytes::from_static(b"probe-1");
    let mut first = store
        .put_whole(key, b1.clone(), &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(b"probe-1"))
        .await;
    if matches!(first, Err(StoreError::PreconditionFailed(_))) {
        sweep(store, key).await;
        first = store
            .put_whole(key, b1, &PutCondition::IfNoneMatchAny, &stamps, crc64_nvme(b"probe-1"))
            .await;
    }
    let m1 = first.map_err(|_| refuse("cannot write the probe object"))?;
    // Everything from here down is the VERSION surface. The
    // conditional-write half above is what `probe_conditional_writes`
    // checks on its own, for callers that arbitrate but do not cite
    // versions.
    let v1 = m1.version_id.clone().ok_or_else(|| {
        refuse("PUT returned no x-amz-version-id (versioning off, or a proxy strips the header)")
    })?;

    let b2 = Bytes::from_static(b"probe-2");
    let m2 = store
        .put_whole(
            key,
            b2,
            &PutCondition::IfMatch(m1.etag.clone()),
            &stamps,
            crc64_nvme(b"probe-2"),
        )
        .await
        .map_err(|_| refuse("cannot supersede the probe object"))?;
    let v2 = m2.version_id.clone().ok_or_else(|| refuse("no version id on the second PUT"))?;
    if v1 == v2 {
        return Err(refuse("the backend reused one version id for two PUTs"));
    }

    // The first version must still be fetchable BY ID — the read any
    // version-pinned citation would depend on.
    let (_, body) = store
        .get_version(key, &v1)
        .await
        .map_err(|_| refuse("version-scoped GET is unavailable"))?;
    if body.as_ref() != b"probe-1" {
        return Err(refuse("version-scoped GET returned the wrong generation"));
    }
    store
        .head_version(key, &v1)
        .await
        .map_err(|_| refuse("version-scoped HEAD is unavailable"))?;
    let listed = store
        .list_versions(key)
        .await
        .map_err(|_| refuse("ListObjectVersions is not permitted"))?;
    if listed.iter().filter(|v| v.key == key && !v.is_delete_marker).count() < 2 {
        return Err(refuse("ListObjectVersions did not report both generations"));
    }
    store
        .delete_version(key, &v1)
        .await
        .map_err(|_| refuse("version-scoped DELETE is unavailable"))?;
    if store.head_version(key, &v1).await.is_ok() {
        return Err(refuse("version-scoped DELETE did not remove the version"));
    }
    let _ = store.delete_version(key, &v2).await;
    // Belt and braces: leave the key with no versions at all, so a
    // partially-failed cleanup cannot wedge the NEXT probe either.
    sweep(store, key).await;
    Ok(())
}

/// Remove every version of the probe key. Errors are ignored on
/// purpose: this is hygiene, and the probe's verdict must come from the
/// probe, not from a cleanup DELETE.
async fn sweep(store: &dyn ObjectStore, key: &str) {
    if let Ok(vs) = store.list_versions(key).await {
        for v in vs.iter().filter(|v| v.key == key) {
            let _ = store.delete_version(key, &v.version_id).await;
        }
    }
}

/// The CONDITIONAL-WRITE surface alone — a strict subset of
/// [`probe_version_surface`], for callers that arbitrate with
/// `If-Match`/`If-None-Match` but never cite a version id.
///
/// The flint-lite tier is exactly that caller. Its A6 arbitration is
/// compare-and-swap: the flusher publishes under `If-Match` on the
/// generation it read, and a 412 is how it tells its own torn flush
/// from a genuinely foreign overwrite. Against a store that ACCEPTS the
/// conditional headers and ignores them, every one of those PUTs
/// succeeds unconditionally and arbitration degrades — silently — to
/// last-writer-wins. Nothing else in the tier would notice.
///
/// Deliberately does NOT require bucket versioning: a lite bucket needs
/// no version ids, and demanding them would refuse a perfectly good
/// deployment. That is the whole reason this is not just a call to the
/// bigger probe.
pub async fn probe_conditional_writes(store: &dyn ObjectStore, key: &str) -> Result<(), String> {
    let stamps = GenerationStamps {
        generation: 0,
        epoch: 0,
        flush_uuid: "cond-probe".into(),
        boundary_source: None,
        posix: None,
    };
    let refuse = |m: &str| m.to_string();
    let body = |b: &'static [u8]| (Bytes::from_static(b), crc64_nvme(b));

    // Leftovers from a crashed probe would 412 the first write below
    // and read as a conformant store. Clear first.
    let _ = store.delete(key).await;

    let (b1, c1) = body(b"cond-probe-1");
    let m1 = store
        .put_whole(key, b1, &PutCondition::IfNoneMatchAny, &stamps, c1)
        .await
        .map_err(|e| refuse(&format!("{PROBE_UNREACHABLE}: {e}")))?;

    // If-None-Match:* over an object that now EXISTS must be refused.
    // A store that ignores the header answers Ok here, and that is the
    // silent degradation this probe exists to catch.
    let (b2, c2) = body(b"cond-probe-2");
    match store.put_whole(key, b2, &PutCondition::IfNoneMatchAny, &stamps, c2).await {
        Err(StoreError::PreconditionFailed(_)) => {}
        Ok(_) => {
            let _ = store.delete(key).await;
            return Err(refuse(
                "If-None-Match:* was ACCEPTED over an existing object — this store does not \
                 enforce conditional writes, so tier arbitration would silently degrade to \
                 last-writer-wins",
            ));
        }
        Err(e) => {
            let _ = store.delete(key).await;
            return Err(refuse(&format!("If-None-Match:* failed unexpectedly: {e}")));
        }
    }

    // If-Match on a STALE etag must be refused...
    let (b3, c3) = body(b"cond-probe-3");
    let stale = PutCondition::IfMatch("\"0000000000000000000000000000dead\"".to_string());
    match store.put_whole(key, b3, &stale, &stamps, c3).await {
        Err(StoreError::PreconditionFailed(_)) => {}
        Ok(_) => {
            let _ = store.delete(key).await;
            return Err(refuse(
                "If-Match on a STALE etag was ACCEPTED — a foreign overwrite would be \
                 indistinguishable from this hub's own publish",
            ));
        }
        Err(e) => {
            let _ = store.delete(key).await;
            return Err(refuse(&format!("If-Match(stale) failed unexpectedly: {e}")));
        }
    }

    // ...and on the CURRENT etag it must succeed, or the tier could
    // never publish a second generation at all. Without this leg a
    // store that refused every conditional PUT would score as
    // conformant.
    let (b4, c4) = body(b"cond-probe-4");
    let r = store.put_whole(key, b4, &PutCondition::IfMatch(m1.etag.clone()), &stamps, c4).await;
    let _ = store.delete(key).await;
    r.map_err(|e| refuse(&format!("If-Match on the CURRENT etag was refused: {e}")))?;
    Ok(())
}

/// The CONDITIONAL-DELETE surface: `DeleteObject` with `If-Match`.
///
/// Lean's garbage collector needs it once several writers share a
/// workspace. A GC that HEADs an object, recognizes the etag and then
/// deletes unconditionally races another writer's lease-free upload of
/// the same path: the upload lands in between, the delete removes it,
/// and that writer's next commit cites bytes that are gone. `If-Match`
/// on the DELETE closes the window — on a backend that enforces it.
/// One that accepts the header and ignores it answers exactly like one
/// that enforces it, until the day it deletes the wrong generation, so
/// this is asked separately from the conditional-PUT probe (the lite
/// tier needs that one and not this).
pub async fn probe_conditional_delete(store: &dyn ObjectStore, key: &str) -> Result<(), String> {
    let stamps = GenerationStamps {
        generation: 0,
        epoch: 0,
        flush_uuid: "cond-delete-probe".into(),
        boundary_source: None,
        posix: None,
    };
    let refuse = |m: &str| m.to_string();
    let _ = store.delete(key).await;
    let m1 = store
        .put_whole(
            key,
            Bytes::from_static(b"cond-delete-probe"),
            &PutCondition::IfNoneMatchAny,
            &stamps,
            crc64_nvme(b"cond-delete-probe"),
        )
        .await
        .map_err(|e| refuse(&format!("{PROBE_UNREACHABLE}: {e}")))?;

    // A STALE etag must be refused, and the object must still be there.
    let stale = "\"0000000000000000000000000000dead\"";
    let r = store.delete_if_match(key, stale).await;
    let survived = store.head(key).await.map(|m| m.etag == m1.etag).unwrap_or(false);
    match (r, survived) {
        (Err(StoreError::PreconditionFailed(_)), true) => {}
        (_, false) => {
            return Err(refuse(
                "DELETE If-Match on a STALE etag removed the object — this store ignores the \
                 condition, so a garbage collector would delete another writer's upload",
            ));
        }
        (Ok(()), true) => {
            let _ = store.delete(key).await;
            return Err(refuse("DELETE If-Match on a STALE etag answered success and deleted nothing"));
        }
        (Err(e), true) => {
            let _ = store.delete(key).await;
            return Err(refuse(&format!("DELETE If-Match (stale) failed unexpectedly: {e}")));
        }
    }

    // On the CURRENT etag it must delete, or the collector could never
    // collect: a store that refused every conditional DELETE would pass
    // the leg above.
    store
        .delete_if_match(key, &m1.etag)
        .await
        .map_err(|e| refuse(&format!("DELETE If-Match on the CURRENT etag was refused: {e}")))?;
    match store.head(key).await {
        Err(StoreError::NotFound(_)) => Ok(()),
        Ok(_) => {
            let _ = store.delete(key).await;
            Err(refuse("DELETE If-Match on the CURRENT etag answered success but the object remains"))
        }
        Err(e) => Err(refuse(&format!("HEAD after the conditional DELETE failed: {e}"))),
    }
}

/// The cross-key copy surface (`copy_object`). Run against a REAL
/// bucket; the memory double cannot tell you whether `CopyObject`
/// honours `x-amz-copy-source-if-match`, whether `MetadataDirective:
/// REPLACE` actually replaces, or whether the MPU arm's
/// `UploadPartCopy` assembles what we think.
///
/// `key` is a PREFIX: the probe writes `<key>.src` and `<key>.dst` and
/// removes both. As with the other probes, distinct keys per principal.
///
/// `force_mpu_over` lets a caller drive the MPU arm without a 5 GiB
/// object by lowering the store's single-request ceiling first — the
/// arm that has never executed anywhere is exactly the one a probe
/// should reach.
pub async fn probe_cross_key_copy(store: &dyn ObjectStore, key: &str) -> Result<(), String> {
    let src = format!("{key}.src");
    let dst = format!("{key}.dst");
    let body = Bytes::from_static(b"cross-key copy probe payload, non-uniform: \x01\x02\x03");
    let src_crc = crc64_nvme(&body);
    let stamps = |g: u64, f: &str| GenerationStamps {
        generation: g,
        epoch: 0,
        flush_uuid: f.into(),
        boundary_source: None,
        posix: None,
    };

    // Clean start: a leftover from a torn probe must not fail the
    // If-None-Match writes below (the leftover-object lesson the
    // conditional-writes probe already learned).
    let _ = store.delete(&src).await;
    let _ = store.delete(&dst).await;

    let s = store
        .put_whole(&src, body.clone(), &PutCondition::IfNoneMatchAny, &stamps(9, "probe-src"), src_crc)
        .await
        .map_err(|e| format!("seed the copy source: {e}"))?;

    // 1. A stale source guard must REFUSE. Nothing may land.
    match store
        .copy_object(&src, Some("\"not-the-etag\""), &dst, &PutCondition::IfNoneMatchAny, &stamps(1, "probe-stale"))
        .await
    {
        Err(StoreError::PreconditionFailed(_)) => {}
        Ok(_) => {
            let _ = store.delete(&dst).await;
            let _ = store.delete(&src).await;
            return Err("copy with a STALE source etag succeeded — the copy-source guard is \
                        not enforced, so a rename can move bytes the caller never read"
                .into());
        }
        Err(e) => {
            let _ = store.delete(&src).await;
            return Err(format!("copy with a stale source etag failed with the wrong error: {e}"));
        }
    }
    if store.head(&dst).await.is_ok() {
        let _ = store.delete(&dst).await;
        let _ = store.delete(&src).await;
        return Err("a REFUSED copy still landed an object at the destination".into());
    }

    // 2. The real copy.
    let d = store
        .copy_object(&src, Some(&s.etag), &dst, &PutCondition::IfNoneMatchAny, &stamps(1, "probe-dst"))
        .await
        .map_err(|e| format!("copy {src} -> {dst}: {e}"))?;

    // 3. Bytes identical, checksum identical.
    let (dmeta, got) = store.get_whole(&dst, None).await.map_err(|e| format!("read back {dst}: {e}"))?;
    if got != body {
        return Err(format!("the copy is not byte-identical: {} bytes vs {}", got.len(), body.len()));
    }
    if crc64_nvme(&got) != src_crc {
        return Err("the copy's CONTENT checksum differs from the source's".into());
    }
    if d.crc64_b64 != s.crc64_b64 {
        return Err(format!(
            "the copy REPORTS a different checksum than the source: {:?} vs {:?} — identical \
             bytes cannot have different CRCs, so one of the two is not describing its object",
            d.crc64_b64, s.crc64_b64
        ));
    }

    // 4. Stamps are the DESTINATION's. This is the one a double cannot
    //    check: `MetadataDirective::REPLACE` is a server behaviour.
    let gen = dmeta.meta.get(GenerationStamps::META_GEN).cloned().unwrap_or_default();
    let flush = dmeta.meta.get(GenerationStamps::META_FLUSH_UUID).cloned().unwrap_or_default();
    if gen != "1" || flush != "probe-dst" {
        return Err(format!(
            "the copy inherited the SOURCE's stamps (gen={gen:?} flush={flush:?}, wanted \
             gen=\"1\" flush=\"probe-dst\") — MetadataDirective did not replace, so one \
             object's publish history is filed under another object's key"
        ));
    }

    // 5. The source survives: a copy, not a move.
    if store.head(&src).await.is_err() {
        return Err("the SOURCE is gone after a copy".into());
    }

    // 6. An occupied destination under If-None-Match must refuse.
    match store
        .copy_object(&src, Some(&s.etag), &dst, &PutCondition::IfNoneMatchAny, &stamps(2, "probe-clobber"))
        .await
    {
        Err(StoreError::PreconditionFailed(_)) => {}
        Ok(_) => {
            return Err("a copy CLOBBERED an existing destination under If-None-Match — a \
                        rename would silently overwrite whatever it landed on"
                .into())
        }
        Err(e) => return Err(format!("copy onto an occupied destination: wrong error: {e}")),
    }

    let _ = store.delete(&dst).await;
    let _ = store.delete(&src).await;
    Ok(())
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::memory::MemoryStore;

    /// The probe must PASS on a store that behaves. This is the leg
    /// that matters most in practice: the failure mode I was closest to
    /// shipping was a probe that over-demanded — `probe_version_surface`
    /// requires bucket VERSIONING, which a lite bucket does not need,
    /// so wiring that one into the hub would have refused perfectly
    /// good deployments at startup.
    ///
    /// The negative direction — a store that ACCEPTS conditional
    /// headers and ignores them — cannot be built here without a full
    /// non-enforcing `ObjectStore` double; it belongs in the real-S3 /
    /// MinIO acceptance drill, pointed at a backend that actually
    /// misbehaves. What stands in for it in-process is the probe's own
    /// fourth leg: `If-Match` on the CURRENT etag must SUCCEED, so a
    /// store that simply refused every conditional PUT could not score
    /// as conformant either.
    #[tokio::test]
    async fn a_conformant_store_passes_the_conditional_write_probe() {
        let store = MemoryStore::new();
        probe_conditional_writes(&store, "t/.flint/probe/conditional-writes")
            .await
            .expect("MemoryStore enforces conditional writes and must pass");
    }

    /// Leftovers from a crashed probe must not read as conformance: the
    /// first `If-None-Match:*` would 412 against them, which is the
    /// same answer a working store gives to leg 2. Without the opening
    /// sweep the probe would pass on a store it never actually tested.
    #[tokio::test]
    async fn a_leftover_probe_object_does_not_fake_a_pass() {
        let store = MemoryStore::new();
        let key = "t/.flint/probe/leftover";
        let stamps = GenerationStamps {
            generation: 0,
            epoch: 0,
            flush_uuid: "leftover".into(),
            boundary_source: None,
            posix: None,
        };
        store
            .put_whole(
                key,
                Bytes::from_static(b"stale"),
                &PutCondition::IfNoneMatchAny,
                &stamps,
                crc64_nvme(b"stale"),
            )
            .await
            .expect("seed the leftover");

        probe_conditional_writes(&store, key).await.expect("must still probe correctly");
    }

    /// The probe must PASS a conformant store...
    #[tokio::test]
    async fn cross_key_copy_probe_passes_on_a_conformant_store() {
        let store = crate::memory::MemoryStore::new();
        probe_cross_key_copy(&store, "probe/copy").await.expect("the double is conformant");
    }

    /// ...and must FAIL a store that gets the one thing wrong that a
    /// memory double can never catch on its own: `MetadataDirective`.
    /// A probe nothing can fail is a probe that proves nothing.
    #[tokio::test]
    async fn cross_key_copy_probe_catches_inherited_stamps() {
        use crate::{
            BootstrapReport, ComposeSpec, EpochLease, EpochState, ListedObject, ObjectMeta,
            PendingUpload, StoreResult,
        };
        struct InheritsStamps(crate::memory::MemoryStore);
        #[async_trait::async_trait]
        impl ObjectStore for InheritsStamps {
            async fn copy_object(
                &self,
                src_key: &str,
                src_if_match: Option<&str>,
                dst_key: &str,
                condition: &PutCondition,
                _stamps: &GenerationStamps,
            ) -> crate::StoreResult<crate::ObjectMeta> {
                // `MetadataDirective: COPY`: the destination wears the
                // SOURCE's generation and flush_uuid.
                let src = self.0.head(src_key).await?;
                let inherited =
                    GenerationStamps::from_meta(&src.meta).unwrap_or(GenerationStamps {
                        generation: 0,
                        epoch: 0,
                        flush_uuid: String::new(),
                        boundary_source: None,
                        posix: None,
                    });
                self.0.copy_object(src_key, src_if_match, dst_key, condition, &inherited).await
            }
            async fn put_whole( &self, key: &str, body: Bytes, condition: &PutCondition, stamps: &GenerationStamps, crc64: u64, ) -> StoreResult<ObjectMeta> {
                self.0.put_whole(key, body, condition, stamps, crc64).await
            }
            async fn compose_generation(&self, spec: &ComposeSpec<'_>) -> StoreResult<ObjectMeta> {
                self.0.compose_generation(spec).await
            }
            async fn head(&self, key: &str) -> StoreResult<ObjectMeta> {
                self.0.head(key).await
            }
            async fn get_whole(&self, key: &str, if_match: Option<&str>) -> StoreResult<(ObjectMeta, Bytes)> {
                self.0.get_whole(key, if_match).await
            }
            async fn get_range( &self, key: &str, offset: u64, len: u64, if_match: &str, ) -> StoreResult<Bytes> {
                self.0.get_range(key, offset, len, if_match).await
            }
            async fn list(&self, prefix: &str) -> StoreResult<Vec<ListedObject>> {
                self.0.list(prefix).await
            }
            async fn delete(&self, key: &str) -> StoreResult<()> {
                self.0.delete(key).await
            }
            async fn list_uploads(&self, prefix: &str) -> StoreResult<Vec<PendingUpload>> {
                self.0.list_uploads(prefix).await
            }
            async fn abort_upload(&self, key: &str, upload_id: &str) -> StoreResult<()> {
                self.0.abort_upload(key, upload_id).await
            }
            async fn bootstrap(&self, prefix: &str) -> StoreResult<BootstrapReport> {
                self.0.bootstrap(prefix).await
            }
            async fn epoch_read(&self, key: &str) -> StoreResult<Option<EpochState>> {
                self.0.epoch_read(key).await
            }
            async fn epoch_acquire( &self, key: &str, holder_id: &str, supersede: Option<&EpochState>, ) -> StoreResult<EpochLease> {
                self.0.epoch_acquire(key, holder_id, supersede).await
            }
            async fn epoch_renew( &self, key: &str, lease: &EpochLease, echo: Option<&str>, ) -> StoreResult<EpochLease> {
                self.0.epoch_renew(key, lease, echo).await
            }
            async fn epoch_release(&self, key: &str, lease: &EpochLease) -> StoreResult<()> {
                self.0.epoch_release(key, lease).await
            }
            fn min_part_size(&self) -> u64 {
                self.0.min_part_size()
            }
            fn max_parts(&self) -> usize {
                self.0.max_parts()
            }
        }

        let store = InheritsStamps(crate::memory::MemoryStore::new());
        let err = probe_cross_key_copy(&store, "probe/copy")
            .await
            .expect_err("inherited stamps must be caught");
        assert!(err.contains("inherited the SOURCE's stamps"), "{err}");
    }

    #[tokio::test]
    async fn a_conformant_store_passes_the_conditional_delete_probe() {
        let store = MemoryStore::new();
        probe_conditional_delete(&store, "t/.flint/probe/conditional-delete")
            .await
            .expect("MemoryStore enforces If-Match on DELETE and must pass");
    }

    /// The failure the probe exists for: a backend that takes the header
    /// and deletes anyway. Without this leg the probe could pass a store
    /// it never actually tested.
    #[tokio::test]
    async fn the_conditional_delete_probe_catches_a_store_that_ignores_the_condition() {
        use crate::{
            BootstrapReport, ComposeSpec, EpochLease, EpochState, ListedObject, ObjectMeta,
            PendingUpload, StoreResult,
        };
        struct IgnoresIfMatch(MemoryStore);
        #[async_trait::async_trait]
        impl ObjectStore for IgnoresIfMatch {
            async fn delete_if_match(&self, key: &str, _etag: &str) -> StoreResult<()> {
                self.0.delete(key).await
            }
            async fn copy_object(
                &self,
                src_key: &str,
                src_if_match: Option<&str>,
                dst_key: &str,
                condition: &PutCondition,
                stamps: &GenerationStamps,
            ) -> StoreResult<ObjectMeta> {
                self.0.copy_object(src_key, src_if_match, dst_key, condition, stamps).await
            }
            async fn put_whole( &self, key: &str, body: Bytes, condition: &PutCondition, stamps: &GenerationStamps, crc64: u64, ) -> StoreResult<ObjectMeta> {
                self.0.put_whole(key, body, condition, stamps, crc64).await
            }
            async fn compose_generation(&self, spec: &ComposeSpec<'_>) -> StoreResult<ObjectMeta> {
                self.0.compose_generation(spec).await
            }
            async fn head(&self, key: &str) -> StoreResult<ObjectMeta> {
                self.0.head(key).await
            }
            async fn get_whole(&self, key: &str, if_match: Option<&str>) -> StoreResult<(ObjectMeta, Bytes)> {
                self.0.get_whole(key, if_match).await
            }
            async fn get_range( &self, key: &str, offset: u64, len: u64, if_match: &str, ) -> StoreResult<Bytes> {
                self.0.get_range(key, offset, len, if_match).await
            }
            async fn list(&self, prefix: &str) -> StoreResult<Vec<ListedObject>> {
                self.0.list(prefix).await
            }
            async fn delete(&self, key: &str) -> StoreResult<()> {
                self.0.delete(key).await
            }
            async fn list_uploads(&self, prefix: &str) -> StoreResult<Vec<PendingUpload>> {
                self.0.list_uploads(prefix).await
            }
            async fn abort_upload(&self, key: &str, upload_id: &str) -> StoreResult<()> {
                self.0.abort_upload(key, upload_id).await
            }
            async fn bootstrap(&self, prefix: &str) -> StoreResult<BootstrapReport> {
                self.0.bootstrap(prefix).await
            }
            async fn epoch_read(&self, key: &str) -> StoreResult<Option<EpochState>> {
                self.0.epoch_read(key).await
            }
            async fn epoch_acquire( &self, key: &str, holder_id: &str, supersede: Option<&EpochState>, ) -> StoreResult<EpochLease> {
                self.0.epoch_acquire(key, holder_id, supersede).await
            }
            async fn epoch_renew( &self, key: &str, lease: &EpochLease, echo: Option<&str>, ) -> StoreResult<EpochLease> {
                self.0.epoch_renew(key, lease, echo).await
            }
            async fn epoch_release(&self, key: &str, lease: &EpochLease) -> StoreResult<()> {
                self.0.epoch_release(key, lease).await
            }
            fn min_part_size(&self) -> u64 {
                self.0.min_part_size()
            }
            fn max_parts(&self) -> usize {
                self.0.max_parts()
            }
        }

        let store = IgnoresIfMatch(MemoryStore::new());
        let err = probe_conditional_delete(&store, "t/.flint/probe/conditional-delete")
            .await
            .expect_err("a store that ignores If-Match on DELETE must be refused");
        assert!(err.contains("ignores the condition"), "{err}");
    }
}