bstack_raii 0.1.0

Typed, RAII-style ownership and lifetime layer over the bstack allocation primitives
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
//! Fallible clone for handles whose duplication touches disk.
//!
//! Two flavours, split by what the duplication needs:
//!
//! * [`TryClone`](crate::TryClone) — a same-type clone that needs **no allocator**.
//!   Implemented by [`crate::BStackRc`] / [`crate::BStackWeak`], whose clone only
//!   bumps an on-disk refcount (they carry their own allocator).
//! * [`TryCloneIn`] — a **deep** clone of a whole block, producing a fresh
//!   independent [`BStackOwned`] copy. A without-allocator block handle
//!   (`X(BStackRange)`) has no allocator to allocate the copy with, so the
//!   allocator is passed in. Generated by `#[bstack_block]` (mirror of the
//!   block's `__bstack_drop_children` teardown, run in reverse):
//!     - POD / `#[bstack_ref]` fields — byte-copied (a ref aliases the same
//!       borrowed target);
//!     - `#[bstack_owned]` — recursively deep-cloned into a fresh child;
//!     - `#[bstack_strong]` / `#[bstack_weak]` — the shared target's refcount is
//!       bumped (the clone stays shared, not copied).
//!
//! # Atomicity model
//!
//! A deep clone mixes two workloads with different atomicity domains:
//! **allocations** (which extend the stack and cannot be part of an atomic
//! in-place batch) and **writes** (in-place payload writes, which can). So
//! [`TryCloneIn`] is built as two hard-separated phases via [`ClonePlan`]:
//!
//! 1. **Plan + allocate.** Recurse the block graph, allocating every new block
//!    up front (each fallible) and recording the payload bytes to write — but
//!    committing *no* write yet. A mid-descent allocation failure rolls back with
//!    zero payload written, so there is no partially-fixed-up state to repair.
//! 2. **Commit.** Once every allocation has succeeded, apply the refcount bumps
//!    and flush every payload write as one crash-atomic [`BStack::set_batched`]
//!    batch (the new blocks are distinct ranges, so the writes never overlap).
//!
//! A crash *between* the phases would leak the allocated-but-uncommitted blocks,
//! never a torn write (the clone result is not yet reachable from any persistent
//! root). When the allocator names a WAL anchor ([`BStackRaiiAllocator::wal_anchor`]
//! returns `Some`), phase 1 is additionally **intention-first**: each allocation is
//! logged `Pending` to the persistent WAL block the instant it is made
//! ([`ClonePlan::alloc_raw`]), so a crash mid-descent is *reclaimed* by
//! [`crate::io_core::wal::finish`] on the next open — down to a one-block window (the block
//! allocated but not yet logged). The phase-2 commit flips that transaction
//! `Complete` inside the same atomic batch, so "clone committed" and "WAL Complete"
//! are one event. Because the WAL block is a single-writer-per-file singleton, the
//! plan holds the file's WAL lock for the whole descent; concurrent deep clones on
//! the *same* file therefore serialize (they still run fully concurrently across
//! different files). An allocator that names no anchor keeps the plain two-phase
//! behaviour (mid-descent crash ⇒ orphan leak).

use std::io;

use bstack::{BStackGenOp, BStackRange};

use crate::BStackRaiiAllocator;
use crate::io_core::bulk::seq_free_many;
use crate::io_core::{WalStatus, WalTxn};
use crate::primitives::{Offset, checked_off};
use crate::types::compiled::owned::BStackOwned;
use crate::types::compiled::rc::{CTRL_WEAK_OFFSET, strong_counter_off};
use crate::types::compiled::vec::{BYTEVEC_HEADER, VecDesc};
use crate::types::traits::{BStackBlock, BStackRef, BStackShared};
use crate::util::io_error;

/// Deep-clone a whole block into a fresh, independent [`BStackOwned`] copy,
/// allocating the copy with the supplied allocator.
///
/// Generated by `#[bstack_block]` for **plain** (uniquely-owned) blocks only.
/// Owned children are deep copied; shared (`#[bstack_strong]` / `#[bstack_weak]`)
/// children stay shared (their refcount is bumped); POD and `#[bstack_ref]`
/// fields are byte-copied.
///
/// A **shared** (`rc` / `rc, weak`) block does *not* implement this — its clone
/// is a handle duplication (a refcount bump) via [`TryClone`](crate::TryClone), not
/// a deep copy, so `try_clone_in` on such a block is a compile error that points you
/// at `BStackRc::try_clone` / `BStackWeak::try_clone` instead. See
/// [`TryClone`](crate::TryClone) for why (in particular, why a weak reference can
/// only ever be cloned as another weak reference).
pub trait TryCloneIn: BStackBlock + Sized {
    /// Deep-clone `self` into a fresh, detached [`BStackOwned`] block: stage the copy
    /// through a [`ClonePlan`] (rolled back on any failure, then committed atomically).
    ///
    /// The default drives the plan; an implementor supplies only the type-specific
    /// [`__bstack_clone_into`](BStackBlock::__bstack_clone_into) walk it calls. Every
    /// deep-clonable block wants this exact driver, so overriding it is almost never
    /// right.
    fn try_clone_in<A: BStackRaiiAllocator>(&self, allocator: &A) -> io::Result<BStackOwned<Self>> {
        let mut plan = ClonePlan::new();
        let dst = match self.__bstack_clone_into(allocator, &mut plan) {
            Ok(range) => range,
            Err(e) => {
                plan.rollback(allocator);
                return Err(e);
            }
        };
        plan.commit(allocator)?;
        // SAFETY: `dst` is a fresh block owned by nobody else.
        Ok(unsafe { BStackOwned::from_raw(Self::from_range(dst)) })
    }
}

/// Accumulates a deep clone's allocations, payload writes, and refcount bumps so
/// the writes commit as one atomic batch after every allocation has succeeded.
///
/// See the module docs for the two-phase model. Built up
/// by the generated `__bstack_clone_into` methods during the recursive descent,
/// then either [`commit`](Self::commit)ted or [`rollback`](Self::rollback)ed.
pub struct ClonePlan {
    /// Which allocation strategy this plan is running (see [`Mode`]).
    mode: Mode,
    /// New blocks the clone will occupy. In [`Direct`](Mode::Direct) these are the
    /// eagerly-allocated blocks (freed in reverse on rollback); in [`Build`](Mode::Build)
    /// this is the pre-allocated pool handed out in order by [`cursor`](Self::cursor).
    allocated: Vec<BStackRange>,
    /// [`Measure`](Mode::Measure) phase only: the size of every block the descent
    /// wants, in descent order — allocated together by [`allocate`](Self::allocate).
    sizes: Vec<u64>,
    /// [`Build`](Mode::Build) phase only: index of the next pre-allocated range in
    /// [`allocated`](Self::allocated) to hand back from [`alloc_raw`](Self::alloc_raw).
    cursor: usize,
    /// [`Measure`](Mode::Measure) phase only: a monotonic, non-zero placeholder
    /// offset handed back for each measured block. Its bytes are never committed
    /// (measure discards all writes), so any distinct non-zero value is sound.
    fake_next: u64,
    /// Pending in-place payload writes `(offset, bytes)`, flushed as one batch.
    writes: Vec<(u64, Vec<u8>)>,
    /// Absolute offsets of `u64` counters to increment by 1 at commit (the
    /// strong/weak counts a `#[bstack_strong]` / `#[bstack_weak]` clone acquires).
    bumps: Vec<u64>,
    /// The WAL transaction protecting the fresh allocations' alloc→commit orphan
    /// window: in [`Direct`](Mode::Direct) it is begun intention-first on the first
    /// [`alloc_raw`](Self::alloc_raw); in [`Build`](Mode::Build) it is staged in one
    /// shot by [`allocate`](Self::allocate) after the atomic bulk alloc. `None` when
    /// the allocator opts out of reclamation ([`wal_anchor`](BStackRaiiAllocator::wal_anchor)
    /// is `None`) or nothing was allocated. Its [`HeldLock`] pins the file's WAL lock
    /// through commit; its logged `Pending` `Alloc` entries are *exactly*
    /// [`allocated`](Self::allocated), so [`finish`](crate::io_core::wal::finish) reclaims
    /// precisely those on abandon.
    wal: Option<WalTxn>,
}

/// How a [`ClonePlan`] turns the descent's allocation requests into real blocks.
#[derive(Clone, Copy, PartialEq, Eq)]
enum Mode {
    /// Single pass: allocate each block eagerly and log it intention-first (the
    /// non-bulk allocators — one descent, a one-block crash window).
    Direct,
    /// Two-pass phase 1 (bulk allocators): gather every block's *size*, allocate
    /// nothing, and discard all writes / skip all cross-file work.
    Measure,
    /// Two-pass phase 2 (bulk allocators): hand out the pre-allocated ranges in
    /// descent order and record writes / refcount bumps as usual.
    Build,
}

impl Default for ClonePlan {
    fn default() -> Self {
        Self::new()
    }
}

impl ClonePlan {
    /// A fresh, empty plan in the single-pass [`Direct`](Mode::Direct) mode (the
    /// two-pass driver [`run_clone`](Self::run_clone) flips it to measure/build).
    pub fn new() -> Self {
        ClonePlan {
            mode: Mode::Direct,
            allocated: Vec::new(),
            sizes: Vec::new(),
            cursor: 0,
            // Non-zero: `0` is the crate's null niche, and a measured payload may
            // embed a child's placeholder offset (harmlessly, since it is discarded).
            fake_next: 1,
            writes: Vec::new(),
            bumps: Vec::new(),
            wal: None,
        }
    }

    /// A plan pre-loaded into the two-pass [`Build`](Mode::Build) phase over a fixed
    /// pool of pre-allocated ranges — test-only, to exercise the Build-mode size guard in
    /// [`alloc_raw`](Self::alloc_raw) directly.
    #[cfg(test)]
    pub(crate) fn for_build_test(allocated: Vec<BStackRange>) -> Self {
        let mut p = Self::new();
        p.mode = Mode::Build;
        p.allocated = allocated;
        p
    }

    /// Whether the plan is in the measure phase of the two-pass bulk clone. Generated
    /// clone code checks this to make eager cross-file (`Foreign`) work **build-only**,
    /// so re-running the descent to measure the home file's sizes never
    /// double-executes a foreign deep-clone or refcount bump.
    pub fn is_measuring(&self) -> bool {
        matches!(self.mode, Mode::Measure)
    }

    /// Allocate a `size`-byte block **without writing anything**, recording it
    /// for rollback. The caller supplies the block's bytes later via
    /// [`write`](Self::write). The allocator's owned-slice handle is not RAII, so
    /// letting it drop here does not free the block.
    ///
    /// This is the single funnel for every deep-clone allocation (plain child
    /// blocks and, via [`stage_bytevec`](Self::stage_bytevec), vec data blocks), so
    /// it is also where the **intention-first WAL** hooks in: when the allocator
    /// names a WAL anchor, the freshly allocated block is logged `Pending` before
    /// this returns, so a crash mid-descent is reclaimed on the next open. The
    /// allocation and its log entry are kept in lockstep — if logging fails the
    /// block is freed again before returning the error — so the WAL's live entries
    /// always equal [`allocated`](Self::allocated).
    pub fn alloc_raw<A: BStackRaiiAllocator>(
        &mut self,
        allocator: &A,
        size: u64,
    ) -> io::Result<BStackRange> {
        match self.mode {
            // Two-pass phase 1: just record the size; hand back a placeholder range
            // (never allocated, never written) so the descent can proceed.
            Mode::Measure => {
                self.sizes.push(size);
                let off = self.fake_next;
                // `saturating_add` (not `+`) on purpose. A real object graph can never
                // sum to 2^64 bytes, but if it somehow did, saturating pins `fake_next`
                // at `u64::MAX` — still non-zero, the one invariant these throwaway
                // offsets hold — whereas wrapping could land it back on `0`, the null
                // niche. Distinctness is lost at that unreachable ceiling, but a measured
                // offset is never committed or dereferenced, so a collision is harmless.
                self.fake_next = self.fake_next.saturating_add(size.max(1));
                Ok(BStackRange::new(off, size))
            }
            // Two-pass phase 2: hand back the next pre-allocated range (allocated in
            // one atomic bulk by `allocate`), in the same order it was measured.
            Mode::Build => {
                // Build must hand back exactly the blocks Measure sized, in order. A
                // count overrun or a size mismatch means the source changed between the
                // two descents — which the API forbids (the descent must be deterministic
                // on the unmutated source). Fail hard rather than let `commit` stage an
                // oversized image past the block: the former `debug_assert_eq!` compiled
                // out in release, turning a violated precondition into a silent
                // out-of-bounds write.
                let range = *self.allocated.get(self.cursor).ok_or_else(|| {
                    io::Error::other(
                        "clone build/measure block-count mismatch (source mutated mid-clone?)",
                    )
                })?;
                self.cursor += 1;
                if range.len() != size {
                    return Err(io::Error::other(
                        "clone build/measure size mismatch (source mutated mid-clone?)",
                    ));
                }
                Ok(range)
            }
            // Single pass: allocate eagerly and log it intention-first, keeping the
            // allocation and its WAL entry in lockstep with `allocated` (shared with
            // the RTTI interpreter's clone).
            Mode::Direct => {
                crate::io_core::alloc_logged(allocator, &mut self.wal, &mut self.allocated, size)
            }
        }
    }

    /// Record a pending in-place write of `bytes` at absolute `offset`. A no-op in
    /// the [`Measure`](Mode::Measure) phase (the destination does not exist yet).
    pub fn write(&mut self, offset: u64, bytes: Vec<u8>) {
        if self.mode != Mode::Measure {
            self.writes.push((offset, bytes));
        }
    }

    /// Stage a fresh `BStackByteVec` data block holding `data` into the plan:
    /// allocate the block (`[len | cap | data]`, `16`-byte header) via
    /// [`alloc_raw`](Self::alloc_raw), stage its full on-disk image into the
    /// commit batch, and return its descriptor. This folds a cloned vector's data
    /// block into the plan's single atomic commit — the block is allocated through
    /// our machinery and its bytes ride the same `inplace_gen` as everything else,
    /// instead of the vector runtime writing it eagerly. `cap == len` (a fresh
    /// clone carries no spare capacity, matching `BStackByteVec::from_slice`).
    pub fn stage_bytevec<A: BStackRaiiAllocator>(
        &mut self,
        allocator: &A,
        data: &[u8],
    ) -> io::Result<VecDesc> {
        let len = data.len() as u64;
        let size = BYTEVEC_HEADER + len;
        let range = self.alloc_raw(allocator, size)?;
        // cap == len: a fresh clone carries no spare capacity. Skip building the
        // image in the measure phase (the write would be discarded anyway).
        if self.mode != Mode::Measure {
            self.write(
                range.start(),
                crate::types::compiled::vec::bytevec_image(len, len, data),
            );
        }
        Ok(VecDesc {
            data_off: Offset::from_raw(range.start()),
            data_size: size,
        })
    }

    /// Record that the strong count of a shared child at `data` must be bumped by
    /// one — the strong reference this clone's `#[bstack_strong]` field acquires.
    pub fn bump_strong<T: BStackShared, A: BStackRaiiAllocator>(
        &mut self,
        data: BStackRef<T>,
        allocator: &A,
    ) -> io::Result<()> {
        // Measure gathers only home-file allocation sizes; the bump (and its locating
        // read) is done for real in the build phase.
        if self.mode == Mode::Measure {
            return Ok(());
        }
        let (data_ref, ctrl) = T::strong_parts(data, allocator)?;
        let off = strong_counter_off(data_ref.into_range().start(), ctrl)?;
        self.bumps.push(off);
        Ok(())
    }

    /// Record that the weak count of the control block at `ctrl_off` must be
    /// bumped by one — the weak reference this clone's `#[bstack_weak]` field
    /// acquires.
    pub fn bump_weak(&mut self, ctrl_off: u64) -> io::Result<()> {
        if self.mode != Mode::Measure {
            self.bumps.push(checked_off(ctrl_off, CTRL_WEAK_OFFSET)?);
        }
        Ok(())
    }

    /// Drive a whole deep clone: run the `descend` closure (a call to the block's
    /// generated `__bstack_clone_into`) under the right allocation strategy, then
    /// commit. Returns the root copy's range.
    ///
    /// * **Bulk allocator** ([`atomic_bulk`](BStackRaiiAllocator::atomic_bulk)) — the
    ///   two-pass path: descend once in [`Measure`](Mode::Measure) to gather every
    ///   home-file block size (allocating nothing, doing no cross-file work), allocate
    ///   them all in one atomic [`alloc_bulk`](bstack::BStackBulkAllocator::alloc_bulk)
    ///   via [`allocate`](Self::allocate), then descend again in [`Build`](Mode::Build)
    ///   against the real addresses and commit. The whole alloc→commit orphan window is
    ///   covered by a single WAL transaction. `descend` must be deterministic on the
    ///   (unmutated) source, since the two passes must agree on the allocation sequence.
    /// * **Non-bulk allocator** — the single-pass [`Direct`](Mode::Direct) path:
    ///   descend once, allocating eagerly and logging each block intention-first (a
    ///   one-block crash window, strictly better than a sequential allocator would get
    ///   from a post-hoc bulk log).
    pub fn run_clone<A, F>(allocator: &A, mut descend: F) -> io::Result<BStackRange>
    where
        A: BStackRaiiAllocator,
        F: FnMut(&mut ClonePlan) -> io::Result<BStackRange>,
    {
        if allocator.atomic_bulk() {
            let mut plan = ClonePlan::new();
            plan.mode = Mode::Measure;
            // Phase 1: gather sizes. Nothing is allocated, so a failure here needs no
            // cleanup (the placeholder ranges are pure bookkeeping).
            descend(&mut plan)?;
            // Allocate every measured block atomically and stage the WAL.
            plan.allocate(allocator)?;
            // Phase 2: build the payloads against the real addresses.
            plan.mode = Mode::Build;
            match descend(&mut plan) {
                Ok(dst) => {
                    plan.commit(allocator)?;
                    Ok(dst)
                }
                Err(e) => {
                    plan.rollback(allocator);
                    Err(e)
                }
            }
        } else {
            let mut plan = ClonePlan::new();
            match descend(&mut plan) {
                Ok(dst) => {
                    plan.commit(allocator)?;
                    Ok(dst)
                }
                Err(e) => {
                    plan.rollback(allocator);
                    Err(e)
                }
            }
        }
    }

    /// Two-pass phase transition: allocate every [`Measure`](Mode::Measure)-gathered
    /// size as one atomic bulk allocation and stage the fresh blocks `Pending` in the
    /// WAL (one `persist_at`, covering the whole alloc→commit orphan window; the
    /// commit flips it `Complete`). On a WAL-staging failure the just-allocated blocks
    /// are freed and the error propagates. Leaves the plan ready for the build phase
    /// (`allocated` = the pool, `cursor` = 0).
    fn allocate<A: BStackRaiiAllocator>(&mut self, allocator: &A) -> io::Result<()> {
        let ranges = allocator.alloc_many(&self.sizes)?;
        if allocator.wal_anchor().is_some() && !ranges.is_empty() {
            match WalTxn::begin_many(allocator, &ranges) {
                Ok(txn) => self.wal = Some(txn),
                Err(e) => {
                    // Couldn't take the lock / stage the WAL (the lock is already
                    // released inside `begin_many` on failure): free the fresh blocks
                    // and abort. SAFETY: `ranges` are this plan's own staged allocations.
                    let _ = unsafe { allocator.free_many(ranges) };
                    return Err(e);
                }
            }
        }
        self.allocated = ranges;
        self.cursor = 0;
        Ok(())
    }

    /// Free everything allocated so far. The error path, taken when planning fails
    /// before [`commit`](Self::commit).
    ///
    /// With an intention-first WAL in flight, the fresh allocations are logged
    /// `Pending`, so this abandons that transaction via [`finish`](crate::io_core::wal::finish)
    /// (freeing exactly the logged allocs — which equal `allocated` — and marking the
    /// persistent block idle), the same path a real crash takes; the WAL lock is held
    /// by `self` and released as it drops. Without a WAL the ranges are freed directly.
    pub fn rollback<A: BStackRaiiAllocator>(self, allocator: &A) {
        // Same policy as the `commit` error path — see [`reclaim`](Self::reclaim). `self`
        // (and its held WAL lock) drops at the end of this call, after reclamation.
        Self::reclaim(&self.wal, self.allocated, allocator);
    }

    /// Commit the whole plan as **one** crash-atomic unit: every refcount bump
    /// (as an atomic read-modify-write) and every staged payload / vec-data write,
    /// via a single [`BStack::inplace_gen`]. Either all of it lands or none of it
    /// does. On failure (I/O, or a refcount overflow) nothing is committed and the
    /// plan's allocations are freed.
    ///
    /// Bumps are de-duplicated to distinct counters (a shared child referenced N
    /// times in the clone is one counter `+N`), then applied as **all reads, then
    /// all writes**: because the counters are distinct, no read depends on another
    /// bump's pending write, so an overflow can be detected after the reads —
    /// before any write is emitted — and abort with nothing committed.
    ///
    /// **Crash reclamation is automatic**: when the allocator names a WAL anchor
    /// ([`BStackRaiiAllocator::wal_anchor`] returns `Some`), the plan's fresh
    /// allocations were already logged `Pending` *during the descent*
    /// ([`alloc_raw`](Self::alloc_raw)) and are reclaimed by [`crate::io_core::wal::finish`]
    /// on the next open if the process dies before the commit lands; here that
    /// transaction is flipped `Complete` *inside* the commit batch, so "clone
    /// committed" and "WAL Complete" are the same atomic event. An allocator that
    /// returns `None` behaves exactly as before (mid-commit crash ⇒ orphan leak).
    pub fn commit<A: BStackRaiiAllocator>(self, allocator: &A) -> io::Result<()> {
        let ClonePlan {
            allocated,
            writes,
            mut bumps,
            wal,
            ..
        } = self;
        let stack = allocator.stack();

        // The WAL transaction (if any) was staged `Pending` incrementally during the
        // descent and its file lock is held by `wal._held`; here we only flip it
        // `Complete` inside the commit batch. A clone with no allocations has no WAL
        // (and needs no lock — the bumps ride bstack's own per-op atomicity).

        // De-duplicate bump offsets into distinct `(counter, delta)`.
        bumps.sort_unstable();
        let mut counters: Vec<(u64, u64)> = Vec::new();
        for off in bumps {
            match counters.last_mut() {
                Some(last) if last.0 == off => last.1 += 1,
                _ => counters.push((off, 1)),
            }
        }

        // Buffers that must outlive the whole `inplace_gen` call (bstack's
        // documented generator pattern): the read-back counter values and the
        // computed new values. `writes` is borrowed for its `Write` data.
        let n = counters.len();
        let mut readbufs: Vec<[u8; 8]> = vec![[0u8; 8]; n];
        let mut newbufs: Vec<[u8; 8]> = vec![[0u8; 8]; n];
        let mut overflow = false;
        // A failed counter `Read` is reported here as the previous op's result. The
        // incremented values are computed from `readbufs`, so a swallowed read error
        // would write back counts derived from stale/zero bytes (or return a false
        // `Ok` for a clone whose refcounts were never bumped → a later double-free).
        // Capture it in the read phase — nothing is staged yet — and route it through
        // the same reclaim path as an error return below.
        let mut read_err: Option<io::Error> = None;

        // The WAL commit marker (`WalHeader.txn_status`, the byte after the u64
        // magic). Written last, so it lands in the same atomic batch as the clone.
        let flip = [WalStatus::Complete as u8];
        let flip_off = wal.as_ref().map(|w| w.commit_marker_off());
        let mut flipped = flip_off.is_none();

        let mut read_i = 0usize;
        let mut computed = false;
        let mut cwrite_i = 0usize;
        let mut write_i = 0usize;

        let result = stack.inplace_gen(|feedback| {
            if let Err(e) = feedback {
                read_err = Some(e);
                return None; // read phase, nothing staged → commits nothing
            }
            // Phase 1 — read every distinct counter.
            if read_i < n {
                let i = read_i;
                read_i += 1;
                // SAFETY: `readbufs` outlives this `inplace_gen` call and each
                // read buffer is used by exactly one in-flight `Read` op.
                let buf: &mut [u8] =
                    unsafe { core::mem::transmute::<&mut [u8], _>(&mut readbufs[i][..]) };
                return Some(BStackGenOp::Read {
                    offset: counters[i].0,
                    buf,
                });
            }
            // Transition — compute new values + check overflow before any write.
            if !computed {
                computed = true;
                for i in 0..n {
                    let cur = u64::from_le_bytes(readbufs[i]);
                    match cur.checked_add(counters[i].1) {
                        Some(v) => newbufs[i] = v.to_le_bytes(),
                        None => overflow = true,
                    }
                }
                if overflow {
                    // No `Write` emitted yet → ending now commits nothing.
                    return None;
                }
            }
            // Phase 2a — write the incremented counters.
            if cwrite_i < n {
                let i = cwrite_i;
                cwrite_i += 1;
                // SAFETY: `newbufs` outlives this call; each slot is written once
                // above and only read here.
                let data: &[u8] = unsafe { core::mem::transmute::<&[u8], _>(&newbufs[i][..]) };
                return Some(BStackGenOp::Write {
                    offset: counters[i].0,
                    data,
                });
            }
            // Phase 2b — write every staged payload / vec-data block.
            if write_i < writes.len() {
                let i = write_i;
                write_i += 1;
                let (off, ref bytes) = writes[i];
                // SAFETY: `writes` outlives this call; each entry is read once.
                let data: &[u8] = unsafe { core::mem::transmute::<&[u8], _>(bytes.as_slice()) };
                return Some(BStackGenOp::Write { offset: off, data });
            }
            // Phase 2c — flip the WAL transaction to `Complete`, last, so it commits
            // atomically with the clone's writes (a no-op when there is no WAL).
            if !flipped {
                flipped = true;
                // SAFETY: `flip` outlives this call and is only read here.
                let data: &[u8] = unsafe { core::mem::transmute::<&[u8], _>(&flip[..]) };
                return Some(BStackGenOp::Write {
                    offset: flip_off.unwrap(),
                    data,
                });
            }
            None
        });

        match result {
            // A counter read failed mid-generator: nothing committed (the read
            // phase stages no writes), and the refcounts were *not* bumped — so
            // reclaim the plan's allocations and surface the read error rather than
            // reporting a false success.
            Ok(()) if read_err.is_some() => {
                Self::reclaim(&wal, allocated, allocator);
                Err(read_err.unwrap())
            }
            Ok(()) if overflow => {
                Self::reclaim(&wal, allocated, allocator);
                Err(io_error!("refcount overflow while committing clone"))
            }
            Ok(()) => {
                // Committed — the WAL was flipped `Complete` in the same batch. A
                // clone logs only `Alloc`s, which a committed txn keeps, so there is
                // nothing to roll forward: just mark the persistent block idle for
                // reuse (a crash before this is harmlessly finished on the next open,
                // freeing nothing). The block itself is never freed.
                if let Some(w) = &wal {
                    let _ = w.set_idle(allocator);
                }
                Ok(())
            }
            // `inplace_gen` is atomic: on error nothing committed. Reclaim the
            // plan's allocations (via the WAL if present, else directly).
            Err(e) => {
                Self::reclaim(&wal, allocated, allocator);
                Err(e)
            }
        }
        // `wal` (holding the file's WAL lock) drops here, after reclamation.
    }

    /// The shared reclamation policy for a failed plan, used by both
    /// [`rollback`](Self::rollback) (whole-plan) and the [`commit`](Self::commit) error
    /// path (which holds the WAL lock as a live local through the call). With an
    /// intention-first WAL in flight, abandoning the still-`Pending` transaction frees
    /// exactly the logged allocs (which equal `allocated`) and marks the persistent
    /// block idle — the same path a real crash's reclamation takes. Without a WAL, the
    /// ranges are freed directly.
    fn reclaim<A: BStackRaiiAllocator>(
        wal: &Option<WalTxn>,
        allocated: Vec<BStackRange>,
        allocator: &A,
    ) {
        match wal {
            // A WAL transaction is in flight: abandon it via the allocator's own
            // anchor (same as a real crash's `finish`); it frees the logged allocs.
            Some(w) => {
                let _ = w.finish(allocator);
            }
            // No WAL: best-effort direct free of the plan's own `alloc_raw` blocks.
            // They are disjoint fresh allocations, so order is immaterial (unlike
            // `teardown`, there are no forged interior pointers to sequence around).
            // `seq_free_many` frees every range and continues past a failure; the error
            // is unactionable here (the commit already failed) — an unfreeable range
            // then just leaks, recorded nowhere.
            None => {
                // SAFETY (deferred to `seq_free_many` / `dealloc_range`): each range
                // came from our own `alloc_raw` and was never handed to another owner.
                let _ = seq_free_many(allocator, allocated);
            }
        }
    }
}