//! Concrete single-writer pager for Phase 5 persistence.
//!
//! `SimplePager` implements [`MvccPager`] with single-writer semantics over a
//! VFS-backed database file and a zero-copy [`ShardedPageCache`].
//! Full concurrent MVCC behavior is layered on top in Phase 6.
#[cfg(target_arch = "x86_64")]
use core::intrinsics::prefetch_read_data;
use std::cell::{Cell, RefCell};
use std::collections::{BTreeSet, HashMap, HashSet, VecDeque};
use std::future::Future;
use std::path::{Component, Path, PathBuf};
use std::pin::Pin;
use std::sync::atomic::{
AtomicBool, AtomicU8, AtomicU32, AtomicU64, AtomicUsize, Ordering as AtomicOrdering,
};
use std::sync::{Arc, Condvar, Mutex, MutexGuard, OnceLock, RwLock, Weak};
use std::time::Duration;
use asupersync::sync::{Notify, RwLock as AsyncRwLock};
use dashmap::DashMap;
use fsqlite_error::{FrankenError, Result};
use fsqlite_observability::PageCacheEfficiencySnapshot;
use fsqlite_types::cx::Cx;
use fsqlite_types::flags::{AccessFlags, SyncFlags, VfsOpenFlags};
use fsqlite_types::sync_primitives::Instant;
use fsqlite_types::{
BTreePageHeader, CommitSeq, DATABASE_HEADER_MAGIC, DATABASE_HEADER_SIZE, DatabaseHeader,
DatabaseHeaderError, FRANKENSQLITE_SQLITE_VERSION_NUMBER, LockLevel, PageData, PageNumber,
PageNumberBuildHasher, PageSize,
};
#[cfg(all(feature = "native", any(unix, windows)))]
use fsqlite_vfs::{
DatabaseNamespaceBinding, NamespaceOpenIntent, PendingNamespaceOpen, WindowsLockSidecarPolicy,
validate_reserved_database_artifacts,
};
use fsqlite_vfs::{
FileIdentity, SyncKind, Vfs, VfsFile, VfsWriteCompletion, VfsWriteCompletionState,
};
use smallvec::SmallVec;
use crate::journal::{JOURNAL_MAGIC, JournalHeader, JournalPageRecord};
use crate::page_buf::{PageBuf, PageBufPool};
use crate::page_cache::{
PageCacheEvictionPolicy, PageCacheMetricsSnapshot, PageCachePageSnapshot, ShardedPageCache,
};
use crate::s3_fifo::S3FifoConfig;
use crate::traits::{
self, JournalMode, MvccPager, TransactionHandle, TransactionMode, WalBackend, WalFuture,
};
fn atomic_usize_checked_update(
counter: &AtomicUsize,
success: AtomicOrdering,
failure: AtomicOrdering,
mut update: impl FnMut(usize) -> Option<usize>,
) -> std::result::Result<usize, usize> {
let mut current = counter.load(failure);
loop {
let Some(next) = update(current) else {
return Err(current);
};
match counter.compare_exchange_weak(current, next, success, failure) {
Ok(previous) => return Ok(previous),
Err(observed) => current = observed,
}
}
}
fn atomic_u64_checked_update(
counter: &AtomicU64,
success: AtomicOrdering,
failure: AtomicOrdering,
mut update: impl FnMut(u64) -> Option<u64>,
) -> std::result::Result<u64, u64> {
let mut current = counter.load(failure);
loop {
let Some(next) = update(current) else {
return Err(current);
};
match counter.compare_exchange_weak(current, next, success, failure) {
Ok(previous) => return Ok(previous),
Err(observed) => current = observed,
}
}
}
/// Identity-hashed `HashMap<PageNumber, V>` used on the INSERT hot path.
///
/// Profile showed `RandomState::hash_one::<&PageNumber>` at ~1.0% self-time.
/// `PageNumberBuildHasher` bypasses SipHash-1-3 — `PageNumber::Hash` already
/// delegates to `write_u32`, so the identity hasher makes lookups a single
/// shift+mask.
type PagePageMap<V> = HashMap<PageNumber, V, PageNumberBuildHasher>;
use fsqlite_wal::{
ConsolidationPhase, FrameSubmission, GLOBAL_CONSOLIDATION_METRICS, GroupCommitConfig,
GroupCommitConsolidator, PARALLEL_WAL_COMPATIBILITY_SELECTOR, PARALLEL_WAL_FLUSH_SCENARIO_ID,
PARALLEL_WAL_LANE_POLICY_VERSION, PARALLEL_WAL_PUBLICATION_SCENARIO_ID,
PARALLEL_WAL_STAGE_SCENARIO_ID, ParallelWalCombinerError, ParallelWalCommitCertificate,
ParallelWalConservativeShadowEvidence, ParallelWalControlSurface,
ParallelWalDurabilityCombiner, ParallelWalDurabilityReceipt, ParallelWalDurabilityRequest,
ParallelWalFallbackReason, ParallelWalFramePayloadDigestBuilder, ParallelWalLaneBatch,
ParallelWalLaneStager, ParallelWalOperatingMode, ParallelWalPendingPublication,
ParallelWalShadowVerdict, ParallelWalVisibilitySnapshot, RecoveryFence, SubmitOutcome,
TransactionConflictPageBaseline, TransactionConflictSnapshot, TransactionFrameBatch,
TransactionFrameBatchContext, WalFile, WalGenerationIdentity, commit_phase_timing_enabled,
detailed_consolidation_metrics_enabled, parallel_wal_fallback_reason_name,
parallel_wal_mode_name, parallel_wal_shadow_verdict_name, parallel_wal_should_shadow_compare,
resolve_parallel_wal_control_surface_from_env,
};
#[cfg(target_arch = "x86_64")]
#[inline]
fn prefetch_l1_read<T>(ptr: *const T) {
if ptr.is_null() {
return;
}
prefetch_read_data::<T, 3>(ptr);
}
#[cfg(not(target_arch = "x86_64"))]
#[inline]
fn prefetch_l1_read<T>(_ptr: *const T) {}
#[inline]
fn elapsed_profile_us(start: Option<Instant>) -> u64 {
start.map_or(0, |start| {
u64::try_from(Instant::now().duration_since(start).as_micros()).unwrap_or(u64::MAX)
})
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct PagerCommitProfileSnapshot {
pub commit_calls: u64,
pub phase_a_time_ns: u64,
pub wal_commit_time_ns: u64,
pub memory_flush_time_ns: u64,
pub journal_commit_time_ns: u64,
pub phase_c_metadata_time_ns: u64,
pub file_size_time_ns: u64,
pub unlock_time_ns: u64,
pub publish_time_ns: u64,
pub cache_finish_time_ns: u64,
}
static PAGER_COMMIT_PROFILE_ENABLED: AtomicBool = AtomicBool::new(false);
static PAGER_COMMIT_CALLS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_PHASE_A_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_WAL_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_MEMORY_FLUSH_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_JOURNAL_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_PHASE_C_METADATA_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_FILE_SIZE_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_UNLOCK_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_PUBLISH_TIME_NS: AtomicU64 = AtomicU64::new(0);
static PAGER_COMMIT_CACHE_FINISH_TIME_NS: AtomicU64 = AtomicU64::new(0);
pub fn set_pager_commit_profile_enabled(enabled: bool) {
PAGER_COMMIT_PROFILE_ENABLED.store(enabled, AtomicOrdering::Relaxed);
}
pub fn reset_pager_commit_profile() {
PAGER_COMMIT_CALLS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_PHASE_A_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_WAL_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_MEMORY_FLUSH_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_JOURNAL_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_PHASE_C_METADATA_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_FILE_SIZE_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_UNLOCK_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_PUBLISH_TIME_NS.store(0, AtomicOrdering::Relaxed);
PAGER_COMMIT_CACHE_FINISH_TIME_NS.store(0, AtomicOrdering::Relaxed);
}
#[must_use]
pub fn pager_commit_profile_snapshot() -> PagerCommitProfileSnapshot {
PagerCommitProfileSnapshot {
commit_calls: PAGER_COMMIT_CALLS.load(AtomicOrdering::Relaxed),
phase_a_time_ns: PAGER_COMMIT_PHASE_A_TIME_NS.load(AtomicOrdering::Relaxed),
wal_commit_time_ns: PAGER_COMMIT_WAL_TIME_NS.load(AtomicOrdering::Relaxed),
memory_flush_time_ns: PAGER_COMMIT_MEMORY_FLUSH_TIME_NS.load(AtomicOrdering::Relaxed),
journal_commit_time_ns: PAGER_COMMIT_JOURNAL_TIME_NS.load(AtomicOrdering::Relaxed),
phase_c_metadata_time_ns: PAGER_COMMIT_PHASE_C_METADATA_TIME_NS
.load(AtomicOrdering::Relaxed),
file_size_time_ns: PAGER_COMMIT_FILE_SIZE_TIME_NS.load(AtomicOrdering::Relaxed),
unlock_time_ns: PAGER_COMMIT_UNLOCK_TIME_NS.load(AtomicOrdering::Relaxed),
publish_time_ns: PAGER_COMMIT_PUBLISH_TIME_NS.load(AtomicOrdering::Relaxed),
cache_finish_time_ns: PAGER_COMMIT_CACHE_FINISH_TIME_NS.load(AtomicOrdering::Relaxed),
}
}
#[inline]
#[must_use]
pub fn pager_commit_profile_enabled() -> bool {
PAGER_COMMIT_PROFILE_ENABLED.load(AtomicOrdering::Relaxed)
}
#[inline]
fn pager_commit_profile_start(enabled: bool) -> Option<Instant> {
enabled.then(Instant::now)
}
#[inline]
fn record_pager_commit_duration(metric: &AtomicU64, start: Option<Instant>) {
if let Some(start) = start {
metric.fetch_add(
u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX),
AtomicOrdering::Relaxed,
);
}
}
#[inline]
fn record_pager_commit_call(enabled: bool) {
if enabled {
PAGER_COMMIT_CALLS.fetch_add(1, AtomicOrdering::Relaxed);
}
}
// The production WAL path deliberately has no bookkeeping for this receipt.
// Unit keepers install the scope around one real transaction commit so they can
// distinguish process-wide registry mutexes from per-pager/per-database
// coordination without turning the release build into an instrumentation build.
#[cfg(test)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
enum CommitFastPathLockClass {
ProcessGlobalRegistry,
PagerInner,
QueueConsolidator,
QueueEpochState,
ExactHandleCoordination,
WalBackendSlot,
WalBackendRead,
WalBackendWrite,
PublishedPagerState,
}
#[cfg(test)]
#[derive(Default)]
struct CommitFastPathLockReceipt {
process_global_registry: AtomicUsize,
pager_inner: AtomicUsize,
queue_consolidator: AtomicUsize,
queue_epoch_state: AtomicUsize,
exact_handle_coordination: AtomicUsize,
wal_backend_slot: AtomicUsize,
wal_backend_read: AtomicUsize,
wal_backend_write: AtomicUsize,
published_pager_state: AtomicUsize,
}
#[cfg(test)]
impl CommitFastPathLockReceipt {
fn record(&self, class: CommitFastPathLockClass) {
let counter = match class {
CommitFastPathLockClass::ProcessGlobalRegistry => &self.process_global_registry,
CommitFastPathLockClass::PagerInner => &self.pager_inner,
CommitFastPathLockClass::QueueConsolidator => &self.queue_consolidator,
CommitFastPathLockClass::QueueEpochState => &self.queue_epoch_state,
CommitFastPathLockClass::ExactHandleCoordination => &self.exact_handle_coordination,
CommitFastPathLockClass::WalBackendSlot => &self.wal_backend_slot,
CommitFastPathLockClass::WalBackendRead => &self.wal_backend_read,
CommitFastPathLockClass::WalBackendWrite => &self.wal_backend_write,
CommitFastPathLockClass::PublishedPagerState => &self.published_pager_state,
};
counter.fetch_add(1, AtomicOrdering::Relaxed);
}
fn count(&self, class: CommitFastPathLockClass) -> usize {
let counter = match class {
CommitFastPathLockClass::ProcessGlobalRegistry => &self.process_global_registry,
CommitFastPathLockClass::PagerInner => &self.pager_inner,
CommitFastPathLockClass::QueueConsolidator => &self.queue_consolidator,
CommitFastPathLockClass::QueueEpochState => &self.queue_epoch_state,
CommitFastPathLockClass::ExactHandleCoordination => &self.exact_handle_coordination,
CommitFastPathLockClass::WalBackendSlot => &self.wal_backend_slot,
CommitFastPathLockClass::WalBackendRead => &self.wal_backend_read,
CommitFastPathLockClass::WalBackendWrite => &self.wal_backend_write,
CommitFastPathLockClass::PublishedPagerState => &self.published_pager_state,
};
counter.load(AtomicOrdering::Acquire)
}
fn scope(receipt: &Arc<Self>) -> CommitFastPathLockReceiptScope {
COMMIT_FAST_PATH_LOCK_RECEIPT.with(|active| CommitFastPathLockReceiptScope {
previous: active.borrow_mut().replace(Arc::clone(receipt)),
})
}
}
#[cfg(test)]
std::thread_local! {
static COMMIT_FAST_PATH_LOCK_RECEIPT: RefCell<Option<Arc<CommitFastPathLockReceipt>>> = const { RefCell::new(None) };
}
#[cfg(test)]
struct CommitFastPathLockReceiptScope {
previous: Option<Arc<CommitFastPathLockReceipt>>,
}
#[cfg(test)]
impl Drop for CommitFastPathLockReceiptScope {
fn drop(&mut self) {
COMMIT_FAST_PATH_LOCK_RECEIPT.with(|active| {
*active.borrow_mut() = self.previous.take();
});
}
}
#[cfg(test)]
fn record_commit_fast_path_lock(class: CommitFastPathLockClass) {
COMMIT_FAST_PATH_LOCK_RECEIPT.with(|active| {
if let Some(receipt) = active.borrow().as_ref() {
receipt.record(class);
}
});
}
// ---------------------------------------------------------------------------
// Group Commit Queue (D1: replaces global WAL_APPEND_GATES mutex)
// ---------------------------------------------------------------------------
//
// The GroupCommitQueue provides same-process WAL write consolidation. Instead
// of serializing all concurrent writers through a global mutex (the old
// `WAL_APPEND_GATES`), writers submit their frame batches to a consolidator.
// The first writer becomes the "flusher" and waits briefly for more writers
// to arrive. Subsequent writers become "waiters" and park on a targeted keyed
// eventcount with a bounded timeout recheck.
// When the flusher decides to flush (batch full OR max delay exceeded), it
// writes all accumulated frames in one consolidated I/O, fsyncs once, and
// wakes all waiters.
//
// This reduces:
// - Lock contention: Mutex<()> serialization → cooperative batching
// - fsync overhead: N commits × fsync → 1 group × fsync
// - Cache-line ping-pong: N lock acquisitions → 1 flusher + N-1 keyed waits
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[allow(dead_code)] // LegacyCondvarTimeout is a compile-time alternative to KeyedEventcount
enum WaitPathMode {
KeyedEventcount,
LegacyCondvarTimeout,
}
impl WaitPathMode {
const fn as_str(self) -> &'static str {
match self {
Self::KeyedEventcount => "keyed_eventcount",
Self::LegacyCondvarTimeout => "legacy_condvar_timeout",
}
}
#[cfg(any(test, feature = "fault-injection"))]
const fn notification_surface(self) -> &'static str {
match self {
Self::KeyedEventcount => "keyed_notify",
Self::LegacyCondvarTimeout => "legacy_condvar",
}
}
}
const GROUP_COMMIT_WAIT_PATH_MODE: WaitPathMode = WaitPathMode::KeyedEventcount;
const PUBLISHED_SEQUENCE_WAIT_PATH_MODE: WaitPathMode = WaitPathMode::KeyedEventcount;
const GROUP_COMMIT_WAIT_TIMEOUT_FALLBACK: Duration = Duration::from_millis(200);
const LEGACY_GROUP_COMMIT_ARRIVAL_WAIT: Duration = Duration::from_micros(20);
const GROUP_COMMIT_SPARSE_ARRIVAL_WAIT: Duration = Duration::from_micros(8);
const GROUP_COMMIT_BALANCED_ARRIVAL_WAIT: Duration = LEGACY_GROUP_COMMIT_ARRIVAL_WAIT;
const GROUP_COMMIT_BURST_ARRIVAL_WAIT: Duration = Duration::from_micros(40);
const GROUP_COMMIT_ARRIVAL_WAIT_POLICY: &str = "bounded_fair_commit_v1";
const SINGLE_WRITER_BATON_SPINS: u32 = 128;
const SINGLE_WRITER_BATON_PARK: Duration = Duration::from_micros(50);
const PHYSICAL_WRITER_LANE_RUN_ID: &str = "physical-writer-batching-lane";
const PHYSICAL_WRITER_CHECKPOINT_RUN_ID: &str = "physical-writer-checkpoint-decoupling";
const PHYSICAL_WRITER_CHECKPOINT_SCENARIO_ID: &str = "parallel_wal_checkpoint_coordination";
// Flush busy handoff: retry on-CPU with a bounded spin budget and yield only
// every `FLUSH_BUSY_HANDOFF_YIELD_EVERY` attempts. The owner-handoff rule is
// that a losing flusher yields rarely and otherwise stays hot so the current
// lock holder can complete without millisecond sleeps or exponential backoff.
const FLUSH_BUSY_HANDOFF_BASE_SPINS: u32 = 64;
const FLUSH_BUSY_HANDOFF_MAX_SPINS: u32 = 2_048;
const FLUSH_BUSY_HANDOFF_YIELD_EVERY: u32 = 4;
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct FlushBusyRetryWait {
attempt: u32,
spin_loops: u32,
yielded: bool,
}
const fn flush_busy_retry_spin_loops(attempt: u32) -> u32 {
let growth = attempt.saturating_sub(1);
let shift = if growth > 5 { 5 } else { growth };
let spins = FLUSH_BUSY_HANDOFF_BASE_SPINS << shift;
if spins > FLUSH_BUSY_HANDOFF_MAX_SPINS {
FLUSH_BUSY_HANDOFF_MAX_SPINS
} else {
spins
}
}
const fn flush_busy_retry_should_yield(attempt: u32) -> bool {
attempt >= FLUSH_BUSY_HANDOFF_YIELD_EVERY
&& attempt.is_multiple_of(FLUSH_BUSY_HANDOFF_YIELD_EVERY)
}
const fn flush_busy_retry_wait(attempt: u32) -> FlushBusyRetryWait {
FlushBusyRetryWait {
attempt,
spin_loops: flush_busy_retry_spin_loops(attempt),
yielded: flush_busy_retry_should_yield(attempt),
}
}
fn perform_flush_busy_retry_handoff(wait: FlushBusyRetryWait) {
for _ in 0..wait.spin_loops {
std::hint::spin_loop();
}
#[cfg(not(target_arch = "wasm32"))]
if wait.yielded {
std::thread::yield_now();
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ArrivalWaitObservation {
pending_batch_count: usize,
should_flush_now: bool,
fill_age: Duration,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CommitServiceMode {
LowLatency,
Balanced,
Throughput,
}
impl CommitServiceMode {
const fn as_u8(self) -> u8 {
match self {
Self::LowLatency => 0,
Self::Balanced => 1,
Self::Throughput => 2,
}
}
const fn from_u8(value: u8) -> Self {
match value {
0 => Self::LowLatency,
2 => Self::Throughput,
_ => Self::Balanced,
}
}
}
fn commit_service_mode_name(mode: CommitServiceMode) -> &'static str {
match mode {
CommitServiceMode::LowLatency => "low_latency",
CommitServiceMode::Balanced => "balanced",
CommitServiceMode::Throughput => "throughput",
}
}
fn duration_from_nanos_saturating(nanos: u128) -> Duration {
Duration::from_nanos(u64::try_from(nanos.min(u128::from(u64::MAX))).unwrap_or(u64::MAX))
}
fn duration_fraction(duration: Duration, numerator: u32, denominator: u32) -> Duration {
debug_assert!(denominator > 0);
duration_from_nanos_saturating(
duration.as_nanos().saturating_mul(u128::from(numerator)) / u128::from(denominator.max(1)),
)
}
fn commit_service_fairness_budget(
control: &ParallelWalControlSurface,
max_wait: Duration,
) -> Duration {
let configured_budget = control
.max_flush_delay_ms
.map(Duration::from_millis)
.unwrap_or(max_wait);
std::cmp::min(configured_budget, max_wait)
}
fn recent_queue_age_p95(fill_age: Duration) -> Duration {
// This runs in the group-commit flusher's scheduling loop. Do not call
// `GLOBAL_CONSOLIDATION_METRICS.snapshot()` here: the report path copies
// and sorts the histogram ring to compute exact percentiles. A decaying
// tail estimate preserves the starvation-pressure signal without turning
// every flush decision into a telemetry aggregation.
let recent_arrival_wait_tail = Duration::from_micros(
GLOBAL_CONSOLIDATION_METRICS
.hist_arrival_wait
.recent_tail_us(),
);
std::cmp::max(fill_age, recent_arrival_wait_tail)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ArrivalWaitDecision {
control_epoch: u64,
wait_budget: Duration,
fairness_budget: Duration,
max_wait: Duration,
mode: CommitServiceMode,
policy: &'static str,
reason: &'static str,
mode_switch_reason: &'static str,
queue_age_p95: Duration,
fill_age: Duration,
used_legacy_fallback: bool,
starvation_prevented: bool,
}
impl ArrivalWaitDecision {
#[allow(clippy::too_many_arguments)]
fn skip(
control_epoch: u64,
reason: &'static str,
fill_age: Duration,
queue_age_p95: Duration,
fairness_budget: Duration,
max_wait: Duration,
mode: CommitServiceMode,
starvation_prevented: bool,
) -> Self {
Self {
control_epoch,
wait_budget: Duration::ZERO,
fairness_budget,
max_wait,
mode,
policy: GROUP_COMMIT_ARRIVAL_WAIT_POLICY,
reason,
mode_switch_reason: reason,
queue_age_p95,
fill_age,
used_legacy_fallback: false,
starvation_prevented,
}
}
#[allow(clippy::too_many_arguments)]
fn wait(
control_epoch: u64,
wait_budget: Duration,
reason: &'static str,
fill_age: Duration,
queue_age_p95: Duration,
fairness_budget: Duration,
max_wait: Duration,
mode: CommitServiceMode,
used_legacy_fallback: bool,
) -> Self {
Self {
control_epoch,
wait_budget,
fairness_budget,
max_wait,
mode,
policy: GROUP_COMMIT_ARRIVAL_WAIT_POLICY,
reason,
mode_switch_reason: reason,
queue_age_p95,
fill_age,
used_legacy_fallback,
starvation_prevented: false,
}
}
fn wait_budget_us(self) -> u64 {
self.wait_budget.as_micros() as u64
}
fn fill_age_us(self) -> u64 {
self.fill_age.as_micros() as u64
}
fn target_wait_ns(self) -> u64 {
self.wait_budget.as_nanos() as u64
}
fn max_wait_ns(self) -> u64 {
self.max_wait.as_nanos() as u64
}
fn fairness_budget_ns(self) -> u64 {
self.fairness_budget.as_nanos() as u64
}
fn queue_age_p95_ns(self) -> u64 {
self.queue_age_p95.as_nanos() as u64
}
fn queue_delay_ns(self) -> u64 {
self.fill_age.as_nanos() as u64
}
}
fn decide_group_commit_arrival_wait(
observation: Option<ArrivalWaitObservation>,
max_wait: Duration,
fairness_budget: Duration,
queue_age_p95: Duration,
previous_mode: CommitServiceMode,
control_epoch: u64,
) -> ArrivalWaitDecision {
let fairness_budget = std::cmp::min(fairness_budget, max_wait);
match observation {
None => ArrivalWaitDecision::skip(
control_epoch,
"promoted_follow_on",
Duration::ZERO,
queue_age_p95,
fairness_budget,
max_wait,
previous_mode,
false,
),
Some(observation) => {
if observation.should_flush_now {
let mode = if matches!(previous_mode, CommitServiceMode::Throughput)
|| observation.pending_batch_count >= 3
{
CommitServiceMode::Throughput
} else {
CommitServiceMode::Balanced
};
let reason = if matches!(mode, CommitServiceMode::Throughput)
&& matches!(previous_mode, CommitServiceMode::Throughput)
{
"throughput_hysteresis"
} else {
"queue_flushable"
};
return ArrivalWaitDecision::skip(
control_epoch,
reason,
observation.fill_age,
queue_age_p95,
fairness_budget,
max_wait,
mode,
false,
);
}
if fairness_budget.is_zero()
|| observation.fill_age >= fairness_budget
|| queue_age_p95 >= fairness_budget
{
let reason = if fairness_budget.is_zero() || observation.fill_age >= fairness_budget
{
"fairness_budget_exhausted"
} else {
"tail_latency_pressure"
};
return ArrivalWaitDecision::skip(
control_epoch,
reason,
observation.fill_age,
queue_age_p95,
fairness_budget,
max_wait,
CommitServiceMode::LowLatency,
true,
);
}
let remaining_budget = fairness_budget.saturating_sub(observation.fill_age);
if observation.pending_batch_count >= 3
|| (matches!(previous_mode, CommitServiceMode::Throughput)
&& observation.pending_batch_count >= 2
&& queue_age_p95 <= duration_fraction(fairness_budget, 1, 2))
{
let reason = if matches!(previous_mode, CommitServiceMode::Throughput)
&& observation.pending_batch_count >= 2
{
"throughput_hysteresis"
} else {
"burst_backlog"
};
return ArrivalWaitDecision::wait(
control_epoch,
std::cmp::min(remaining_budget, GROUP_COMMIT_BURST_ARRIVAL_WAIT),
reason,
observation.fill_age,
queue_age_p95,
fairness_budget,
max_wait,
CommitServiceMode::Throughput,
false,
);
}
if observation.pending_batch_count >= 2 {
return ArrivalWaitDecision::wait(
control_epoch,
std::cmp::min(remaining_budget, GROUP_COMMIT_BALANCED_ARRIVAL_WAIT),
"mixed_backlog",
observation.fill_age,
queue_age_p95,
fairness_budget,
max_wait,
CommitServiceMode::Balanced,
false,
);
}
ArrivalWaitDecision::wait(
control_epoch,
std::cmp::min(remaining_budget, GROUP_COMMIT_SPARSE_ARRIVAL_WAIT),
"sparse_queue",
observation.fill_age,
queue_age_p95,
fairness_budget,
max_wait,
CommitServiceMode::LowLatency,
true,
)
}
}
}
fn physical_writer_batch_membership(batches: &[TransactionFrameBatch]) -> String {
batches
.iter()
.map(|batch| batch.context.batch_id.to_string())
.collect::<Vec<_>>()
.join(",")
}
fn physical_writer_primary_batch_id(batches: &[TransactionFrameBatch]) -> u64 {
batches
.first()
.map(|batch| batch.context.batch_id)
.unwrap_or_default()
}
fn physical_writer_rollback_mode_active(
mode: ParallelWalOperatingMode,
fallback_reason: Option<ParallelWalFallbackReason>,
) -> bool {
matches!(mode, ParallelWalOperatingMode::Conservative) || fallback_reason.is_some()
}
fn physical_writer_fsync_boundary(sync_policy: WalCommitSyncPolicy) -> &'static str {
if sync_policy.should_sync_on_commit() {
"commit_sync"
} else {
"deferred_sync"
}
}
fn physical_writer_ordering_phase(arrival_wait_reason: &'static str) -> &'static str {
if arrival_wait_reason == "promoted_follow_on" {
"promoted_follow_on_flush"
} else {
"group_flush"
}
}
fn group_commit_phase_name(phase: ConsolidationPhase) -> &'static str {
match phase {
ConsolidationPhase::Filling => "filling",
ConsolidationPhase::Flushing => "flushing",
ConsolidationPhase::Complete => "complete",
}
}
fn transaction_mode_name(mode: TransactionMode) -> &'static str {
match mode {
TransactionMode::ReadOnly => "read_only",
TransactionMode::Deferred => "deferred",
TransactionMode::Immediate => "immediate",
TransactionMode::Exclusive => "exclusive",
TransactionMode::Concurrent => "concurrent",
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct CheckpointCoordinationQueueSnapshot {
queue_phase: &'static str,
queue_epoch: u64,
pending_batch_count: usize,
}
fn checkpoint_coordination_queue_snapshot(
queue: &GroupCommitQueueRef,
) -> CheckpointCoordinationQueueSnapshot {
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
CheckpointCoordinationQueueSnapshot {
queue_phase: group_commit_phase_name(consolidator.phase()),
queue_epoch: consolidator.epoch(),
pending_batch_count: consolidator.pending_batch_count(),
}
}
#[allow(clippy::too_many_arguments)]
fn log_checkpoint_coordination(
cx: &Cx,
queue: &GroupCommitQueueRef,
checkpoint_phase: &'static str,
foreground_phase: &'static str,
foreground_action: &'static str,
interaction_rule: &'static str,
stall_avoided: bool,
active_transactions: u32,
checkpoint_active: bool,
) {
let queue_snapshot = checkpoint_coordination_queue_snapshot(queue);
tracing::debug!(
target: "fsqlite::wal::checkpoint_coordination",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_CHECKPOINT_RUN_ID,
scenario_id = PHYSICAL_WRITER_CHECKPOINT_SCENARIO_ID,
checkpoint_phase,
foreground_phase,
foreground_action,
interaction_rule,
stall_avoided,
active_transactions,
checkpoint_active,
queue_phase = queue_snapshot.queue_phase,
queue_epoch = queue_snapshot.queue_epoch,
pending_batch_count = queue_snapshot.pending_batch_count,
"checkpoint/foreground coordination event"
);
}
fn pager_group_commit_queue<V: Vfs>(pager: &SimplePager<V>) -> GroupCommitQueueRef {
Arc::clone(&pager.group_commit_queue)
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum KeyedWaitResult {
Signaled,
RecoveredAfterTimeout,
TimedOut,
}
/// Owns one resolved epoch wake until it is classified against queue state.
///
/// Identity-wide finalization can suspend after the keyed wait resolves. If
/// the outer waiter future is dropped during that suspension, `Drop` records
/// the wake as nonterminal instead of silently losing the accounting event.
#[must_use = "a resolved epoch wake must be classified exactly once"]
struct PendingEpochWake<'a> {
queue: &'a GroupCommitQueue,
target_epoch: u64,
wait_result: Option<KeyedWaitResult>,
}
impl<'a> PendingEpochWake<'a> {
fn new(queue: &'a GroupCommitQueue, target_epoch: u64, wait_result: KeyedWaitResult) -> Self {
Self {
queue,
target_epoch,
wait_result: Some(wait_result),
}
}
fn observe_failure(&mut self) -> Option<FrankenError> {
let failure = self.queue.observe_failed_epoch(self.target_epoch);
if failure.is_some() {
self.wait_result = None;
}
failure
}
fn observe(
mut self,
guard: &mut std::sync::MutexGuard<'_, GroupCommitConsolidator>,
) -> Result<Option<WaitForEpochOutcome>> {
let wait_result = self
.wait_result
.expect("pending epoch wake must remain armed until observation");
let outcome = self
.queue
.observe_epoch_outcome(guard, self.target_epoch, Some(wait_result));
// `observe_epoch_outcome` classifies every return path. Disarm before
// propagating its stored Result so an epoch error is not double-counted
// by this value's Drop implementation.
self.wait_result = None;
outcome
}
}
impl Drop for PendingEpochWake<'_> {
fn drop(&mut self) {
if let Some(wait_result) = self.wait_result.take() {
#[cfg(test)]
self.queue
.unaccounted_epoch_wake_drops
.fetch_add(1, AtomicOrdering::AcqRel);
GroupCommitQueue::record_nonterminal_epoch_wake(self.target_epoch, wait_result);
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EpochWakeReason {
Notify,
Timeout,
FlusherTakeover,
FailedEpoch,
BusyRetry,
}
impl EpochWakeReason {
const fn as_str(self) -> &'static str {
match self {
Self::Notify => "notify",
Self::Timeout => "timeout",
Self::FlusherTakeover => "flusher_takeover",
Self::FailedEpoch => "failed_epoch",
Self::BusyRetry => "busy_retry",
}
}
}
fn completed_epoch_wake_reason(pending_wake: Option<KeyedWaitResult>) -> EpochWakeReason {
match pending_wake.unwrap_or(KeyedWaitResult::Signaled) {
KeyedWaitResult::Signaled => EpochWakeReason::Notify,
KeyedWaitResult::RecoveredAfterTimeout | KeyedWaitResult::TimedOut => {
EpochWakeReason::Timeout
}
}
}
fn nonterminal_epoch_wake_reason(wait_result: KeyedWaitResult) -> EpochWakeReason {
match wait_result {
KeyedWaitResult::Signaled => EpochWakeReason::BusyRetry,
KeyedWaitResult::RecoveredAfterTimeout | KeyedWaitResult::TimedOut => {
EpochWakeReason::Timeout
}
}
}
#[derive(Debug)]
struct KeyedWaitSlot {
state: Mutex<u64>,
cv: Condvar,
notify: Notify,
#[cfg(test)]
drop_next_async_notify: AtomicBool,
#[cfg(test)]
active_async_waiters: AtomicUsize,
#[cfg(test)]
timeout_recoveries: AtomicUsize,
#[cfg(test)]
async_wait_rendezvous: Mutex<Option<Arc<KeyedWaitTestRendezvous>>>,
}
#[cfg(test)]
#[derive(Debug, Default)]
struct KeyedWaitTestRendezvousState {
entered: bool,
released: bool,
}
#[cfg(test)]
#[derive(Debug, Default)]
struct KeyedWaitTestRendezvous {
state: Mutex<KeyedWaitTestRendezvousState>,
cv: Condvar,
}
#[cfg(test)]
impl KeyedWaitTestRendezvous {
fn wait_at_boundary(&self) -> bool {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.entered = true;
self.cv.notify_all();
let (state, _) = self
.cv
.wait_timeout_while(state, Duration::from_secs(10), |state| !state.released)
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.released
}
fn wait_until_entered(&self, timeout: Duration) -> bool {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let (state, _) = self
.cv
.wait_timeout_while(state, timeout, |state| !state.entered)
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.entered
}
fn release(&self) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.released = true;
self.cv.notify_all();
}
}
#[cfg(test)]
struct ActiveAsyncKeyedWaiter<'a> {
count: &'a AtomicUsize,
}
#[cfg(test)]
impl<'a> ActiveAsyncKeyedWaiter<'a> {
fn new(count: &'a AtomicUsize) -> Self {
count.fetch_add(1, AtomicOrdering::AcqRel);
Self { count }
}
}
#[cfg(test)]
impl Drop for ActiveAsyncKeyedWaiter<'_> {
fn drop(&mut self) {
let previous = self.count.fetch_sub(1, AtomicOrdering::AcqRel);
debug_assert!(previous > 0, "active keyed waiter count must not underflow");
}
}
impl Default for KeyedWaitSlot {
fn default() -> Self {
Self {
state: Mutex::new(0),
cv: Condvar::new(),
notify: Notify::new(),
#[cfg(test)]
drop_next_async_notify: AtomicBool::new(false),
#[cfg(test)]
active_async_waiters: AtomicUsize::new(0),
#[cfg(test)]
timeout_recoveries: AtomicUsize::new(0),
#[cfg(test)]
async_wait_rendezvous: Mutex::new(None),
}
}
}
impl KeyedWaitSlot {
fn generation(&self) -> u64 {
*self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
}
fn wait_for_change(&self, observed_generation: u64, timeout: Duration) -> KeyedWaitResult {
let guard = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if *guard != observed_generation {
return KeyedWaitResult::Signaled;
}
let (_guard, timeout_result) = self
.cv
.wait_timeout_while(guard, timeout, |generation| {
*generation == observed_generation
})
.unwrap_or_else(std::sync::PoisonError::into_inner);
if timeout_result.timed_out() {
KeyedWaitResult::TimedOut
} else {
KeyedWaitResult::Signaled
}
}
async fn wait_for_change_async(&self, observed_generation: u64) -> KeyedWaitResult {
if self.generation() != observed_generation {
return KeyedWaitResult::Signaled;
}
#[cfg(test)]
if self
.drop_next_async_notify
.swap(false, AtomicOrdering::AcqRel)
{
self.advance_generation_without_notify();
}
#[cfg(test)]
let _active_waiter = ActiveAsyncKeyedWaiter::new(&self.active_async_waiters);
#[cfg(test)]
let rendezvous = self
.async_wait_rendezvous
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
#[cfg(test)]
if let Some(rendezvous) = rendezvous {
assert!(
rendezvous.wait_at_boundary(),
"keyed-wait test rendezvous must be released within its watchdog"
);
}
let wait_result = asupersync::time::timeout(
asupersync::time::wall_now(),
GROUP_COMMIT_WAIT_TIMEOUT_FALLBACK,
self.notify.notified(),
)
.await;
match wait_result {
Ok(()) => KeyedWaitResult::Signaled,
Err(_) if self.generation() != observed_generation => {
#[cfg(test)]
self.timeout_recoveries.fetch_add(1, AtomicOrdering::AcqRel);
KeyedWaitResult::RecoveredAfterTimeout
}
Err(_) => KeyedWaitResult::TimedOut,
}
}
#[cfg(any(test, feature = "fault-injection"))]
fn advance_generation_without_notify(&self) {
let mut generation = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*generation = generation.wrapping_add(1);
}
#[cfg(test)]
fn arm_drop_next_async_notify(&self) {
self.drop_next_async_notify
.store(true, AtomicOrdering::Release);
}
#[cfg(test)]
fn active_async_waiter_count(&self) -> usize {
self.active_async_waiters.load(AtomicOrdering::Acquire)
}
#[cfg(test)]
fn timeout_recovery_count(&self) -> usize {
self.timeout_recoveries.load(AtomicOrdering::Acquire)
}
#[cfg(test)]
fn arm_async_wait_rendezvous(&self, rendezvous: Arc<KeyedWaitTestRendezvous>) {
let mut armed = self
.async_wait_rendezvous
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
armed.replace(rendezvous).is_none(),
"one keyed wait slot may hold only one test rendezvous"
);
}
fn signal(&self) {
let mut generation = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*generation = generation.wrapping_add(1);
self.cv.notify_all();
self.notify.notify_waiters();
}
}
#[derive(Debug, Default)]
struct KeyedWaitRegistry {
slots: Mutex<HashMap<u64, Weak<KeyedWaitSlot>>>,
}
impl KeyedWaitRegistry {
fn new() -> Self {
Self {
slots: Mutex::new(HashMap::new()),
}
}
fn slot(&self, key: u64) -> Arc<KeyedWaitSlot> {
let mut slots = self
.slots
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
slots.retain(|_, slot| slot.strong_count() > 0);
if let Some(slot) = slots.get(&key).and_then(Weak::upgrade) {
return slot;
}
let slot = Arc::new(KeyedWaitSlot::default());
slots.insert(key, Arc::downgrade(&slot));
slot
}
fn signal(&self, key: u64) -> bool {
let slot = {
let mut slots = self
.slots
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
slots.retain(|_, slot| slot.strong_count() > 0);
slots.get(&key).and_then(Weak::upgrade)
};
if let Some(slot) = slot {
slot.signal();
true
} else {
false
}
}
#[cfg(any(test, feature = "fault-injection"))]
fn advance_generation_without_notify(&self, key: u64) -> bool {
let slot = {
let mut slots = self
.slots
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
slots.retain(|_, slot| slot.strong_count() > 0);
slots.get(&key).and_then(Weak::upgrade)
};
if let Some(slot) = slot {
slot.advance_generation_without_notify();
true
} else {
false
}
}
#[cfg(test)]
fn has_slot(&self, key: u64) -> bool {
self.slots
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&key)
.is_some_and(|slot| slot.strong_count() > 0)
}
}
/// Per-database group commit queue for WAL write consolidation.
#[derive(Debug, Clone, Default)]
struct GroupCommitFinalizationBinding {
paths: HashSet<PathBuf>,
identity: Option<FileIdentity>,
}
struct RootedPendingEpochResolution {
durability: GroupCommitFlushDurability,
durable_io_completions: Vec<Arc<AtomicBool>>,
root_attempt: ProcessRootFinalizationAttempt,
}
struct PendingEpochResolutionClaim {
queue: Arc<GroupCommitQueue>,
epoch: u64,
record: Option<RootedPendingEpochResolution>,
in_flight: bool,
}
#[derive(Default)]
struct GroupCommitExternalLockCoordination {
physical_lock_windows: HashSet<SharedDbFileKey>,
logical_exit_in_flight: HashSet<SharedDbFileKey>,
}
trait PendingGroupCommitTxnAttemptOperation: Send + Sync {
fn pager_inner_identity(&self) -> *const ();
fn allocator_delta(&self) -> PendingGroupCommitAllocatorDelta;
fn complete_authorized_global(
&self,
authorization: ParallelWalPublicationAuthorization,
complete_group_pages: &HashMap<PageNumber, PageData>,
group_allocator_delta: &PendingGroupCommitAllocatorDelta,
apply_allocator_delta: bool,
) -> Result<()>;
fn complete_not_committed_global(&self) -> Result<()>;
}
struct PendingGroupCommitTxnAttemptRegistration {
epoch: u64,
operation: Arc<dyn PendingGroupCommitTxnAttemptOperation>,
}
struct GroupCommitQueue {
/// Stable process-local identity. Pointer addresses are unsuitable because
/// allocator reuse can alias an old finalization record after its queue is
/// dropped.
queue_id: u64,
/// Purely lexical path aliases plus the concrete open-file identity used
/// to find process-root finalization work before a new opener inspects
/// storage.
finalization_binding: Mutex<GroupCommitFinalizationBinding>,
/// Fast-path admission fence. Ordinary begins read only this atomic; the
/// process-root registry mutex is touched only while exceptional
/// finalization work actually exists.
rooted_finalization_attempts: AtomicUsize,
/// bd-b4mwn rework: settle's claim-contention retry budget in
/// milliseconds, tracking the largest busy_timeout any connection on
/// this path has published (default: the engine's 5000ms default). The
/// fixed 250ms envelope was timing-dependent — green on a 64-core host,
/// red on a 16-core one where resolution bursts run longer.
settle_budget_ms: AtomicU64,
/// bd-b4mwn rework #2: logical cleanups currently CLAIMED and being
/// resolved by a settler. `pending_logical_cleanups` excludes claimed
/// entries, so "queue empty + root armed" is ambiguous between a true
/// wedge and live resolution in progress — on slow-fsync hosts a peer's
/// resolve holds its claim for seconds and every other settler
/// misdiagnosed that window as a wedge and refused BusyRecovery.
claimed_logical_cleanups: AtomicUsize,
/// The consolidator managing FILLING→FLUSHING→COMPLETE phases.
consolidator: Mutex<GroupCommitConsolidator>,
/// Condvar for waiters to park on until flush completes.
flush_complete: Condvar,
/// Atomic epoch counter for lock-free waiter polling.
/// Updated by flusher after complete_flush(), read by waiters.
completed_epoch: AtomicU64,
/// Failure outcomes by epoch. Kept so late-scheduled waiters cannot miss
/// a failed flush after a newer epoch completes successfully.
failed_epochs: Mutex<HashMap<u64, GroupCommitEpochFailure>>,
/// Certificate-backed durable membership by completed epoch.
///
/// Unlike the former trace-only map, this is part of the publication
/// handoff: every waiter must bind its batch id to the certificate before
/// Phase C may expose pager visibility.
persisted_epochs: Mutex<HashMap<u64, PersistedGroupCommitEpoch>>,
/// Active batch owners that can still consume terminal evidence by epoch.
///
/// Registration happens while the consolidator mutex still owns admission,
/// so publication cannot overtake a newly admitted consumer. The matching
/// RAII lease releases on success, error, or cancellation. Terminal
/// evidence is reclaimed only after this count reaches zero.
epoch_consumer_counts: Mutex<HashMap<u64, usize>>,
/// Stable per-queue ordering for deferred finalization lanes. A record
/// keeps its first sequence across claims and cancellation requeues.
next_finalization_sequence: AtomicU64,
/// Logical Phase-C owners keyed by the exact physical batch id.
///
/// A single physical flush can contain transactions from several pager
/// handles. Recovery therefore cannot safely finalize only the flusher's
/// local transaction state; it must address each admitted batch member.
pending_txn_attempts: Mutex<HashMap<u64, PendingGroupCommitTxnAttemptRegistration>>,
/// Lazily seeded from the pager's current visible commit clock at the
/// first physical flush for this database identity.
durability_combiner: Mutex<Option<Arc<ParallelWalDurabilityCombiner>>>,
/// Narrow per-target-epoch wake slots for waiter coordination.
epoch_waiters: KeyedWaitRegistry,
/// Monotonic control-decision epoch for service-policy traces.
commit_service_control_epoch: AtomicU64,
/// Last applied bounded-latency service mode for hysteresis.
commit_service_mode: AtomicU8,
/// WAL-owned lane-local staging state for prepared batches.
parallel_wal_lanes: ParallelWalLaneStager<traits::PreparedWalFrameBatch>,
/// External database-lock restorations whose owning flusher future was
/// dropped while the shared file handle was contended.
///
/// Each entry is type-erased above the concrete `VfsFile` so the
/// identity-bound queue can retain the cleanup obligation. A claimant
/// removes exactly one entry and returns it on Drop until restoration is
/// terminal; no detached cleanup task is required.
pending_external_unlocks: Mutex<VecDeque<PendingExternalUnlock>>,
/// External-unlock records temporarily owned by async claimants.
///
/// A claim is removed from `pending_external_unlocks` while it reconciles
/// durability and restores the flusher's exact file handle. Logical
/// transaction cleanup must remain fenced during that interval or it can
/// release a snapshot first and then have the physical claimant restore a
/// stronger lock afterward.
external_unlock_claims_in_flight: AtomicUsize,
/// Identity-wide external restorations temporarily owned by claimants.
///
/// This is a subset of `external_unlock_claims_in_flight`. Publishing the
/// subset while the pending-queue mutex is held prevents an exact-handle
/// settler from racing past a global maintenance or partial-acquisition
/// restoration after that record has been removed from the visible queue.
identity_wide_external_unlock_claims_in_flight: AtomicUsize,
/// Exact file handles whose oldest queued restoration is currently leased.
///
/// Queue order alone is insufficient once a claimant removes the oldest
/// record: without this set, a second claimant could lease the next record
/// for the same handle and restore the two baselines concurrently. Claims
/// for unrelated handles remain independent.
exact_external_unlock_claims_in_flight: Mutex<HashSet<SharedDbFileKey>>,
/// Serializes external-lock transitions on each exact open file handle.
///
/// The flusher clones its `SharedDbFile` while briefly holding
/// `PagerInner`, drops that guard, then publishes exclusive physical
/// ownership for the handle before RESERVED is acquired. Logical
/// transitions on that same handle wait until direct or queued restoration
/// is terminal. Distinct handles for one file identity remain concurrent.
external_lock_coordination: Mutex<GroupCommitExternalLockCoordination>,
/// Cancel-safe wake generation for exact-handle external-lock ownership.
///
/// Normal admissions and already-authorized logical exits wait here rather
/// than surfacing `BusyRecovery` merely because a compatible peer reached
/// the coordination boundary first.
external_lock_waiters: KeyedWaitSlot,
/// Epochs whose flusher was dropped after durable mutation started but
/// before the lower I/O layer reported a terminal durable result.
///
/// These remain fail-closed in FLUSHING. The shared completion signal is
/// retained so a waiter, subsequent commit, or future lower-layer
/// reconciler can publish the epoch once durability becomes terminal.
in_doubt_epochs: Mutex<HashMap<u64, RootedPendingEpochResolution>>,
/// Resolution records temporarily removed from `in_doubt_epochs` while a
/// claimant performs the terminal consolidator transition.
epoch_resolution_claims_in_flight: AtomicUsize,
/// Transaction objects abandoned after physical admission but before
/// logical Phase C. Each record carries its own process-root attempt and
/// remains queued across cancellation until transaction exit is terminal.
pending_logical_cleanups: Mutex<VecDeque<PendingGroupCommitLogicalCleanup>>,
/// Test-local proof that a resolved wake dropped during post-wake
/// finalization is classified instead of being silently discarded.
#[cfg(test)]
unaccounted_epoch_wake_drops: AtomicUsize,
}
type LaneStagedPreparedBatch = ParallelWalLaneBatch<traits::PreparedWalFrameBatch>;
static GROUP_COMMIT_TRACE_ENABLED: OnceLock<bool> = OnceLock::new();
static GROUP_COMMIT_TRACE_FSYNC_SEQ: AtomicU64 = AtomicU64::new(0);
fn group_commit_trace_enabled() -> bool {
*GROUP_COMMIT_TRACE_ENABLED.get_or_init(|| {
std::env::var_os("FSQLITE_TRACE_GROUP_COMMIT").is_some_and(|value| {
let value = value.to_string_lossy();
!value.is_empty() && value != "0" && !value.eq_ignore_ascii_case("false")
})
})
}
fn trace_group_commit(args: std::fmt::Arguments<'_>) {
if group_commit_trace_enabled() {
eprintln!("[fsqlite_group_commit] {args}");
}
}
#[derive(Debug, Clone)]
struct PersistedGroupCommitEpoch {
members: HashSet<u64>,
frames_start: u64,
frames_end: u64,
fsync_seq: u64,
durability_receipt: ParallelWalDurabilityReceipt,
}
struct GroupCommitEpochConsumer {
queue: Weak<GroupCommitQueue>,
epoch: u64,
tracked: bool,
}
impl Drop for GroupCommitEpochConsumer {
fn drop(&mut self) {
if self.tracked
&& let Some(queue) = self.queue.upgrade()
{
queue.release_epoch_consumer(self.epoch);
}
}
}
struct PersistedGroupCommitInput<'a> {
trace_id: u64,
epoch: u64,
batches: &'a [TransactionFrameBatch],
frames_start: u64,
frames_end: u64,
fsync_seq: u64,
initial_visible_commit_seq: CommitSeq,
db_size_pages: u32,
page_set_size: usize,
checkpoint_active: bool,
fallback_reason: Option<ParallelWalFallbackReason>,
authorized_seed: Option<ParallelWalCommitCertificate>,
wal_frame_payload_digest: [u8; 32],
}
struct PreparedPersistedGroupCommitEpoch {
epoch: u64,
members: HashSet<u64>,
frames_start: u64,
frames_end: u64,
fsync_seq: u64,
combiner: Arc<ParallelWalDurabilityCombiner>,
pending_publication: ParallelWalPendingPublication,
}
struct PendingGroupCommitPublicationState {
prepared: Option<PreparedPersistedGroupCommitEpoch>,
receipt: Option<ParallelWalDurabilityReceipt>,
interval: Option<(u64, u64)>,
}
struct PendingGroupCommitPublication {
state: Mutex<PendingGroupCommitPublicationState>,
}
impl PendingGroupCommitPublication {
fn new(prepared: PreparedPersistedGroupCommitEpoch) -> Self {
let interval = Some((prepared.frames_start, prepared.frames_end));
Self {
state: Mutex::new(PendingGroupCommitPublicationState {
prepared: Some(prepared),
receipt: None,
interval,
}),
}
}
fn certificate(&self) -> Result<ParallelWalCommitCertificate> {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state
.prepared
.as_ref()
.map(|prepared| prepared.pending_publication.certificate().clone())
.or_else(|| {
state
.receipt
.as_ref()
.map(|receipt| receipt.certificate.clone())
})
.ok_or_else(|| FrankenError::internal("parallel WAL publication was already aborted"))
}
fn interval(&self) -> Result<(u64, u64)> {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.interval.ok_or_else(|| {
FrankenError::internal("parallel WAL publication has no recoverable interval")
})
}
fn finalize(&self, queue: &GroupCommitQueue) -> Result<ParallelWalDurabilityReceipt> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if let Some(receipt) = state.receipt.as_ref() {
return Ok(receipt.clone());
}
let prepared = state.prepared.as_ref().ok_or_else(|| {
FrankenError::internal("parallel WAL publication was already aborted")
})?;
let durability_receipt = prepared
.combiner
.finalize_pending_publication(&prepared.pending_publication)
.map_err(|error| {
FrankenError::internal(format!(
"parallel WAL pending publication failed for epoch {}: {error}",
prepared.epoch
))
})?;
let mut members_display = prepared.members.iter().copied().collect::<Vec<_>>();
members_display.sort_unstable();
let members_display = members_display
.into_iter()
.map(|member| member.to_string())
.collect::<Vec<_>>()
.join(",");
queue
.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
prepared.epoch,
PersistedGroupCommitEpoch {
members: prepared.members.clone(),
frames_start: prepared.frames_start,
frames_end: prepared.frames_end,
fsync_seq: prepared.fsync_seq,
durability_receipt: durability_receipt.clone(),
},
);
if group_commit_trace_enabled() {
trace_group_commit(format_args!(
"batch epoch={} members=[{members_display}] frames_written_range={}..={} fsync_seq={} commit_certificate={} durability_seq={} publication_generation={} ordered_region_ns={} batch_size={} lookup_mode={:?} control_mode={} shadow_certificate_verdict={} compatibility_selector={} fallback_reason={}",
prepared.epoch,
prepared.frames_start,
prepared.frames_end,
prepared.fsync_seq,
durability_receipt.certificate.certificate_crc32c,
durability_receipt.durability_seq,
durability_receipt.publication_generation,
durability_receipt.ordered_region_ns,
durability_receipt.batch_size,
durability_receipt.lookup_mode,
parallel_wal_mode_name(durability_receipt.control_mode),
parallel_wal_shadow_verdict_name(durability_receipt.shadow_certificate_verdict),
PARALLEL_WAL_COMPATIBILITY_SELECTOR,
parallel_wal_fallback_reason_name(durability_receipt.fallback_reason),
));
}
state.receipt = Some(durability_receipt.clone());
state.prepared = None;
Ok(durability_receipt)
}
fn abort(&self) -> Result<()> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.receipt.is_some() {
return Err(FrankenError::internal(
"cannot abort an already-published parallel WAL interval",
));
}
let Some(prepared) = state.prepared.as_ref() else {
return Ok(());
};
prepared
.combiner
.abort_pending_publication(&prepared.pending_publication)
.map_err(|error| {
FrankenError::internal(format!(
"parallel WAL pending publication abort failed for epoch {}: {error}",
prepared.epoch
))
})?;
state.prepared.take();
state.interval = None;
Ok(())
}
}
#[derive(Debug)]
enum WaitForEpochOutcome {
Completed,
TakeOverFlusher {
batches: Vec<TransactionFrameBatch>,
flush_epoch: u64,
},
}
#[derive(Debug, Clone)]
enum GroupCommitEpochFailure {
Abort,
Busy,
BusyRecovery,
BusySnapshot { conflicting_pages: String },
Other(String),
}
impl GroupCommitEpochFailure {
fn from_error(error: &FrankenError) -> Self {
match error {
FrankenError::Abort => Self::Abort,
FrankenError::Busy => Self::Busy,
FrankenError::BusyRecovery => Self::BusyRecovery,
FrankenError::BusySnapshot { conflicting_pages } => Self::BusySnapshot {
conflicting_pages: conflicting_pages.clone(),
},
_ => Self::Other(error.to_string()),
}
}
fn into_error(self, target_epoch: u64) -> FrankenError {
match self {
Self::Abort => FrankenError::Abort,
Self::Busy => FrankenError::Busy,
Self::BusyRecovery => FrankenError::BusyRecovery,
Self::BusySnapshot { conflicting_pages } => {
FrankenError::BusySnapshot { conflicting_pages }
}
Self::Other(detail) => FrankenError::internal(format!(
"group commit flush failed for epoch {target_epoch}: {detail}"
)),
}
}
}
#[cfg(test)]
static PARALLEL_WAL_CONTROL_OVERRIDE: OnceLock<Mutex<Option<ParallelWalControlSurface>>> =
OnceLock::new();
fn resolve_parallel_wal_control_surface() -> ParallelWalControlSurface {
#[cfg(test)]
let test_override = PARALLEL_WAL_CONTROL_OVERRIDE
.get_or_init(|| Mutex::new(None))
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
#[cfg(test)]
if let Some(control) = test_override {
return control;
}
resolve_parallel_wal_control_surface_from_env()
}
#[cfg(test)]
fn set_parallel_wal_control_override(control: Option<ParallelWalControlSurface>) {
*PARALLEL_WAL_CONTROL_OVERRIDE
.get_or_init(|| Mutex::new(None))
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = control;
}
impl GroupCommitQueue {
fn bind_finalization_path(self: &Arc<Self>, path: &Path) {
let path = lexical_normalize_path(path.to_path_buf());
{
let mut binding = self
.finalization_binding
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
binding.paths.insert(path);
}
refresh_process_root_finalization_binding(self);
}
fn bind_finalization_identity(self: &Arc<Self>, identity: FileIdentity) {
{
let mut binding = self
.finalization_binding
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match binding.identity {
Some(bound) => debug_assert_eq!(
bound, identity,
"one group-commit queue must not span file identities"
),
None => binding.identity = Some(identity),
}
}
refresh_process_root_finalization_binding(self);
}
fn has_process_root_finalization_attempt(&self) -> bool {
self.rooted_finalization_attempts
.load(AtomicOrdering::Acquire)
!= 0
}
fn has_identity_wide_process_root(&self) -> bool {
self.has_process_root_scope_with_hook(None, || {})
}
fn has_relevant_process_root(&self, handle_key: SharedDbFileKey) -> bool {
self.has_process_root_scope_with_hook(Some(handle_key), || {})
}
fn has_process_root_scope_with_hook(
&self,
handle_key: Option<SharedDbFileKey>,
after_fast_observation: impl FnOnce(),
) -> bool {
if !self.has_process_root_finalization_attempt() {
return false;
}
after_fast_observation();
let registry = process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
// Terminal release decrements the atomic and removes the registry
// entry while holding this mutex. Revalidate after acquiring it so a
// stale optimistic observation cannot become a false invariant error
// or a spurious BusyRecovery result. Registration uses the same mutex,
// so a still-nonzero count must have an authoritative registry entry.
if !self.has_process_root_finalization_attempt() {
return false;
}
process_root_finalization_scope_is_relevant_in_registry(
®istry,
self.queue_id,
handle_key,
)
}
fn new(config: GroupCommitConfig) -> Self {
Self::with_parallel_wal_control(config, resolve_parallel_wal_control_surface())
}
fn with_parallel_wal_control(
config: GroupCommitConfig,
parallel_wal_control: ParallelWalControlSurface,
) -> Self {
Self {
queue_id: next_process_root_finalization_id(&NEXT_GROUP_COMMIT_QUEUE_ID),
finalization_binding: Mutex::new(GroupCommitFinalizationBinding::default()),
rooted_finalization_attempts: AtomicUsize::new(0),
settle_budget_ms: AtomicU64::new(5000),
claimed_logical_cleanups: AtomicUsize::new(0),
consolidator: Mutex::new(GroupCommitConsolidator::new(config)),
flush_complete: Condvar::new(),
completed_epoch: AtomicU64::new(0),
failed_epochs: Mutex::new(HashMap::new()),
persisted_epochs: Mutex::new(HashMap::new()),
epoch_consumer_counts: Mutex::new(HashMap::new()),
next_finalization_sequence: AtomicU64::new(1),
pending_txn_attempts: Mutex::new(HashMap::new()),
durability_combiner: Mutex::new(None),
epoch_waiters: KeyedWaitRegistry::new(),
commit_service_control_epoch: AtomicU64::new(0),
commit_service_mode: AtomicU8::new(CommitServiceMode::Balanced.as_u8()),
parallel_wal_lanes: ParallelWalLaneStager::new(parallel_wal_control),
pending_external_unlocks: Mutex::new(VecDeque::new()),
external_unlock_claims_in_flight: AtomicUsize::new(0),
identity_wide_external_unlock_claims_in_flight: AtomicUsize::new(0),
exact_external_unlock_claims_in_flight: Mutex::new(HashSet::new()),
external_lock_coordination: Mutex::new(GroupCommitExternalLockCoordination::default()),
external_lock_waiters: KeyedWaitSlot::default(),
in_doubt_epochs: Mutex::new(HashMap::new()),
epoch_resolution_claims_in_flight: AtomicUsize::new(0),
pending_logical_cleanups: Mutex::new(VecDeque::new()),
#[cfg(test)]
unaccounted_epoch_wake_drops: AtomicUsize::new(0),
}
}
fn parallel_wal_control(&self) -> &ParallelWalControlSurface {
self.parallel_wal_lanes.control()
}
fn next_parallel_wal_batch_id(&self) -> u64 {
self.parallel_wal_lanes.next_batch_id()
}
fn next_finalization_sequence(&self) -> u64 {
next_process_root_finalization_id(&self.next_finalization_sequence)
}
fn current_parallel_wal_lane_id(&self) -> u16 {
self.parallel_wal_lanes.current_lane_id()
}
fn current_lane_backlog(&self, lane_id: u16) -> usize {
self.parallel_wal_lanes.current_lane_backlog(lane_id)
}
fn current_commit_service_mode(&self) -> CommitServiceMode {
CommitServiceMode::from_u8(self.commit_service_mode.load(AtomicOrdering::Relaxed))
}
fn next_commit_service_control_epoch(&self) -> u64 {
self.commit_service_control_epoch
.fetch_add(1, AtomicOrdering::Relaxed)
.saturating_add(1)
}
fn store_commit_service_mode(&self, mode: CommitServiceMode) {
self.commit_service_mode
.store(mode.as_u8(), AtomicOrdering::Relaxed);
}
fn record_prepared_batch(&self, prepared_batch: LaneStagedPreparedBatch) -> usize {
self.parallel_wal_lanes.record_batch(prepared_batch)
}
fn take_prepared_batches_for_flush(
&self,
batches: &[TransactionFrameBatch],
) -> Option<HashMap<u64, LaneStagedPreparedBatch>> {
let contexts = batches
.iter()
.map(|batch| batch.context)
.collect::<Vec<_>>();
self.parallel_wal_lanes.take_batches_for_flush(&contexts)
}
fn discard_prepared_batches_for_flush(&self, batches: &[TransactionFrameBatch]) -> usize {
let contexts = batches
.iter()
.map(|batch| batch.context)
.collect::<Vec<_>>();
self.parallel_wal_lanes.discard_batches_for_flush(&contexts)
}
fn durability_combiner(
&self,
initial_visible_commit_seq: CommitSeq,
initial_db_size: u32,
) -> Arc<ParallelWalDurabilityCombiner> {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueEpochState);
let mut slot = self
.durability_combiner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let combiner = Arc::clone(slot.get_or_insert_with(|| {
Arc::new(ParallelWalDurabilityCombiner::new(
ParallelWalVisibilitySnapshot {
visible_commit_seq: initial_visible_commit_seq,
db_size_pages: initial_db_size,
..ParallelWalVisibilitySnapshot::default()
},
))
}));
drop(slot);
// The queue is process-global and can outlive a pager refresh or WAL
// checkpoint generation. Never let its certificate allocator trail the
// durable pager identity observed under the external writer gate.
combiner.reconcile_durable_visibility_floor(initial_visible_commit_seq);
combiner
}
async fn prepare_persisted_epoch(
&self,
cx: &Cx,
input: PersistedGroupCommitInput<'_>,
) -> Result<Arc<PendingGroupCommitPublication>> {
let members = input
.batches
.iter()
.map(|batch| batch.context.batch_id)
.collect::<HashSet<_>>();
let max_lane_id = input
.batches
.iter()
.map(|batch| batch.context.lane_id)
.max()
.unwrap_or(0);
let mut lane_record_counts = vec![0_u32; usize::from(max_lane_id) + 1];
for batch in input.batches {
let lane_record_count = &mut lane_record_counts[usize::from(batch.context.lane_id)];
*lane_record_count = lane_record_count
.saturating_add(u32::try_from(batch.frames.len()).unwrap_or(u32::MAX));
}
let batch_ids = input
.batches
.iter()
.map(|batch| batch.context.batch_id)
.collect::<Vec<_>>();
let combiner =
self.durability_combiner(input.initial_visible_commit_seq, input.db_size_pages);
if let Some(certificate) = input.authorized_seed.as_ref() {
combiner
.reconcile_authorized_seed(certificate)
.map_err(|error| {
FrankenError::internal(format!(
"parallel WAL authorized tail reconciliation failed: {error}"
))
})?;
}
let control_mode = self.parallel_wal_control().mode;
let request = ParallelWalDurabilityRequest {
trace_id: input.trace_id,
scenario_id: PARALLEL_WAL_PUBLICATION_SCENARIO_ID.to_owned(),
// Allocate both clocks after reconciling the authorized
// durable tail. The group-commit epoch is process-local and
// therefore cannot serve as a cross-process certificate id.
certificate_epoch: 0,
durable_segment_epoch: 0,
batch_size: u32::try_from(input.batches.len()).unwrap_or(u32::MAX),
batch_ids,
lane_record_counts,
db_size_pages: input.db_size_pages,
page_set_size: u32::try_from(input.page_set_size).unwrap_or(u32::MAX),
control_mode,
fallback_reason: input.fallback_reason,
checkpoint_active: input.checkpoint_active,
wal_frame_payload_digest: input.wal_frame_payload_digest,
};
let conservative_shadow_evidence =
matches!(control_mode, ParallelWalOperatingMode::ShadowCompare).then(|| {
let raw_max_lane_id = input
.batches
.iter()
.map(|batch| batch.context.lane_id)
.max()
.unwrap_or(0);
let mut raw_lane_record_counts =
vec![0_u32; usize::from(raw_max_lane_id).saturating_add(1)];
for batch in input.batches {
raw_lane_record_counts[usize::from(batch.context.lane_id)] =
raw_lane_record_counts[usize::from(batch.context.lane_id)]
.saturating_add(u32::try_from(batch.frames.len()).unwrap_or(u32::MAX));
}
ParallelWalConservativeShadowEvidence {
certificate_epoch: 0,
durable_segment_epoch: 0,
batch_ids: input
.batches
.iter()
.map(|batch| batch.context.batch_id)
.collect(),
lane_record_counts: raw_lane_record_counts,
db_size_pages: input
.batches
.iter()
.flat_map(|batch| batch.frames.iter())
.filter_map(|frame| {
(frame.db_size_if_commit > 0).then_some(frame.db_size_if_commit)
})
.max()
.unwrap_or(input.db_size_pages),
page_set_size: input
.batches
.iter()
.map(|batch| u32::try_from(batch.frames.len()).unwrap_or(u32::MAX))
.fold(0_u32, u32::saturating_add),
control_mode,
fallback_reason: input.fallback_reason,
checkpoint_active: input.checkpoint_active,
wal_frame_start: input.frames_start,
wal_frame_end: input.frames_end,
wal_frame_payload_digest: input.wal_frame_payload_digest,
}
});
let pending_publication = if let Some(evidence) = conservative_shadow_evidence {
combiner
.prepare_pending_publication_with_conservative_shadow(cx, request, evidence)
.await
} else {
combiner.prepare_pending_publication(cx, request).await
}
.map_err(|error| match error {
ParallelWalCombinerError::Cancelled => FrankenError::Abort,
error => FrankenError::internal(format!(
"parallel WAL publication preparation failed for epoch {}: {error}",
input.epoch
)),
})?;
Ok(Arc::new(PendingGroupCommitPublication::new(
PreparedPersistedGroupCommitEpoch {
epoch: input.epoch,
members,
frames_start: input.frames_start,
frames_end: input.frames_end,
fsync_seq: input.fsync_seq,
combiner,
pending_publication,
},
)))
}
fn persisted_epoch_for(&self, epoch: u64) -> Option<PersistedGroupCommitEpoch> {
self.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&epoch)
.cloned()
}
fn register_epoch_consumer(self: &Arc<Self>, epoch: u64) -> Arc<GroupCommitEpochConsumer> {
let tracked = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueEpochState);
let mut counts = self
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let count = counts.entry(epoch).or_default();
if *count == usize::MAX {
// Fail closed by retaining this epoch forever in the
// unrepresentable case rather than risking early evidence
// reclamation.
tracing::error!(epoch, "group-commit epoch consumer count overflow");
false
} else {
*count += 1;
true
}
};
Arc::new(GroupCommitEpochConsumer {
queue: Arc::downgrade(self),
epoch,
tracked,
})
}
fn register_txn_attempt(
&self,
epoch: u64,
batch_id: u64,
operation: Arc<dyn PendingGroupCommitTxnAttemptOperation>,
) -> Result<()> {
use std::collections::hash_map::Entry;
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueEpochState);
let mut attempts = self
.pending_txn_attempts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match attempts.entry(batch_id) {
Entry::Vacant(entry) => {
entry.insert(PendingGroupCommitTxnAttemptRegistration { epoch, operation });
Ok(())
}
Entry::Occupied(_) => Err(FrankenError::internal(format!(
"group-commit batch {batch_id} registered two logical Phase-C owners"
))),
}
}
fn unregister_txn_attempt(&self, batch_id: u64) {
self.pending_txn_attempts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&batch_id);
}
fn complete_txn_attempts_authorized(
&self,
epoch: u64,
batches: &[TransactionFrameBatch],
durability_receipt: &ParallelWalDurabilityReceipt,
complete_group_pages: &HashMap<PageNumber, PageData>,
) -> Result<()> {
let (attempts, registered_for_epoch) = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueEpochState);
let attempts = self
.pending_txn_attempts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let registered_for_epoch = attempts
.values()
.filter(|registration| registration.epoch == epoch)
.count();
let matching_attempts = batches
.iter()
.filter_map(|batch| {
let batch_id = batch.context.batch_id;
attempts.get(&batch_id).map(|registration| {
(
batch_id,
registration.epoch,
Arc::clone(®istration.operation),
)
})
})
.collect::<Vec<_>>();
(matching_attempts, registered_for_epoch)
};
if attempts.len() != registered_for_epoch {
return Err(FrankenError::internal(format!(
"group-commit epoch {epoch} omitted a registered logical owner from its durable batch set"
)));
}
if !attempts.is_empty() && attempts.len() != batches.len() {
return Err(FrankenError::internal(format!(
"group-commit epoch {epoch} mixed owned and ownerless durable batches"
)));
}
let mut group_allocator_delta = PendingGroupCommitAllocatorDelta::default();
for (_, _, operation) in &attempts {
group_allocator_delta.extend(operation.allocator_delta());
}
group_allocator_delta.normalize();
let mut normalized_pager_inners = HashSet::new();
for (batch_id, registered_epoch, operation) in &attempts {
if *registered_epoch != epoch {
return Err(FrankenError::internal(format!(
"group-commit batch {batch_id} was registered for epoch {registered_epoch}, \
not recovered epoch {epoch}"
)));
}
let assigned_commit_seq = durability_receipt
.commit_seq_for_batch(*batch_id)
.ok_or_else(|| {
FrankenError::internal(format!(
"authorized group-commit receipt has no sequence for batch {batch_id}"
))
})?;
let apply_allocator_delta =
normalized_pager_inners.insert(operation.pager_inner_identity());
operation.complete_authorized_global(
ParallelWalPublicationAuthorization {
durability_receipt: durability_receipt.clone(),
batch_id: *batch_id,
assigned_commit_seq,
},
complete_group_pages,
&group_allocator_delta,
apply_allocator_delta,
)?;
}
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueEpochState);
let mut registered = self
.pending_txn_attempts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for (batch_id, _, _) in attempts {
registered.remove(&batch_id);
}
Ok(())
}
fn complete_txn_attempts_not_committed(&self, epoch: u64) -> Result<()> {
let attempts = {
let attempts = self
.pending_txn_attempts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
attempts
.iter()
.filter(|(_, registration)| registration.epoch == epoch)
.map(|(&batch_id, registration)| {
(
batch_id,
registration.epoch,
Arc::clone(®istration.operation),
)
})
.collect::<Vec<_>>()
};
for (batch_id, registered_epoch, operation) in &attempts {
if *registered_epoch != epoch {
return Err(FrankenError::internal(format!(
"group-commit batch {batch_id} was registered for epoch {registered_epoch}, \
not rejected epoch {epoch}"
)));
}
operation.complete_not_committed_global()?;
}
let mut registered = self
.pending_txn_attempts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for (batch_id, _, _) in attempts {
registered.remove(&batch_id);
}
Ok(())
}
fn enqueue_pending_logical_cleanup(
self: &Arc<Self>,
mut cleanup: PendingGroupCommitLogicalCleanup,
) {
if cleanup.sequence.is_none() {
cleanup.sequence = Some(self.next_finalization_sequence());
}
if cleanup.root_attempt.is_none() {
// Publish process-root ownership before the queue record. A
// concurrent entry gate in this short interval observes the root
// and fails closed instead of mistaking the database for clean.
cleanup.root_attempt = Some(ProcessRootFinalizationAttempt::register_scope(
self,
cleanup.scope,
));
}
self.pending_logical_cleanups
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push_back(cleanup);
}
fn requeue_pending_logical_cleanup(
self: &Arc<Self>,
mut cleanup: PendingGroupCommitLogicalCleanup,
) {
if cleanup.sequence.is_none() {
tracing::error!(
"requeued logical group-commit cleanup lost its FIFO sequence; \
assigning a fail-closed tail sequence"
);
cleanup.sequence = Some(self.next_finalization_sequence());
}
if cleanup.root_attempt.is_none() {
tracing::error!(
"requeued logical group-commit cleanup lost its process-root token; \
installing a replacement fail-closed owner"
);
cleanup.root_attempt = Some(ProcessRootFinalizationAttempt::register_scope(
self,
cleanup.scope,
));
}
let mut pending = self
.pending_logical_cleanups
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
insert_pending_logical_cleanup_by_sequence(&mut pending, cleanup);
}
fn pending_logical_cleanup_count(&self) -> usize {
self.pending_logical_cleanups
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len()
}
fn pending_logical_cleanup_count_for_handle(&self, handle_key: SharedDbFileKey) -> usize {
self.pending_logical_cleanups
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.iter()
.filter(|cleanup| {
cleanup.scope == ProcessRootFinalizationScope::ExactHandle(handle_key)
})
.count()
}
fn claim_pending_logical_cleanup(
self: &Arc<Self>,
) -> Option<PendingGroupCommitLogicalCleanupClaim> {
self.claim_pending_logical_cleanup_for(ProcessRootFinalizationSelector::Any)
}
fn claim_pending_logical_cleanup_for_handle(
self: &Arc<Self>,
handle_key: SharedDbFileKey,
) -> Option<PendingGroupCommitLogicalCleanupClaim> {
self.claim_pending_logical_cleanup_for(ProcessRootFinalizationSelector::ExactHandle(
handle_key,
))
}
fn claim_pending_logical_cleanup_for(
self: &Arc<Self>,
selector: ProcessRootFinalizationSelector,
) -> Option<PendingGroupCommitLogicalCleanupClaim> {
let mut pending_logical_cleanups = self
.pending_logical_cleanups
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut seen_handles = HashSet::new();
let (position, logical_exit_claim) =
pending_logical_cleanups
.iter()
.enumerate()
.find_map(|(position, cleanup)| {
if !selector.matches(cleanup.scope) {
return None;
}
let handle_key = cleanup.operation.handle_key();
debug_assert_eq!(
cleanup.scope,
ProcessRootFinalizationScope::ExactHandle(handle_key),
"logical cleanup scope must match its pinned file handle"
);
if !seen_handles.insert(handle_key) {
// Preserve FIFO within one exact-handle lane. If its
// oldest entry cannot claim the handle, a younger entry
// for that handle must not overtake it.
return None;
}
GroupCommitLogicalExitClaim::try_register(self, handle_key)
.map(|claim| (position, claim))
})?;
let cleanup = pending_logical_cleanups
.remove(position)
.expect("selected cleanup must remain present while its queue lock is held");
// bd-b4mwn rework #2: count the claim while the queue lock is still
// held so no observer can see the entry vanish from the pending
// queue before it appears in the claimed count.
self.claimed_logical_cleanups
.fetch_add(1, AtomicOrdering::AcqRel);
drop(pending_logical_cleanups);
Some(PendingGroupCommitLogicalCleanupClaim {
queue: Arc::clone(self),
cleanup: Some(cleanup),
logical_exit_claim: Some(logical_exit_claim),
})
}
async fn resolve_one_pending_logical_cleanup(self: &Arc<Self>) -> Result<bool> {
if self.has_pending_or_claimed_identity_wide_external_unlock()
|| self.has_unresolved_in_doubt_epoch()
{
return Ok(false);
}
let Some(mut claim) = self.claim_pending_logical_cleanup() else {
return Ok(false);
};
if !claim.resolve().await? {
return Ok(false);
}
let mut cleanup = claim.finish();
cleanup.release_root_after_terminal();
// bd-b4mwn rework #2 (ordering): the success-path claimed decrement
// happens only after the root is released, so no observer can see
// pending==0 && claimed==0 with the root still armed (the false-wedge
// window the trj holder receipt caught).
self.claimed_logical_cleanups
.fetch_sub(1, AtomicOrdering::AcqRel);
Ok(true)
}
async fn resolve_one_pending_logical_cleanup_for_handle(
self: &Arc<Self>,
handle_key: SharedDbFileKey,
) -> Result<bool> {
if self.has_pending_or_claimed_identity_wide_external_unlock()
|| self.has_unresolved_in_doubt_epoch()
{
return Ok(false);
}
let Some(mut claim) = self.claim_pending_logical_cleanup_for_handle(handle_key) else {
return Ok(false);
};
if !claim.resolve().await? {
return Ok(false);
}
let mut cleanup = claim.finish();
cleanup.release_root_after_terminal();
// bd-b4mwn rework #2 (ordering): the success-path claimed decrement
// happens only after the root is released, so no observer can see
// pending==0 && claimed==0 with the root still armed (the false-wedge
// window the trj holder receipt caught).
self.claimed_logical_cleanups
.fetch_sub(1, AtomicOrdering::AcqRel);
Ok(true)
}
fn release_epoch_consumer(&self, epoch: u64) {
let mut counts = self
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(count) = counts.get_mut(&epoch) else {
tracing::error!(
epoch,
"group-commit epoch consumer released without registration"
);
return;
};
if *count > 1 {
*count -= 1;
return;
}
counts.remove(&epoch);
self.remove_epoch_metadata(epoch);
}
fn reclaim_epoch_metadata_if_unowned(&self, epoch: u64) {
let counts = self
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if counts.contains_key(&epoch) {
return;
}
self.remove_epoch_metadata(epoch);
}
fn remove_epoch_metadata(&self, epoch: u64) {
self.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&epoch);
self.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&epoch);
}
/// Publish a completed epoch and wake all waiters.
///
/// We take the consolidator mutex before publishing so a waiter cannot
/// observe an incomplete epoch, race with notify, and then go to sleep
/// forever on a lost wakeup.
fn publish_completed_epoch(&self, epoch: u64, wake_next_epoch: bool) {
let _guard = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
self.completed_epoch.store(epoch, AtomicOrdering::Release);
// H11 fault hook: preserve the completed-epoch store and keyed
// generation advance, but suppress direct waiter delivery. Both the
// active keyed path and the legacy Condvar path must recover by
// rechecking the published epoch after their bounded timeout.
#[cfg(any(test, feature = "fault-injection"))]
let suppress_waiter_notify = crate::fault_hooks::maybe_inject_drop_waiter_notify(
epoch,
GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
GROUP_COMMIT_WAIT_PATH_MODE.notification_surface(),
);
#[cfg(not(any(test, feature = "fault-injection")))]
let suppress_waiter_notify = false;
if suppress_waiter_notify {
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
published_epoch = epoch,
wake_next_epoch,
fallback = "timeout_recheck",
"suppressed direct waiter wake after completion publish"
);
}
self.signal_completed_epoch_waiters(epoch, wake_next_epoch, !suppress_waiter_notify);
self.reclaim_epoch_metadata_if_unowned(epoch);
}
/// Publish a failed epoch and wake all waiters.
///
/// This uses the same mutex discipline as `publish_completed_epoch` so
/// waiter condition checks and condvar parking stay synchronized.
#[cfg(test)]
fn publish_failed_epoch(&self, epoch: u64, error: &FrankenError, wake_next_epoch: bool) {
let guard = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut failed_epochs = self
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
failed_epochs.insert(epoch, GroupCommitEpochFailure::from_error(error));
drop(failed_epochs);
drop(guard);
if let Err(logical_error) = self.complete_txn_attempts_not_committed(epoch) {
tracing::error!(
epoch,
%logical_error,
"failed group-commit epoch retained unfinished logical owners"
);
}
self.signal_failed_epoch_waiters(epoch, wake_next_epoch);
self.reclaim_epoch_metadata_if_unowned(epoch);
}
fn abort_flushing_epoch_as_failed(&self, epoch: u64, error: &FrankenError) -> Result<bool> {
let wake_next_epoch = {
let mut consolidator = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if consolidator.phase() != ConsolidationPhase::Flushing || consolidator.epoch() != epoch
{
let already_failed = self
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&epoch);
drop(consolidator);
if already_failed {
self.complete_txn_attempts_not_committed(epoch)?;
return Ok(false);
}
return Err(FrankenError::internal(format!(
"cannot abort group-commit epoch {epoch}: it is not the active FLUSHING epoch"
)));
}
consolidator.abort_flush()?;
let wake_next_epoch = consolidator.has_flusher_vacancy();
self.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(epoch, GroupCommitEpochFailure::from_error(error));
wake_next_epoch
};
self.complete_txn_attempts_not_committed(epoch)?;
self.signal_failed_epoch_waiters(epoch, wake_next_epoch);
self.reclaim_epoch_metadata_if_unowned(epoch);
Ok(wake_next_epoch)
}
fn abort_cancelled_flush(&self, epoch: u64) -> Result<()> {
self.abort_flushing_epoch_as_failed(epoch, &FrankenError::Abort)?;
Ok(())
}
fn complete_cancelled_durable_flush(&self, epoch: u64) -> Result<()> {
let has_promoted = {
let mut consolidator = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if consolidator.phase() != ConsolidationPhase::Flushing || consolidator.epoch() != epoch
{
drop(consolidator);
if self.is_epoch_complete(epoch) {
return Ok(());
}
return Err(FrankenError::internal(format!(
"cannot complete durable group-commit epoch {epoch}: it is not the active FLUSHING epoch"
)));
}
consolidator.complete_flush()?
};
// The certificate, frames, and requested sync are already durable.
// Never reinterpret that commit as an Abort merely because its caller
// was cancelled during local cleanup/publication.
self.publish_completed_epoch(epoch, has_promoted);
Ok(())
}
fn enqueue_pending_external_unlock(self: &Arc<Self>, mut pending: PendingExternalUnlock) {
if pending.sequence.is_none() {
pending.sequence = Some(self.next_finalization_sequence());
}
if pending.root_attempt.is_none() {
// Install process-root ownership before publishing the queue item.
// A concurrent opener that sees this admitted-but-not-yet-queued
// interval fails closed with BusyRecovery.
pending.root_attempt = Some(ProcessRootFinalizationAttempt::register_scope(
self,
pending.scope,
));
}
let epoch = pending.epoch;
self.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push_back(pending);
// A waiter already parked on the stranded epoch is the preferred
// structured cleanup owner. The bounded eventcount fallback still
// covers a signal that races registration.
if let Some(epoch) = epoch {
let _ = self.epoch_waiters.signal(epoch);
if GROUP_COMMIT_WAIT_PATH_MODE == WaitPathMode::LegacyCondvarTimeout {
self.flush_complete.notify_all();
}
}
}
fn requeue_pending_external_unlock(self: &Arc<Self>, mut pending: PendingExternalUnlock) {
if pending.sequence.is_none() {
tracing::error!(
epoch = ?pending.epoch,
"requeued group-commit finalization lost its FIFO sequence; assigning a fail-closed tail sequence"
);
pending.sequence = Some(self.next_finalization_sequence());
}
if pending.root_attempt.is_none() {
tracing::error!(
epoch = ?pending.epoch,
"requeued group-commit finalization lost its process-root token; installing a replacement fail-closed owner"
);
pending.root_attempt = Some(ProcessRootFinalizationAttempt::register_scope(
self,
pending.scope,
));
}
let epoch = pending.epoch;
let mut pending_external_unlocks = self
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
insert_pending_external_unlock_by_sequence(&mut pending_external_unlocks, pending);
drop(pending_external_unlocks);
if let Some(epoch) = epoch {
let _ = self.epoch_waiters.signal(epoch);
if GROUP_COMMIT_WAIT_PATH_MODE == WaitPathMode::LegacyCondvarTimeout {
self.flush_complete.notify_all();
}
}
}
fn claim_pending_external_unlock(self: &Arc<Self>) -> Option<PendingExternalUnlockClaim> {
self.claim_pending_external_unlock_for(ProcessRootFinalizationSelector::Any)
}
fn claim_pending_external_unlock_for(
self: &Arc<Self>,
selector: ProcessRootFinalizationSelector,
) -> Option<PendingExternalUnlockClaim> {
let mut pending_external_unlocks = self
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if selector != ProcessRootFinalizationSelector::Any
&& matches!(selector, ProcessRootFinalizationSelector::ExactHandle(_))
&& pending_external_unlocks
.iter()
.any(|pending| pending.scope == ProcessRootFinalizationScope::IdentityWide)
{
// An admitted global restoration fences every exact handle even
// before a claimant removes it from the visible queue.
return None;
}
let identity_wide_claims = self
.identity_wide_external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire);
let total_claims = self
.external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire);
let position = if selector == ProcessRootFinalizationSelector::Any {
// Identity-wide restoration is the conservative priority lane.
// It fences every exact handle, so an Any claimant must not run a
// later exact restoration merely because the global record is not
// at the deque front. When no global record exists, skip a
// handle whose oldest record is already leased so independent
// handles do not convoy behind it.
pending_external_unlocks
.iter()
.position(|pending| pending.scope == ProcessRootFinalizationScope::IdentityWide)
.or_else(|| {
let exact_claims = self
.exact_external_unlock_claims_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
pending_external_unlocks.iter().position(|pending| {
let ProcessRootFinalizationScope::ExactHandle(handle_key) = pending.scope
else {
return false;
};
!exact_claims.contains(&handle_key)
})
})?
} else {
pending_external_unlocks
.iter()
.position(|pending| selector.matches(pending.scope))?
};
let scope = pending_external_unlocks[position].scope;
let epoch = pending_external_unlocks[position].epoch;
match scope {
ProcessRootFinalizationScope::IdentityWide if total_claims != 0 => return None,
ProcessRootFinalizationScope::ExactHandle(handle_key)
if identity_wide_claims != 0
|| self
.exact_external_unlock_claims_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains(&handle_key) =>
{
return None;
}
ProcessRootFinalizationScope::IdentityWide
| ProcessRootFinalizationScope::ExactHandle(_) => {}
}
// Publish the in-flight claim while the queue lock is still held.
// Otherwise a settler can observe both an empty queue and a zero
// claim count after removal but before fetch_add, and run logical
// transaction exit ahead of the physical lock restoration.
if !self.publish_external_unlock_claim(scope) {
tracing::error!(
epoch = ?epoch,
"group-commit external-unlock claim ownership could not be published; left queued fail closed"
);
return None;
}
let pending = pending_external_unlocks
.remove(position)
.expect("selected item must remain present while the queue lock is held");
drop(pending_external_unlocks);
Some(PendingExternalUnlockClaim {
queue: Arc::clone(self),
pending: Some(pending),
scope,
in_flight: true,
})
}
fn publish_external_unlock_claim(&self, scope: ProcessRootFinalizationScope) -> bool {
if let ProcessRootFinalizationScope::ExactHandle(handle_key) = scope {
let mut exact_claims = self
.exact_external_unlock_claims_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !exact_claims.insert(handle_key) {
return false;
}
}
if atomic_usize_checked_update(
&self.external_unlock_claims_in_flight,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|count| count.checked_add(1),
)
.is_err()
{
if let ProcessRootFinalizationScope::ExactHandle(handle_key) = scope {
self.exact_external_unlock_claims_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&handle_key);
}
return false;
}
if scope == ProcessRootFinalizationScope::IdentityWide
&& atomic_usize_checked_update(
&self.identity_wide_external_unlock_claims_in_flight,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|count| count.checked_add(1),
)
.is_err()
{
self.release_external_unlock_claim_total();
return false;
}
true
}
fn release_external_unlock_claim(&self, scope: ProcessRootFinalizationScope) {
match scope {
ProcessRootFinalizationScope::IdentityWide => {
if atomic_usize_checked_update(
&self.identity_wide_external_unlock_claims_in_flight,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|count| count.checked_sub(1),
)
.is_err()
{
tracing::error!("identity-wide external-unlock claim count underflow");
}
}
ProcessRootFinalizationScope::ExactHandle(handle_key) => {
if !self
.exact_external_unlock_claims_in_flight
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.remove(&handle_key)
{
tracing::error!(
handle_key = handle_key.0,
"exact-handle external-unlock claim released without ownership"
);
}
}
}
self.release_external_unlock_claim_total();
}
fn release_external_unlock_claim_total(&self) {
if atomic_usize_checked_update(
&self.external_unlock_claims_in_flight,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|count| count.checked_sub(1),
)
.is_err()
{
tracing::error!("group-commit external-unlock claim count underflow");
}
}
fn has_pending_or_claimed_external_unlock(&self) -> bool {
let pending_external_unlocks = self
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !pending_external_unlocks.is_empty() {
return true;
}
// Read the claim count only after acquiring the queue mutex. A reader
// that loaded the count first could observe zero, wait for a claimant
// to increment-and-pop under this mutex, then see an empty queue while
// retaining the stale zero.
self.external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire)
!= 0
}
fn has_pending_or_claimed_identity_wide_external_unlock(&self) -> bool {
let pending_external_unlocks = self
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if pending_external_unlocks
.iter()
.any(|pending| pending.scope == ProcessRootFinalizationScope::IdentityWide)
{
return true;
}
self.identity_wide_external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire)
!= 0
}
fn identity_wide_finalization_is_quiescent(&self) -> bool {
// Every pending/claimed identity-wide record and every in-doubt epoch
// owns a process-root token before it is published. Keep the ordinary
// wake path to one Acquire load when no exceptional work exists.
if !self.has_process_root_finalization_attempt() {
return true;
}
// Check the process-root fence last. Registration publishes that
// fence before either its queue record or epoch signal, so it closes
// the admitted-but-not-yet-queued interval across the earlier checks.
!self.has_pending_or_claimed_identity_wide_external_unlock()
&& !self.has_unresolved_in_doubt_epoch()
&& !self.has_identity_wide_process_root()
}
async fn resolve_one_pending_external_unlock(self: &Arc<Self>) -> Result<bool> {
self.resolve_one_pending_external_unlock_for(ProcessRootFinalizationSelector::Any)
.await
}
async fn resolve_one_pending_external_unlock_for_handle(
self: &Arc<Self>,
handle_key: SharedDbFileKey,
) -> Result<bool> {
self.resolve_one_pending_external_unlock_for(ProcessRootFinalizationSelector::ExactHandle(
handle_key,
))
.await
}
async fn resolve_one_pending_external_unlock_for(
self: &Arc<Self>,
selector: ProcessRootFinalizationSelector,
) -> Result<bool> {
let Some(mut claim) = self.claim_pending_external_unlock_for(selector) else {
return Ok(false);
};
if claim.durability_state() == GroupCommitFlushDurability::InDoubt {
// The dropped callback's lower I/O may still own and mutate the
// ordered WAL residue. The recovery object first waits for both
// source tokens, then validates the exact on-disk interval while
// RESERVED remains held.
if claim.reconcile_durability().await? == GroupCommitFlushDurability::InDoubt {
return Ok(false);
}
}
claim.restore().await?;
self.resolve_cancelled_flush_after_external_unlock(claim.pending_mut())?;
let mut restored = claim.finish();
restored.release_after_terminal();
Ok(true)
}
fn try_resolve_one_pending_external_unlock(self: &Arc<Self>) -> Result<bool> {
let Some(mut claim) = self.claim_pending_external_unlock() else {
return Ok(false);
};
if claim.durability_state() == GroupCommitFlushDurability::InDoubt {
// Synchronous Drop paths must obey the same cross-process
// fail-closed rule as async cleanup claimants.
return Ok(false);
}
if !claim.try_restore()? {
return Ok(false);
}
self.resolve_cancelled_flush_after_external_unlock(claim.pending_mut())?;
let mut restored = claim.finish();
restored.release_after_terminal();
Ok(true)
}
fn resolve_cancelled_flush_after_external_unlock(
self: &Arc<Self>,
pending: &mut PendingExternalUnlock,
) -> Result<()> {
let Some(epoch) = pending.epoch else {
return Ok(());
};
match pending.durability_state() {
GroupCommitFlushDurability::PreDurable => {
self.abort_cancelled_flush(epoch)?;
}
GroupCommitFlushDurability::Durable => {
self.complete_cancelled_durable_flush(epoch)?;
}
GroupCommitFlushDurability::InDoubt => {
let root_attempt = pending.root_attempt.take().ok_or_else(|| {
FrankenError::internal(
"in-doubt group-commit finalization lost its process-root token",
)
})?;
self.defer_pending_epoch_resolution(
epoch,
GroupCommitFlushDurability::InDoubt,
Arc::clone(&pending.durable_io_completed),
Some(root_attempt),
);
}
}
Ok(())
}
fn defer_pending_epoch_resolution(
self: &Arc<Self>,
epoch: u64,
durability: GroupCommitFlushDurability,
durable_io_completed: Arc<AtomicBool>,
root_attempt: Option<ProcessRootFinalizationAttempt>,
) {
let root_attempt =
root_attempt.unwrap_or_else(|| ProcessRootFinalizationAttempt::register(self));
self.requeue_pending_epoch_resolution(
epoch,
RootedPendingEpochResolution {
durability,
durable_io_completions: if durability == GroupCommitFlushDurability::InDoubt {
vec![durable_io_completed]
} else {
Vec::new()
},
root_attempt,
},
);
tracing::warn!(
epoch,
?durability,
"deferred dropped group-commit epoch resolution to a process-root owner"
);
}
fn requeue_pending_epoch_resolution(&self, epoch: u64, record: RootedPendingEpochResolution) {
let redundant_root = {
let mut pending = self
.in_doubt_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match pending.entry(epoch) {
std::collections::hash_map::Entry::Vacant(entry) => {
entry.insert(record);
None
}
std::collections::hash_map::Entry::Occupied(mut entry) => {
let existing = entry.get_mut();
let explicitly_durable = record.durability
== GroupCommitFlushDurability::Durable
|| existing.durability == GroupCommitFlushDurability::Durable;
if explicitly_durable {
existing.durability = GroupCommitFlushDurability::Durable;
} else if existing.durability == GroupCommitFlushDurability::InDoubt
|| record.durability == GroupCommitFlushDurability::InDoubt
{
existing.durability = GroupCommitFlushDurability::InDoubt;
}
for completion in &record.durable_io_completions {
if !existing
.durable_io_completions
.iter()
.any(|known| Arc::ptr_eq(known, completion))
{
existing.durable_io_completions.push(Arc::clone(completion));
}
}
Some(record.root_attempt)
}
}
};
if let Some(root_attempt) = redundant_root {
// A concurrent deferrer installed another owner while this record
// was claimed. Preserve that live record and explicitly retire
// only the now-redundant process-root token.
root_attempt.release_after_terminal();
}
}
fn claim_pending_epoch_resolution(
self: &Arc<Self>,
epoch: u64,
) -> Option<PendingEpochResolutionClaim> {
let mut pending = self
.in_doubt_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
pending.get(&epoch)?;
if atomic_usize_checked_update(
&self.epoch_resolution_claims_in_flight,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|count| count.checked_add(1),
)
.is_err()
{
tracing::error!(
epoch,
"group-commit epoch-resolution claim count overflowed; left queued fail closed"
);
return None;
}
let record = pending
.remove(&epoch)
.expect("epoch record must remain present while its map lock is held");
drop(pending);
Some(PendingEpochResolutionClaim {
queue: Arc::clone(self),
epoch,
record: Some(record),
in_flight: true,
})
}
fn release_epoch_resolution_claim(&self) {
if atomic_usize_checked_update(
&self.epoch_resolution_claims_in_flight,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|count| count.checked_sub(1),
)
.is_err()
{
tracing::error!("group-commit epoch-resolution claim count underflow");
}
}
fn resolve_pending_epoch_resolutions(self: &Arc<Self>) -> Result<()> {
let epochs = self
.in_doubt_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.keys()
.copied()
.collect::<Vec<_>>();
for epoch in epochs {
let Some(mut claim) = self.claim_pending_epoch_resolution(epoch) else {
continue;
};
let record = claim
.record
.as_ref()
.expect("epoch-resolution claim must retain its record");
let target = match record.durability {
GroupCommitFlushDurability::PreDurable => GroupCommitFlushDurability::PreDurable,
GroupCommitFlushDurability::Durable => GroupCommitFlushDurability::Durable,
GroupCommitFlushDurability::InDoubt
if !record.durable_io_completions.is_empty()
&& record
.durable_io_completions
.iter()
.all(|signal| signal.load(AtomicOrdering::Acquire)) =>
{
GroupCommitFlushDurability::Durable
}
GroupCommitFlushDurability::InDoubt => {
drop(claim);
continue;
}
};
let terminal_result = match target {
GroupCommitFlushDurability::PreDurable => self.abort_cancelled_flush(epoch),
GroupCommitFlushDurability::Durable => self.complete_cancelled_durable_flush(epoch),
GroupCommitFlushDurability::InDoubt => unreachable!(),
};
match terminal_result {
Ok(()) => claim.finish_terminal(),
Err(error) => return Err(error),
}
}
Ok(())
}
fn has_unresolved_in_doubt_epoch(&self) -> bool {
let pending_epoch_resolutions = self
.in_doubt_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !pending_epoch_resolutions.is_empty() {
return true;
}
if self
.epoch_resolution_claims_in_flight
.load(AtomicOrdering::Acquire)
!= 0
{
return true;
}
drop(pending_epoch_resolutions);
self.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.iter()
.any(|pending| pending.durability_state() == GroupCommitFlushDurability::InDoubt)
}
fn abort_cancelled_filling(&self, target_epoch: u64) {
let failed_epoch = {
let mut consolidator = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match consolidator.abort_filling(target_epoch) {
Ok(failed_epoch) => {
self.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
failed_epoch,
GroupCommitEpochFailure::from_error(&FrankenError::Abort),
);
failed_epoch
}
Err(error) => {
let already_failed = self
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&target_epoch);
drop(consolidator);
if already_failed {
if let Err(logical_error) =
self.complete_txn_attempts_not_committed(target_epoch)
{
tracing::error!(
target_epoch,
%logical_error,
"cancelled filling epoch retained unfinished logical owners"
);
}
return;
}
tracing::debug!(
target_epoch,
%error,
"cancelled group-commit filling obligation was already resolved"
);
return;
}
}
};
if let Err(error) = self.complete_txn_attempts_not_committed(failed_epoch) {
tracing::error!(
failed_epoch,
%error,
"cancelled filling epoch retained unfinished logical owners"
);
}
self.signal_failed_epoch_waiters(failed_epoch, false);
self.reclaim_epoch_metadata_if_unowned(failed_epoch);
}
/// Check if a given epoch has completed (for waiters).
fn is_epoch_complete(&self, epoch: u64) -> bool {
self.completed_epoch.load(AtomicOrdering::Acquire) >= epoch
}
fn signal_completed_epoch_waiters(
&self,
epoch: u64,
wake_next_epoch: bool,
deliver_notification: bool,
) {
let signal_epoch = |target_epoch| {
if deliver_notification {
self.epoch_waiters.signal(target_epoch)
} else {
#[cfg(any(test, feature = "fault-injection"))]
{
self.epoch_waiters
.advance_generation_without_notify(target_epoch)
}
#[cfg(not(any(test, feature = "fault-injection")))]
{
false
}
}
};
let target_epoch_slot_present = signal_epoch(epoch);
let next_epoch_slot_present =
wake_next_epoch && epoch.checked_add(1).is_some_and(signal_epoch);
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
published_epoch = epoch,
wake_next_epoch,
delivery_suppressed = !deliver_notification,
target_epoch_slot_present,
next_epoch_slot_present,
"published completed epoch to targeted waiters"
);
if deliver_notification && GROUP_COMMIT_WAIT_PATH_MODE == WaitPathMode::LegacyCondvarTimeout
{
self.flush_complete.notify_all();
}
}
fn signal_failed_epoch_waiters(&self, epoch: u64, wake_next_epoch: bool) {
let woke_failed_epoch = self.epoch_waiters.signal(epoch);
let woke_next_epoch = wake_next_epoch
&& epoch
.checked_add(1)
.is_some_and(|next_epoch| self.epoch_waiters.signal(next_epoch));
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
failed_epoch = epoch,
wake_next_epoch,
woke_failed_epoch,
woke_next_epoch,
"published failed epoch to targeted waiters"
);
if GROUP_COMMIT_WAIT_PATH_MODE == WaitPathMode::LegacyCondvarTimeout {
self.flush_complete.notify_all();
}
}
fn record_epoch_wake(reason: EpochWakeReason) {
let counter = match reason {
EpochWakeReason::Notify => &GLOBAL_CONSOLIDATION_METRICS.wake_reasons.notify,
EpochWakeReason::Timeout => &GLOBAL_CONSOLIDATION_METRICS.wake_reasons.timeout,
EpochWakeReason::FlusherTakeover => {
&GLOBAL_CONSOLIDATION_METRICS.wake_reasons.flusher_takeover
}
EpochWakeReason::FailedEpoch => &GLOBAL_CONSOLIDATION_METRICS.wake_reasons.failed_epoch,
EpochWakeReason::BusyRetry => &GLOBAL_CONSOLIDATION_METRICS.wake_reasons.busy_retry,
};
counter.fetch_add(1, AtomicOrdering::Relaxed);
}
fn record_nonterminal_epoch_wake(target_epoch: u64, wait_result: KeyedWaitResult) {
let wake_reason = nonterminal_epoch_wake_reason(wait_result);
Self::record_epoch_wake(wake_reason);
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
wake_reason = wake_reason.as_str(),
target_epoch,
fallback = "nonterminal_recheck",
generation_advanced_after_timeout =
wait_result == KeyedWaitResult::RecoveredAfterTimeout,
"epoch waiter woke without observing a terminal outcome"
);
}
fn observe_failed_epoch(&self, target_epoch: u64) -> Option<FrankenError> {
let failed_detail = self
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&target_epoch)
.cloned()?;
Self::record_epoch_wake(EpochWakeReason::FailedEpoch);
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
wake_reason = EpochWakeReason::FailedEpoch.as_str(),
target_epoch,
"waiter observed failed epoch"
);
Some(failed_detail.into_error(target_epoch))
}
fn observe_epoch_outcome(
&self,
guard: &mut std::sync::MutexGuard<'_, GroupCommitConsolidator>,
target_epoch: u64,
pending_wake: Option<KeyedWaitResult>,
) -> Result<Option<WaitForEpochOutcome>> {
if let Some(error) = self.observe_failed_epoch(target_epoch) {
return Err(error);
}
// Failure observation is read-only and retains first precedence.
// Successful completion and promoted-flusher takeover, however, must
// not cross identity-wide physical or in-doubt finalization admitted
// while this waiter was parked.
if !self.identity_wide_finalization_is_quiescent() {
if let Some(wait_result) = pending_wake {
Self::record_nonterminal_epoch_wake(target_epoch, wait_result);
}
return Ok(None);
}
if self.is_epoch_complete(target_epoch) {
let wake_reason = completed_epoch_wake_reason(pending_wake);
Self::record_epoch_wake(wake_reason);
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
wake_reason = wake_reason.as_str(),
generation_advanced_after_timeout =
pending_wake == Some(KeyedWaitResult::RecoveredAfterTimeout),
target_epoch,
completed_epoch = self.completed_epoch.load(AtomicOrdering::Acquire),
"waiter observed completed epoch"
);
return Ok(Some(WaitForEpochOutcome::Completed));
}
if guard.has_flusher_vacancy()
&& guard.epoch().checked_add(1) == Some(target_epoch)
&& guard.claim_flusher_vacancy()
{
Self::record_epoch_wake(EpochWakeReason::FlusherTakeover);
tracing::trace!(
target: "fsqlite::wal::epoch_wait",
wait_strategy = GROUP_COMMIT_WAIT_PATH_MODE.as_str(),
wake_reason = EpochWakeReason::FlusherTakeover.as_str(),
target_epoch,
current_epoch = guard.epoch(),
"waiter claimed promoted flusher vacancy"
);
let batches = guard.begin_flush()?;
return Ok(Some(WaitForEpochOutcome::TakeOverFlusher {
flush_epoch: guard.epoch(),
batches,
}));
}
if let Some(wait_result) = pending_wake {
Self::record_nonterminal_epoch_wake(target_epoch, wait_result);
}
Ok(None)
}
#[cfg(test)]
fn wait_for_epoch_outcome_legacy(
&self,
mut guard: std::sync::MutexGuard<'_, GroupCommitConsolidator>,
target_epoch: u64,
) -> Result<WaitForEpochOutcome> {
let mut pending_wake = None;
loop {
if let Some(outcome) =
self.observe_epoch_outcome(&mut guard, target_epoch, pending_wake.take())?
{
return Ok(outcome);
}
let (new_guard, timeout_result) = self
.flush_complete
.wait_timeout(guard, GROUP_COMMIT_WAIT_TIMEOUT_FALLBACK)
.unwrap_or_else(std::sync::PoisonError::into_inner);
guard = new_guard;
pending_wake = Some(if timeout_result.timed_out() {
KeyedWaitResult::TimedOut
} else {
KeyedWaitResult::Signaled
});
}
}
#[cfg(test)]
fn wait_for_epoch_outcome_keyed<'a>(
&'a self,
mut guard: std::sync::MutexGuard<'a, GroupCommitConsolidator>,
target_epoch: u64,
) -> Result<WaitForEpochOutcome> {
let mut pending_wake = None;
loop {
if let Some(outcome) =
self.observe_epoch_outcome(&mut guard, target_epoch, pending_wake.take())?
{
return Ok(outcome);
}
let slot = self.epoch_waiters.slot(target_epoch);
let observed_generation = slot.generation();
if let Some(outcome) =
self.observe_epoch_outcome(&mut guard, target_epoch, pending_wake.take())?
{
return Ok(outcome);
}
drop(guard);
let wait_result =
slot.wait_for_change(observed_generation, GROUP_COMMIT_WAIT_TIMEOUT_FALLBACK);
guard = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
pending_wake = Some(wait_result);
}
}
/// Wait for the target epoch to either complete successfully, fail, or be
/// taken over by this waiter if the promoted epoch lost its original flusher.
#[cfg(test)]
fn wait_for_epoch_outcome(
&self,
guard: std::sync::MutexGuard<'_, GroupCommitConsolidator>,
target_epoch: u64,
) -> Result<WaitForEpochOutcome> {
match GROUP_COMMIT_WAIT_PATH_MODE {
WaitPathMode::KeyedEventcount => self.wait_for_epoch_outcome_keyed(guard, target_epoch),
WaitPathMode::LegacyCondvarTimeout => {
self.wait_for_epoch_outcome_legacy(guard, target_epoch)
}
}
}
/// Cancellation-safe production wait path. The generation is sampled
/// before checking the epoch state, so a publication between the check
/// and registration is observed by `wait_for_change_async` without a lost
/// wake. No executor thread blocks on a condition variable.
async fn wait_for_epoch_outcome_async(
self: &Arc<Self>,
cx: &Cx,
target_epoch: u64,
) -> Result<WaitForEpochOutcome> {
loop {
// A pending exact-handle logical exit can itself depend on this
// epoch reaching a terminal verdict. Requiring every exact lane
// to settle before observing the epoch creates a progress cycle
// when the promoted flusher was dropped. Identity-wide physical
// and in-doubt work remains a queue-wide prerequisite; exact
// physical ownership is enforced later by the flusher's handle
// coordination window.
if self.has_process_root_finalization_attempt() {
if let Some(error) = self.observe_failed_epoch(target_epoch) {
return Err(error);
}
let settlement_result = settle_identity_wide_group_commit_finalization(self).await;
if let Err(settlement_error) = settlement_result {
if let Some(error) = self.observe_failed_epoch(target_epoch) {
return Err(error);
}
return Err(settlement_error);
}
}
let slot = self.epoch_waiters.slot(target_epoch);
let observed_generation = slot.generation();
let outcome = {
let mut guard = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
self.observe_epoch_outcome(&mut guard, target_epoch, None)?
};
if let Some(outcome) = outcome {
return Ok(outcome);
}
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
let wait_result = slot.wait_for_change_async(observed_generation).await;
let mut pending_wake = PendingEpochWake::new(self, target_epoch, wait_result);
if self.has_process_root_finalization_attempt() {
let settlement_result = settle_identity_wide_group_commit_finalization(self).await;
if let Err(settlement_error) = settlement_result {
// Exact target failure is read-only terminal evidence and
// keeps precedence over an unrelated BusyRecovery discovered
// while the queue-wide prerequisite was being settled.
if let Some(error) = pending_wake.observe_failure() {
return Err(error);
}
return Err(settlement_error);
}
}
let outcome = {
let mut guard = self
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
pending_wake.observe(&mut guard)?
};
if let Some(outcome) = outcome {
return Ok(outcome);
}
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
}
}
}
impl PendingEpochResolutionClaim {
fn finish_terminal(&mut self) {
let record = self
.record
.take()
.expect("terminal epoch-resolution claim must own its record");
record.root_attempt.release_after_terminal();
if self.in_flight {
self.queue.release_epoch_resolution_claim();
self.in_flight = false;
}
}
}
impl Drop for PendingEpochResolutionClaim {
fn drop(&mut self) {
if let Some(record) = self.record.take() {
self.queue
.requeue_pending_epoch_resolution(self.epoch, record);
}
// Requeue before clearing the claim publication so a concurrent
// settler cannot observe both an empty map and a zero claim count.
if self.in_flight {
self.queue.release_epoch_resolution_claim();
self.in_flight = false;
}
}
}
struct GroupCommitFillingObligation {
queue: Arc<GroupCommitQueue>,
target_epoch: u64,
armed: bool,
}
impl GroupCommitFillingObligation {
fn new(queue: &Arc<GroupCommitQueue>, target_epoch: u64) -> Self {
Self {
queue: Arc::clone(queue),
target_epoch,
armed: true,
}
}
fn disarm(&mut self) {
self.armed = false;
}
}
impl Drop for GroupCommitFillingObligation {
fn drop(&mut self) {
if self.armed {
self.queue.abort_cancelled_filling(self.target_epoch);
}
}
}
struct GroupCommitFlushObligation {
queue: Arc<GroupCommitQueue>,
epoch: u64,
phase: GroupCommitFlushObligationPhase,
durability_started: Arc<AtomicBool>,
durable_io_completed: Arc<AtomicBool>,
external_lock_restored: Arc<AtomicBool>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum GroupCommitFlushObligationPhase {
PreDurable,
Durable,
Completed,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum GroupCommitFlushDurability {
/// The callback had not reached its first side-effecting await.
PreDurable,
/// Durable mutation started, but the lower I/O layer is not terminal.
InDoubt,
/// WAL append and the selected sync boundary are terminally durable.
Durable,
}
impl GroupCommitFlushObligation {
fn new(queue: &Arc<GroupCommitQueue>, epoch: u64) -> Self {
Self {
queue: Arc::clone(queue),
epoch,
phase: GroupCommitFlushObligationPhase::PreDurable,
durability_started: Arc::new(AtomicBool::new(false)),
durable_io_completed: Arc::new(AtomicBool::new(false)),
external_lock_restored: Arc::new(AtomicBool::new(true)),
}
}
fn durability_started_signal(&self) -> Arc<AtomicBool> {
Arc::clone(&self.durability_started)
}
fn durable_io_signal(&self) -> Arc<AtomicBool> {
Arc::clone(&self.durable_io_completed)
}
fn external_lock_state(&self) -> Arc<AtomicBool> {
Arc::clone(&self.external_lock_restored)
}
#[cfg(test)]
fn mark_durable(&mut self) {
debug_assert_eq!(
self.phase,
GroupCommitFlushObligationPhase::PreDurable,
"group-commit flush may become durable only once"
);
self.durable_io_completed
.store(true, AtomicOrdering::Release);
self.phase = GroupCommitFlushObligationPhase::Durable;
}
fn is_durable(&self) -> bool {
self.phase == GroupCommitFlushObligationPhase::Durable
|| self.durable_io_completed.load(AtomicOrdering::Acquire)
}
fn durability_state(&self) -> GroupCommitFlushDurability {
if self.durable_io_completed.load(AtomicOrdering::Acquire) {
GroupCommitFlushDurability::Durable
} else if self.durability_started.load(AtomicOrdering::Acquire) {
GroupCommitFlushDurability::InDoubt
} else {
GroupCommitFlushDurability::PreDurable
}
}
fn disarm(&mut self) {
self.phase = GroupCommitFlushObligationPhase::Completed;
}
}
impl Drop for GroupCommitFlushObligation {
fn drop(&mut self) {
if self.phase == GroupCommitFlushObligationPhase::Completed {
return;
}
if !self.external_lock_restored.load(AtomicOrdering::Acquire) {
// GroupCommitDbLockObligation queued the type-erased restoration
// before this outer obligation was dropped. Its eventual claimant
// owns both lock cleanup and epoch resolution.
tracing::warn!(
epoch = self.epoch,
durability_state = ?self.durability_state(),
"dropped group-commit future deferred epoch resolution until external lock restoration"
);
return;
}
let durability = self.durability_state();
let transition_result = match durability {
GroupCommitFlushDurability::PreDurable => self.queue.abort_cancelled_flush(self.epoch),
GroupCommitFlushDurability::InDoubt => {
self.queue.defer_pending_epoch_resolution(
self.epoch,
GroupCommitFlushDurability::InDoubt,
Arc::clone(&self.durable_io_completed),
None,
);
Ok(())
}
GroupCommitFlushDurability::Durable => {
self.queue.complete_cancelled_durable_flush(self.epoch)
}
};
if let Err(error) = transition_result {
tracing::error!(
%error,
epoch = self.epoch,
?durability,
"dropped group-commit epoch transition failed; retained for process-root retry"
);
self.queue.defer_pending_epoch_resolution(
self.epoch,
durability,
Arc::clone(&self.durable_io_completed),
None,
);
}
}
}
trait PendingGroupCommitRecoveryOperation: Send + Sync {
fn reconcile(&self) -> LocalPagerFuture<'_, GroupCommitFlushDurability>;
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum PendingGroupCommitRecoveryResolution {
AuthorizedPendingLogical(ParallelWalDurabilityReceipt),
Authorized(ParallelWalDurabilityReceipt),
NotCommittedPendingLogical,
NotCommitted,
}
struct PendingGroupCommitRecovery<F: VfsFile + 'static> {
queue: Weak<GroupCommitQueue>,
epoch: u64,
_epoch_consumer: Arc<GroupCommitEpochConsumer>,
publication: Arc<PendingGroupCommitPublication>,
/// Exact backend instance that accepted the certified frame interval.
/// The pager's outer `SharedWalBackend` slot is replaceable and therefore
/// cannot identify recovery work after the initiating future is dropped.
wal_backend: WalBackendHandle,
inner: Arc<Mutex<PagerInner<F>>>,
published: Option<Arc<PublishedPagerState>>,
batches: Vec<TransactionFrameBatch>,
final_db_size: u32,
sync: bool,
sidecar_completion: Arc<Mutex<Option<VfsWriteCompletion>>>,
wal_completion: Arc<Mutex<Option<VfsWriteCompletion>>>,
cleanup_cx: Cx,
durability_started: Arc<AtomicBool>,
durable_io_completed: Arc<AtomicBool>,
resolution: Mutex<Option<PendingGroupCommitRecoveryResolution>>,
}
impl<F: VfsFile + 'static> PendingGroupCommitRecovery<F> {
fn writes_are_terminal(&self) -> bool {
let sidecar_terminal = self
.sidecar_completion
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.is_none_or(|completion| completion.state() != VfsWriteCompletionState::Pending);
let wal_terminal = self
.wal_completion
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.is_none_or(|completion| completion.state() != VfsWriteCompletionState::Pending);
sidecar_terminal && wal_terminal
}
fn complete_authorized(&self, cx: &Cx) -> Result<ParallelWalDurabilityReceipt> {
let recorded_resolution = self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
match recorded_resolution {
Some(PendingGroupCommitRecoveryResolution::Authorized(receipt)) => {
return Ok(receipt);
}
Some(
PendingGroupCommitRecoveryResolution::NotCommitted
| PendingGroupCommitRecoveryResolution::NotCommittedPendingLogical,
) => {
return Err(FrankenError::internal(
"cannot publish a parallel WAL interval already proven not committed",
));
}
Some(PendingGroupCommitRecoveryResolution::AuthorizedPendingLogical(_)) | None => {}
}
// Validate and materialize the full page plane before consuming the
// combiner's exact pending handle. Once `finalize` succeeds, every
// later retry can reconstruct the same map from the retained batches.
let complete_group_pages =
PublishedPagerState::prepare_parallel_wal_group_pages(&self.batches)?;
let queue = self.queue.upgrade().ok_or_else(|| {
FrankenError::internal(
"group-commit queue dropped before pending durability recovery finalized",
)
})?;
let receipt = match recorded_resolution {
Some(PendingGroupCommitRecoveryResolution::AuthorizedPendingLogical(receipt)) => {
receipt
}
None => {
let receipt = self.publication.finalize(&queue)?;
*self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(
PendingGroupCommitRecoveryResolution::AuthorizedPendingLogical(receipt.clone()),
);
receipt
}
Some(
PendingGroupCommitRecoveryResolution::Authorized(_)
| PendingGroupCommitRecoveryResolution::NotCommitted
| PendingGroupCommitRecoveryResolution::NotCommittedPendingLogical,
) => unreachable!("terminal recovery states returned above"),
};
queue.complete_txn_attempts_authorized(
self.epoch,
&self.batches,
&receipt,
&complete_group_pages,
)?;
let publish_update = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let mut inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
// Concurrent transactions may reserve later EOF pages while the
// physical writer is in flight. Never erase those reservations.
inner.db_size = inner.db_size.max(self.final_db_size);
let next_unallocated_page = if inner.db_size >= 2 {
inner.db_size.saturating_add(1)
} else {
2
};
inner.next_page = inner.next_page.max(next_unallocated_page);
inner.record_local_wal_commit_at(receipt.certificate.commit_seq_hi);
PublishedPagerUpdate {
visible_commit_seq: receipt.certificate.commit_seq_hi,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
}
};
if let Some(published) = self.published.as_ref() {
published.publish_prepared_parallel_wal_group(
cx,
publish_update,
complete_group_pages,
receipt.certificate.commit_seq_lo,
);
}
*self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(
PendingGroupCommitRecoveryResolution::Authorized(receipt.clone()),
);
self.durable_io_completed
.store(true, AtomicOrdering::Release);
Ok(receipt)
}
}
impl<F: VfsFile + 'static> PendingGroupCommitRecoveryOperation for PendingGroupCommitRecovery<F> {
fn reconcile(&self) -> LocalPagerFuture<'_, GroupCommitFlushDurability> {
Box::pin(async move {
let recorded_resolution = self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone();
match recorded_resolution.as_ref() {
Some(PendingGroupCommitRecoveryResolution::Authorized(_)) => {
return Ok(GroupCommitFlushDurability::Durable);
}
Some(PendingGroupCommitRecoveryResolution::NotCommitted) => {
return Ok(GroupCommitFlushDurability::PreDurable);
}
Some(PendingGroupCommitRecoveryResolution::AuthorizedPendingLogical(_)) => {
self.complete_authorized(&self.cleanup_cx)?;
return Ok(GroupCommitFlushDurability::Durable);
}
Some(PendingGroupCommitRecoveryResolution::NotCommittedPendingLogical) => {
let queue = self.queue.upgrade().ok_or_else(|| {
FrankenError::internal(
"group-commit queue dropped before logical rejection finalized",
)
})?;
queue.complete_txn_attempts_not_committed(self.epoch)?;
*self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(PendingGroupCommitRecoveryResolution::NotCommitted);
return Ok(GroupCommitFlushDurability::PreDurable);
}
None => {}
}
if !self.writes_are_terminal() {
return Ok(GroupCommitFlushDurability::InDoubt);
}
// Recovery owns a masked child context for its entire external-lock
// and storage-reconciliation window. Parent cancellation may wake
// the claimant, but it must not prevent the claimant from reaching
// a terminal durability verdict and releasing RESERVED.
let _recovery_mask = self.cleanup_cx.masked();
let certificate = self.publication.certificate()?;
let (frames_start, frames_end) = self.publication.interval()?;
let mut wal =
async_rwlock_write(&self.wal_backend, &self.cleanup_cx, "WAL recovery backend")
.await?;
let verdict = wal
.reconcile_parallel_wal_commit(
&self.cleanup_cx,
&certificate,
frames_start,
frames_end,
self.sync,
)
.await?;
drop(wal);
match verdict {
traits::ParallelWalCommitReconciliation::Authorized => {
self.complete_authorized(&self.cleanup_cx)?;
Ok(GroupCommitFlushDurability::Durable)
}
traits::ParallelWalCommitReconciliation::NotCommitted => {
self.publication.abort()?;
*self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(PendingGroupCommitRecoveryResolution::NotCommittedPendingLogical);
let queue = self.queue.upgrade().ok_or_else(|| {
FrankenError::internal(
"group-commit queue dropped before logical rejection finalized",
)
})?;
queue.complete_txn_attempts_not_committed(self.epoch)?;
*self
.resolution
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(PendingGroupCommitRecoveryResolution::NotCommitted);
self.durability_started
.store(false, AtomicOrdering::Release);
self.durable_io_completed
.store(false, AtomicOrdering::Release);
Ok(GroupCommitFlushDurability::PreDurable)
}
}
})
}
}
trait PendingExternalUnlockOperation: Send {
fn restore(&mut self) -> LocalPagerFuture<'_, ()>;
/// Try the synchronous portion of restoration without waiting for the
/// shared file handle. `Ok(false)` means the handle is still contended.
fn try_restore(&mut self) -> Result<bool>;
}
struct GroupCommitPhysicalLockWindow {
queue: Arc<GroupCommitQueue>,
handle_key: SharedDbFileKey,
active: bool,
}
impl GroupCommitPhysicalLockWindow {
#[cfg(test)]
fn register(queue: &Arc<GroupCommitQueue>, handle_key: SharedDbFileKey) -> Result<Self> {
Self::try_register(queue, handle_key).ok_or(FrankenError::BusyRecovery)
}
fn try_register(queue: &Arc<GroupCommitQueue>, handle_key: SharedDbFileKey) -> Option<Self> {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ExactHandleCoordination);
let mut coordination = queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if coordination.logical_exit_in_flight.contains(&handle_key)
|| coordination.physical_lock_windows.contains(&handle_key)
{
return None;
}
coordination.physical_lock_windows.insert(handle_key);
drop(coordination);
Some(Self {
queue: Arc::clone(queue),
handle_key,
active: true,
})
}
async fn acquire(
queue: &Arc<GroupCommitQueue>,
handle_key: SharedDbFileKey,
cx: &Cx,
) -> Result<Self> {
loop {
let observed_generation = queue.external_lock_waiters.generation();
if let Some(window) = Self::try_register(queue, handle_key) {
return Ok(window);
}
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
let _ = queue
.external_lock_waiters
.wait_for_change_async(observed_generation)
.await;
}
}
fn release(&mut self) {
if !self.active {
return;
}
let mut coordination = self
.queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !coordination.physical_lock_windows.remove(&self.handle_key) {
tracing::error!(
handle_key = self.handle_key.0,
"group-commit physical lock window released without ownership"
);
}
drop(coordination);
self.active = false;
self.queue.external_lock_waiters.signal();
}
}
impl Drop for GroupCommitPhysicalLockWindow {
fn drop(&mut self) {
self.release();
}
}
struct GroupCommitLogicalExitClaim {
queue: Arc<GroupCommitQueue>,
handle_key: SharedDbFileKey,
active: bool,
}
impl GroupCommitLogicalExitClaim {
fn try_register(queue: &Arc<GroupCommitQueue>, handle_key: SharedDbFileKey) -> Option<Self> {
let mut coordination = queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if coordination.physical_lock_windows.contains(&handle_key)
|| coordination.logical_exit_in_flight.contains(&handle_key)
{
return None;
}
coordination.logical_exit_in_flight.insert(handle_key);
drop(coordination);
Some(Self {
queue: Arc::clone(queue),
handle_key,
active: true,
})
}
async fn acquire(
queue: &Arc<GroupCommitQueue>,
handle_key: SharedDbFileKey,
cx: &Cx,
) -> Result<Self> {
loop {
let observed_generation = queue.external_lock_waiters.generation();
if let Some(claim) = Self::try_register(queue, handle_key) {
return Ok(claim);
}
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
let _ = queue
.external_lock_waiters
.wait_for_change_async(observed_generation)
.await;
}
}
fn release(&mut self) {
if !self.active {
return;
}
let mut coordination = self
.queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !coordination.logical_exit_in_flight.remove(&self.handle_key) {
tracing::error!(
handle_key = self.handle_key.0,
"group-commit logical exit claim released without ownership"
);
}
drop(coordination);
self.active = false;
self.queue.external_lock_waiters.signal();
}
}
impl Drop for GroupCommitLogicalExitClaim {
fn drop(&mut self) {
self.release();
}
}
enum GroupCommitExternalLockOwner {
Physical(GroupCommitPhysicalLockWindow),
Exclusive(GroupCommitLogicalExitClaim),
}
impl GroupCommitExternalLockOwner {
fn release(self) {
match self {
Self::Physical(window) => drop(window),
Self::Exclusive(claim) => drop(claim),
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PendingExternalUnlockTarget {
/// Restore the ordinary SQLite lock state while other transactions remain.
LockLevel(LockLevel),
/// Release the last transaction's external snapshot fence. This is
/// distinct from `LockLevel::None` on VFSes that track the fence's prior
/// lock level separately.
ExternalSnapshot,
/// Release the cross-process maintenance epoch, including WAL writer and
/// checkpoint slots when recovery entered from WAL mode.
ExternalMaintenance,
}
impl PendingExternalUnlockTarget {
fn restore<F: VfsFile>(self, file: &mut F, cx: &Cx) -> Result<()> {
match self {
Self::LockLevel(level) => file.unlock(cx, level),
Self::ExternalSnapshot => file.restore_external_shared_snapshot_attempt(cx),
Self::ExternalMaintenance => file.restore_external_maintenance_attempt(cx),
}
}
}
struct BeginExternalLockState<F: VfsFile + 'static> {
queue: Arc<GroupCommitQueue>,
db_file: SharedDbFile<F>,
cleanup_cx: Cx,
restore_target: Option<PendingExternalUnlockTarget>,
restore_scope: Option<ProcessRootFinalizationScope>,
}
impl<F: VfsFile + 'static> BeginExternalLockState<F> {
fn new(queue: &Arc<GroupCommitQueue>, db_file: SharedDbFile<F>, cx: &Cx) -> Self {
Self {
queue: Arc::clone(queue),
db_file,
cleanup_cx: cleanup_child_cx(cx),
restore_target: None,
restore_scope: None,
}
}
async fn acquire_snapshot(&mut self, cx: &Cx) -> Result<()> {
debug_assert!(
self.restore_target.is_none(),
"snapshot acquisition requires no previously armed external lock"
);
self.restore_target = Some(PendingExternalUnlockTarget::ExternalSnapshot);
self.restore_scope = Some(ProcessRootFinalizationScope::IdentityWide);
let result = shared_db_lock_external_snapshot(&self.db_file, cx).await;
if result.is_ok() {
self.restore_scope = Some(ProcessRootFinalizationScope::ExactHandle(
shared_db_file_key(&self.db_file),
));
}
result
}
async fn acquire_maintenance(&mut self, cx: &Cx, wal_mode: bool) -> Result<()> {
debug_assert!(
self.restore_target.is_none(),
"maintenance acquisition requires no previously armed external lock"
);
self.restore_target = Some(PendingExternalUnlockTarget::ExternalMaintenance);
self.restore_scope = Some(ProcessRootFinalizationScope::IdentityWide);
shared_db_lock_external_maintenance(&self.db_file, cx, wal_mode).await
}
async fn restore(&mut self) -> Result<()> {
let Some(restore_target) = self.restore_target else {
return Ok(());
};
let cleanup_cx = self.cleanup_cx.clone();
let _cleanup_mask = cleanup_cx.masked();
let mut file = shared_db_file_write(&self.db_file, &cleanup_cx).await?;
restore_target.restore(&mut *file, &cleanup_cx)?;
self.restore_target = None;
self.restore_scope = None;
Ok(())
}
fn arm_lock_level(&mut self, restore_level: LockLevel) {
debug_assert!(
self.restore_target.is_none(),
"lock-level restoration requires no previously armed external lock"
);
self.restore_target = Some(PendingExternalUnlockTarget::LockLevel(restore_level));
self.restore_scope = Some(ProcessRootFinalizationScope::IdentityWide);
}
fn mark_lock_level_acquired(&mut self) {
let Some(PendingExternalUnlockTarget::LockLevel(restore_level)) = self.restore_target
else {
debug_assert!(
false,
"lock acquisition marker requires a lock-level target"
);
return;
};
self.restore_scope = Some(if restore_level <= LockLevel::Reserved {
ProcessRootFinalizationScope::ExactHandle(shared_db_file_key(&self.db_file))
} else {
ProcessRootFinalizationScope::IdentityWide
});
}
fn disarm(&mut self) {
self.restore_target = None;
self.restore_scope = None;
}
fn is_armed(&self) -> bool {
self.restore_target.is_some()
}
}
impl<F: VfsFile + 'static> Drop for BeginExternalLockState<F> {
fn drop(&mut self) {
let Some(restore_target) = self.restore_target.take() else {
return;
};
let restore_scope = self.restore_scope.take().unwrap_or_else(|| {
tracing::error!(
"armed external-lock attempt lost its finalization scope; retaining an identity-wide root"
);
ProcessRootFinalizationScope::IdentityWide
});
let restored = Arc::new(AtomicBool::new(false));
let operation = SharedDbPendingExternalUnlock {
db_file: Arc::clone(&self.db_file),
cleanup_cx: self.cleanup_cx.clone(),
restore_target,
restored: Arc::clone(&restored),
};
let mut pending = PendingExternalUnlock {
sequence: None,
scope: restore_scope,
epoch: None,
durability_started: Arc::new(AtomicBool::new(false)),
durable_io_completed: Arc::new(AtomicBool::new(false)),
coordination_owner: None,
recovery: None,
root_attempt: None,
operation: Box::new(operation),
};
match pending.operation.try_restore() {
Ok(true) => {}
Ok(false) => self.queue.enqueue_pending_external_unlock(pending),
Err(error) => {
tracing::error!(
%error,
"drop-time external-lock attempt restoration failed; queued for structured retry"
);
self.queue.enqueue_pending_external_unlock(pending);
}
}
}
}
struct SharedDbPendingExternalUnlock<F: VfsFile> {
db_file: SharedDbFile<F>,
cleanup_cx: Cx,
restore_target: PendingExternalUnlockTarget,
restored: Arc<AtomicBool>,
}
impl<F: VfsFile> SharedDbPendingExternalUnlock<F> {
fn mark_restored(&self) {
self.restored.store(true, AtomicOrdering::Release);
}
}
impl<F: VfsFile + 'static> PendingExternalUnlockOperation for SharedDbPendingExternalUnlock<F> {
fn restore(&mut self) -> LocalPagerFuture<'_, ()> {
Box::pin(async move {
if self.restored.load(AtomicOrdering::Acquire) {
return Ok(());
}
let restore_result = {
let _cleanup_mask = self.cleanup_cx.masked();
let mut file = shared_db_file_write(&self.db_file, &self.cleanup_cx).await?;
let result = self.restore_target.restore(&mut *file, &self.cleanup_cx);
if result.is_ok() {
self.mark_restored();
}
result
};
restore_result
})
}
fn try_restore(&mut self) -> Result<bool> {
if self.restored.load(AtomicOrdering::Acquire) {
return Ok(true);
}
let _cleanup_mask = self.cleanup_cx.masked();
let mut file = match self.db_file.try_write() {
Ok(file) => file,
Err(asupersync::sync::TryWriteError::Locked) => return Ok(false),
Err(asupersync::sync::TryWriteError::Poisoned) => {
return Err(FrankenError::internal(
"pending group-commit database-file lock is poisoned",
));
}
};
self.restore_target.restore(&mut *file, &self.cleanup_cx)?;
self.mark_restored();
Ok(true)
}
}
struct BeginAdmissionPendingExternalUnlock<F: VfsFile + 'static> {
inner: Arc<Mutex<PagerInner<F>>>,
db_file: SharedDbFile<F>,
writer_idle: Arc<Condvar>,
cleanup_cx: Cx,
restore_target: PendingExternalUnlockTarget,
writer_baton_owned: bool,
maintenance_lease: Option<PagerMaintenanceLease>,
external_restored: bool,
completed: bool,
}
impl<F: VfsFile + 'static> BeginAdmissionPendingExternalUnlock<F> {
fn finish_restored_state(&mut self, inner: &mut PagerInner<F>) -> bool {
let notify_writer_idle = self.writer_baton_owned && release_single_writer_baton(inner);
self.writer_baton_owned = false;
self.maintenance_lease.take();
self.completed = true;
notify_writer_idle
}
}
impl<F: VfsFile + 'static> PendingExternalUnlockOperation
for BeginAdmissionPendingExternalUnlock<F>
{
#[allow(clippy::await_holding_lock)]
fn restore(&mut self) -> LocalPagerFuture<'_, ()> {
Box::pin(async move {
if self.completed {
return Ok(());
}
let cleanup_cx = self.cleanup_cx.clone();
let _cleanup_mask = cleanup_cx.masked();
if !self.external_restored {
let db_file = Arc::clone(&self.db_file);
let mut file = shared_db_file_write(&db_file, &cleanup_cx).await?;
self.restore_target.restore(&mut *file, &cleanup_cx)?;
drop(file);
self.external_restored = true;
}
loop {
let inner_arc = Arc::clone(&self.inner);
match inner_arc.try_lock() {
Ok(mut inner) => {
let notify_writer_idle = self.finish_restored_state(&mut inner);
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
return Ok(());
}
Err(std::sync::TryLockError::WouldBlock) => {
asupersync::runtime::yield_now().await;
}
Err(std::sync::TryLockError::Poisoned(error)) => {
tracing::error!(
"cancelled begin recovered a poisoned pager guard for fail-closed cleanup"
);
let mut inner = error.into_inner();
let notify_writer_idle = self.finish_restored_state(&mut inner);
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
return Ok(());
}
}
}
})
}
fn try_restore(&mut self) -> Result<bool> {
if self.completed {
return Ok(true);
}
let cleanup_cx = self.cleanup_cx.clone();
let _cleanup_mask = cleanup_cx.masked();
if !self.external_restored {
let db_file = Arc::clone(&self.db_file);
let mut file = match db_file.try_write() {
Ok(file) => file,
Err(asupersync::sync::TryWriteError::Locked) => return Ok(false),
Err(asupersync::sync::TryWriteError::Poisoned) => {
return Err(FrankenError::internal(
"cancelled begin database-file lock is poisoned",
));
}
};
self.restore_target.restore(&mut *file, &cleanup_cx)?;
drop(file);
self.external_restored = true;
}
let inner_arc = Arc::clone(&self.inner);
let mut inner = match inner_arc.try_lock() {
Ok(inner) => inner,
Err(std::sync::TryLockError::WouldBlock) => return Ok(false),
Err(std::sync::TryLockError::Poisoned(error)) => {
tracing::error!(
"cancelled begin recovered a poisoned pager guard for fail-closed cleanup"
);
error.into_inner()
}
};
let notify_writer_idle = self.finish_restored_state(&mut inner);
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
Ok(true)
}
}
struct BeginAdmission<F: VfsFile + 'static> {
queue: Arc<GroupCommitQueue>,
inner: Arc<Mutex<PagerInner<F>>>,
external_lock: BeginExternalLockState<F>,
writer_idle: Arc<Condvar>,
maintenance_lease: Option<PagerMaintenanceLease>,
coordination_owner: Option<GroupCommitExternalLockOwner>,
writer_baton_owned: bool,
completed: bool,
}
impl<F: VfsFile + 'static> BeginAdmission<F> {
fn new(
queue: &Arc<GroupCommitQueue>,
inner: Arc<Mutex<PagerInner<F>>>,
db_file: SharedDbFile<F>,
writer_idle: Arc<Condvar>,
maintenance_lease: PagerMaintenanceLease,
logical_claim: Option<GroupCommitLogicalExitClaim>,
cx: &Cx,
) -> Self {
Self {
queue: Arc::clone(queue),
inner,
external_lock: BeginExternalLockState::new(queue, db_file, cx),
writer_idle,
maintenance_lease: Some(maintenance_lease),
coordination_owner: logical_claim.map(GroupCommitExternalLockOwner::Exclusive),
writer_baton_owned: false,
completed: false,
}
}
fn mark_writer_baton_owned(&mut self) {
self.writer_baton_owned = true;
}
fn complete(&mut self) -> Result<PagerMaintenanceLease> {
let maintenance_lease = self.maintenance_lease.take().ok_or_else(|| {
FrankenError::internal("completed begin admission lost its maintenance lease")
})?;
self.external_lock.disarm();
self.writer_baton_owned = false;
self.coordination_owner.take();
self.completed = true;
Ok(maintenance_lease)
}
}
impl<F: VfsFile + 'static> Drop for BeginAdmission<F> {
fn drop(&mut self) {
if self.completed {
return;
}
let Some(restore_target) = self.external_lock.restore_target.take() else {
self.coordination_owner.take();
self.maintenance_lease.take();
return;
};
let restore_scope = self.external_lock.restore_scope.take().unwrap_or_else(|| {
tracing::error!(
"armed begin admission lost its finalization scope; retaining an identity-wide root"
);
ProcessRootFinalizationScope::IdentityWide
});
let operation = BeginAdmissionPendingExternalUnlock {
inner: Arc::clone(&self.inner),
db_file: Arc::clone(&self.external_lock.db_file),
writer_idle: Arc::clone(&self.writer_idle),
cleanup_cx: self.external_lock.cleanup_cx.clone(),
restore_target,
writer_baton_owned: self.writer_baton_owned,
maintenance_lease: self.maintenance_lease.take(),
external_restored: false,
completed: false,
};
let mut pending = PendingExternalUnlock {
sequence: None,
scope: restore_scope,
epoch: None,
durability_started: Arc::new(AtomicBool::new(false)),
durable_io_completed: Arc::new(AtomicBool::new(false)),
coordination_owner: self.coordination_owner.take(),
recovery: None,
root_attempt: None,
operation: Box::new(operation),
};
match pending.operation.try_restore() {
Ok(true) => pending.release_after_terminal(),
Ok(false) => self.queue.enqueue_pending_external_unlock(pending),
Err(error) => {
tracing::error!(
%error,
"drop-time begin-admission lock restoration failed; queued for structured retry"
);
self.queue.enqueue_pending_external_unlock(pending);
}
}
}
}
struct PendingExternalUnlock {
/// Stable FIFO sequence retained across claim cancellation.
sequence: Option<u64>,
/// Admission scope of this physical restoration.
scope: ProcessRootFinalizationScope,
/// Physical group-commit epoch, when restoration also terminates a flush.
/// `None` denotes a pure exact-handle restoration such as a cancelled
/// admission; it must not mutate consolidator state.
epoch: Option<u64>,
durability_started: Arc<AtomicBool>,
durable_io_completed: Arc<AtomicBool>,
coordination_owner: Option<GroupCommitExternalLockOwner>,
recovery: Option<Arc<dyn PendingGroupCommitRecoveryOperation>>,
root_attempt: Option<ProcessRootFinalizationAttempt>,
operation: Box<dyn PendingExternalUnlockOperation>,
}
impl PendingExternalUnlock {
fn durability_state(&self) -> GroupCommitFlushDurability {
if self.durable_io_completed.load(AtomicOrdering::Acquire) {
GroupCommitFlushDurability::Durable
} else if self.durability_started.load(AtomicOrdering::Acquire) {
GroupCommitFlushDurability::InDoubt
} else {
GroupCommitFlushDurability::PreDurable
}
}
fn release_after_terminal(&mut self) {
if let Some(owner) = self.coordination_owner.take() {
owner.release();
}
if let Some(root_attempt) = self.root_attempt.take() {
root_attempt.release_after_terminal();
}
}
}
struct PendingExternalUnlockClaim {
queue: Arc<GroupCommitQueue>,
pending: Option<PendingExternalUnlock>,
scope: ProcessRootFinalizationScope,
in_flight: bool,
}
impl PendingExternalUnlockClaim {
fn durability_state(&self) -> GroupCommitFlushDurability {
self.pending
.as_ref()
.expect("pending external unlock claim must own its operation")
.durability_state()
}
fn pending_mut(&mut self) -> &mut PendingExternalUnlock {
self.pending
.as_mut()
.expect("pending external unlock claim must own its operation")
}
async fn restore(&mut self) -> Result<()> {
self.pending
.as_mut()
.expect("pending external unlock claim must own its operation")
.operation
.restore()
.await
}
async fn reconcile_durability(&self) -> Result<GroupCommitFlushDurability> {
let pending = self
.pending
.as_ref()
.expect("pending external unlock claim must own its operation");
let Some(recovery) = pending.recovery.as_ref() else {
return Ok(pending.durability_state());
};
recovery.reconcile().await
}
fn try_restore(&mut self) -> Result<bool> {
self.pending
.as_mut()
.expect("pending external unlock claim must own its operation")
.operation
.try_restore()
}
fn finish(mut self) -> PendingExternalUnlock {
let pending = self
.pending
.take()
.expect("finished external unlock claim must own its operation");
if self.in_flight {
self.queue.release_external_unlock_claim(self.scope);
self.in_flight = false;
}
pending
}
}
impl Drop for PendingExternalUnlockClaim {
fn drop(&mut self) {
if let Some(pending) = self.pending.take() {
self.queue.requeue_pending_external_unlock(pending);
}
if self.in_flight {
self.queue.release_external_unlock_claim(self.scope);
self.in_flight = false;
}
}
}
fn insert_pending_external_unlock_by_sequence(
pending: &mut VecDeque<PendingExternalUnlock>,
unlock: PendingExternalUnlock,
) {
let sequence = unlock
.sequence
.expect("queued external unlock must have a stable sequence");
let scope = unlock.scope;
let insert_at = pending.iter().position(|queued| {
queued.scope == scope
&& queued
.sequence
.is_none_or(|queued_sequence| queued_sequence > sequence)
});
if let Some(insert_at) = insert_at {
pending.insert(insert_at, unlock);
} else {
pending.push_back(unlock);
}
}
struct GroupCommitDbLockObligation<F: VfsFile + 'static> {
queue: Arc<GroupCommitQueue>,
epoch: u64,
db_file: SharedDbFile<F>,
cleanup_cx: Cx,
restore_lock_level: LockLevel,
durability_started: Arc<AtomicBool>,
durable_io_completed: Arc<AtomicBool>,
restored: Arc<AtomicBool>,
physical_lock_window: Option<GroupCommitPhysicalLockWindow>,
recovery: Option<Arc<dyn PendingGroupCommitRecoveryOperation>>,
armed: bool,
}
impl<F: VfsFile + 'static> GroupCommitDbLockObligation<F> {
// The obligation snapshot deliberately captures each coordination flag
// as its own argument; bundling them into a struct would only move the
// field list.
#[allow(clippy::too_many_arguments)]
fn new(
queue: &Arc<GroupCommitQueue>,
epoch: u64,
db_file: &SharedDbFile<F>,
cx: &Cx,
restore_lock_level: LockLevel,
durability_started: Arc<AtomicBool>,
durable_io_completed: Arc<AtomicBool>,
restored: Arc<AtomicBool>,
physical_lock_window: GroupCommitPhysicalLockWindow,
) -> Self {
restored.store(false, AtomicOrdering::Release);
Self {
queue: Arc::clone(queue),
epoch,
db_file: Arc::clone(db_file),
cleanup_cx: cleanup_child_cx(cx),
restore_lock_level,
durability_started,
durable_io_completed,
restored,
physical_lock_window: Some(physical_lock_window),
recovery: None,
armed: true,
}
}
fn set_recovery(&mut self, recovery: Arc<dyn PendingGroupCommitRecoveryOperation>) {
self.recovery = Some(recovery);
}
async fn restore(&mut self) -> Result<()> {
let restore_result = {
let _cleanup_mask = self.cleanup_cx.masked();
shared_db_unlock(&self.db_file, &self.cleanup_cx, self.restore_lock_level).await
};
if restore_result.is_ok() {
self.restored.store(true, AtomicOrdering::Release);
self.physical_lock_window.take();
self.armed = false;
}
restore_result
}
}
impl<F: VfsFile + 'static> Drop for GroupCommitDbLockObligation<F> {
fn drop(&mut self) {
if !self.armed {
return;
}
let restore_target = PendingExternalUnlockTarget::LockLevel(self.restore_lock_level);
let operation = SharedDbPendingExternalUnlock {
db_file: Arc::clone(&self.db_file),
cleanup_cx: self.cleanup_cx.clone(),
restore_target,
restored: Arc::clone(&self.restored),
};
let mut pending = PendingExternalUnlock {
sequence: None,
scope: ProcessRootFinalizationScope::IdentityWide,
epoch: Some(self.epoch),
durability_started: Arc::clone(&self.durability_started),
durable_io_completed: Arc::clone(&self.durable_io_completed),
coordination_owner: self
.physical_lock_window
.take()
.map(GroupCommitExternalLockOwner::Physical),
recovery: self.recovery.clone(),
root_attempt: None,
operation: Box::new(operation),
};
if pending.durability_state() == GroupCommitFlushDurability::InDoubt {
// Do not even attempt to acquire the shared handle here. The
// lower write may still be running after its parent future was
// dropped, so RESERVED remains the cross-process ownership token
// until durable completion can be reconciled.
self.queue.enqueue_pending_external_unlock(pending);
self.armed = false;
return;
}
match pending.operation.try_restore() {
Ok(true) => {
self.armed = false;
}
Ok(false) => {
self.queue.enqueue_pending_external_unlock(pending);
self.armed = false;
}
Err(error) => {
tracing::error!(
%error,
epoch = self.epoch,
"drop-time group-commit lock restoration failed; queued for structured retry"
);
self.queue.enqueue_pending_external_unlock(pending);
self.armed = false;
}
}
}
}
type GroupCommitQueueRef = Arc<GroupCommitQueue>;
// ---------------------------------------------------------------------------
// Shared WAL Backend (D1-CRITICAL: enables split-lock commit)
// ---------------------------------------------------------------------------
//
// The WAL backend is held in a separate Arc<RwLock<...>> to enable split-lock
// commit. This allows Thread B to start its prepare phase (which needs
// inner.lock()) while Thread A is doing WAL I/O (which needs wal_backend.write()
// but NOT inner.lock()).
//
// Before: inner.lock() held for ~100us (prepare + WAL I/O + publish)
// After: inner.lock() held for ~20us (prepare only)
// wal_backend.lock() held for ~50us (WAL I/O only)
// inner.lock() re-acquired for ~10us (post-commit only)
/// Thread-safe shared WAL backend for split-lock commit protocol.
///
/// The `RwLock` enables split-lock access: page-lookup paths that support
/// pinned reads take a shared (read) lock, while mutation paths (append,
/// sync, begin_transaction) take an exclusive (write) lock.
///
/// # bd-db300.3.8.7: write-lock-scope narrowing
///
/// Before this change, all WAL access went through `with_wal_backend` which
/// always took the write lock. Now, `with_wal_backend_read` takes only the
/// read lock for `read_page_pinned` when the backend supports pinned reads.
type WalBackendHandle = Arc<AsyncRwLock<Box<dyn WalBackend>>>;
pub type SharedWalBackend = Arc<std::sync::RwLock<Option<WalBackendHandle>>>;
type SharedDbFile<F> = Arc<AsyncRwLock<F>>;
type LocalPagerFuture<'a, T> = Pin<Box<dyn Future<Output = Result<T>> + 'a>>;
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
struct SharedDbFileKey(usize);
fn shared_db_file_key<F>(file: &SharedDbFile<F>) -> SharedDbFileKey {
SharedDbFileKey(Arc::as_ptr(file).cast::<()>() as usize)
}
async fn async_rwlock_read<'a, T>(
lock: &'a AsyncRwLock<T>,
cx: &Cx,
label: &str,
) -> Result<asupersync::sync::RwLockReadGuard<'a, T>> {
#[cfg(feature = "native")]
if let Some(native_cx) = cx.attached_native_cx() {
return lock
.read(&native_cx)
.await
.map_err(|error| FrankenError::internal(format!("{label} read lock failed: {error}")));
}
loop {
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
match lock.try_read() {
Ok(guard) => return Ok(guard),
Err(asupersync::sync::TryReadError::Locked) => {
asupersync::runtime::yield_now().await;
}
Err(asupersync::sync::TryReadError::Poisoned) => {
return Err(FrankenError::internal(format!(
"{label} read lock failed: rwlock poisoned"
)));
}
}
}
}
async fn async_rwlock_write<'a, T>(
lock: &'a AsyncRwLock<T>,
cx: &Cx,
label: &str,
) -> Result<asupersync::sync::RwLockWriteGuard<'a, T>> {
#[cfg(feature = "native")]
if let Some(native_cx) = cx.attached_native_cx() {
return lock.write(&native_cx).await.map_err(|error| {
FrankenError::internal(format!("{label} write lock failed: {error}"))
});
}
loop {
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
match lock.try_write() {
Ok(guard) => return Ok(guard),
Err(asupersync::sync::TryWriteError::Locked) => {
asupersync::runtime::yield_now().await;
}
Err(asupersync::sync::TryWriteError::Poisoned) => {
return Err(FrankenError::internal(format!(
"{label} write lock failed: rwlock poisoned"
)));
}
}
}
}
async fn shared_db_file_read<'a, F: VfsFile>(
file: &'a SharedDbFile<F>,
cx: &Cx,
) -> Result<asupersync::sync::RwLockReadGuard<'a, F>> {
async_rwlock_read(file, cx, "database-file").await
}
async fn shared_db_file_write<'a, F: VfsFile>(
file: &'a SharedDbFile<F>,
cx: &Cx,
) -> Result<asupersync::sync::RwLockWriteGuard<'a, F>> {
async_rwlock_write(file, cx, "database-file").await
}
async fn shared_db_lock_external_snapshot<F: VfsFile>(
file: &SharedDbFile<F>,
cx: &Cx,
) -> Result<()> {
shared_db_file_write(file, cx)
.await?
.lock_external_shared_snapshot(cx)
}
async fn shared_db_restore_external_snapshot_attempt<F: VfsFile>(
file: &SharedDbFile<F>,
cx: &Cx,
) -> Result<()> {
shared_db_file_write(file, cx)
.await?
.restore_external_shared_snapshot_attempt(cx)
}
async fn shared_db_lock_external_maintenance<F: VfsFile>(
file: &SharedDbFile<F>,
cx: &Cx,
wal_mode: bool,
) -> Result<()> {
shared_db_file_write(file, cx)
.await?
.lock_external_maintenance(cx, wal_mode)
}
async fn shared_db_lock<F: VfsFile>(
file: &SharedDbFile<F>,
cx: &Cx,
level: LockLevel,
) -> Result<()> {
shared_db_file_write(file, cx).await?.lock(cx, level)
}
async fn shared_db_unlock<F: VfsFile>(
file: &SharedDbFile<F>,
cx: &Cx,
level: LockLevel,
) -> Result<()> {
shared_db_file_write(file, cx).await?.unlock(cx, level)
}
/// Create a new empty shared WAL backend.
fn new_shared_wal_backend() -> SharedWalBackend {
Arc::new(std::sync::RwLock::new(None))
}
fn wal_backend_handle(wal_backend: &SharedWalBackend) -> Result<WalBackendHandle> {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::WalBackendSlot);
wal_backend
.read()
.map_err(|_| FrankenError::internal("SharedWalBackend registry lock poisoned"))?
.as_ref()
.cloned()
.ok_or_else(|| FrankenError::internal("WAL mode active but no WAL backend installed"))
}
/// Read access to WAL backend (read_page_pinned, frame_count).
///
/// Takes only a shared (read) lock on the WAL backend RwLock. This allows
/// multiple concurrent readers without blocking the append path, and the
/// append path without blocking readers.
///
/// # bd-db300.3.8.7
async fn with_wal_backend_read<T>(
wal_backend: &SharedWalBackend,
cx: &Cx,
f: impl for<'a> FnOnce(&'a dyn WalBackend, &'a Cx) -> LocalPagerFuture<'a, T>,
) -> Result<T> {
let backend = wal_backend_handle(wal_backend)?;
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::WalBackendRead);
let guard = async_rwlock_read(&backend, cx, "WAL backend").await?;
f(guard.as_ref(), cx).await
}
async fn capture_wal_conflict_snapshot_at_begin(
wal_backend: &SharedWalBackend,
cx: &Cx,
snapshot_initialized: bool,
other_transactions_active: bool,
) -> Result<Option<traits::WalPublicationSnapshot>> {
// bd-dk9ra reconciliation of two contracts def2ed8c5 collapsed into one
// active-count test: (A) every pager begin drives wal.begin_transaction —
// a parked WAL begin must be reachable and cancellable with exact
// ownership restore; (B) bd-1fc2c — a second begin must not disturb an
// earlier transaction's begin snapshot. The true coherence condition for
// skipping the WAL begin is an ALREADY-PINNED read lineage while other
// transactions are active: re-beginning would re-pin and shift the first
// transaction's view. Backends with no pinned lineage (nothing to
// disturb) always take the begin, preserving contract (A).
let skip_wal_begin = snapshot_initialized
|| (other_transactions_active
&& with_wal_backend_read(wal_backend, cx, |wal, _| {
Box::pin(async move { Ok(wal.pinned_read_snapshot().is_some()) })
})
.await?);
if !skip_wal_begin {
with_wal_backend(wal_backend, cx, |wal, cx| wal.begin_transaction(cx)).await?;
}
with_wal_backend_read(wal_backend, cx, |wal, _| {
Box::pin(async move {
Ok(wal
.pinned_read_snapshot()
.or_else(|| wal.published_snapshot()))
})
})
.await
}
enum WalReadLookup {
Ready(Option<Vec<u8>>),
NeedsWriteFallback,
}
async fn read_page_from_wal_backend(
wal_backend: &SharedWalBackend,
cx: &Cx,
page_no: PageNumber,
) -> Result<Option<Vec<u8>>> {
match with_wal_backend_read(wal_backend, cx, |wal, cx| {
Box::pin(async move {
if wal.supports_pinned_reads() {
wal.read_page_pinned(cx, page_no.get())
.await
.map(WalReadLookup::Ready)
} else {
Ok(WalReadLookup::NeedsWriteFallback)
}
})
})
.await?
{
WalReadLookup::Ready(data) => Ok(data),
WalReadLookup::NeedsWriteFallback => {
with_wal_backend(wal_backend, cx, |wal, cx| wal.read_page(cx, page_no.get())).await
}
}
}
/// Write access to WAL backend (append_frames, sync, set_wal_backend).
async fn with_wal_backend<T>(
wal_backend: &SharedWalBackend,
cx: &Cx,
f: impl for<'a> FnOnce(&'a mut dyn WalBackend, &'a Cx) -> LocalPagerFuture<'a, T>,
) -> Result<T> {
let backend = wal_backend_handle(wal_backend)?;
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::WalBackendWrite);
let mut guard = async_rwlock_write(&backend, cx, "WAL backend").await?;
f(guard.as_mut(), cx).await
}
fn has_wal_backend(wal_backend: &SharedWalBackend) -> Result<bool> {
let guard = wal_backend
.read()
.map_err(|_| FrankenError::internal("SharedWalBackend lock poisoned"))?;
Ok(guard.is_some())
}
static GROUP_COMMIT_QUEUES: OnceLock<Mutex<HashMap<PathBuf, GroupCommitQueueRef>>> =
OnceLock::new();
static GROUP_COMMIT_IDENTITY_QUEUES: OnceLock<Mutex<IdentityWeakRegistry<GroupCommitQueue>>> =
OnceLock::new();
static NEXT_GROUP_COMMIT_QUEUE_ID: AtomicU64 = AtomicU64::new(1);
static NEXT_GROUP_COMMIT_FINALIZATION_ATTEMPT_ID: AtomicU64 = AtomicU64::new(1);
fn next_process_root_finalization_id(counter: &AtomicU64) -> u64 {
atomic_u64_checked_update(
counter,
AtomicOrdering::Relaxed,
AtomicOrdering::Relaxed,
|current| current.checked_add(1),
)
.expect("process-root pager finalization identifier space exhausted")
}
struct RootedGroupCommitQueue {
queue: GroupCommitQueueRef,
attempts: HashMap<u64, ProcessRootFinalizationScope>,
paths: HashSet<PathBuf>,
identity: Option<FileIdentity>,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
enum ProcessRootFinalizationScope {
IdentityWide,
ExactHandle(SharedDbFileKey),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ProcessRootFinalizationSelector {
Any,
IdentityWide,
ExactHandle(SharedDbFileKey),
}
impl ProcessRootFinalizationSelector {
fn matches(self, scope: ProcessRootFinalizationScope) -> bool {
match (self, scope) {
(Self::Any, _) | (Self::IdentityWide, ProcessRootFinalizationScope::IdentityWide) => {
true
}
(Self::ExactHandle(selected), ProcessRootFinalizationScope::ExactHandle(candidate)) => {
selected == candidate
}
(Self::IdentityWide, ProcessRootFinalizationScope::ExactHandle(_))
| (Self::ExactHandle(_), ProcessRootFinalizationScope::IdentityWide) => false,
}
}
}
#[allow(clippy::struct_field_names)]
#[derive(Default)]
struct ProcessRootFinalizationRegistry {
by_queue: HashMap<u64, RootedGroupCommitQueue>,
by_path: HashMap<PathBuf, HashSet<u64>>,
by_identity: HashMap<FileIdentity, HashSet<u64>>,
}
static PROCESS_ROOT_FINALIZATION_REGISTRY: OnceLock<Mutex<ProcessRootFinalizationRegistry>> =
OnceLock::new();
fn process_root_finalization_registry() -> &'static Mutex<ProcessRootFinalizationRegistry> {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
PROCESS_ROOT_FINALIZATION_REGISTRY
.get_or_init(|| Mutex::new(ProcessRootFinalizationRegistry::default()))
}
struct ProcessRootFinalizationAttempt {
queue_id: u64,
attempt_id: u64,
released: bool,
}
impl ProcessRootFinalizationAttempt {
fn register(queue: &GroupCommitQueueRef) -> Self {
Self::register_identity_wide(queue)
}
fn register_identity_wide(queue: &GroupCommitQueueRef) -> Self {
Self::register_with_scope_and_publication_hook(
queue,
ProcessRootFinalizationScope::IdentityWide,
|| {},
)
}
fn register_exact_handle(queue: &GroupCommitQueueRef, handle_key: SharedDbFileKey) -> Self {
Self::register_with_scope_and_publication_hook(
queue,
ProcessRootFinalizationScope::ExactHandle(handle_key),
|| {},
)
}
fn register_scope(queue: &GroupCommitQueueRef, scope: ProcessRootFinalizationScope) -> Self {
match scope {
ProcessRootFinalizationScope::IdentityWide => Self::register_identity_wide(queue),
ProcessRootFinalizationScope::ExactHandle(handle_key) => {
Self::register_exact_handle(queue, handle_key)
}
}
}
#[cfg(test)]
fn register_with_publication_hook(
queue: &GroupCommitQueueRef,
after_queue_fence: impl FnOnce(),
) -> Self {
Self::register_with_scope_and_publication_hook(
queue,
ProcessRootFinalizationScope::IdentityWide,
after_queue_fence,
)
}
fn register_with_scope_and_publication_hook(
queue: &GroupCommitQueueRef,
scope: ProcessRootFinalizationScope,
after_queue_fence: impl FnOnce(),
) -> Self {
let attempt_id =
next_process_root_finalization_id(&NEXT_GROUP_COMMIT_FINALIZATION_ATTEMPT_ID);
let queue_id = queue.queue_id;
// Lock order is binding -> process-root registry everywhere. Holding
// both across the queue-local fence and every global index insertion
// makes the Release increment the single publication point:
//
// * same-queue admission observes the atomic and fails closed;
// * same-path/replacement admission blocks on the registry mutex until
// the old queue is present in every applicable index.
//
// In particular, there must be no observable atomic->registry gap.
let binding = queue
.finalization_binding
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut registry = process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
atomic_usize_checked_update(
&queue.rooted_finalization_attempts,
AtomicOrdering::Release,
AtomicOrdering::Relaxed,
|current| current.checked_add(1),
)
.expect("process-root pager finalization attempt count exhausted");
after_queue_fence();
{
let rooted =
registry
.by_queue
.entry(queue_id)
.or_insert_with(|| RootedGroupCommitQueue {
queue: Arc::clone(queue),
attempts: HashMap::new(),
paths: HashSet::new(),
identity: binding.identity,
});
debug_assert!(
Arc::ptr_eq(&rooted.queue, queue),
"stable queue id must identify exactly one group-commit queue"
);
if let (Some(bound), Some(incoming)) = (rooted.identity, binding.identity) {
debug_assert_eq!(
bound, incoming,
"rooted group-commit queue identity must remain stable"
);
} else if rooted.identity.is_none() {
rooted.identity = binding.identity;
}
rooted.paths.extend(binding.paths.iter().cloned());
rooted.attempts.insert(attempt_id, scope);
}
let (paths, identity) = registry
.by_queue
.get(&queue_id)
.map(|rooted| (rooted.paths.clone(), rooted.identity))
.expect("new process-root finalization queue must remain registered");
for path in paths {
registry.by_path.entry(path).or_default().insert(queue_id);
}
if let Some(identity) = identity {
registry
.by_identity
.entry(identity)
.or_default()
.insert(queue_id);
}
drop(registry);
drop(binding);
Self {
queue_id,
attempt_id,
released: false,
}
}
fn release_after_terminal(mut self) {
self.released = release_process_root_finalization_attempt(self.queue_id, self.attempt_id);
}
}
impl Drop for ProcessRootFinalizationAttempt {
fn drop(&mut self) {
if !self.released {
tracing::error!(
queue_id = self.queue_id,
attempt_id = self.attempt_id,
"process-root pager finalization token dropped before terminal release; retaining fail-closed root"
);
}
}
}
fn process_root_finalization_scope_is_relevant_in_registry(
registry: &ProcessRootFinalizationRegistry,
queue_id: u64,
handle_key: Option<SharedDbFileKey>,
) -> bool {
let Some(rooted) = registry.by_queue.get(&queue_id) else {
tracing::error!(
queue_id,
"process-root scope registry is missing a queue whose atomic root count is nonzero"
);
return true;
};
rooted.attempts.values().any(|scope| match scope {
ProcessRootFinalizationScope::IdentityWide => true,
ProcessRootFinalizationScope::ExactHandle(rooted_key) => {
handle_key.is_some_and(|handle_key| *rooted_key == handle_key)
}
})
}
fn refresh_process_root_finalization_binding(queue: &GroupCommitQueueRef) {
// Match register's binding -> registry lock order. If registration is
// waiting for this binding, it will snapshot the new alias after we
// release it. If registration already published its atomic fence, this
// refresh holds the binding until the alias is globally indexed.
let binding = queue
.finalization_binding
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !queue.has_process_root_finalization_attempt() {
return;
}
let Some(registry) = PROCESS_ROOT_FINALIZATION_REGISTRY.get() else {
return;
};
let mut registry = registry
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(rooted) = registry.by_queue.get_mut(&queue.queue_id) else {
return;
};
rooted.paths.extend(binding.paths.iter().cloned());
if let Some(identity) = binding.identity {
if let Some(bound) = rooted.identity {
debug_assert_eq!(
bound, identity,
"rooted group-commit queue identity must remain stable"
);
} else {
rooted.identity = Some(identity);
}
}
let paths = rooted.paths.clone();
let identity = rooted.identity;
let queue_id = queue.queue_id;
for path in paths {
registry.by_path.entry(path).or_default().insert(queue_id);
}
if let Some(identity) = identity {
registry
.by_identity
.entry(identity)
.or_default()
.insert(queue_id);
}
}
fn release_process_root_finalization_attempt(queue_id: u64, attempt_id: u64) -> bool {
let mut registry = process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let Some(rooted) = registry.by_queue.get_mut(&queue_id) else {
tracing::error!(
queue_id,
attempt_id,
"terminal pager finalization queue root is missing"
);
return false;
};
if rooted.attempts.remove(&attempt_id).is_none() {
tracing::error!(
queue_id,
attempt_id,
"terminal pager finalization attempted to release an unknown root token; retaining fail-closed root"
);
return false;
}
atomic_usize_checked_update(
&rooted.queue.rooted_finalization_attempts,
AtomicOrdering::Release,
AtomicOrdering::Relaxed,
|current| current.checked_sub(1),
)
.expect("process-root pager finalization attempt count underflowed");
let remove_queue = rooted.attempts.is_empty();
let released_queue = Arc::clone(&rooted.queue);
if !remove_queue {
drop(registry);
drop(released_queue);
return true;
}
let Some(rooted) = registry.by_queue.remove(&queue_id) else {
tracing::error!(
queue_id,
attempt_id,
"terminal pager finalization queue disappeared during release"
);
return false;
};
for path in &rooted.paths {
if let Some(queue_ids) = registry.by_path.get_mut(path) {
queue_ids.remove(&queue_id);
if queue_ids.is_empty() {
registry.by_path.remove(path);
}
}
}
if let Some(identity) = rooted.identity
&& let Some(queue_ids) = registry.by_identity.get_mut(&identity)
{
queue_ids.remove(&queue_id);
if queue_ids.is_empty() {
registry.by_identity.remove(&identity);
}
}
drop(registry);
drop(released_queue);
drop(rooted);
true
}
fn process_root_finalization_queues_for_path(path: &Path) -> Vec<GroupCommitQueueRef> {
let key = lexical_normalize_path(path.to_path_buf());
let registry = process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
registry
.by_path
.get(&key)
.map_or_else(Vec::new, |queue_ids| {
queue_ids
.iter()
.filter_map(|queue_id| {
registry
.by_queue
.get(queue_id)
.map(|rooted| Arc::clone(&rooted.queue))
})
.collect()
})
}
fn process_root_finalization_queues_for_identity(
identity: FileIdentity,
) -> Vec<GroupCommitQueueRef> {
let registry = process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
registry
.by_identity
.get(&identity)
.map_or_else(Vec::new, |queue_ids| {
queue_ids
.iter()
.filter_map(|queue_id| {
registry
.by_queue
.get(queue_id)
.map(|rooted| Arc::clone(&rooted.queue))
})
.collect()
})
}
// ---------------------------------------------------------------------------
// bd-yfdb6: Recovery fence registry
// ---------------------------------------------------------------------------
//
// Keyed by canonical DB path so every `SimplePager::open_with_cx*` call for
// the same file contends on the same `RecoveryFence`. Connection-open
// acquires the fence while running the hot-journal / WAL recovery probe,
// with bounded backoff (100 ms × 10 = 1 s) before surfacing a soft
// `BusyRecovery` to the caller.
static RECOVERY_FENCES: OnceLock<Mutex<HashMap<PathBuf, Arc<RecoveryFence>>>> = OnceLock::new();
static RECOVERY_IDENTITY_FENCES: OnceLock<Mutex<IdentityWeakRegistry<RecoveryFence>>> =
OnceLock::new();
// VACUUM publication mutates the already-open database inode in place. Native
// file locks exclude other processes, but POSIX record locks are process-wide:
// a second pager in this process must therefore be fenced explicitly. The
// gate also covers pager bootstrap so a same-process opener cannot discover
// and replay the publisher's deliberately-hot rollback journal.
static MAINTENANCE_GATES: OnceLock<Mutex<HashMap<PathBuf, Arc<PagerMaintenanceGate>>>> =
OnceLock::new();
static MAINTENANCE_IDENTITY_GATES: OnceLock<Mutex<IdentityWeakRegistry<PagerMaintenanceGate>>> =
OnceLock::new();
/// Process-local coordination keyed by a concrete open-file generation.
///
/// Weak values prevent an identity registry from extending the lifetime of a
/// pager's coordination state. Expired keys are reclaimed by a geometrically
/// spaced sweep, keeping insertion amortized O(1) instead of scanning every
/// live database on every open.
#[derive(Debug)]
struct IdentityWeakRegistry<T> {
entries: HashMap<FileIdentity, Weak<T>>,
mutations_since_sweep: usize,
sweep_after_mutations: usize,
}
impl<T> Default for IdentityWeakRegistry<T> {
fn default() -> Self {
Self {
entries: HashMap::new(),
mutations_since_sweep: 0,
sweep_after_mutations: 64,
}
}
}
impl<T> IdentityWeakRegistry<T> {
fn get_or_insert_with(
&mut self,
identity: FileIdentity,
create: impl FnOnce() -> Arc<T>,
) -> Arc<T> {
if let Some(value) = self.entries.get(&identity).and_then(Weak::upgrade) {
return value;
}
let value = create();
self.entries.insert(identity, Arc::downgrade(&value));
self.mutations_since_sweep = self.mutations_since_sweep.saturating_add(1);
if self.mutations_since_sweep >= self.sweep_after_mutations {
self.entries.retain(|_, entry| entry.strong_count() > 0);
self.mutations_since_sweep = 0;
self.sweep_after_mutations = self.entries.len().max(64);
}
value
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct RollbackRecoveryOwnerId(usize);
impl RollbackRecoveryOwnerId {
#[must_use]
const fn new(raw: usize) -> Option<Self> {
if raw == 0 { None } else { Some(Self(raw)) }
}
#[must_use]
const fn get(self) -> usize {
self.0
}
}
#[derive(Debug, Clone)]
struct RollbackRecoveryNamespace {
db_path: PathBuf,
journal_path: PathBuf,
db_identity: Option<FileIdentity>,
journal_mode: JournalMode,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: Option<Arc<DatabaseNamespaceBinding>>,
}
#[derive(Debug, Clone)]
struct OrphanedRollbackRecovery {
owner: RollbackRecoveryOwnerId,
recovery_state: RollbackJournalRecoveryState,
namespace: RollbackRecoveryNamespace,
}
struct RollbackJournalOpenRecoveryContext<'a, F> {
group_commit_queue: &'a Arc<GroupCommitQueue>,
db_file: &'a SharedDbFile<F>,
current_db_path: &'a Path,
current_db_identity: Option<FileIdentity>,
journal_path: &'a Path,
}
struct OrphanedRollbackRecoveryOpenClaim {
gate: Arc<PagerMaintenanceGate>,
recovery: Option<OrphanedRollbackRecovery>,
}
impl OrphanedRollbackRecoveryOpenClaim {
fn recovery(&self) -> &OrphanedRollbackRecovery {
self.recovery
.as_ref()
.expect("live orphaned-recovery open claim must retain its receipt")
}
fn recovery_mut(&mut self) -> &mut OrphanedRollbackRecovery {
self.recovery
.as_mut()
.expect("live orphaned-recovery open claim must retain its receipt")
}
fn finish(&mut self) -> OrphanedRollbackRecovery {
self.recovery
.take()
.expect("finished orphaned-recovery open claim must retain its receipt")
}
}
impl Drop for OrphanedRollbackRecoveryOpenClaim {
fn drop(&mut self) {
let Some(recovery) = self.recovery.take() else {
return;
};
let mut state = match self.gate.state.lock() {
Ok(state) => state,
Err(error) => {
tracing::error!(
"orphaned-recovery open claim recovered a poisoned maintenance gate"
);
error.into_inner()
}
};
if self.gate.rollback_recovery_owner() != Some(recovery.owner)
|| state.orphaned_rollback_recovery.is_some()
{
tracing::error!(
owner = recovery.owner.get(),
"failed open could not restore its exact orphaned-recovery receipt"
);
return;
}
state.orphaned_rollback_recovery = Some(recovery);
}
}
#[derive(Debug, Default)]
struct PagerMaintenanceState {
active_openers: usize,
active_transactions: usize,
maintenance_active: bool,
/// Last exact rollback-recovery owner issued for this file identity.
/// Zero is reserved for the clean/no-owner atomic representation.
next_rollback_recovery_owner: usize,
/// Recovery state transferred from a PagerInner that was dropped before
/// its exact owner reached a terminal state. The identity-wide atomic
/// remains armed; once every pre-existing transaction/opener drains, one
/// surviving pager may adopt this receipt and run canonical recovery.
orphaned_rollback_recovery: Option<OrphanedRollbackRecovery>,
}
#[derive(Debug, Default)]
struct PagerMaintenanceGate {
state: Mutex<PagerMaintenanceState>,
/// Exact identity-bound recovery owner (`0` means clean). Separately opened
/// pagers share this atomic, and only the holder of the matching monotonic
/// owner id may transition or clear the recovery barrier.
rollback_recovery_pending: Arc<AtomicUsize>,
}
#[derive(Debug, Clone, Copy)]
enum PagerMaintenanceLeaseKind {
Open,
Transaction,
Exclusive,
}
#[derive(Debug)]
struct PagerMaintenanceLease {
gate: Arc<PagerMaintenanceGate>,
kind: PagerMaintenanceLeaseKind,
/// Original lease kind retained while an open or transaction lease is
/// upgraded for recovery. This receipt belongs to the persistent lease,
/// not to one async recovery future: dropping that future must leave the
/// gate fail closed, while a later retry must still know which admission
/// count to restore after terminal recovery.
exclusive_upgrade_prior: Option<PagerMaintenanceLeaseKind>,
}
impl PagerMaintenanceGate {
fn rollback_recovery_owner(&self) -> Option<RollbackRecoveryOwnerId> {
RollbackRecoveryOwnerId::new(self.rollback_recovery_pending.load(AtomicOrdering::Acquire))
}
fn rollback_recovery_pending(&self) -> bool {
self.rollback_recovery_owner().is_some()
}
fn claim_rollback_recovery_owner(&self) -> Result<RollbackRecoveryOwnerId> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if state.active_openers != 0 || self.rollback_recovery_owner().is_some() {
return Err(FrankenError::BusyRecovery);
}
let next = state
.next_rollback_recovery_owner
.checked_add(1)
.filter(|next| *next != 0)
.ok_or_else(|| FrankenError::internal("rollback-recovery owner id overflow"))?;
state.next_rollback_recovery_owner = next;
let owner = RollbackRecoveryOwnerId(next);
self.rollback_recovery_pending
.store(owner.get(), AtomicOrdering::Release);
Ok(owner)
}
fn release_rollback_recovery_owner(&self, owner: RollbackRecoveryOwnerId) -> Result<()> {
let _state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.rollback_recovery_owner() != Some(owner) {
return Err(FrankenError::internal(
"rollback-recovery owner release did not match the identity owner",
));
}
self.rollback_recovery_pending
.store(0, AtomicOrdering::Release);
Ok(())
}
fn orphan_rollback_recovery_owner(
&self,
owner: RollbackRecoveryOwnerId,
recovery_state: RollbackJournalRecoveryState,
namespace: RollbackRecoveryNamespace,
) -> Result<()> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.rollback_recovery_owner() != Some(owner)
|| !recovery_state.is_pager_takeover_eligible()
{
return Err(FrankenError::internal(
"orphaned rollback recovery did not match the identity owner",
));
}
if state.orphaned_rollback_recovery.is_some() {
return Err(FrankenError::internal(
"identity already retained an orphaned rollback recovery",
));
}
state.orphaned_rollback_recovery = Some(OrphanedRollbackRecovery {
owner,
recovery_state,
namespace,
});
Ok(())
}
fn try_adopt_orphaned_rollback_recovery(&self) -> Result<Option<OrphanedRollbackRecovery>> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
let Some(orphaned) = state.orphaned_rollback_recovery.as_ref() else {
return Ok(None);
};
if state.active_openers != 0 || state.active_transactions != 0 || state.maintenance_active {
return Ok(None);
}
if self.rollback_recovery_owner() != Some(orphaned.owner)
|| !orphaned.recovery_state.is_pending()
{
return Err(FrankenError::internal(
"orphaned rollback recovery lost its identity owner",
));
}
Ok(state.orphaned_rollback_recovery.take())
}
fn lock_clean_writer_upgrade(&self) -> Result<MutexGuard<'_, PagerMaintenanceState>> {
let state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.rollback_recovery_owner().is_some() || state.orphaned_rollback_recovery.is_some() {
return Err(FrankenError::BusyRecovery);
}
Ok(state)
}
fn enter_open(self: &Arc<Self>) -> Result<PagerMaintenanceLease> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.rollback_recovery_pending() {
return Err(FrankenError::BusyRecovery);
}
if state.maintenance_active {
return Err(FrankenError::Busy);
}
state.active_openers = state.active_openers.saturating_add(1);
drop(state);
Ok(PagerMaintenanceLease {
gate: Arc::clone(self),
kind: PagerMaintenanceLeaseKind::Open,
exclusive_upgrade_prior: None,
})
}
fn enter_readwrite_open_for_orphan_recovery(
self: &Arc<Self>,
) -> Result<(
PagerMaintenanceLease,
Option<OrphanedRollbackRecoveryOpenClaim>,
)> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if state.maintenance_active {
return Err(FrankenError::Busy);
}
let recovery_claim = if let Some(owner) = self.rollback_recovery_owner() {
if state.active_openers != 0 || state.active_transactions != 0 {
return Err(FrankenError::BusyRecovery);
}
let retained = state
.orphaned_rollback_recovery
.as_ref()
.ok_or(FrankenError::BusyRecovery)?;
// A new open can resolve to a replacement path generation. If the
// origin VFS supplied no stable identity, lexical path equality
// cannot authorize replaying the old generation's journal into
// the newly opened file. A surviving PagerInner may still settle
// the receipt, but re-open adoption must remain fail closed.
if retained.namespace.db_identity.is_none() {
return Err(FrankenError::BusyRecovery);
}
let recovery = state
.orphaned_rollback_recovery
.take()
.expect("validated orphaned recovery must remain in the locked gate");
if recovery.owner != owner || !recovery.recovery_state.is_pager_takeover_eligible() {
state.orphaned_rollback_recovery = Some(recovery);
return Err(FrankenError::BusyRecovery);
}
Some(OrphanedRollbackRecoveryOpenClaim {
gate: Arc::clone(self),
recovery: Some(recovery),
})
} else {
if state.orphaned_rollback_recovery.is_some() {
return Err(FrankenError::internal(
"clean maintenance gate retained an orphaned recovery receipt",
));
}
None
};
state.active_openers = state.active_openers.saturating_add(1);
drop(state);
Ok((
PagerMaintenanceLease {
gate: Arc::clone(self),
kind: PagerMaintenanceLeaseKind::Open,
exclusive_upgrade_prior: None,
},
recovery_claim,
))
}
fn enter_transaction(self: &Arc<Self>) -> Result<PagerMaintenanceLease> {
self.enter_transaction_with_recovery_owner(None)
}
fn enter_transaction_with_recovery_owner(
self: &Arc<Self>,
expected_recovery_owner: Option<RollbackRecoveryOwnerId>,
) -> Result<PagerMaintenanceLease> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.rollback_recovery_owner() != expected_recovery_owner {
// bd-b4mwn round 5: name the silent refusal — an armed
// rollback-recovery owner no live pager can satisfy previously
// surfaced only as an anonymous BusyRecovery at every
// transaction admission on the path.
tracing::warn!(
target: "fsqlite.pager.maintenance_gate",
armed_owner = ?self.rollback_recovery_owner().map(RollbackRecoveryOwnerId::get),
expected_owner = ?expected_recovery_owner.map(RollbackRecoveryOwnerId::get),
active_openers = state.active_openers,
active_transactions = state.active_transactions,
orphan_retained = state.orphaned_rollback_recovery.is_some(),
"transaction admission refused: rollback-recovery owner mismatch"
);
return Err(FrankenError::BusyRecovery);
}
if state.maintenance_active {
return Err(FrankenError::Busy);
}
state.active_transactions = state.active_transactions.saturating_add(1);
drop(state);
Ok(PagerMaintenanceLease {
gate: Arc::clone(self),
kind: PagerMaintenanceLeaseKind::Transaction,
exclusive_upgrade_prior: None,
})
}
fn enter_exclusive_maintenance(self: &Arc<Self>) -> Result<PagerMaintenanceLease> {
let mut state = self
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.rollback_recovery_pending() {
return Err(FrankenError::BusyRecovery);
}
if state.maintenance_active || state.active_openers != 0 || state.active_transactions != 0 {
return Err(FrankenError::Busy);
}
state.maintenance_active = true;
drop(state);
Ok(PagerMaintenanceLease {
gate: Arc::clone(self),
kind: PagerMaintenanceLeaseKind::Exclusive,
exclusive_upgrade_prior: None,
})
}
}
impl PagerMaintenanceLease {
/// Temporarily turn this lease into the sole same-process maintenance
/// owner. Recovery callers use this only after releasing every VFS lock,
/// so the cross-process acquisition can follow the canonical
/// WAL-slots-before-main-file order.
fn upgrade_to_exclusive(
&mut self,
expected_recovery_owner: Option<RollbackRecoveryOwnerId>,
) -> Result<PagerMaintenanceLeaseKind> {
let mut state = self
.gate
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
if self.gate.rollback_recovery_owner() != expected_recovery_owner {
// bd-b4mwn round 6: name the silent refusal (see the sibling
// gate receipts).
tracing::warn!(
target: "fsqlite.pager.maintenance_gate",
armed_owner = ?self.gate.rollback_recovery_owner().map(RollbackRecoveryOwnerId::get),
expected_owner = ?expected_recovery_owner.map(RollbackRecoveryOwnerId::get),
"exclusive upgrade refused: rollback-recovery owner mismatch"
);
return Err(FrankenError::BusyRecovery);
}
if matches!(self.kind, PagerMaintenanceLeaseKind::Exclusive) {
if !state.maintenance_active {
return Err(FrankenError::internal(
"exclusive pager maintenance lease lost its active gate",
));
}
// A cancelled recovery may be retried with this same persistent
// lease. Return its original typed receipt rather than treating
// the lease's own fail-closed exclusive ownership as contention.
return Ok(self
.exclusive_upgrade_prior
.unwrap_or(PagerMaintenanceLeaseKind::Exclusive));
}
if state.maintenance_active {
tracing::warn!(
target: "fsqlite.pager.maintenance_gate",
"exclusive upgrade refused: maintenance already active"
);
return Err(FrankenError::BusyRecovery);
}
let prior = self.kind;
let sole_owner = match prior {
PagerMaintenanceLeaseKind::Open => {
state.active_openers == 1 && state.active_transactions == 0
}
PagerMaintenanceLeaseKind::Transaction => {
state.active_transactions == 1 && state.active_openers == 0
}
PagerMaintenanceLeaseKind::Exclusive => return Ok(prior),
};
if !sole_owner {
// bd-b4mwn round 5: name the silent refusal — recovery-requiring
// work (e.g. a leftover hot journal from a dropped peer) demands
// sole ownership that a churning multi-writer path never grants.
tracing::warn!(
target: "fsqlite.pager.maintenance_gate",
lease_kind = ?prior,
active_openers = state.active_openers,
active_transactions = state.active_transactions,
"exclusive upgrade refused: not sole owner"
);
return Err(FrankenError::BusyRecovery);
}
match prior {
PagerMaintenanceLeaseKind::Open => state.active_openers = 0,
PagerMaintenanceLeaseKind::Transaction => state.active_transactions = 0,
PagerMaintenanceLeaseKind::Exclusive => unreachable!(),
}
state.maintenance_active = true;
self.kind = PagerMaintenanceLeaseKind::Exclusive;
self.exclusive_upgrade_prior = Some(prior);
Ok(prior)
}
fn downgrade_from_exclusive(&mut self, prior: PagerMaintenanceLeaseKind) -> Result<()> {
if !matches!(self.kind, PagerMaintenanceLeaseKind::Exclusive) {
return Ok(());
}
if matches!(prior, PagerMaintenanceLeaseKind::Exclusive) {
return if self.exclusive_upgrade_prior.is_none() {
Ok(())
} else {
Err(FrankenError::internal(
"pager maintenance lease downgrade omitted its recorded prior kind",
))
};
}
let Some(recorded_prior) = self.exclusive_upgrade_prior else {
return Err(FrankenError::internal(
"upgraded pager maintenance lease lost its prior-kind receipt",
));
};
if !matches!(
(recorded_prior, prior),
(
PagerMaintenanceLeaseKind::Open,
PagerMaintenanceLeaseKind::Open
) | (
PagerMaintenanceLeaseKind::Transaction,
PagerMaintenanceLeaseKind::Transaction
)
) {
return Err(FrankenError::internal(
"pager maintenance lease downgrade receipt does not match its upgrade",
));
}
let mut state = self
.gate
.state
.lock()
.map_err(|_| FrankenError::internal("pager maintenance gate poisoned"))?;
state.maintenance_active = false;
match prior {
PagerMaintenanceLeaseKind::Open => {
state.active_openers = state.active_openers.saturating_add(1);
}
PagerMaintenanceLeaseKind::Transaction => {
state.active_transactions = state.active_transactions.saturating_add(1);
}
PagerMaintenanceLeaseKind::Exclusive => unreachable!(),
}
self.kind = prior;
self.exclusive_upgrade_prior = None;
Ok(())
}
}
impl Drop for PagerMaintenanceLease {
fn drop(&mut self) {
let mut state = self
.gate
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match self.kind {
PagerMaintenanceLeaseKind::Open => {
state.active_openers = state.active_openers.saturating_sub(1);
}
PagerMaintenanceLeaseKind::Transaction => {
state.active_transactions = state.active_transactions.saturating_sub(1);
}
PagerMaintenanceLeaseKind::Exclusive => {
state.maintenance_active = false;
}
}
}
}
fn lexical_normalize_path(path: PathBuf) -> PathBuf {
let mut normalized = PathBuf::new();
for component in path.components() {
match component {
Component::CurDir => {}
Component::ParentDir => {
if !normalized.pop() && !normalized.has_root() {
normalized.push(component.as_os_str());
}
}
Component::Prefix(_) | Component::RootDir | Component::Normal(_) => {
normalized.push(component.as_os_str());
}
}
}
normalized
}
fn shared_file_state_key(db_path: &Path) -> PathBuf {
std::fs::canonicalize(db_path).unwrap_or_else(|_| {
let absolute = if db_path.is_absolute() {
db_path.to_path_buf()
} else {
std::env::current_dir()
.map(|cwd| cwd.join(db_path))
.unwrap_or_else(|_| db_path.to_path_buf())
};
lexical_normalize_path(absolute)
})
}
fn recovery_fence_for_path(db_path: &Path) -> Arc<RecoveryFence> {
let key = shared_file_state_key(db_path);
let fences = RECOVERY_FENCES.get_or_init(|| Mutex::new(HashMap::new()));
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
let mut fences = fences
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Arc::clone(
fences
.entry(key)
.or_insert_with(|| Arc::new(RecoveryFence::new())),
)
}
fn recovery_fence_for_backend<V: Vfs>(vfs: &V, db_path: &Path) -> Arc<RecoveryFence> {
if vfs.is_memory() {
// In-memory databases are connection-local; a fresh fence per open
// keeps isolation intact.
Arc::new(RecoveryFence::new())
} else {
recovery_fence_for_path(db_path)
}
}
fn recovery_fence_for_identity(identity: FileIdentity) -> Arc<RecoveryFence> {
let fences =
RECOVERY_IDENTITY_FENCES.get_or_init(|| Mutex::new(IdentityWeakRegistry::default()));
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
let mut fences = fences
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
fences.get_or_insert_with(identity, || Arc::new(RecoveryFence::new()))
}
fn identity_bound_recovery_fence<V: Vfs>(
vfs: &V,
db_path: &Path,
file: &V::File,
) -> Result<Arc<RecoveryFence>> {
file.file_identity()?.map_or_else(
|| Ok(recovery_fence_for_backend(vfs, db_path)),
|identity| Ok(recovery_fence_for_identity(identity)),
)
}
fn maintenance_gate_for_path(db_path: &Path) -> Arc<PagerMaintenanceGate> {
let key = shared_file_state_key(db_path);
let gates = MAINTENANCE_GATES.get_or_init(|| Mutex::new(HashMap::new()));
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
let mut gates = gates
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Arc::clone(
gates
.entry(key)
.or_insert_with(|| Arc::new(PagerMaintenanceGate::default())),
)
}
fn maintenance_gate_for_backend<V: Vfs>(vfs: &V, db_path: &Path) -> Arc<PagerMaintenanceGate> {
if vfs.is_memory() {
Arc::new(PagerMaintenanceGate::default())
} else {
maintenance_gate_for_path(db_path)
}
}
fn maintenance_gate_for_identity(identity: FileIdentity) -> Arc<PagerMaintenanceGate> {
let gates =
MAINTENANCE_IDENTITY_GATES.get_or_init(|| Mutex::new(IdentityWeakRegistry::default()));
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
let mut gates = gates
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
gates.get_or_insert_with(identity, || Arc::new(PagerMaintenanceGate::default()))
}
fn identity_bound_maintenance_gate<V: Vfs>(
vfs: &V,
path_gate: &Arc<PagerMaintenanceGate>,
file: &V::File,
) -> Result<Arc<PagerMaintenanceGate>> {
let _ = vfs;
file.file_identity()?.map_or_else(
|| Ok(Arc::clone(path_gate)),
|identity| Ok(maintenance_gate_for_identity(identity)),
)
}
fn group_commit_queue_for_path(db_path: &Path) -> GroupCommitQueueRef {
let key = shared_file_state_key(db_path);
let queues = GROUP_COMMIT_QUEUES.get_or_init(|| Mutex::new(HashMap::new()));
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
let mut queues = queues
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let queue = Arc::clone(
queues
.entry(key)
.or_insert_with(|| Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()))),
);
drop(queues);
queue.bind_finalization_path(db_path);
queue
}
fn group_commit_queue_for_backend<V: Vfs>(vfs: &V, db_path: &Path) -> GroupCommitQueueRef {
if vfs.is_memory() {
// Private :memory: databases are connection-local, so sharing a
// global queue by the synthetic "/:memory:" path would cross-wire
// unrelated databases. Use a fresh queue instead.
Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()))
} else {
group_commit_queue_for_path(db_path)
}
}
fn group_commit_queue_for_identity(
identity: FileIdentity,
db_path: &Path,
bind_path: bool,
) -> GroupCommitQueueRef {
let queues =
GROUP_COMMIT_IDENTITY_QUEUES.get_or_init(|| Mutex::new(IdentityWeakRegistry::default()));
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::ProcessGlobalRegistry);
let mut queues = queues
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let queue = queues.get_or_insert_with(identity, || {
Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()))
});
drop(queues);
queue.bind_finalization_identity(identity);
if bind_path {
queue.bind_finalization_path(db_path);
}
queue
}
fn identity_bound_group_commit_queue<V: Vfs>(
vfs: &V,
db_path: &Path,
file: &V::File,
) -> Result<GroupCommitQueueRef> {
file.file_identity()?.map_or_else(
|| Ok(group_commit_queue_for_backend(vfs, db_path)),
|identity| {
Ok(group_commit_queue_for_identity(
identity,
db_path,
!vfs.is_memory(),
))
},
)
}
/// bd-b4mwn rounds 7+rework: in-settle retries when the armed root has live,
/// claimable cleanup work (another settler holds the per-handle exit claim,
/// or a resolve round simply lost the claim race). Bursts of concurrent
/// settlers previously made every non-winning settler refuse BusyRecovery
/// despite active progress. The original fixed ~250ms ceiling was
/// timing-dependent (green on a 64-core host, red on 16 cores where
/// resolution bursts run longer); the envelope now scales with the largest
/// busy_timeout any connection on the path has published
/// (`GroupCommitQueue::settle_budget_ms`, default 5000ms). True wedges still
/// surface promptly: the loop exits as soon as no claimable work remains.
const SETTLE_CLAIM_CONTENTION_BACKOFF: Duration = Duration::from_millis(10);
async fn settle_pending_group_commit_finalization(queue: &GroupCommitQueueRef) -> Result<()> {
if !queue.has_process_root_finalization_attempt() {
return Ok(());
}
let budget = Duration::from_millis(queue.settle_budget_ms.load(AtomicOrdering::Acquire).max(1));
let started = Instant::now();
let mut rounds: u32 = 0;
let mut cleanups_resolved_total: usize = 0;
loop {
rounds = rounds.saturating_add(1);
let before = queue.pending_logical_cleanup_count();
match settle_pending_group_commit_finalization_round(queue).await {
Ok(()) => {
if !queue.has_process_root_finalization_attempt() {
return Ok(());
}
}
// A refused round is retryable exactly while claimable cleanup
// work remains; any other error is terminal.
Err(FrankenError::BusyRecovery) => {}
Err(error) => return Err(error),
}
let after = queue.pending_logical_cleanup_count();
cleanups_resolved_total =
cleanups_resolved_total.saturating_add(before.saturating_sub(after));
let claimed_in_flight = queue.claimed_logical_cleanups.load(AtomicOrdering::Acquire);
if after == 0 && claimed_in_flight == 0 {
// No claimable work remains AND nothing is being resolved by a
// peer — the residual root is a true wedge; fall through to the
// final round's fail-closed verdict. (A nonzero claimed count is
// LIVE progress: a peer settler is resolving right now — on
// slow-fsync hosts that resolve can hold its claim for seconds,
// which every other settler previously misdiagnosed as a wedge.)
break;
}
if started.elapsed() >= budget {
// bd-b4mwn rework #2: the PROGRESS LEDGER — when the budget burns
// out, say exactly what recovery was (not) progressing on so a
// multi-second "recovery in progress" window on any host names
// its own starvation, instead of tuning waits blind.
tracing::warn!(
target: "fsqlite.pager.group_commit",
rounds,
cleanups_resolved_total,
pending_cleanups_remaining = after,
pending_or_claimed_external_unlock =
queue.has_pending_or_claimed_external_unlock(),
unresolved_in_doubt_epoch = queue.has_unresolved_in_doubt_epoch(),
elapsed_ms = u64::try_from(started.elapsed().as_millis()).unwrap_or(u64::MAX),
"settle budget exhausted with cleanup work still pending"
);
break;
}
// Live cleanups remain (claimed by a peer settler or lost claim
// races): yield briefly and retry instead of refusing.
std::thread::sleep(SETTLE_CLAIM_CONTENTION_BACKOFF);
}
settle_pending_group_commit_finalization_round(queue).await
}
async fn settle_pending_group_commit_finalization_round(queue: &GroupCommitQueueRef) -> Result<()> {
if !queue.has_process_root_finalization_attempt() {
return Ok(());
}
while queue.resolve_one_pending_external_unlock().await? {}
queue.resolve_pending_epoch_resolutions()?;
if queue.has_pending_or_claimed_external_unlock() || queue.has_unresolved_in_doubt_epoch() {
// bd-b4mwn: fail-closed refusals must name themselves — a token a
// dead connection can no longer resolve otherwise surfaces only as
// an anonymous BusyRecovery at some later caller.
tracing::warn!(
target: "fsqlite.pager.group_commit",
pending_or_claimed_external_unlock = queue.has_pending_or_claimed_external_unlock(),
unresolved_in_doubt_epoch = queue.has_unresolved_in_doubt_epoch(),
"settle refused: unresolved external unlock or in-doubt epoch"
);
return Err(FrankenError::BusyRecovery);
}
let logical_cleanup_count = queue.pending_logical_cleanup_count();
let mut first_logical_cleanup_error = None;
for _ in 0..logical_cleanup_count {
if let Err(error) = queue.resolve_one_pending_logical_cleanup().await
&& first_logical_cleanup_error.is_none()
{
first_logical_cleanup_error = Some(error);
}
}
if let Some(error) = first_logical_cleanup_error {
return Err(error);
}
// bd-b4mwn (terminal): the residual refusal previously keyed on ANY
// armed root (has_process_root_finalization_attempt) — but exact-handle
// roots are normal, short-lived receipts of some OTHER connection's
// in-flight operation, individually milliseconds wide and collectively
// near-continuous under 8-writer churn. The generic settle refusing on a
// FOREIGN handle's root manufactured the multi-second 'recovery in
// progress' relay (trj registry receipts: distinct attempt ids, always
// ExactHandle, always zero pending/claimed). Exact-handle roots keep
// gating their OWN handle's settles via has_relevant_process_root in the
// per-handle settle; the generic settle refuses only on state that
// genuinely blocks everyone: an in-doubt epoch or an IDENTITY-WIDE root.
if queue.has_unresolved_in_doubt_epoch() || queue.has_identity_wide_process_root() {
// Ground truth on the holder — enumerate the registry's live
// attempts for this queue instead of inferring from queue-side
// counters.
let live_attempts: Vec<String> = {
let registry = process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
registry
.by_queue
.get(&queue.queue_id)
.map_or_else(Vec::new, |rooted| {
rooted
.attempts
.iter()
.map(|(attempt_id, scope)| format!("{attempt_id}:{scope:?}"))
.collect()
})
};
tracing::warn!(
target: "fsqlite.pager.group_commit",
unresolved_in_doubt_epoch = queue.has_unresolved_in_doubt_epoch(),
process_root_finalization_attempt = queue.has_process_root_finalization_attempt(),
live_attempts = ?live_attempts,
pending_cleanups = queue.pending_logical_cleanup_count(),
claimed_cleanups = queue
.claimed_logical_cleanups
.load(AtomicOrdering::Acquire),
"settle refused: residual in-doubt epoch or process-root attempt after cleanup"
);
return Err(FrankenError::BusyRecovery);
}
Ok(())
}
async fn settle_identity_wide_group_commit_finalization(queue: &GroupCommitQueueRef) -> Result<()> {
// Every identity-wide record and in-doubt epoch owns a process-root token
// before publication. Keep the ordinary waiter path free of recovery-map
// and pending-queue mutex traffic.
if !queue.has_process_root_finalization_attempt() {
return Ok(());
}
if !queue.has_identity_wide_process_root() && !queue.has_unresolved_in_doubt_epoch() {
return Ok(());
}
while queue
.resolve_one_pending_external_unlock_for(ProcessRootFinalizationSelector::IdentityWide)
.await?
{}
queue.resolve_pending_epoch_resolutions()?;
if queue.has_pending_or_claimed_identity_wide_external_unlock()
|| queue.has_unresolved_in_doubt_epoch()
|| queue.has_identity_wide_process_root()
{
// bd-b4mwn round 4: name the refusal (see the sibling settle fn).
tracing::warn!(
target: "fsqlite.pager.group_commit",
pending_or_claimed_identity_wide_unlock =
queue.has_pending_or_claimed_identity_wide_external_unlock(),
unresolved_in_doubt_epoch = queue.has_unresolved_in_doubt_epoch(),
identity_wide_process_root = queue.has_identity_wide_process_root(),
"identity-wide settle refused"
);
return Err(FrankenError::BusyRecovery);
}
Ok(())
}
async fn settle_pending_group_commit_finalization_for_handle(
queue: &GroupCommitQueueRef,
handle_key: SharedDbFileKey,
) -> Result<()> {
if !queue.has_identity_wide_process_root() && !queue.has_relevant_process_root(handle_key) {
return Ok(());
}
settle_identity_wide_group_commit_finalization(queue).await?;
while queue
.resolve_one_pending_external_unlock_for_handle(handle_key)
.await?
{}
if queue.has_pending_or_claimed_identity_wide_external_unlock()
|| queue.has_unresolved_in_doubt_epoch()
{
return Err(FrankenError::BusyRecovery);
}
let logical_cleanup_count = queue.pending_logical_cleanup_count_for_handle(handle_key);
let mut first_logical_cleanup_error = None;
for _ in 0..logical_cleanup_count {
if let Err(error) = queue
.resolve_one_pending_logical_cleanup_for_handle(handle_key)
.await
&& first_logical_cleanup_error.is_none()
{
first_logical_cleanup_error = Some(error);
}
}
if let Some(error) = first_logical_cleanup_error {
return Err(error);
}
if queue.has_identity_wide_process_root() || queue.has_relevant_process_root(handle_key) {
// bd-b4mwn round 4: name the refusal (see the sibling settle fns).
tracing::warn!(
target: "fsqlite.pager.group_commit",
identity_wide_process_root = queue.has_identity_wide_process_root(),
relevant_process_root = queue.has_relevant_process_root(handle_key),
"per-handle settle refused"
);
return Err(FrankenError::BusyRecovery);
}
Ok(())
}
async fn settle_process_root_finalizations_for_path(path: &Path) -> Result<()> {
let queues = process_root_finalization_queues_for_path(path);
for queue in queues {
settle_pending_group_commit_finalization(&queue).await?;
}
Ok(())
}
async fn settle_process_root_finalizations_for_identity(identity: FileIdentity) -> Result<()> {
let queues = process_root_finalization_queues_for_identity(identity);
for queue in queues {
settle_pending_group_commit_finalization(&queue).await?;
}
Ok(())
}
/// Remove the group commit queue for the given database path.
///
/// Called when the last connection using this path closes, to prevent stale
/// consolidator state (epoch, db_size) from leaking into future connections
/// that open a different file at the same path.
pub fn remove_group_commit_queue(db_path: &Path) {
if let Some(queues) = GROUP_COMMIT_QUEUES.get() {
let key = shared_file_state_key(db_path);
let mut queues = queues
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
queues.remove(&key);
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum WalCommitSyncPolicy {
Deferred,
PerCommit,
}
impl WalCommitSyncPolicy {
#[must_use]
const fn should_sync_on_commit(self) -> bool {
matches!(self, Self::PerCommit)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum PagerAccessMode {
ReadWrite,
ReadOnly,
}
impl PagerAccessMode {
#[must_use]
const fn is_readonly(self) -> bool {
matches!(self, Self::ReadOnly)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RollbackJournalRecoveryState {
Clean,
/// Exact ownership was reserved before creating or mutating the shared
/// rollback-journal artifact. The main database is still untouched, so
/// recovery may discard absent or proven non-hot construction debris. A
/// valid hot journal must nevertheless be replayed defensively: after the
/// native lock epoch ends, another process may replace the pathname with
/// its own recovery record. Malformed hot-looking data stays fail-closed.
JournalConstructionPending,
/// The complete pre-image payload is durable and final-header activation
/// has started, but no main-database write can have begun. Recovery must
/// discard an absent or proven non-hot activation artifact. A valid hot
/// journal must still be replayed defensively: after the native lock epoch
/// ends, another process may replace the pathname with its own recovery
/// record. Malformed hot-looking data stays fail-closed. This provenance
/// remains armed until metadata and external finalization are terminal.
JournalActivationPending,
/// A locally failed commit may have changed main-database bytes and still
/// requires a hot-journal replay.
ReplayPending,
/// Replay durably restored the main image, but connection-local metadata
/// has not yet been rebuilt and must be retried without requiring the now
/// non-hot journal to remain present.
MetadataRefreshPending,
/// The database image is durably committed and the journal is non-hot,
/// but transaction-local metadata publication and logical exit have not
/// both reached a terminal state. Recovery must preserve the committed
/// image; it may refresh metadata but must never replay this transaction's
/// pre-image journal.
DurableCommitFinalizationPending,
/// Durable replay and metadata refresh have completed, but the recovery
/// caller still owes the full external-finalization sequence: release the
/// maintenance fence, restore its typed same-process lease, reacquire the
/// transaction/snapshot fence, and verify that no new hot journal won the
/// intervening race. This state must survive future cancellation because
/// neither a clean journal nor a restored main image proves those lock
/// ownership obligations reached a terminal state.
ExternalFinalizationPending,
}
impl RollbackJournalRecoveryState {
#[must_use]
const fn is_pending(self) -> bool {
!matches!(self, Self::Clean)
}
/// Whether a pager-level begin/refresh may take over this state after the
/// transaction that created it has completed or been dropped.
///
/// A durable commit is different: its live transaction (or rooted detached
/// finalizer) owns the exact logical-exit and publication receipt. Treating
/// that state as generic recovery would let a sibling acknowledge another
/// transaction's commit and strand the real owner.
#[must_use]
const fn is_pager_takeover_eligible(self) -> bool {
matches!(
self,
Self::JournalConstructionPending
| Self::JournalActivationPending
| Self::ReplayPending
| Self::MetadataRefreshPending
| Self::ExternalFinalizationPending
)
}
#[must_use]
const fn needs_replay(self) -> bool {
matches!(self, Self::ReplayPending)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RollbackJournalPrefixState {
NonHot,
Hot,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum RollbackJournalReplayOutcome {
NonHot,
Replayed(PageSize),
}
const LOCAL_JOURNAL_MARKER: [u8; 16] = *b"FSQLITE-JRNL-v1\0";
fn mark_local_journal_header(header: &mut [u8]) {
let marker_start = crate::journal::JOURNAL_HEADER_SIZE;
let marker_end = marker_start + LOCAL_JOURNAL_MARKER.len();
debug_assert!(header.len() >= marker_end);
header[marker_start..marker_end].copy_from_slice(&LOCAL_JOURNAL_MARKER);
}
async fn local_journal_marker_present<F: VfsFile>(
cx: &Cx,
journal_file: &F,
padded_header_size: u64,
) -> Result<bool> {
let marker_start =
u64::try_from(crate::journal::JOURNAL_HEADER_SIZE).expect("journal header size fits u64");
let marker_span =
u64::try_from(LOCAL_JOURNAL_MARKER.len()).expect("journal marker size fits u64");
if padded_header_size < marker_start + marker_span {
return Ok(false);
}
let mut marker = [0_u8; LOCAL_JOURNAL_MARKER.len()];
let bytes_read = journal_file.read(cx, &mut marker, marker_start).await?;
if bytes_read != marker.len() {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short rollback-journal local marker read: got {bytes_read} of {} bytes",
marker.len()
),
});
}
Ok(marker == LOCAL_JOURNAL_MARKER)
}
#[derive(Debug, Clone, Copy)]
enum JournalInvalidation {
ZeroMagic,
Truncate,
}
async fn durable_invalidate_journal<F: VfsFile>(
cx: &Cx,
journal_file: &mut F,
invalidation: JournalInvalidation,
) -> Result<()> {
match invalidation {
JournalInvalidation::ZeroMagic => {
journal_file
.write(cx, &[0_u8; JOURNAL_MAGIC.len()], 0)
.await?;
journal_file.durable_sync(cx, SyncKind::FullDurable)?;
let mut observed = [0_u8; JOURNAL_MAGIC.len()];
let bytes_read = journal_file.read(cx, &mut observed, 0).await?;
if bytes_read != observed.len() || observed.iter().any(|byte| *byte != 0) {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback-journal invalidation did not persist zero magic: read {bytes_read} bytes, prefix={observed:02x?}"
),
});
}
}
JournalInvalidation::Truncate => {
journal_file.truncate(cx, 0)?;
journal_file.durable_sync(cx, SyncKind::FullDurable)?;
let observed_size = journal_file.file_size(cx)?;
if observed_size != 0 {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback-journal truncation reported success but retained {observed_size} bytes"
),
});
}
}
}
Ok(())
}
async fn durable_write_and_verify_journal_header<F: VfsFile>(
cx: &Cx,
journal_file: &mut F,
header: &[u8],
) -> Result<()> {
journal_file.write(cx, header, 0).await?;
journal_file.durable_sync(cx, SyncKind::FullDurable)?;
let mut observed = vec![0_u8; header.len()];
let bytes_read = journal_file.read(cx, &mut observed, 0).await?;
if bytes_read != header.len() || observed != header {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback-journal hot-header write was not durable: read {bytes_read} of {} bytes",
header.len()
),
});
}
Ok(())
}
async fn classify_rollback_journal_prefix<F: VfsFile>(
cx: &Cx,
journal_file: &F,
) -> Result<(RollbackJournalPrefixState, u64)> {
let journal_size = journal_file.file_size(cx)?;
if journal_size == 0 {
return Ok((RollbackJournalPrefixState::NonHot, 0));
}
let prefix_len = usize::try_from(journal_size.min(JOURNAL_MAGIC.len() as u64))
.expect("rollback-journal magic length fits usize");
let mut prefix = [0_u8; JOURNAL_MAGIC.len()];
let bytes_read = journal_file.read(cx, &mut prefix[..prefix_len], 0).await?;
if bytes_read != prefix_len {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short rollback-journal magic read: got {bytes_read} of {prefix_len} bytes"
),
});
}
// SQLite's PERSIST commit marker is the first journal-header byte. Once
// that byte is zero the journal is non-hot, even if a torn sector write
// left later bytes from the old magic in place. A nonzero malformed
// prefix remains fail-closed.
if prefix[0] == 0 {
return Ok((RollbackJournalPrefixState::NonHot, journal_size));
}
if prefix_len < JOURNAL_MAGIC.len() {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("rollback journal has a nonzero truncated magic: {journal_size} bytes"),
});
}
if prefix == JOURNAL_MAGIC {
Ok((RollbackJournalPrefixState::Hot, journal_size))
} else {
Err(FrankenError::DatabaseCorrupt {
detail: format!("rollback journal has invalid magic: {prefix:02x?}"),
})
}
}
async fn with_main_shared_lock<F, S, T>(
cx: &Cx,
queue: &Arc<GroupCommitQueue>,
db_file: &SharedDbFile<F>,
state: &mut S,
operation: impl for<'a> FnOnce(&'a Cx, &'a F, &'a mut S) -> LocalPagerFuture<'a, T>,
) -> Result<T>
where
F: VfsFile + 'static,
{
let mut attempt = BeginExternalLockState::new(queue, Arc::clone(db_file), cx);
attempt.acquire_snapshot(cx).await?;
let file = shared_db_file_read(db_file, cx).await?;
let operation_result = operation(cx, &*file, state).await;
drop(file);
let unlock_result = attempt.restore().await;
drop(attempt);
match (operation_result, unlock_result) {
(Ok(value), Ok(())) => Ok(value),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(operation_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"database snapshot failed and could not release the main-file SHARED lock: operation={operation_error}; unlock={unlock_error}"
))),
}
}
// ---------------------------------------------------------------------------
// Immutable committed-state snapshot (bd-db300.5.3.3.1 / Card 1: M6)
// ---------------------------------------------------------------------------
/// Frozen read-only snapshot of pager committed state.
///
/// Published atomically on every commit via `RwLock<Arc<...>>`. Readers
/// clone the `Arc` (nanosecond RwLock-read hold) then inspect fields without
/// touching the `PagerInner` Mutex. This eliminates the #1 hot-path Mutex
/// acquisition for read-only begin checks and staleness probes.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PagerCommittedSnapshot {
/// Monotonic commit sequence at publication time.
pub commit_seq: CommitSeq,
/// Database size in pages.
pub db_size: u32,
/// Active journal mode.
pub journal_mode: JournalMode,
/// Number of pages on the freelist.
pub freelist_count: usize,
/// Whether a checkpoint was active when this snapshot was taken.
pub checkpoint_active: bool,
/// Whether a writer transaction was active when this snapshot was taken.
pub writer_active: bool,
/// File size in bytes at snapshot time (for staleness detection).
pub db_file_size_bytes: u64,
}
impl PagerCommittedSnapshot {
/// Build a snapshot from the current `PagerInner` state.
/// Caller must hold the PagerInner Mutex.
fn from_inner<F: VfsFile>(inner: &PagerInner<F>) -> Self {
Self {
commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
writer_active: inner.writer_active,
db_file_size_bytes: inner.committed_db_file_size_bytes,
}
}
}
/// The inner mutable pager state protected by a mutex.
pub(crate) struct PagerInner<F: VfsFile> {
/// Handle to the main database file.
db_file: SharedDbFile<F>,
/// Stable main-file namespace and identity captured when this pager opened.
/// An exact recovery receipt snapshots these values before its first
/// journal side effect so later adopters can validate the origin.
database_path: PathBuf,
database_identity: Option<FileIdentity>,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: Option<Arc<DatabaseNamespaceBinding>>,
/// Page size for this database.
page_size: PageSize,
/// Current database size in pages.
db_size: u32,
/// Next page to allocate (1-based).
next_page: u32,
/// Whether a writer transaction is currently active.
writer_active: bool,
/// Number of active transactions (readers + writers).
active_transactions: u32,
/// Whether a checkpoint is currently running.
checkpoint_active: bool,
/// Whether this pager was opened read-only (skip freelist
/// scans during refresh since we never allocate pages).
access_mode: PagerAccessMode,
/// Deallocated pages available for reuse.
freelist: Vec<PageNumber>,
/// Current journal mode (rollback journal vs WAL).
journal_mode: JournalMode,
/// WAL commit sync policy derived from `PRAGMA synchronous`.
wal_commit_sync_policy: WalCommitSyncPolicy,
/// Whether this pager has a locally failed rollback-journal commit that
/// must be repaired before the handle can be reused.
rollback_journal_recovery_state: RollbackJournalRecoveryState,
/// Exact monotonic identity-wide owner for the non-clean state above.
rollback_journal_recovery_owner: Option<RollbackRecoveryOwnerId>,
/// Exact main/journal namespace belonging to the recovery receipt. File
/// identities can have multiple lexical aliases, so an adopted owner must
/// validate and recover the origin namespace rather than deriving one from
/// the surviving pager.
rollback_journal_recovery_namespace: Option<RollbackRecoveryNamespace>,
/// Strong ownership of the canonical identity gate for as long as this
/// pager state (including any transaction or detached finalizer retaining
/// it) remains live. The identity registry stores only a `Weak`, so keeping
/// merely the counter alive would allow a reopen to install a fresh gate
/// and bypass an outstanding recovery receipt.
maintenance_gate: Arc<PagerMaintenanceGate>,
/// Lock-free mirror of every non-clean rollback-recovery state. Active
/// transactions clone this `Arc` so sibling handles fail closed without
/// taking `PagerInner` on each page operation.
rollback_recovery_pending: Arc<AtomicUsize>,
// NOTE: wal_backend moved to SharedWalBackend on SimplePager/SimpleTransaction (D1-CRITICAL)
/// Monotonic commit sequence for MVCC version tracking.
commit_seq: CommitSeq,
/// Main database-file size observed when committed metadata was last fully
/// refreshed. A stable `(commit_seq, file_size)` pair lets later begins
/// skip the expensive committed-page metadata reload when no durable state
/// changed underneath this pager.
committed_db_file_size_bytes: u64,
/// Durable database-header change counter from the last committed-state
/// metadata probe. In WAL mode this is the stable base added to the
/// currently visible WAL commit count.
committed_db_change_counter: u64,
/// WAL generation paired with `committed_db_change_counter`. If the WAL
/// generation changes, the main DB header must be read again before the
/// cached base counter can be trusted.
committed_wal_generation: Option<WalGenerationIdentity>,
/// Visible WAL commit count paired with the cached base counter. External
/// checkpoints can move commits from WAL into the main database while the
/// summed visible commit sequence stays the same; this keeps that physical
/// composition change observable to cache invalidation.
committed_wal_visible_commit_count: u64,
}
impl<F: VfsFile> Drop for PagerInner<F> {
fn drop(&mut self) {
if let Some(owner) = self.rollback_journal_recovery_owner {
// Never expose a clean identity atomic merely because this one
// PagerInner disappeared. Other handles may still have transactions
// that rely on the shared barrier to avoid reading through a hot or
// partially applied journal. Transfer the exact receipt to the
// identity gate; a surviving pager may adopt it only after every
// pre-existing identity lease drains.
let orphan_result = self
.rollback_journal_recovery_namespace
.take()
.ok_or_else(|| {
FrankenError::internal(
"PagerInner Drop recovery owner lost its origin namespace",
)
})
.and_then(|namespace| {
self.maintenance_gate.orphan_rollback_recovery_owner(
owner,
self.rollback_journal_recovery_state,
namespace,
)
});
if let Err(error) = orphan_result {
tracing::error!(
%error,
"PagerInner Drop could not retain its exact orphaned rollback-recovery owner"
);
}
}
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct CommittedStateRefresh {
wal_snapshot_initialized: bool,
page_cache_invalidated: bool,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct VisibleCommitProbe {
visible_commit_seq: CommitSeq,
file_size: u64,
wal_snapshot_initialized: bool,
durable_identity_changed: bool,
db_change_counter: u64,
wal_generation: Option<WalGenerationIdentity>,
wal_visible_commit_count: u64,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum CommittedStateRefreshMode {
Normal,
/// A hot rollback journal has already restored the durable bytes. Cached
/// candidate metadata is never authoritative in this mode: bypass the
/// identity fast path and allow the database extent to shrink exactly.
PostRecovery,
}
#[derive(Debug, Clone, PartialEq, Eq)]
enum PendingGroupCommitTxnResolution {
Pending,
Authorized(ParallelWalPublicationAuthorization),
NotCommitted,
}
#[derive(Clone, Default)]
struct PendingReturnedAllocations {
from_freelist: Vec<PageNumber>,
from_eof: Vec<PageNumber>,
page_lease: Vec<PageNumber>,
}
impl PendingReturnedAllocations {
fn all_pages(&self) -> Vec<PageNumber> {
self.from_freelist
.iter()
.chain(&self.from_eof)
.chain(&self.page_lease)
.copied()
.collect()
}
}
#[derive(Clone, Default)]
struct PendingGroupCommitAllocatorDelta {
live_committed_allocations: Vec<PageNumber>,
returned_or_freed_pages: Vec<PageNumber>,
}
impl PendingGroupCommitAllocatorDelta {
fn new(
live_committed_allocations: Vec<PageNumber>,
returned_allocations: &PendingReturnedAllocations,
pending_freed_pages: &[PageNumber],
) -> Self {
let mut delta = Self {
live_committed_allocations,
returned_or_freed_pages: returned_allocations
.all_pages()
.into_iter()
.chain(pending_freed_pages.iter().copied())
.collect(),
};
delta.normalize();
delta
}
fn extend(&mut self, other: Self) {
self.live_committed_allocations
.extend(other.live_committed_allocations);
self.returned_or_freed_pages
.extend(other.returned_or_freed_pages);
}
fn normalize(&mut self) {
self.live_committed_allocations.sort_unstable();
self.live_committed_allocations.dedup();
self.returned_or_freed_pages
.retain(|page| self.live_committed_allocations.binary_search(page).is_err());
self.returned_or_freed_pages.sort_unstable();
self.returned_or_freed_pages.dedup();
}
fn apply_to_freelist(&self, freelist: &mut Vec<PageNumber>) {
freelist.retain(|page| self.live_committed_allocations.binary_search(page).is_err());
return_pages_to_freelist(freelist, self.returned_or_freed_pages.iter().copied());
}
}
#[derive(Default)]
struct PhaseAWriteSetUndo {
entries: Mutex<HashMap<PageNumber, Option<PageData>>>,
}
impl PhaseAWriteSetUndo {
fn capture<S: std::hash::BuildHasher>(
&self,
write_set: &HashMap<PageNumber, StagedPage, S>,
page_no: PageNumber,
) {
let mut entries = self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
entries
.entry(page_no)
.or_insert_with(|| write_set.get(&page_no).map(StagedPage::published_page));
}
fn restore<S: std::hash::BuildHasher>(
&self,
write_set: &mut HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &mut Vec<PageNumber>,
) {
let entries = std::mem::take(
&mut *self
.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
);
for (page_no, original) in entries {
match original {
Some(page) => {
write_set.insert(page_no, StagedPage::from_page_data(page));
insert_page_sorted(write_pages_sorted, page_no);
}
None => {
write_set.remove(&page_no);
remove_page_sorted(write_pages_sorted, page_no);
}
}
}
}
fn clear(&self) {
self.entries
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clear();
}
}
struct PendingGroupCommitNotCommittedState {
returned_allocations: PendingReturnedAllocations,
pending_freed_pages: Vec<PageNumber>,
}
struct PendingGroupCommitTxnAttemptState {
consumer: Option<Arc<GroupCommitEpochConsumer>>,
batch_id: Option<u64>,
resolution: PendingGroupCommitTxnResolution,
returned_allocations: Option<PendingReturnedAllocations>,
pending_freed_pages: Vec<PageNumber>,
publication_intent: Option<ParallelWalPublicationIntent>,
publication_applied: bool,
transaction_exit_complete: bool,
terminal: bool,
}
struct PendingGroupCommitTxnAttempt<F: VfsFile> {
queue: Weak<GroupCommitQueue>,
inner: Arc<Mutex<PagerInner<F>>>,
/// Retains the exact open handle for the lifetime of every deferred
/// logical-exit claim. This makes `SharedDbFileKey` immune to allocator
/// address reuse while the claim or its queued operation is live.
db_file: SharedDbFile<F>,
committed_snapshot: Arc<RwLock<Arc<PagerCommittedSnapshot>>>,
published: Arc<PublishedPagerState>,
writer_idle: Arc<Condvar>,
cleanup_cx: Cx,
committed_db_size: u32,
mode: TransactionMode,
is_writer: bool,
staged_page_high_water: u32,
allocator_delta: PendingGroupCommitAllocatorDelta,
phase_a_undo: PhaseAWriteSetUndo,
state: Mutex<PendingGroupCommitTxnAttemptState>,
}
impl<F: VfsFile + 'static> PendingGroupCommitTxnAttempt<F> {
#[allow(clippy::too_many_arguments)]
fn new(
queue: &GroupCommitQueueRef,
inner: Arc<Mutex<PagerInner<F>>>,
db_file: SharedDbFile<F>,
committed_snapshot: Arc<RwLock<Arc<PagerCommittedSnapshot>>>,
published: Arc<PublishedPagerState>,
writer_idle: Arc<Condvar>,
cleanup_cx: Cx,
committed_db_size: u32,
mode: TransactionMode,
is_writer: bool,
staged_page_high_water: u32,
returned_allocations: PendingReturnedAllocations,
pending_freed_pages: Vec<PageNumber>,
live_committed_allocations: Vec<PageNumber>,
) -> Self {
let allocator_delta = PendingGroupCommitAllocatorDelta::new(
live_committed_allocations,
&returned_allocations,
&pending_freed_pages,
);
Self {
queue: Arc::downgrade(queue),
inner,
db_file,
committed_snapshot,
published,
writer_idle,
cleanup_cx,
committed_db_size,
mode,
is_writer,
staged_page_high_water,
allocator_delta,
phase_a_undo: PhaseAWriteSetUndo::default(),
state: Mutex::new(PendingGroupCommitTxnAttemptState {
consumer: None,
batch_id: None,
resolution: PendingGroupCommitTxnResolution::Pending,
returned_allocations: Some(returned_allocations),
pending_freed_pages,
publication_intent: None,
publication_applied: false,
transaction_exit_complete: false,
terminal: false,
}),
}
}
fn admit(&self, consumer: Arc<GroupCommitEpochConsumer>, batch_id: u64) -> Result<()> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.consumer.is_some() || state.batch_id.is_some() {
return Err(FrankenError::internal(
"group-commit transaction attempt was admitted twice",
));
}
state.consumer = Some(consumer);
state.batch_id = Some(batch_id);
Ok(())
}
fn evidence_key(&self) -> Option<(u64, u64)> {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Some((state.consumer.as_ref()?.epoch, state.batch_id?))
}
fn resolution(&self) -> PendingGroupCommitTxnResolution {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.resolution
.clone()
}
fn publication_intent(&self) -> Result<ParallelWalPublicationIntent> {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.publication_intent
.ok_or_else(|| {
FrankenError::internal(
"authorized group-commit transaction has no publication intent",
)
})
}
fn projected_db_size_with_inner(&self, inner: &PagerInner<F>) -> u32 {
self.allocator_delta
.live_committed_allocations
.iter()
.map(|page| page.get())
.max()
.unwrap_or(self.committed_db_size)
.max(self.committed_db_size)
.max(inner.db_size)
}
fn projected_db_size(&self) -> u32 {
self.inner.lock().map_or(self.committed_db_size, |inner| {
self.projected_db_size_with_inner(&inner)
})
}
fn transaction_visible_db_size_bound(&self, snapshot_db_size: u32) -> u32 {
self.allocator_delta
.live_committed_allocations
.iter()
.map(|page| page.get())
.max()
.unwrap_or(snapshot_db_size)
.max(snapshot_db_size)
.max(self.staged_page_high_water)
}
fn projected_live_freelist(&self) -> Vec<PageNumber> {
self.inner.lock().map_or_else(
|_| Vec::new(),
|inner| {
let projected_db_size = self.projected_db_size_with_inner(&inner);
let upper_bound = inner.next_page.saturating_sub(1).max(projected_db_size);
let mut freelist = inner.freelist.clone();
self.allocator_delta.apply_to_freelist(&mut freelist);
normalize_freelist(&freelist, upper_bound)
.into_iter()
.filter(|page| page.get() <= projected_db_size)
.collect()
},
)
}
fn reconcile_global_from_queue(&self) -> Result<PendingGroupCommitTxnResolution> {
if !matches!(self.resolution(), PendingGroupCommitTxnResolution::Pending) {
return Ok(self.resolution());
}
let queue = self.queue.upgrade().ok_or_else(|| {
FrankenError::internal(
"group-commit queue dropped before logical transaction finalization",
)
})?;
let Some((epoch, batch_id)) = self.evidence_key() else {
self.complete_not_committed_global()?;
return Ok(self.resolution());
};
if let Some(persisted) = queue.persisted_epoch_for(epoch) {
if !persisted.members.contains(&batch_id) {
return Err(FrankenError::internal(format!(
"persisted group-commit epoch {epoch} omits admitted batch {batch_id}"
)));
}
// Certificate persistence precedes logical Phase C by design.
// The receipt proves durability, but only the physical recovery
// owner has the complete consolidated page plane needed to apply
// the Authorized verdict. Keep the logical attempt pending until
// that owner publishes the page plane and terminal resolution.
return Ok(PendingGroupCommitTxnResolution::Pending);
} else if queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&epoch)
{
self.complete_not_committed_global()?;
queue.unregister_txn_attempt(batch_id);
}
Ok(self.resolution())
}
fn take_not_committed_state(&self) -> Result<PendingGroupCommitNotCommittedState> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !matches!(
state.resolution,
PendingGroupCommitTxnResolution::NotCommitted
) {
return Err(FrankenError::internal(
"cannot restore transaction state before a NotCommitted verdict",
));
}
Ok(PendingGroupCommitNotCommittedState {
returned_allocations: state.returned_allocations.take().ok_or_else(|| {
FrankenError::internal(
"NotCommitted group-commit allocations were already restored",
)
})?,
pending_freed_pages: std::mem::take(&mut state.pending_freed_pages),
})
}
fn restore_phase_a_write_set<S: std::hash::BuildHasher>(
&self,
write_set: &mut HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &mut Vec<PageNumber>,
) {
self.phase_a_undo.restore(write_set, write_pages_sorted);
}
#[allow(clippy::await_holding_lock)]
async fn finish_txn_exit(
&self,
logical_exit_claim: &GroupCommitLogicalExitClaim,
) -> Result<()> {
let queue = self.queue.upgrade().ok_or_else(|| {
FrankenError::internal("group-commit queue dropped before logical transaction exit")
})?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.transaction_exit_complete {
return Ok(());
}
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let mut inner = match self.inner.lock() {
Ok(inner) => inner,
Err(error) => {
tracing::error!(
"group-commit transaction exit recovered a poisoned PagerInner for fail-closed cleanup"
);
error.into_inner()
}
};
let releases_writer_baton = self.is_writer && self.mode != TransactionMode::Concurrent;
let notify_writer_idle = coordinated_transaction_exit(
&queue,
&self.cleanup_cx,
&mut inner,
releases_writer_baton,
logical_exit_claim,
)
.await?;
state.transaction_exit_complete = true;
drop(inner);
drop(state);
if notify_writer_idle {
self.writer_idle.notify_one();
}
Ok(())
}
fn finish_terminal(&self, transaction_released: bool) -> Result<()> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.terminal {
return Ok(());
}
if matches!(state.resolution, PendingGroupCommitTxnResolution::Pending) {
return Err(FrankenError::internal(
"group-commit transaction finalized before a terminal verdict",
));
}
if matches!(
state.resolution,
PendingGroupCommitTxnResolution::Authorized(_)
) && !state.publication_applied
{
return Err(FrankenError::internal(
"authorized group-commit transaction finalized before publication",
));
}
if transaction_released && !state.transaction_exit_complete {
return Err(FrankenError::internal(
"group-commit transaction finalized before exit and snapshot release",
));
}
state.consumer.take();
state.terminal = true;
self.phase_a_undo.clear();
Ok(())
}
}
impl<F: VfsFile + 'static> PendingGroupCommitTxnAttemptOperation
for PendingGroupCommitTxnAttempt<F>
{
fn pager_inner_identity(&self) -> *const () {
Arc::as_ptr(&self.inner).cast()
}
fn allocator_delta(&self) -> PendingGroupCommitAllocatorDelta {
self.allocator_delta.clone()
}
fn complete_authorized_global(
&self,
authorization: ParallelWalPublicationAuthorization,
complete_group_pages: &HashMap<PageNumber, PageData>,
group_allocator_delta: &PendingGroupCommitAllocatorDelta,
apply_allocator_delta: bool,
) -> Result<()> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match &state.resolution {
PendingGroupCommitTxnResolution::Authorized(existing) => {
if existing == &authorization {
return Ok(());
}
return Err(FrankenError::internal(
"group-commit transaction received two different authorized verdicts",
));
}
PendingGroupCommitTxnResolution::NotCommitted => {
return Err(FrankenError::internal(
"cannot authorize a group-commit transaction already proven NotCommitted",
));
}
PendingGroupCommitTxnResolution::Pending => {}
}
if state
.batch_id
.is_some_and(|batch_id| batch_id != authorization.batch_id)
{
return Err(FrankenError::internal(
"authorized group-commit batch does not match its logical owner",
));
}
let mut inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut committed_freelist = inner.freelist.clone();
if apply_allocator_delta {
group_allocator_delta.apply_to_freelist(&mut committed_freelist);
}
state.returned_allocations.as_ref().ok_or_else(|| {
FrankenError::internal(
"authorized group-commit returned allocations were already finalized",
)
})?;
let committed_db_size = inner.db_size.max(self.committed_db_size);
let publication_intent = parallel_wal_publication_intent(
&authorization,
committed_db_size,
inner.journal_mode,
committed_freelist.len(),
inner.checkpoint_active,
)?;
state.returned_allocations.take();
if apply_allocator_delta {
inner.freelist = committed_freelist;
}
state.pending_freed_pages.clear();
inner.db_size = publication_intent.db_size;
let next_unallocated_page = if inner.db_size >= 2 {
inner.db_size.saturating_add(1)
} else {
2
};
inner.next_page = inner.next_page.max(next_unallocated_page);
let certificate_commit_seq_lo = authorization.durability_receipt.certificate.commit_seq_lo;
inner.record_local_wal_commit_at(publication_intent.visible_commit_seq);
let committed_snapshot = Arc::new(PagerCommittedSnapshot::from_inner(&inner));
*self
.committed_snapshot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = committed_snapshot;
let update = PublishedPagerUpdate {
visible_commit_seq: publication_intent.visible_commit_seq,
db_size: publication_intent.db_size,
journal_mode: publication_intent.journal_mode,
freelist_count: publication_intent.freelist_count,
checkpoint_active: publication_intent.checkpoint_active,
};
state.publication_intent = Some(publication_intent);
state.resolution = PendingGroupCommitTxnResolution::Authorized(authorization);
drop(inner);
self.published.publish_prepared_parallel_wal_group(
&self.cleanup_cx,
update,
complete_group_pages.clone(),
certificate_commit_seq_lo,
);
self.published
.bind_parallel_wal_publication(publication_intent);
state.publication_applied = true;
self.phase_a_undo.clear();
Ok(())
}
fn complete_not_committed_global(&self) -> Result<()> {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match state.resolution {
PendingGroupCommitTxnResolution::NotCommitted => return Ok(()),
PendingGroupCommitTxnResolution::Authorized(_) => {
return Err(FrankenError::internal(
"cannot reject a group-commit transaction already authorized",
));
}
PendingGroupCommitTxnResolution::Pending => {}
}
state.resolution = PendingGroupCommitTxnResolution::NotCommitted;
Ok(())
}
}
trait PendingGroupCommitLogicalCleanupOperation: Send {
/// Exact open-handle key whose transition this cleanup will perform.
///
/// Every implementor must retain the corresponding `SharedDbFile` for at
/// least as long as the operation so the pointer-derived key cannot be
/// reused for a different handle.
fn handle_key(&self) -> SharedDbFileKey;
fn resolve<'a>(
&'a mut self,
logical_exit_claim: &'a GroupCommitLogicalExitClaim,
) -> LocalPagerFuture<'a, bool>;
}
/// bd-b4mwn round 4: settles observed with the attempt still `Pending` and
/// its epoch absent from BOTH the persisted and failed maps before the epoch
/// is declared abandoned and force-failed. Multiple observations tolerate an
/// in-flight leader that has simply not landed the certificate yet.
const ABANDONED_EPOCH_SETTLE_OBSERVATIONS: u32 = 3;
struct DetachedPendingGroupCommitTxnCleanup<F: VfsFile + 'static> {
attempt: Arc<PendingGroupCommitTxnAttempt<F>>,
maintenance_lease: Option<PagerMaintenanceLease>,
allocated_from_freelist: Vec<PageNumber>,
allocated_from_eof: Vec<PageNumber>,
page_lease: Vec<PageNumber>,
allocation_cleanup_applied: bool,
/// bd-b4mwn round 4: count of settle passes that found this detached
/// attempt `Pending` with a pre-persistence (abandonable) epoch.
stale_pending_observations: u32,
}
impl<F: VfsFile + 'static> PendingGroupCommitLogicalCleanupOperation
for DetachedPendingGroupCommitTxnCleanup<F>
{
fn handle_key(&self) -> SharedDbFileKey {
shared_db_file_key(&self.attempt.db_file)
}
fn resolve<'a>(
&'a mut self,
logical_exit_claim: &'a GroupCommitLogicalExitClaim,
) -> LocalPagerFuture<'a, bool> {
Box::pin(async move {
match self.attempt.reconcile_global_from_queue()? {
PendingGroupCommitTxnResolution::Pending => {
// bd-b4mwn round 4: a worker dropped mid-commit leaves
// its epoch non-terminal, and this detached cleanup
// previously re-queued forever with its process-root
// token armed — refusing every settle on the path with
// BusyRecovery. Distinguish the two Pending flavors:
//
// * POST-persistence (certificate durable, page plane
// unpublished): force-failing would un-commit durable
// data — stay conservative and warn; recovery or the
// publication owner must finish it.
// * PRE-persistence (epoch in NEITHER the persisted nor
// the failed map): no durable certificate exists, so
// NotCommitted is the correct verdict. After several
// settle observations (tolerating an in-flight leader
// that has not landed the certificate yet), declare
// the epoch abandoned and publish it failed; the next
// settle's reconcile then reaches the NotCommitted
// arm, completes the cleanup, and releases the root.
if let Some(queue) = self.attempt.queue.upgrade()
&& let Some((epoch, batch_id)) = self.attempt.evidence_key()
{
if queue.persisted_epoch_for(epoch).is_some() {
tracing::warn!(
target: "fsqlite.pager.group_commit",
epoch,
batch_id,
"detached commit attempt awaits page-plane \
publication of a durable epoch; retaining \
fail-closed root"
);
} else if !queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&epoch)
{
self.stale_pending_observations =
self.stale_pending_observations.saturating_add(1);
if self.stale_pending_observations
>= ABANDONED_EPOCH_SETTLE_OBSERVATIONS
{
tracing::warn!(
target: "fsqlite.pager.group_commit",
epoch,
batch_id,
observations = self.stale_pending_observations,
"declaring pre-persistence group-commit \
epoch abandoned by its dropped owner; \
aborting the flush so the detached \
cleanup can reach NotCommitted"
);
// Production fail path: only aborts the epoch
// if it is the ACTIVE FLUSHING one (the
// dead-mid-flush leader shape); any other
// phase errors closed — log and stay pending
// rather than force state we cannot prove.
if let Err(abort_error) = queue.abort_flushing_epoch_as_failed(
epoch,
&FrankenError::internal(
"group-commit epoch abandoned by dropped owner \
before certificate persistence",
),
) {
tracing::warn!(
target: "fsqlite.pager.group_commit",
epoch,
batch_id,
%abort_error,
"abandoned-epoch abort refused; \
retaining fail-closed root"
);
}
}
}
}
return Ok(false);
}
PendingGroupCommitTxnResolution::Authorized(_) => {
if !self.allocation_cleanup_applied {
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
self.page_lease.clear();
self.allocation_cleanup_applied = true;
}
}
PendingGroupCommitTxnResolution::NotCommitted => {
if !self.allocation_cleanup_applied {
let not_committed = self.attempt.take_not_committed_state()?;
let mut inner = self
.attempt
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
return_pages_to_freelist(
&mut inner.freelist,
self.allocated_from_freelist.drain(..),
);
return_pages_to_freelist(
&mut inner.freelist,
not_committed.returned_allocations.from_freelist,
);
// Deferred cleanup may run after later transactions
// reserved higher EOF pages. Never rewind next_page;
// quarantine every abandoned EOF reservation in the
// in-memory freelist instead.
return_pages_to_freelist(
&mut inner.freelist,
self.allocated_from_eof.drain(..),
);
return_pages_to_freelist(&mut inner.freelist, self.page_lease.drain(..));
return_pages_to_freelist(
&mut inner.freelist,
not_committed.returned_allocations.from_eof,
);
return_pages_to_freelist(
&mut inner.freelist,
not_committed.returned_allocations.page_lease,
);
drop(not_committed.pending_freed_pages);
self.allocation_cleanup_applied = true;
}
}
}
self.attempt.finish_txn_exit(logical_exit_claim).await?;
self.maintenance_lease.take();
self.attempt.finish_terminal(true)?;
Ok(true)
})
}
}
struct DetachedTransactionExit<F: VfsFile + 'static> {
queue: Arc<GroupCommitQueue>,
inner: Arc<Mutex<PagerInner<F>>>,
/// Owns the exact handle for the entire deferred-exit lifetime.
db_file: SharedDbFile<F>,
writer_idle: Arc<Condvar>,
cleanup_cx: Cx,
mode: TransactionMode,
is_writer: bool,
maintenance_lease: Option<PagerMaintenanceLease>,
}
impl<F: VfsFile + 'static> PendingGroupCommitLogicalCleanupOperation
for DetachedTransactionExit<F>
{
fn handle_key(&self) -> SharedDbFileKey {
shared_db_file_key(&self.db_file)
}
// The exact-handle logical-exit claim excludes every physical transition
// while this guard protects the matching PagerInner state update.
#[allow(clippy::await_holding_lock)]
fn resolve<'a>(
&'a mut self,
logical_exit_claim: &'a GroupCommitLogicalExitClaim,
) -> LocalPagerFuture<'a, bool> {
Box::pin(async move {
let _cleanup_mask = self.cleanup_cx.masked();
let mut inner = match self.inner.lock() {
Ok(inner) => inner,
Err(error) => {
tracing::error!(
"detached transaction exit recovered a poisoned PagerInner for fail-closed cleanup"
);
error.into_inner()
}
};
let releases_writer_baton = self.is_writer && self.mode != TransactionMode::Concurrent;
let notify_writer_idle = coordinated_transaction_exit(
&self.queue,
&self.cleanup_cx,
&mut inner,
releases_writer_baton,
logical_exit_claim,
)
.await?;
drop(inner);
self.maintenance_lease.take();
if notify_writer_idle {
self.writer_idle.notify_one();
}
Ok(true)
})
}
}
/// Rooted cleanup for a rollback-journal commit whose durable decision and
/// in-memory metadata application are complete, but whose commit future was
/// dropped while logical transaction exit was still pending.
///
/// Staged pages are deliberately not retained by this rare path. The database
/// file is already the durable authority, so terminal cleanup invalidates both
/// volatile page surfaces and forces later readers to refill from that image.
/// This keeps the detached receipt compact while preserving exact-once commit
/// sequence and transaction-exit accounting.
struct DetachedDurableRollbackCommitExit<F: VfsFile + 'static> {
queue: Arc<GroupCommitQueue>,
inner: Arc<Mutex<PagerInner<F>>>,
/// Owns the exact handle for the entire deferred-exit lifetime.
db_file: SharedDbFile<F>,
writer_idle: Arc<Condvar>,
cleanup_cx: Cx,
mode: TransactionMode,
maintenance_lease: Option<PagerMaintenanceLease>,
recovery_owner: RollbackRecoveryOwnerId,
/// A cancelled/requeued cleanup must not decrement the transaction count
/// or restore the external snapshot twice after that phase has completed.
logical_exit_completed: bool,
cache: Arc<ShardedPageCache>,
published: Arc<PublishedPagerState>,
committed_snapshot: Arc<RwLock<Arc<PagerCommittedSnapshot>>>,
}
impl<F: VfsFile + 'static> PendingGroupCommitLogicalCleanupOperation
for DetachedDurableRollbackCommitExit<F>
{
fn handle_key(&self) -> SharedDbFileKey {
shared_db_file_key(&self.db_file)
}
#[allow(clippy::await_holding_lock)]
fn resolve<'a>(
&'a mut self,
logical_exit_claim: &'a GroupCommitLogicalExitClaim,
) -> LocalPagerFuture<'a, bool> {
Box::pin(async move {
let _cleanup_mask = self.cleanup_cx.masked();
let mut inner = match self.inner.lock() {
Ok(inner) => inner,
Err(error) => {
tracing::error!(
"detached durable rollback-commit exit recovered a poisoned PagerInner"
);
error.into_inner()
}
};
if !matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
) || inner.rollback_journal_recovery_owner != Some(self.recovery_owner)
|| inner.maintenance_gate.rollback_recovery_owner() != Some(self.recovery_owner)
{
return Err(FrankenError::internal(
"detached durable rollback-commit exit lost its exact recovery receipt",
));
}
let notify_writer_idle = if self.logical_exit_completed {
false
} else {
let notify = coordinated_transaction_exit(
&self.queue,
&self.cleanup_cx,
&mut inner,
self.mode != TransactionMode::Concurrent,
logical_exit_claim,
)
.await?;
self.logical_exit_completed = true;
notify
};
let update = PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
};
let committed_snapshot = Arc::new(PagerCommittedSnapshot::from_inner(&inner));
*self
.committed_snapshot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = committed_snapshot;
drop(inner);
self.maintenance_lease.take();
if notify_writer_idle {
self.writer_idle.notify_one();
}
// The identity-wide exact owner remains armed while the
// maintenance lease is released, so no sibling can enter between
// external exit and publication of the durable image.
self.cache.clear();
self.published
.publish_clear_if(&self.cleanup_cx, update, true);
let mut inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.finish_rollback_journal_recovery(self.recovery_owner)?;
drop(inner);
Ok(true)
})
}
}
struct PendingGroupCommitLogicalCleanup {
sequence: Option<u64>,
scope: ProcessRootFinalizationScope,
root_attempt: Option<ProcessRootFinalizationAttempt>,
operation: Box<dyn PendingGroupCommitLogicalCleanupOperation>,
}
impl PendingGroupCommitLogicalCleanup {
fn new(
root_attempt: Option<ProcessRootFinalizationAttempt>,
operation: Box<dyn PendingGroupCommitLogicalCleanupOperation>,
) -> Self {
let scope = ProcessRootFinalizationScope::ExactHandle(operation.handle_key());
Self {
sequence: None,
scope,
root_attempt,
operation,
}
}
fn release_root_after_terminal(&mut self) {
if let Some(root_attempt) = self.root_attempt.take() {
root_attempt.release_after_terminal();
}
}
}
fn insert_pending_logical_cleanup_by_sequence(
pending: &mut VecDeque<PendingGroupCommitLogicalCleanup>,
cleanup: PendingGroupCommitLogicalCleanup,
) {
let sequence = cleanup
.sequence
.expect("queued logical cleanup must have a stable sequence");
let scope = cleanup.scope;
let insert_at = pending.iter().position(|queued| {
queued.scope == scope
&& queued
.sequence
.is_none_or(|queued_sequence| queued_sequence > sequence)
});
if let Some(insert_at) = insert_at {
pending.insert(insert_at, cleanup);
} else {
pending.push_back(cleanup);
}
}
struct PendingGroupCommitLogicalCleanupClaim {
queue: Arc<GroupCommitQueue>,
cleanup: Option<PendingGroupCommitLogicalCleanup>,
logical_exit_claim: Option<GroupCommitLogicalExitClaim>,
}
impl PendingGroupCommitLogicalCleanupClaim {
async fn resolve(&mut self) -> Result<bool> {
let logical_exit_claim = self
.logical_exit_claim
.as_ref()
.expect("logical cleanup claim must retain its queue transition claim");
self.cleanup
.as_mut()
.expect("logical group-commit cleanup claim must own its operation")
.operation
.resolve(logical_exit_claim)
.await
}
fn finish(mut self) -> PendingGroupCommitLogicalCleanup {
let cleanup = self
.cleanup
.take()
.expect("finished logical group-commit cleanup claim must own its operation");
self.logical_exit_claim.take();
cleanup
}
}
impl Drop for PendingGroupCommitLogicalCleanupClaim {
fn drop(&mut self) {
if let Some(cleanup) = self.cleanup.take() {
self.queue.requeue_pending_logical_cleanup(cleanup);
// bd-b4mwn rework #2 (ordering): decrement claimed only AFTER the
// requeue re-inserted the entry, and only on the requeue path —
// the success path decrements after release_root_after_terminal
// in the resolve fns. Decrementing here on the success path
// opened a window (claimed already 0, pending 0, root still
// armed until release ran) that the wedge test sampled as a
// false wedge -> spurious BusyRecovery refusal (the trj
// exact-handle holder receipt).
self.queue
.claimed_logical_cleanups
.fetch_sub(1, AtomicOrdering::AcqRel);
}
// Requeue by stable lane sequence before releasing the exact-handle
// claim so a second settler cannot overtake a cancelled operation.
self.logical_exit_claim.take();
}
}
impl<F: VfsFile> PagerInner<F> {
fn adopt_orphaned_rollback_journal_recovery(&mut self) -> Result<()> {
// Read-only pagers intentionally cannot perform durable journal replay.
// Leave the identity-owned orphan receipt in the gate so a surviving
// read-write pager can adopt it instead of stranding recovery here.
if self.access_mode.is_readonly() {
return Ok(());
}
if self.rollback_journal_recovery_state.is_pending()
|| self.rollback_journal_recovery_owner.is_some()
{
return Ok(());
}
if let Some(orphaned) = self
.maintenance_gate
.try_adopt_orphaned_rollback_recovery()?
{
self.rollback_journal_recovery_owner = Some(orphaned.owner);
self.rollback_journal_recovery_state = orphaned.recovery_state;
self.rollback_journal_recovery_namespace = Some(orphaned.namespace);
}
Ok(())
}
/// Reserve the identity's exact recovery owner before the first shared
/// journal side effect. A competing pager receives `BusyRecovery` and never
/// obtains a receipt that could authorize it to transition or clear state.
fn claim_rollback_journal_recovery(
&mut self,
initial_state: RollbackJournalRecoveryState,
journal_path: &Path,
) -> Result<RollbackRecoveryOwnerId> {
debug_assert!(Arc::ptr_eq(
&self.rollback_recovery_pending,
&self.maintenance_gate.rollback_recovery_pending,
));
if !initial_state.is_pending() {
return Err(FrankenError::internal(
"rollback-recovery claim requires a non-clean initial state",
));
}
if self.rollback_journal_recovery_state.is_pending()
|| self.rollback_journal_recovery_owner.is_some()
|| self.rollback_journal_recovery_namespace.is_some()
{
return Err(FrankenError::BusyRecovery);
}
let owner = self.maintenance_gate.claim_rollback_recovery_owner()?;
self.rollback_journal_recovery_owner = Some(owner);
self.rollback_journal_recovery_state = initial_state;
self.rollback_journal_recovery_namespace = Some(RollbackRecoveryNamespace {
db_path: self.database_path.clone(),
journal_path: journal_path.to_owned(),
db_identity: self.database_identity,
journal_mode: self.journal_mode,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: self.namespace_binding.clone(),
});
Ok(owner)
}
fn transition_rollback_journal_recovery(
&mut self,
owner: RollbackRecoveryOwnerId,
next_state: RollbackJournalRecoveryState,
) -> Result<()> {
if !next_state.is_pending() {
return Err(FrankenError::internal(
"rollback-recovery transition to Clean must use exact-owner finish",
));
}
if self.rollback_journal_recovery_owner != Some(owner)
|| self.maintenance_gate.rollback_recovery_owner() != Some(owner)
|| !self.rollback_journal_recovery_state.is_pending()
|| self.rollback_journal_recovery_namespace.is_none()
{
return Err(FrankenError::internal(
"rollback-recovery transition did not own the exact recovery receipt",
));
}
self.rollback_journal_recovery_state = next_state;
Ok(())
}
fn finish_rollback_journal_recovery(&mut self, owner: RollbackRecoveryOwnerId) -> Result<()> {
if self.rollback_journal_recovery_owner != Some(owner)
|| !self.rollback_journal_recovery_state.is_pending()
|| self.rollback_journal_recovery_namespace.is_none()
{
return Err(FrankenError::internal(
"rollback-recovery finish did not own the exact recovery receipt",
));
}
let prior_state = self.rollback_journal_recovery_state;
let prior_namespace = self.rollback_journal_recovery_namespace.take();
self.rollback_journal_recovery_state = RollbackJournalRecoveryState::Clean;
self.rollback_journal_recovery_owner = None;
if let Err(error) = self.maintenance_gate.release_rollback_recovery_owner(owner) {
self.rollback_journal_recovery_state = prior_state;
self.rollback_journal_recovery_owner = Some(owner);
self.rollback_journal_recovery_namespace = prior_namespace;
return Err(error);
}
Ok(())
}
/// Read a page through WAL (if present) → cache → disk and return an owned copy.
async fn read_page_copy(
&self,
cx: &Cx,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
page_no: PageNumber,
) -> Result<Vec<u8>> {
// In WAL mode, check the WAL for the latest version of the page first.
// bd-db300.3.8.7: try shared-lock path when the backend supports pinned reads.
if self.journal_mode == JournalMode::Wal
&& let Some(data) = read_page_from_wal_backend(wal_backend, cx, page_no).await?
{
// Served from the WAL, bypassing the ARC buffer pool: count it as a
// cache miss so PRAGMA cache_stats reflects the read (bd-dk9ra
// counter-stats ruling, option a). Byte-identical serve.
cache.record_external_read();
return Ok(data);
}
// Reads of yet-unallocated pages should observe zero-filled content.
// This is relied upon by savepoint rollback semantics for pages that
// were allocated and then rolled back before commit.
if page_no.get() > self.db_size {
return Ok(vec![0_u8; self.page_size.as_usize()]);
}
if let Some(data) = cache.get_copy(page_no) {
return Ok(data);
}
let first_read = {
let db_file = shared_db_file_read(&self.db_file, cx).await?;
cache.read_page_copy(cx, &*db_file, page_no).await
};
match first_read {
Ok(data) => Ok(data),
Err(FrankenError::OutOfMemory) => {
if cache.evict_clean_any() {
let retry = {
let db_file = shared_db_file_read(&self.db_file, cx).await?;
cache.read_page_copy(cx, &*db_file, page_no).await
};
match retry {
Err(FrankenError::OutOfMemory) => {
self.read_page_copy_uncached(cx, page_no).await
}
result => result,
}
} else {
self.read_page_copy_uncached(cx, page_no).await
}
}
Err(err) => Err(err),
}
}
async fn read_page_copy_uncached(&self, cx: &Cx, page_no: PageNumber) -> Result<Vec<u8>> {
let page_size = self.page_size.as_usize();
let offset = u64::from(page_no.get() - 1) * page_size as u64;
let mut out = vec![0_u8; page_size];
let db_file = shared_db_file_read(&self.db_file, cx).await?;
let bytes_read = db_file.read(cx, &mut out, offset).await?;
if bytes_read < page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read fetching page {page}: got {bytes_read} of {page_size}",
page = page_no.get()
),
});
}
Ok(out)
}
/// Read a page from the latest committed database state without consulting
/// the local cache.
///
/// This is used to refresh connection-local pager metadata after another
/// connection has committed. The local cache may still reflect an older
/// generation, so committed-state refresh must bypass it.
async fn read_committed_page_copy(
&self,
cx: &Cx,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
page_no: PageNumber,
) -> Result<Vec<u8>> {
// bd-db300.3.8.7: try shared-lock path first for WAL reads.
if self.journal_mode == JournalMode::Wal
&& let Some(data) = read_page_from_wal_backend(wal_backend, cx, page_no).await?
{
// Committed-refresh serve bypasses the ARC buffer pool by design;
// count it as a cache miss so PRAGMA cache_stats reflects the read
// (bd-dk9ra counter-stats ruling, option a). Metrics-only: bytes
// returned unchanged.
cache.record_external_read();
return Ok(data);
}
let page_size = self.page_size.as_usize();
let offset = u64::from(page_no.get().saturating_sub(1)) * page_size as u64;
let db_file = shared_db_file_read(&self.db_file, cx).await?;
let file_size = db_file.file_size(cx)?;
if offset >= file_size {
return Ok(vec![0_u8; page_size]);
}
let mut out = vec![0_u8; page_size];
let bytes_read = db_file.read(cx, &mut out, offset).await?;
if bytes_read < page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read fetching committed page {page}: got {bytes_read} of {page_size}",
page = page_no.get()
),
});
}
Ok(out)
}
/// Read just the database header bytes directly from the main database
/// file, bypassing WAL state.
async fn read_database_file_header_bytes(
&self,
cx: &Cx,
file_size: u64,
) -> Result<[u8; DATABASE_HEADER_SIZE]> {
if file_size == 0 {
return Ok([0_u8; DATABASE_HEADER_SIZE]);
}
let mut out = [0_u8; DATABASE_HEADER_SIZE];
let db_file = shared_db_file_read(&self.db_file, cx).await?;
let bytes_read = db_file.read(cx, &mut out, 0).await?;
if bytes_read < DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read fetching database-file header: got {bytes_read} of {DATABASE_HEADER_SIZE}"
),
});
}
Ok(out)
}
/// Probe the latest visible commit sequence using only durable header/WAL
/// metadata.
///
/// This is intentionally cheaper than a full committed-state refresh: it
/// avoids page-1 materialization and freelist reconstruction unless the
/// visible state actually changed.
async fn probe_visible_commit_seq(
&self,
cx: &Cx,
wal_backend: &SharedWalBackend,
) -> Result<VisibleCommitProbe> {
let file_size = shared_db_file_read(&self.db_file, cx)
.await?
.file_size(cx)?;
let previous_file_size = self.committed_db_file_size_bytes;
let previous_db_change_counter = self.committed_db_change_counter;
let previous_wal_generation = self.committed_wal_generation;
let previous_wal_visible_commit_count = self.committed_wal_visible_commit_count;
let previous_base_commit_seq = self
.commit_seq
.get()
.saturating_sub(previous_wal_visible_commit_count);
let (physical_wal_visible_commit_count, wal_generation, logical_visible_commit_seq) =
if self.journal_mode == JournalMode::Wal {
with_wal_backend(wal_backend, cx, |wal, cx| {
Box::pin(async move {
wal.begin_transaction(cx).await?;
let snapshot = wal.pinned_read_snapshot();
let physical_commit_count = if let Some(snapshot) = snapshot {
snapshot.commit_count
} else {
wal.committed_txn_count(cx).await?
};
let logical_visible_commit_seq =
match (snapshot, wal.pinned_logical_read_snapshot(cx).await?) {
(Some(pinned), Some(logical))
if logical.generation == pinned.generation
&& logical.last_commit_frame == pinned.last_commit_frame =>
{
Some(logical.visible_commit_seq)
}
(Some(_) | None, Some(_)) => {
return Err(FrankenError::WalCorrupt {
detail: "logical WAL reader horizon does not match the pinned snapshot"
.to_owned(),
});
}
(_, None) => None,
};
Ok((
physical_commit_count,
snapshot.map(|s| s.generation),
logical_visible_commit_seq,
))
})
})
.await?
} else {
(0, None, None)
};
let wal_snapshot_initialized = self.journal_mode == JournalMode::Wal;
let cached_wal_base_is_current = self.journal_mode == JournalMode::Wal
// With no visible WAL frames, the main header is authoritative and
// may have been replaced at the same length by another process
// (notably WAL-mode VACUUM). File size + WAL generation alone
// cannot prove that base unchanged.
&& physical_wal_visible_commit_count != 0
&& file_size == self.committed_db_file_size_bytes
&& wal_generation.is_some()
&& self.committed_wal_generation == wal_generation;
let (base_commit_seq, raw_base_change_counter) = if cached_wal_base_is_current {
(previous_base_commit_seq, self.committed_db_change_counter)
} else {
let base_header_bytes = self.read_database_file_header_bytes(cx, file_size).await?;
let raw_base_change_counter = if self.journal_mode == JournalMode::Wal {
match DatabaseHeader::from_bytes(&base_header_bytes) {
Ok(base_header) => base_header.change_counter,
Err(error) => u32::try_from(
stale_main_header_change_counter_under_wal(&base_header_bytes, &error)
.ok_or_else(|| FrankenError::DatabaseCorrupt {
detail: format!(
"invalid database-file header during WAL refresh: {error}"
),
})?,
)
.map_err(|_| {
FrankenError::internal("stale WAL header change counter did not fit u32")
})?,
}
} else {
DatabaseHeader::from_bytes(&base_header_bytes)
.map_err(|error| FrankenError::DatabaseCorrupt {
detail: format!("invalid database header during pager refresh: {error}"),
})?
.change_counter
};
let previous_raw_change_counter =
u32::try_from(previous_db_change_counter).map_err(|_| {
FrankenError::internal("cached database change counter did not fit u32")
})?;
let base_commit_seq = previous_base_commit_seq.saturating_add(u64::from(
raw_base_change_counter.wrapping_sub(previous_raw_change_counter),
));
(base_commit_seq, u64::from(raw_base_change_counter))
};
// A parallel WAL group may collapse multiple logical commits behind a
// single physical marker. A current-generation certificate can widen
// that count only when it was bound above to this exact pinned read
// snapshot. The physical count remains the conservative fallback for
// backends that cannot prove a logical horizon.
let wal_visible_commit_count = logical_visible_commit_seq
.map(|logical_visible_commit_seq| {
physical_wal_visible_commit_count.max(
logical_visible_commit_seq
.get()
.saturating_sub(base_commit_seq),
)
})
.unwrap_or(physical_wal_visible_commit_count);
let visible_commit_seq =
CommitSeq::new(base_commit_seq.saturating_add(wal_visible_commit_count));
let durable_identity_changed = file_size != previous_file_size
|| raw_base_change_counter != previous_db_change_counter
|| wal_generation != previous_wal_generation
|| wal_visible_commit_count != previous_wal_visible_commit_count;
Ok(VisibleCommitProbe {
visible_commit_seq,
file_size,
wal_snapshot_initialized,
durable_identity_changed,
db_change_counter: raw_base_change_counter,
wal_generation,
wal_visible_commit_count,
})
}
/// Record a commit performed through this pager in both the aggregate
/// commit clock and the durable-identity components cached by the next
/// committed-state probe.
///
/// The component update is essential even though `commit_seq` already
/// advances here. External WAL checkpoints are detected by changes in
/// the `(base change counter, WAL generation, visible WAL commits)`
/// composition when their sum stays constant. If a local commit advances
/// only the sum, the next `begin` mistakes that same commit for an external
/// composition change and unnecessarily discards cache, publication, and
/// volatile freelist state.
fn record_local_commit(&mut self) {
self.commit_seq = self.commit_seq.next();
if self.journal_mode == JournalMode::Wal {
self.committed_wal_visible_commit_count = self
.committed_wal_visible_commit_count
.checked_add(1)
.expect("visible WAL commit count overflow after 2^64 commits");
} else {
// The SQLite header stores a wrapping u32 change counter. Keep
// the cached base identical to the bytes written at commit time.
self.committed_db_change_counter = self.commit_seq.get() & u64::from(u32::MAX);
self.committed_wal_generation = None;
self.committed_wal_visible_commit_count = 0;
}
}
/// Catch this pager up to an already-certified WAL group horizon without
/// deriving commit order from Phase C thread arrival.
fn record_local_wal_commit_at(&mut self, certified_commit_seq: CommitSeq) {
debug_assert_eq!(self.journal_mode, JournalMode::Wal);
let newly_visible_commits = certified_commit_seq
.get()
.saturating_sub(self.commit_seq.get());
self.commit_seq = self.commit_seq.max(certified_commit_seq);
self.committed_wal_visible_commit_count = self
.committed_wal_visible_commit_count
.checked_add(newly_visible_commits)
.expect("visible WAL commit count overflow after 2^64 commits");
}
/// Refresh connection-local pager metadata from the latest committed state.
///
/// Reports whether WAL snapshot setup was performed during the refresh and
/// whether the durable identity change invalidated cached page/publication
/// entries.
async fn refresh_committed_state(
&mut self,
cx: &Cx,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
) -> Result<CommittedStateRefresh> {
self.refresh_committed_state_with_mode(
cx,
cache,
wal_backend,
CommittedStateRefreshMode::Normal,
)
.await
}
async fn refresh_committed_state_after_recovery(
&mut self,
cx: &Cx,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
) -> Result<CommittedStateRefresh> {
self.refresh_committed_state_with_mode(
cx,
cache,
wal_backend,
CommittedStateRefreshMode::PostRecovery,
)
.await
}
async fn refresh_committed_state_with_mode(
&mut self,
cx: &Cx,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
mode: CommittedStateRefreshMode,
) -> Result<CommittedStateRefresh> {
let probe = self.probe_visible_commit_seq(cx, wal_backend).await?;
if mode == CommittedStateRefreshMode::Normal
&& probe.visible_commit_seq == self.commit_seq
&& probe.file_size == self.committed_db_file_size_bytes
&& !probe.durable_identity_changed
{
self.committed_db_change_counter = probe.db_change_counter;
self.committed_wal_generation = probe.wal_generation;
self.committed_wal_visible_commit_count = probe.wal_visible_commit_count;
return Ok(CommittedStateRefresh {
wal_snapshot_initialized: probe.wal_snapshot_initialized,
page_cache_invalidated: false,
});
}
// The probe above is pure with respect to pager metadata. Publish its
// identity fields only after every awaited page/freelist read below
// succeeds, so cancellation cannot make a partial refresh look
// accepted to the next transaction.
let full_refresh_result = (async {
let page1 = self
.read_committed_page_copy(cx, cache, wal_backend, PageNumber::ONE)
.await?;
if page1.len() < DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"committed page 1 too small for database header: got {}, need {}",
page1.len(),
DATABASE_HEADER_SIZE
),
});
}
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
header_bytes.copy_from_slice(&page1[..DATABASE_HEADER_SIZE]);
let header = DatabaseHeader::from_bytes(&header_bytes).map_err(|error| {
FrankenError::DatabaseCorrupt {
detail: format!("invalid database header during pager refresh: {error}"),
}
})?;
if header.page_size != self.page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database page size changed from {} to {} while this pager was open; reopen the database before continuing",
self.page_size.get(),
header.page_size.get()
),
});
}
// Always cross-check header.page_count against the actual file size.
// A crash between growing the file and updating the header leaves
// page_count stale even when the stale marker is not set. Using
// max(header, file) ensures newly-committed pages are visible and
// avoids BusySnapshot errors on startup (see GH issue #49).
let file_size = shared_db_file_read(&self.db_file, cx)
.await?
.file_size(cx)?;
let file_derived = header
.page_count_from_file_size(file_size)
.unwrap_or(header.page_count);
let db_size = header.page_count.max(file_derived).max(1);
// Skip freelist scan for read-only pagers -- the freelist is only
// needed for page allocation during writes.
let freelist = if self.access_mode.is_readonly() {
Vec::new()
} else {
load_freelist_from_committed_state(
cx,
self,
cache,
wal_backend,
db_size,
header.freelist_trunk,
header.freelist_count,
)
.await?
};
Ok((db_size, freelist))
})
.await;
let (db_size, freelist) = match full_refresh_result {
Ok(refreshed) => refreshed,
Err(err) => return Err(err),
};
// Cross-process monotonicity (#70): never shrink self.db_size from a
// stale page-1 header we just happened to read from WAL. Another
// connection (or our own flusher) may already know about a larger
// committed extent; overwriting back to the header value turns those
// pages into phantom reads and causes `page N > snapshot db_size`
// BusySnapshot errors in peer readers.
self.db_size = if mode == CommittedStateRefreshMode::PostRecovery
|| self.journal_mode != JournalMode::Wal
{
db_size
} else {
self.db_size.max(db_size)
};
let effective_db_size = self.db_size;
self.next_page = if effective_db_size >= 2 {
effective_db_size.saturating_add(1)
} else {
2
};
self.freelist = freelist;
// Only clear the cache if the database was modified by another
// connection. In WAL mode this uses the latest visible page-1
// durable header baseline plus the visible WAL commit horizon.
let page_cache_invalidated =
probe.visible_commit_seq != self.commit_seq || probe.durable_identity_changed;
if page_cache_invalidated {
cache.clear();
}
// Cross-process monotonicity (bd-rjc): an external WAL checkpoint of our
// own already-committed frames is durability-increasing and
// visibility-preserving — it must never regress this connection's
// committed visibility. After such a reset the delta-based probe collapses
// visible_commit_seq back to a stale WAL base (e.g. 3 -> 1), so in
// steady-state (Normal) WAL refresh clamp to a monotonic floor exactly like
// `self.db_size` above (#70). PostRecovery (rollback-journal recovery, which
// can legitimately rewind) and non-WAL mode keep the probe value verbatim.
self.commit_seq = if mode == CommittedStateRefreshMode::PostRecovery
|| self.journal_mode != JournalMode::Wal
{
probe.visible_commit_seq
} else {
self.commit_seq.max(probe.visible_commit_seq)
};
self.committed_db_file_size_bytes = probe.file_size;
self.committed_db_change_counter = probe.db_change_counter;
self.committed_wal_generation = probe.wal_generation;
self.committed_wal_visible_commit_count = probe.wal_visible_commit_count;
Ok(CommittedStateRefresh {
wal_snapshot_initialized: probe.wal_snapshot_initialized,
page_cache_invalidated,
})
}
}
fn normalize_freelist(pages: &[PageNumber], db_size: u32) -> Vec<PageNumber> {
let mut normalized: Vec<PageNumber> = pages
.iter()
.copied()
.filter(|p| {
let raw = p.get();
raw > 1 && raw <= db_size
})
.collect();
if normalized
.windows(2)
.all(|window| window[0].get() > window[1].get())
{
return normalized;
}
// Keep the in-memory freelist descending so pop() yields the lowest page
// number first. That preserves compact file growth and lets returned pages
// merge back into large freelists without repeatedly sorting the full list.
normalized.sort_unstable_by_key(|page| std::cmp::Reverse(page.get()));
normalized.dedup_by_key(|p| p.get());
normalized
}
fn merge_descending_unique_freelists(left: &[PageNumber], right: &[PageNumber]) -> Vec<PageNumber> {
let mut merged = Vec::with_capacity(left.len() + right.len());
let mut left_index = 0;
let mut right_index = 0;
while left_index < left.len() && right_index < right.len() {
let left_page = left[left_index];
let right_page = right[right_index];
match left_page.get().cmp(&right_page.get()) {
std::cmp::Ordering::Greater => {
merged.push(left_page);
left_index += 1;
}
std::cmp::Ordering::Less => {
merged.push(right_page);
right_index += 1;
}
std::cmp::Ordering::Equal => {
merged.push(left_page);
left_index += 1;
right_index += 1;
}
}
}
merged.extend_from_slice(&left[left_index..]);
merged.extend_from_slice(&right[right_index..]);
merged
}
fn return_pages_to_freelist(
freelist: &mut Vec<PageNumber>,
pages: impl IntoIterator<Item = PageNumber>,
) {
let mut returned_pages: Vec<PageNumber> = pages.into_iter().collect();
if returned_pages.is_empty() {
return;
}
// Sort descending so that pop() and rposition() yield the lowest page numbers first,
// which keeps the database file compact and reduces file size growth.
returned_pages.sort_unstable_by_key(|page| std::cmp::Reverse(page.get()));
returned_pages.dedup_by_key(|page| page.get());
if freelist
.windows(2)
.all(|window| window[0].get() > window[1].get())
{
*freelist = merge_descending_unique_freelists(freelist, &returned_pages);
} else {
freelist.extend(returned_pages);
freelist.sort_unstable_by_key(|page| std::cmp::Reverse(page.get()));
freelist.dedup_by_key(|page| page.get());
}
}
async fn load_freelist_from_disk<F: VfsFile>(
cx: &Cx,
db_file: &F,
page_size: PageSize,
db_size: u32,
first_trunk: u32,
freelist_count: u32,
) -> Result<Vec<PageNumber>> {
if first_trunk == 0 || freelist_count == 0 {
return Ok(Vec::new());
}
let ps = page_size.as_usize();
let mut visited: HashSet<u32> = HashSet::new();
let mut out: Vec<PageNumber> = Vec::with_capacity(freelist_count as usize);
let mut trunk = first_trunk;
while trunk != 0 && out.len() < freelist_count as usize {
if trunk > db_size {
// Trunk page is beyond the database file — can't read it.
// Stop the chain here; we'll use whatever valid pages we've
// collected so far. normalize_freelist cleans up below.
break;
}
if !visited.insert(trunk) {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("freelist loop detected at trunk page {trunk}"),
});
}
let trunk_page = PageNumber::new(trunk).ok_or_else(|| FrankenError::DatabaseCorrupt {
detail: format!("invalid freelist trunk page number {trunk}"),
})?;
out.push(trunk_page);
let mut buf = vec![0u8; ps];
let offset = u64::from(trunk.saturating_sub(1)) * ps as u64;
let bytes_read = db_file.read(cx, &mut buf, offset).await?;
if bytes_read < ps {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read loading freelist trunk page {trunk}: got {bytes_read} of {ps}"
),
});
}
let next_trunk = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
let leaf_count = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]) as usize;
let max_leaf_entries = (ps / 4).saturating_sub(2);
if leaf_count > max_leaf_entries {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"freelist trunk {trunk} leaf_count {leaf_count} exceeds max {max_leaf_entries}"
),
});
}
for idx in 0..leaf_count {
if out.len() >= freelist_count as usize {
break;
}
let base = 8 + idx * 4;
let leaf = u32::from_be_bytes([buf[base], buf[base + 1], buf[base + 2], buf[base + 3]]);
if leaf == 0 || leaf > db_size {
// Skip invalid entries: zero pages (shouldn't appear) and
// pages beyond the database file (stale/corrupt freelist
// state). normalize_freelist also filters these, but we
// skip early to avoid counting them toward freelist_count.
continue;
}
let leaf_page = PageNumber::new(leaf).ok_or_else(|| FrankenError::DatabaseCorrupt {
detail: format!("invalid freelist leaf page number {leaf}"),
})?;
out.push(leaf_page);
}
trunk = next_trunk;
}
out.truncate(freelist_count as usize);
Ok(normalize_freelist(&out, db_size))
}
async fn load_freelist_from_committed_state<F: VfsFile>(
cx: &Cx,
inner: &PagerInner<F>,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
db_size: u32,
first_trunk: u32,
freelist_count: u32,
) -> Result<Vec<PageNumber>> {
if first_trunk == 0 || freelist_count == 0 {
return Ok(Vec::new());
}
let ps = inner.page_size.as_usize();
let mut visited: HashSet<u32> = HashSet::new();
let mut out: Vec<PageNumber> = Vec::with_capacity(freelist_count as usize);
let mut trunk = first_trunk;
while trunk != 0 && out.len() < freelist_count as usize {
if trunk > db_size {
// Trunk page is beyond the committed database size — can't
// read it. Stop the chain here and use whatever valid pages
// we've collected so far.
break;
}
if !visited.insert(trunk) {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("freelist loop detected at trunk page {trunk}"),
});
}
let trunk_page = PageNumber::new(trunk).ok_or_else(|| FrankenError::DatabaseCorrupt {
detail: format!("invalid freelist trunk page number {trunk}"),
})?;
out.push(trunk_page);
let buf = inner
.read_committed_page_copy(cx, cache, wal_backend, trunk_page)
.await?;
if buf.len() < ps {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read loading committed freelist trunk page {trunk}: got {} of {ps}",
buf.len()
),
});
}
let next_trunk = u32::from_be_bytes([buf[0], buf[1], buf[2], buf[3]]);
let leaf_count = u32::from_be_bytes([buf[4], buf[5], buf[6], buf[7]]) as usize;
let max_leaf_entries = (ps / 4).saturating_sub(2);
if leaf_count > max_leaf_entries {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"freelist trunk {trunk} leaf_count {leaf_count} exceeds max {max_leaf_entries}"
),
});
}
for idx in 0..leaf_count {
if out.len() >= freelist_count as usize {
break;
}
let base = 8 + idx * 4;
let leaf = u32::from_be_bytes([buf[base], buf[base + 1], buf[base + 2], buf[base + 3]]);
if leaf == 0 || leaf > db_size {
// Skip invalid entries: zero pages and pages beyond the
// committed database size (stale/corrupt freelist state).
continue;
}
let leaf_page = PageNumber::new(leaf).ok_or_else(|| FrankenError::DatabaseCorrupt {
detail: format!("invalid freelist leaf page number {leaf}"),
})?;
out.push(leaf_page);
}
trunk = next_trunk;
}
out.truncate(freelist_count as usize);
Ok(normalize_freelist(&out, db_size))
}
#[allow(clippy::too_many_arguments)]
async fn serialize_freelist_to_write_set<F: VfsFile, S: std::hash::BuildHasher>(
cx: &Cx,
inner: &mut PagerInner<F>,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
pool: &PageBufPool,
write_set: &mut HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &mut Vec<PageNumber>,
committed_db_size: u32,
pending_free_pages: &[PageNumber],
phase_a_undo: Option<&PhaseAWriteSetUndo>,
) -> Result<()> {
if committed_db_size == 0 {
inner.freelist.clear();
return Ok(());
}
// Use next_page as the normalization bound so we keep valid in-memory EOF
// pages returned by aborted concurrent transactions. Those pages were
// never part of the durable file image, so they must not be serialized
// into page-1 freelist metadata until db_size grows to include them.
let upper_bound = inner.next_page.saturating_sub(1).max(committed_db_size);
// Build the predicted freelist from the committed freelist plus pages
// that will become free if this commit succeeds WITHOUT mutating
// inner.freelist. This prevents concurrent transactions from observing
// uncommitted freelist changes during the window between Phase A (prepare)
// and Phase B (WAL I/O) of the split-lock commit path. Newly free pages
// are only promoted into inner.freelist after Phase B succeeds (in Phase C).
// See beads_rust#138.
let mut predicted_freelist = inner.freelist.clone();
return_pages_to_freelist(&mut predicted_freelist, pending_free_pages.iter().copied());
let predicted_normalized = normalize_freelist(&predicted_freelist, upper_bound);
// NOTE: Do NOT normalize inner.freelist here — this runs during Phase A
// where inner.lock() may be released before Phase B. Mutating the shared
// freelist would leak a side-effect visible to concurrent transactions
// even if this commit fails. Normalization of inner.freelist (if needed)
// should be deferred to Phase C after successful commit.
let durable_freelist: Vec<PageNumber> = predicted_normalized
.iter()
.copied()
.filter(|page| page.get() <= committed_db_size)
.collect();
let ps = inner.page_size.as_usize();
let total_free = durable_freelist.len() as u32;
let (first_trunk, trunk_pages) = if durable_freelist.is_empty() {
(0u32, Vec::<u32>::new())
} else {
let max_leaf_entries = (ps / 4).saturating_sub(2).max(1);
let trunk_count = durable_freelist.len().div_ceil(max_leaf_entries + 1);
let trunks: Vec<u32> = durable_freelist
.iter()
.take(trunk_count)
.map(|p| p.get())
.collect();
(trunks[0], trunks)
};
if !trunk_pages.is_empty() {
let mut leaf_index = trunk_pages.len();
let max_leaf_entries = (ps / 4).saturating_sub(2).max(1);
for (idx, trunk_pg) in trunk_pages.iter().enumerate() {
let next = trunk_pages.get(idx + 1).copied().unwrap_or(0);
let remaining = durable_freelist.len().saturating_sub(leaf_index);
let take = remaining.min(max_leaf_entries);
let mut buf =
acquire_page_buf_with_clean_cache_recovery(pool, cache, "freelist_trunk_stage")?;
// Zero the entire page to avoid leaking stale data from the
// pool in the unused tail of the trunk page.
buf.fill(0);
buf[0..4].copy_from_slice(&next.to_be_bytes());
buf[4..8].copy_from_slice(&(take as u32).to_be_bytes());
for i in 0..take {
let leaf = durable_freelist[leaf_index + i].get();
let base = 8 + i * 4;
buf[base..base + 4].copy_from_slice(&leaf.to_be_bytes());
}
leaf_index += take;
if let Some(pg) = PageNumber::new(*trunk_pg) {
if let Some(undo) = phase_a_undo {
undo.capture(write_set, pg);
}
insert_staged_page(write_set, write_pages_sorted, pg, StagedPage::from_buf(buf));
}
}
}
if let Some(undo) = phase_a_undo {
undo.capture(write_set, PageNumber::ONE);
}
let mut page1 =
ensure_page_one_in_write_set(cx, inner, cache, wal_backend, pool, write_set).await?;
{
let page1_bytes = page1.as_page_bytes_mut();
page1_bytes[32..36].copy_from_slice(&first_trunk.to_be_bytes());
page1_bytes[36..40].copy_from_slice(&total_free.to_be_bytes());
}
insert_staged_page(write_set, write_pages_sorted, PageNumber::ONE, page1);
Ok(())
}
async fn ensure_page_one_in_write_set<F: VfsFile, S: std::hash::BuildHasher>(
cx: &Cx,
inner: &PagerInner<F>,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
pool: &PageBufPool,
write_set: &mut HashMap<PageNumber, StagedPage, S>,
) -> Result<StagedPage> {
if let Some(staged) = write_set.get_mut(&PageNumber::ONE) {
// Keep page 1 resident in the write set until the fallible detach has
// succeeded. A capacity error must leave the transaction retryable and
// roll-backable with its original staged header bytes intact.
staged.make_unpublished_for_mutation(pool, cache, "page_one_commit_mutation")?;
return Ok(write_set
.remove(&PageNumber::ONE)
.expect("page one remained staged after successful detach"));
}
let page1_vec = inner
.read_page_copy(cx, cache, wal_backend, PageNumber::ONE)
.await?;
Ok(StagedPage::from_page_data(PageData::from_vec(page1_vec)))
}
fn insert_page_sorted(pages: &mut Vec<PageNumber>, page_no: PageNumber) {
match pages.binary_search_by_key(&page_no.get(), |page| page.get()) {
Ok(_) => {}
Err(idx) => pages.insert(idx, page_no),
}
}
fn remove_page_sorted(pages: &mut Vec<PageNumber>, page_no: PageNumber) {
if let Ok(idx) = pages.binary_search_by_key(&page_no.get(), |page| page.get()) {
pages.remove(idx);
}
}
fn insert_staged_page<S: std::hash::BuildHasher>(
write_set: &mut HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &mut Vec<PageNumber>,
page_no: PageNumber,
staged: StagedPage,
) {
if write_set.insert(page_no, staged).is_none() {
insert_page_sorted(write_pages_sorted, page_no);
}
}
/// Global counter of same-page-within-transaction writes that reused the
/// existing `StagedPage` buffer in place instead of allocating a fresh
/// [`PageBuf`] from the pool and dropping the old one.
///
/// Consumers can snapshot this counter via
/// [`staged_page_overwrite_steals_total`] to verify the allocation
/// reduction landed on a given workload.
static STAGED_PAGE_OVERWRITE_STEALS_TOTAL: AtomicUsize = AtomicUsize::new(0);
/// Snapshot the running total of in-place staged-page overwrites.
///
/// Used by microbenchmarks and allocation regression tests. The counter is
/// monotonic and uses `Relaxed` ordering — call sites MUST NOT derive
/// correctness invariants from its value.
#[must_use]
pub fn staged_page_overwrite_steals_total() -> u64 {
u64::try_from(STAGED_PAGE_OVERWRITE_STEALS_TOTAL.load(AtomicOrdering::Relaxed))
.unwrap_or(u64::MAX)
}
/// Reset the running total of staged-page overwrite steals.
///
/// Tests and microbenchmarks use this to establish a clean baseline before
/// measuring a specific workload.
pub fn reset_staged_page_overwrite_steals_total() {
STAGED_PAGE_OVERWRITE_STEALS_TOTAL.store(0, AtomicOrdering::Relaxed);
}
#[cfg(test)]
struct WalCommitBatch<'a> {
new_db_size: u32,
frames: Vec<traits::WalFrameRef<'a>>,
}
#[cfg(test)]
fn collect_wal_commit_batch<'a, S: std::hash::BuildHasher>(
current_db_size: u32,
write_set: &'a HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &[PageNumber],
) -> Result<Option<WalCommitBatch<'a>>> {
if write_pages_sorted.is_empty() {
return Ok(None);
}
let max_written = write_pages_sorted.last().map_or(0, |page| page.get());
let new_db_size = current_db_size.max(max_written);
let frame_count = write_pages_sorted.len();
let mut frames = Vec::with_capacity(frame_count);
for (idx, page_no) in write_pages_sorted.iter().enumerate() {
let staged_page = write_set.get(page_no).ok_or_else(|| {
FrankenError::internal(format!(
"WAL commit batch missing page {} from write_set",
page_no.get()
))
})?;
let db_size_if_commit = if idx + 1 == frame_count {
new_db_size
} else {
0
};
frames.push(traits::WalFrameRef {
page_number: page_no.get(),
page_data: staged_page.as_page_bytes(),
db_size_if_commit,
});
}
Ok(Some(WalCommitBatch {
new_db_size,
frames,
}))
}
/// Build a [`TransactionFrameBatch`] with OWNED frame data for group commit.
///
/// Unlike the borrowed-frame helper used by unit tests, this function clones
/// each page's bytes into the batch. This is necessary for group commit
/// because the batch must outlive the caller's write_set while waiting for the
/// flusher to write all batched frames.
///
/// Returns `(batch, new_db_size)` or `None` if there are no pages to commit.
fn build_group_commit_batch<S: std::hash::BuildHasher>(
current_db_size: u32,
write_set: &HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &[PageNumber],
) -> Result<Option<(TransactionFrameBatch, u32)>> {
if write_pages_sorted.is_empty() {
return Ok(None);
}
let max_written = write_pages_sorted.last().map_or(0, |page| page.get());
let new_db_size = current_db_size.max(max_written);
let frame_count = write_pages_sorted.len();
let mut frames = Vec::with_capacity(frame_count);
for (idx, page_no) in write_pages_sorted.iter().enumerate() {
let staged_page = write_set.get(page_no).ok_or_else(|| {
FrankenError::internal(format!(
"group commit batch missing page {} from write_set",
page_no.get()
))
})?;
let db_size_if_commit = if idx + 1 == frame_count {
new_db_size
} else {
0
};
frames.push(FrameSubmission {
page_number: page_no.get(),
page_data: staged_page.as_page_bytes().to_vec(), // Clone data for ownership
db_size_if_commit,
});
}
Ok(Some((TransactionFrameBatch::new(frames), new_db_size)))
}
fn transaction_conflict_snapshot_from_wal(
snapshot: traits::WalPublicationSnapshot,
) -> TransactionConflictSnapshot {
TransactionConflictSnapshot {
generation: snapshot.generation,
last_commit_frame: snapshot.last_commit_frame,
commit_count: snapshot.commit_count,
}
}
fn attach_group_commit_conflict_metadata(
mut batch: TransactionFrameBatch,
conflict_pages: &[PageNumber],
conflict_snapshot: Option<traits::WalPublicationSnapshot>,
conflict_page_baselines: &[TransactionConflictPageBaseline],
) -> TransactionFrameBatch {
let conflict_pages = conflict_pages
.iter()
.map(|page| page.get())
.collect::<Vec<_>>();
let conflict_snapshot = conflict_snapshot.map(transaction_conflict_snapshot_from_wal);
batch = batch.with_conflict_snapshot(conflict_pages, conflict_snapshot);
batch = batch.with_conflict_page_baselines(conflict_page_baselines.to_vec());
batch
}
async fn conflicting_pages_since_batch_snapshots(
cx: &Cx,
wal: &mut dyn WalBackend,
batches: &[TransactionFrameBatch],
) -> Result<Vec<u32>> {
let mut conflicts = Vec::<u32>::new();
for batch in batches {
let Some(snapshot) = batch.conflict_snapshot else {
continue;
};
let batch_conflicts = wal
.conflicting_pages_since_snapshot(
cx,
snapshot,
&batch.conflict_pages,
&batch.conflict_page_baselines,
)
.await?;
conflicts.extend(batch_conflicts);
}
conflicts.sort_unstable();
conflicts.dedup();
Ok(conflicts)
}
fn group_commit_final_db_size(current_db_size: u32, batches: &[TransactionFrameBatch]) -> u32 {
batches
.iter()
.flat_map(|batch| batch.frames.iter())
.fold(current_db_size, |db_size, frame| {
db_size.max(frame.db_size_if_commit)
})
}
fn promote_group_commit_page_one_headers(
batches: &mut [TransactionFrameBatch],
final_db_size: u32,
) -> bool {
let mut promoted = false;
for batch in batches {
for frame in &mut batch.frames {
if frame.page_number != 1 || frame.page_data.len() < DATABASE_HEADER_SIZE {
continue;
}
let mut existing_page_count_bytes = [0_u8; 4];
existing_page_count_bytes.copy_from_slice(&frame.page_data[28..32]);
let existing_page_count = u32::from_be_bytes(existing_page_count_bytes);
let promoted_page_count = existing_page_count.max(final_db_size);
if promoted_page_count != existing_page_count {
frame.page_data[28..32].copy_from_slice(&promoted_page_count.to_be_bytes());
promoted = true;
}
}
}
promoted
}
fn flatten_group_commit_batches<'a>(
current_db_size: u32,
batches: &'a [TransactionFrameBatch],
) -> (Vec<traits::WalFrameRef<'a>>, u32) {
let total_frames: usize = batches.iter().map(|batch| batch.frames.len()).sum();
let mut frame_refs: Vec<traits::WalFrameRef<'a>> = Vec::with_capacity(total_frames);
let final_db_size = group_commit_final_db_size(current_db_size, batches);
let mut last_commit_frame_idx = None;
for batch in batches {
for frame in &batch.frames {
if frame.db_size_if_commit != 0 {
last_commit_frame_idx = Some(frame_refs.len());
}
frame_refs.push(traits::WalFrameRef {
page_number: frame.page_number,
page_data: &frame.page_data,
db_size_if_commit: 0,
});
}
}
if let Some(last_commit_frame_idx) = last_commit_frame_idx {
frame_refs[last_commit_frame_idx].db_size_if_commit = final_db_size;
}
(frame_refs, final_db_size)
}
fn group_commit_batch_frame_refs(batch: &TransactionFrameBatch) -> Vec<traits::WalFrameRef<'_>> {
batch
.frames
.iter()
.map(|frame| traits::WalFrameRef {
page_number: frame.page_number,
page_data: &frame.page_data,
db_size_if_commit: frame.db_size_if_commit,
})
.collect()
}
fn group_commit_batch_staged_bytes(batch: &TransactionFrameBatch) -> u64 {
batch.frames.iter().fold(0_u64, |acc, frame| {
acc.saturating_add(u64::try_from(frame.page_data.len()).unwrap_or(u64::MAX))
.saturating_add(u64::try_from(fsqlite_wal::WAL_FRAME_HEADER_SIZE).unwrap_or(u64::MAX))
})
}
fn prepared_batch_matches_frame_refs(
prepared: &traits::PreparedWalFrameBatch,
frame_refs: &[traits::WalFrameRef<'_>],
) -> bool {
prepared.frame_count() == frame_refs.len()
&& prepared
.frame_metas
.iter()
.zip(frame_refs)
.enumerate()
.all(|(index, (meta, frame))| {
meta.page_number == frame.page_number
&& meta.db_size_if_commit == frame.db_size_if_commit
&& prepared.page_data(index) == frame.page_data
})
}
fn should_shadow_compare_batches(
control: &ParallelWalControlSurface,
batches: &[TransactionFrameBatch],
) -> bool {
batches
.iter()
.any(|batch| parallel_wal_should_shadow_compare(control, batch.context.batch_id))
}
async fn prepare_group_commit_batch_for_lane(
cx: &Cx,
wal_backend: &SharedWalBackend,
batch: &TransactionFrameBatch,
batch_id: u64,
lane_id: u16,
control: &ParallelWalControlSurface,
) -> Result<Option<LaneStagedPreparedBatch>> {
if matches!(control.mode, ParallelWalOperatingMode::Conservative) {
return Ok(None);
}
if let Some(limit) = control.max_parallel_commit_bytes
&& group_commit_batch_staged_bytes(batch) > limit
{
return Ok(None);
}
let frame_refs = group_commit_batch_frame_refs(batch);
if frame_refs.is_empty() {
return Ok(None);
}
let started = Instant::now();
let backend = wal_backend_handle(wal_backend)?;
let wal = async_rwlock_read(&backend, cx, "WAL backend").await?;
let mut prepared = wal.prepare_append_frames(&frame_refs)?;
if let Some(prepared) = prepared.as_mut() {
wal.finalize_prepared_frames(cx, prepared)?;
}
drop(wal);
let Some(prepared) = prepared.take() else {
return Ok(None);
};
Ok(Some(LaneStagedPreparedBatch {
batch_id,
lane_id,
staged_frame_count: u32::try_from(frame_refs.len()).unwrap_or(u32::MAX),
staging_elapsed_ns: u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX),
shadow_verdict: ParallelWalShadowVerdict::NotRun,
payload: prepared,
}))
}
fn merge_prepared_group_commit_batches(
prepared_batches: Vec<LaneStagedPreparedBatch>,
final_db_size: u32,
) -> Result<traits::PreparedWalFrameBatch> {
if prepared_batches.is_empty() {
return Err(FrankenError::internal(
"cannot merge empty prepared group-commit batch set",
));
}
if prepared_batches.len() == 1 {
let mut iter = prepared_batches.into_iter();
let staged = iter
.next()
.ok_or_else(|| FrankenError::internal("missing prepared group-commit batch"))?;
if prepared_batch_is_canonical_single_commit(&staged.payload, final_db_size) {
return Ok(staged.payload);
}
return merge_prepared_group_commit_batches_uncanonicalized(vec![staged], final_db_size);
}
merge_prepared_group_commit_batches_uncanonicalized(prepared_batches, final_db_size)
}
fn merge_prepared_group_commit_batches_uncanonicalized(
prepared_batches: Vec<LaneStagedPreparedBatch>,
final_db_size: u32,
) -> Result<traits::PreparedWalFrameBatch> {
let Some(first) = prepared_batches.first() else {
return Err(FrankenError::internal(
"cannot merge empty prepared group-commit batch set",
));
};
let frame_size = first.payload.frame_size;
let page_data_offset = first.payload.page_data_offset;
let big_endian_checksum = first.payload.big_endian_checksum;
for staged in &prepared_batches {
let prepared = &staged.payload;
if prepared.frame_size != frame_size
|| prepared.page_data_offset != page_data_offset
|| prepared.big_endian_checksum != big_endian_checksum
{
return Err(FrankenError::internal(format!(
"incompatible prepared WAL batch merge for lane {} batch {}",
staged.lane_id, staged.batch_id
)));
}
}
let total_frames = prepared_batches
.iter()
.map(|batch| batch.payload.frame_count())
.sum::<usize>();
let total_bytes = prepared_batches
.iter()
.map(|batch| batch.payload.frame_bytes.len())
.sum::<usize>();
let mut iter = prepared_batches.into_iter();
let first = iter
.next()
.ok_or_else(|| FrankenError::internal("missing prepared group-commit batch"))?;
let mut merged = first.payload;
merged
.frame_metas
.reserve(total_frames.saturating_sub(merged.frame_metas.len()));
merged
.frame_bytes
.reserve(total_bytes.saturating_sub(merged.frame_bytes.len()));
let mut saw_commit = false;
for meta in &mut merged.frame_metas {
saw_commit |= meta.db_size_if_commit != 0;
meta.db_size_if_commit = 0;
}
for staged in iter {
let prepared = staged.payload;
merged
.frame_metas
.extend(prepared.frame_metas.into_iter().map(|mut meta| {
saw_commit |= meta.db_size_if_commit != 0;
meta.db_size_if_commit = 0;
meta
}));
merged.frame_bytes.extend_from_slice(&prepared.frame_bytes);
}
merged.checksum_transforms.clear();
merged.last_commit_frame_offset = None;
merged.finalized_for = None;
merged.finalized_running_checksum = None;
for frame_index in 0..merged.frame_count() {
merged.set_db_size_if_commit(frame_index, 0);
}
if saw_commit {
let last_frame_index = merged.frame_metas.len().saturating_sub(1);
merged.last_commit_frame_offset = Some(last_frame_index);
merged.set_db_size_if_commit(last_frame_index, final_db_size);
}
merged.recompute_checksum_transforms()?;
Ok(merged)
}
fn prepared_batch_is_canonical_single_commit(
prepared: &traits::PreparedWalFrameBatch,
final_db_size: u32,
) -> bool {
let frame_count = prepared.frame_count();
if frame_count == 0 {
return false;
}
let mut commit_index = None;
for (index, meta) in prepared.frame_metas.iter().enumerate() {
let frame = prepared.frame_slice(index);
if frame.len() < 8 {
return false;
}
let frame_db_size = u32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]]);
if meta.db_size_if_commit == 0 {
if frame_db_size != 0 {
return false;
}
continue;
}
if commit_index.is_some() || index + 1 != frame_count {
return false;
}
if meta.db_size_if_commit != final_db_size {
return false;
}
if frame_db_size != final_db_size {
return false;
}
commit_index = Some(index);
}
prepared.last_commit_frame_offset == commit_index
}
fn aggregate_shadow_verdict(
prepared_batches: &[LaneStagedPreparedBatch],
) -> ParallelWalShadowVerdict {
if prepared_batches
.iter()
.any(|batch| matches!(batch.shadow_verdict, ParallelWalShadowVerdict::Diverged))
{
ParallelWalShadowVerdict::Diverged
} else if prepared_batches
.iter()
.any(|batch| matches!(batch.shadow_verdict, ParallelWalShadowVerdict::Clean))
{
ParallelWalShadowVerdict::Clean
} else {
ParallelWalShadowVerdict::NotRun
}
}
fn lane_flush_stats(
queue: &GroupCommitQueue,
batches: &[TransactionFrameBatch],
) -> Vec<(u16, usize, u32, u64)> {
let mut by_lane = HashMap::<u16, (u32, u64)>::new();
for batch in batches {
let entry = by_lane.entry(batch.context.lane_id).or_insert((0, 0));
entry.0 = entry.0.saturating_add(batch.context.staged_frame_count);
entry.1 = entry.1.saturating_add(batch.context.staging_elapsed_ns);
}
let mut stats = by_lane
.into_iter()
.map(|(lane_id, (staged_frame_count, elapsed_ns))| {
(
lane_id,
queue.current_lane_backlog(lane_id),
staged_frame_count,
elapsed_ns,
)
})
.collect::<Vec<_>>();
stats.sort_unstable_by_key(|(lane_id, _, _, _)| *lane_id);
stats
}
fn conflicting_pages_across_group_commit_batches(batches: &[TransactionFrameBatch]) -> Vec<u32> {
let mut first_batch_by_page = HashMap::<u32, usize>::new();
let mut conflicts = HashSet::<u32>::new();
let mut page_one_conflict_batches = Vec::<usize>::new();
let mut page_one_frame_batches = Vec::<usize>::new();
for (batch_idx, batch) in batches.iter().enumerate() {
let mut seen_in_batch = HashSet::<u32>::new();
let mut batch_has_page_one_frame = false;
let mut batch_has_page_one_conflict = false;
for &page_number in &batch.conflict_pages {
if page_number == 1 {
batch_has_page_one_conflict = true;
}
if !seen_in_batch.insert(page_number) {
continue;
}
match first_batch_by_page.get(&page_number).copied() {
Some(previous_batch_idx) if previous_batch_idx != batch_idx => {
conflicts.insert(page_number);
}
None => {
first_batch_by_page.insert(page_number, batch_idx);
}
Some(_) => {}
}
}
for frame in &batch.frames {
if frame.page_number == 1 {
// Page 1 carries the shared database header and legitimately
// appears in disjoint commits; treating it as a hard overlap
// would spuriously abort safe group-commit epochs. Explicit
// Page 1 rewrites still enter the semantic conflict surface
// through `batch.conflict_pages`, and are checked below
// against synthetic Page 1 frames from other batches.
batch_has_page_one_frame = true;
continue;
}
if !seen_in_batch.insert(frame.page_number) {
continue;
}
match first_batch_by_page.get(&frame.page_number).copied() {
Some(previous_batch_idx) if previous_batch_idx != batch_idx => {
conflicts.insert(frame.page_number);
}
None => {
first_batch_by_page.insert(frame.page_number, batch_idx);
}
Some(_) => {}
}
}
if batch_has_page_one_conflict {
page_one_conflict_batches.push(batch_idx);
}
if batch_has_page_one_frame {
page_one_frame_batches.push(batch_idx);
}
}
if page_one_conflict_batches.iter().any(|conflict_batch_idx| {
page_one_frame_batches
.iter()
.any(|frame_batch_idx| frame_batch_idx != conflict_batch_idx)
}) {
conflicts.insert(1);
}
let mut conflicts = conflicts.into_iter().collect::<Vec<_>>();
conflicts.sort_unstable();
conflicts
}
const SNAPSHOT_PUBLICATION_MODE: &str = "seqlock_published_pages";
const PUBLISHED_SNAPSHOT_WAIT_SLICE: Duration = Duration::from_micros(50);
/// Maximum retries for optimistic published-page reads before falling back to
/// the slow path. 64 iterations covers typical publish latency on x86 (1-3 µs
/// per seqlock retry). Not runtime-configurable — tuned for low-contention
/// steady state.
const PUBLISHED_READ_FAST_RETRY_LIMIT: usize = 64;
/// Number of counter stripes for published page version tracking. Power-of-2
/// for masking. Matches typical server core counts (up to 64 cores).
const PUBLISHED_COUNTER_STRIPE_COUNT: usize = 64;
static NEXT_PUBLISHED_COUNTER_STRIPE: AtomicUsize = AtomicUsize::new(0);
std::thread_local! {
static PUBLISHED_COUNTER_STRIPE_INDEX: usize =
NEXT_PUBLISHED_COUNTER_STRIPE.fetch_add(1, AtomicOrdering::Relaxed)
% PUBLISHED_COUNTER_STRIPE_COUNT;
}
#[derive(Debug)]
#[repr(align(64))]
struct CacheAlignedAtomicU64(AtomicU64);
impl CacheAlignedAtomicU64 {
const fn new(value: u64) -> Self {
Self(AtomicU64::new(value))
}
fn fetch_add(&self, value: u64, ordering: AtomicOrdering) {
self.0.fetch_add(value, ordering);
}
fn load(&self, ordering: AtomicOrdering) -> u64 {
self.0.load(ordering)
}
}
#[derive(Debug)]
struct StripedCounter64 {
stripes: [CacheAlignedAtomicU64; PUBLISHED_COUNTER_STRIPE_COUNT],
}
impl StripedCounter64 {
fn new() -> Self {
Self {
stripes: std::array::from_fn(|_| CacheAlignedAtomicU64::new(0)),
}
}
fn increment(&self) {
PUBLISHED_COUNTER_STRIPE_INDEX.with(|stripe| {
self.stripes[*stripe].fetch_add(1, AtomicOrdering::Relaxed);
});
}
fn load(&self) -> u64 {
self.stripes.iter().fold(0_u64, |sum, stripe| {
sum.saturating_add(stripe.load(AtomicOrdering::Acquire))
})
}
}
const ATOMIC_PUBLISHED_PAGE_LIMIT: u32 = 65_535;
/// Minimum size of the direct-index slot plane regardless of the initial
/// database size. Each slot pays one `pthread_mutex_t` / `AtomicBool` /
/// `AtomicUsize` (~80-100 bytes + a pthread_mutex_init syscall) at
/// construction AND a matching destruction cost at
/// [`Arc<PublishedPagerState>::drop_slow`]. On mt_mvcc_bench 8t the
/// 2026-04-23 profile showed this pair (`PublishedPagerState::new` 4.65%
/// and `drop_slow` 4.24%) as ~9% of self-time — dominated by the fixed
/// floor of 4,096 slots per pager on the small-DB bench, even though each
/// worker only publishes ~10-50 pages.
///
/// 512 is enough direct-slot capacity for every FrankenSQLite bench we
/// ship today (mt_mvcc_bench, e2e_bench, mixed_oltp_bench,
/// write_throughput_bench, read_heavy_bench all stay well below 512
/// pages). Pager opens on larger databases still clamp upward via
/// `initial_db_size` — the floor only governs the fresh-connection
/// / small-DB case.
///
/// Pages beyond the direct-slot capacity fall through to the
/// [`ConcurrentPublishedPages`] overflow plane (DashMap) — correctness
/// is unchanged; only hit-path cost shifts from a direct atomic load to
/// a sharded hash lookup for pages in the 512..db_size range.
const ATOMIC_PUBLISHED_MIN_SLOT_COUNT: usize = 512;
const ATOMIC_PUBLISHED_MAX_SLOT_COUNT: usize = ATOMIC_PUBLISHED_PAGE_LIMIT as usize;
/// Direct-index slot for concurrently published pages below
/// [`ATOMIC_PUBLISHED_PAGE_LIMIT`].
#[derive(Debug)]
struct AtomicPublishedPageSlot {
present: AtomicBool,
page: Mutex<Option<PageData>>,
/// Position of this slot within
/// [`AtomicPublishedPages::active_indices`]. Protected by the
/// `active_indices` mutex; only valid when `present` is `true`. Kept
/// as an `AtomicUsize` to avoid an `UnsafeCell`/`Cell` non-`Sync`
/// wrapper — every read/write happens while the `active_indices`
/// lock is held, so `Relaxed` is sufficient.
active_pos: AtomicUsize,
}
/// Lock-free-on-miss publication plane for the low page-number hot set.
///
/// Writes are serialized by [`PublishedPagerState::publish_lock`], so the
/// atomic state word only needs to coordinate readers with the active writer.
#[derive(Debug)]
struct AtomicPublishedPages {
slots: Box<[AtomicPublishedPageSlot]>,
active_indices: Mutex<Vec<usize>>,
page_count: AtomicUsize,
}
impl AtomicPublishedPages {
fn new(initial_db_size: u32) -> Self {
let initial_pages =
usize::try_from(initial_db_size).unwrap_or(ATOMIC_PUBLISHED_MAX_SLOT_COUNT);
let slot_count = initial_pages.clamp(
ATOMIC_PUBLISHED_MIN_SLOT_COUNT,
ATOMIC_PUBLISHED_MAX_SLOT_COUNT,
);
let slots = (0..slot_count)
.map(|_| AtomicPublishedPageSlot {
present: AtomicBool::new(false),
page: Mutex::new(None),
active_pos: AtomicUsize::new(0),
})
.collect::<Vec<_>>()
.into_boxed_slice();
Self {
slots,
active_indices: Mutex::new(Vec::new()),
page_count: AtomicUsize::new(0),
}
}
#[inline]
fn slot_index(&self, page_no: PageNumber) -> Option<usize> {
let raw = page_no.get();
if raw > ATOMIC_PUBLISHED_PAGE_LIMIT {
return None;
}
let idx = usize::try_from(raw.saturating_sub(1)).ok()?;
(idx < self.slots.len()).then_some(idx)
}
#[inline]
fn accepts(&self, page_no: PageNumber) -> bool {
self.slot_index(page_no).is_some()
}
#[cfg(test)]
fn slot_count(&self) -> usize {
self.slots.len()
}
#[cfg(test)]
fn active_slot_count(&self) -> usize {
self.active_indices
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len()
}
fn get(&self, page_no: PageNumber) -> Option<PageData> {
let idx = self.slot_index(page_no)?;
let slot = &self.slots[idx];
if !slot.present.load(AtomicOrdering::Acquire) {
return None;
}
slot.page
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.clone()
}
fn insert(&self, page_no: PageNumber, page: PageData) -> bool {
let Some(idx) = self.slot_index(page_no) else {
return false;
};
let mut active_indices = self
.active_indices
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let slot = &self.slots[idx];
let mut guard = slot
.page
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let inserted = guard.is_none();
*guard = Some(page);
if inserted {
slot.active_pos
.store(active_indices.len(), AtomicOrdering::Relaxed);
active_indices.push(idx);
}
drop(guard);
slot.present.store(true, AtomicOrdering::Release);
if inserted {
self.page_count.fetch_add(1, AtomicOrdering::Relaxed);
}
inserted
}
fn remove(&self, page_no: PageNumber) -> bool {
let Some(idx) = self.slot_index(page_no) else {
return false;
};
let mut active_indices = self
.active_indices
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let slot = &self.slots[idx];
if !slot.present.swap(false, AtomicOrdering::AcqRel) {
return false;
}
let removed = slot
.page
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take()
.is_some();
if removed {
let pos = slot.active_pos.load(AtomicOrdering::Relaxed);
if pos < active_indices.len() && active_indices[pos] == idx {
// O(1) swap_remove using the stored back-pointer. If the
// swapped-in element (formerly the tail) lands at `pos`,
// refresh its own back-pointer so its next `remove`
// stays O(1) as well.
active_indices.swap_remove(pos);
if let Some(&moved_idx) = active_indices.get(pos) {
self.slots[moved_idx]
.active_pos
.store(pos, AtomicOrdering::Relaxed);
}
} else if let Some(fallback_pos) = active_indices
.iter()
.position(|&active_idx| active_idx == idx)
{
// Defensive fallback: back-pointer was stale (should not
// happen, but keep the structure consistent if it ever
// does rather than leaving a dangling active-index).
active_indices.swap_remove(fallback_pos);
if let Some(&moved_idx) = active_indices.get(fallback_pos) {
self.slots[moved_idx]
.active_pos
.store(fallback_pos, AtomicOrdering::Relaxed);
}
}
self.page_count.fetch_sub(1, AtomicOrdering::Relaxed);
}
removed
}
fn clear(&self) {
// Under MT-writer contention the publication plane is cleared on
// every transaction rollback/retry. Keep clear proportional to the
// actually published pages rather than the direct-slot capacity.
if self.page_count.load(AtomicOrdering::Acquire) == 0 {
return;
}
let mut active_indices = self
.active_indices
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for idx in active_indices.drain(..) {
let slot = &self.slots[idx];
if slot.present.swap(false, AtomicOrdering::AcqRel) {
let _ = slot
.page
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
}
}
self.page_count.store(0, AtomicOrdering::Release);
}
fn retain<F>(&self, f: F)
where
F: Fn(&PageNumber) -> bool,
{
let mut active_indices = self
.active_indices
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut retained_indices = Vec::with_capacity(active_indices.len());
for idx in active_indices.drain(..) {
let slot = &self.slots[idx];
if !slot.present.load(AtomicOrdering::Acquire) {
continue;
}
let page_no = PageNumber::new(u32::try_from(idx + 1).unwrap_or(u32::MAX))
.expect("atomic publication slot index must map to a valid page number");
if f(&page_no) {
// Refresh the back-pointer to match the rebuilt position
// so subsequent remove() stays O(1).
slot.active_pos
.store(retained_indices.len(), AtomicOrdering::Relaxed);
retained_indices.push(idx);
continue;
}
if slot.present.swap(false, AtomicOrdering::AcqRel) {
let _ = slot
.page
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
}
}
let retained_total = retained_indices.len();
*active_indices = retained_indices;
self.page_count
.store(retained_total, AtomicOrdering::Release);
}
fn insert_batch<I>(&self, pages: I)
where
I: IntoIterator<Item = (PageNumber, PageData)>,
{
let mut active_indices = self
.active_indices
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let mut total_added = 0_usize;
for (page_no, page) in pages {
let Some(idx) = self.slot_index(page_no) else {
continue;
};
let slot = &self.slots[idx];
let mut guard = slot
.page
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let inserted = guard.is_none();
*guard = Some(page);
if inserted {
slot.active_pos
.store(active_indices.len(), AtomicOrdering::Relaxed);
active_indices.push(idx);
total_added = total_added.saturating_add(1);
}
drop(guard);
slot.present.store(true, AtomicOrdering::Release);
}
if total_added > 0 {
self.page_count
.fetch_add(total_added, AtomicOrdering::Relaxed);
}
}
fn len(&self) -> usize {
self.page_count.load(AtomicOrdering::Acquire)
}
fn prefetch_hint(&self, page_no: PageNumber) {
let Some(idx) = self.slot_index(page_no) else {
return;
};
let slot = &self.slots[idx];
prefetch_l1_read(std::ptr::from_ref(slot));
if slot.present.load(AtomicOrdering::Relaxed)
&& let Ok(guard) = slot.page.try_lock()
&& let Some(page) = guard.as_ref()
{
prefetch_l1_read(page.as_bytes().as_ptr());
}
}
}
/// Concurrent overflow publication plane for pages outside the direct-index atomic range.
#[derive(Debug)]
struct ConcurrentPublishedPages {
pages: DashMap<PageNumber, PageData, foldhash::fast::FixedState>,
page_count: AtomicUsize,
}
impl ConcurrentPublishedPages {
fn new() -> Self {
Self {
pages: DashMap::with_hasher(foldhash::fast::FixedState::default()),
page_count: AtomicUsize::new(0),
}
}
/// Get a page from the concurrent overflow plane.
fn get(&self, page_no: PageNumber) -> Option<PageData> {
self.pages.get(&page_no).map(|page| page.value().clone())
}
/// Insert a page into the concurrent overflow plane.
fn insert(&self, page_no: PageNumber, page: PageData) -> bool {
let inserted = self.pages.insert(page_no, page).is_none();
if inserted {
self.page_count.fetch_add(1, AtomicOrdering::Relaxed);
}
inserted
}
/// Remove a page from the concurrent overflow plane.
fn remove(&self, page_no: PageNumber) -> bool {
let removed = self.pages.remove(&page_no).is_some();
if removed {
self.page_count.fetch_sub(1, AtomicOrdering::Relaxed);
}
removed
}
/// Clear the concurrent overflow plane.
fn clear(&self) {
// All publish-side mutations on this plane (insert / remove / retain /
// insert_batch / clear) are serialized by `PublishedPagerState::publish_lock`,
// so `page_count == 0` observed here implies the DashMap is empty and
// we can skip the per-shard sweep `DashMap::clear` performs unconditionally.
// Mirrors `AtomicPublishedPages::clear`'s identical short-circuit on its
// own `page_count` and the `ShardedPageCache::clear` shards-dirty short-circuit.
if self.page_count.load(AtomicOrdering::Acquire) == 0 {
return;
}
self.pages.clear();
self.page_count.store(0, AtomicOrdering::Release);
}
/// Retain pages matching the predicate across the overflow plane.
fn retain<F>(&self, f: F)
where
F: Fn(&PageNumber) -> bool,
{
self.pages.retain(|page_no, _| f(page_no));
self.page_count
.store(self.pages.len(), AtomicOrdering::Release);
}
/// Insert multiple pages into the overflow plane.
fn insert_batch<I>(&self, pages: I)
where
I: IntoIterator<Item = (PageNumber, PageData)>,
{
let mut total_added = 0_usize;
for (page_no, page) in pages {
if self.pages.insert(page_no, page).is_none() {
total_added = total_added.saturating_add(1);
}
}
if total_added > 0 {
self.page_count
.fetch_add(total_added, AtomicOrdering::Relaxed);
}
}
/// Total number of pages in the overflow plane.
fn len(&self) -> usize {
self.page_count.load(AtomicOrdering::Acquire)
}
fn prefetch_hint(&self, page_no: PageNumber) {
prefetch_l1_read(std::ptr::from_ref(&self.pages));
if let Some(page) = self.pages.get(&page_no) {
prefetch_l1_read(page.value().as_bytes().as_ptr());
}
}
}
/// Hybrid published-page store: direct-index atomic slots for the hot
/// `< 64K` page range, plus a concurrent overflow map for larger page
/// numbers.
#[derive(Debug)]
struct PublishedPages {
atomic: AtomicPublishedPages,
overflow: ConcurrentPublishedPages,
}
impl PublishedPages {
fn new(initial_db_size: u32) -> Self {
Self {
atomic: AtomicPublishedPages::new(initial_db_size),
overflow: ConcurrentPublishedPages::new(),
}
}
fn get(&self, page_no: PageNumber) -> Option<PageData> {
if self.atomic.accepts(page_no) {
self.atomic.get(page_no)
} else {
self.overflow.get(page_no)
}
}
fn insert(&self, page_no: PageNumber, page: PageData) -> bool {
if self.atomic.accepts(page_no) {
self.atomic.insert(page_no, page)
} else {
self.overflow.insert(page_no, page)
}
}
fn remove(&self, page_no: PageNumber) -> bool {
if self.atomic.accepts(page_no) {
self.atomic.remove(page_no)
} else {
self.overflow.remove(page_no)
}
}
fn clear(&self) {
self.atomic.clear();
self.overflow.clear();
}
fn retain<F>(&self, f: F)
where
F: Fn(&PageNumber) -> bool,
{
self.atomic.retain(&f);
self.overflow.retain(f);
}
fn insert_batch<I>(&self, pages: I)
where
I: IntoIterator<Item = (PageNumber, PageData)>,
{
let mut direct_pages = Vec::new();
let mut overflow_pages = Vec::new();
for (page_no, page) in pages {
if self.atomic.accepts(page_no) {
direct_pages.push((page_no, page));
} else {
overflow_pages.push((page_no, page));
}
}
if !direct_pages.is_empty() {
self.atomic.insert_batch(direct_pages);
}
if !overflow_pages.is_empty() {
self.overflow.insert_batch(overflow_pages);
}
}
fn len(&self) -> usize {
self.atomic.len().saturating_add(self.overflow.len())
}
fn prefetch_hint(&self, page_no: PageNumber) {
if self.atomic.accepts(page_no) {
self.atomic.prefetch_hint(page_no);
} else {
self.overflow.prefetch_hint(page_no);
}
}
#[cfg(test)]
fn atomic_slot_count(&self) -> usize {
self.atomic.slot_count()
}
#[cfg(test)]
fn atomic_active_slot_count(&self) -> usize {
self.atomic.active_slot_count()
}
}
/// Minimal metadata handoff from the parallel WAL ordered residue into the
/// pager publication plane.
///
/// D1.a constrains the handoff to data that is already covered by a durable
/// commit certificate. Later implementation beads may change how the handoff is
/// transported, but not which facts must be carried or the rule that the pager
/// only publishes them after the certificate becomes durable.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct ParallelWalPublicationIntent {
pub certificate_epoch: u64,
pub visible_commit_seq: CommitSeq,
pub page_plane_visible_commit_seq: CommitSeq,
pub db_size: u32,
pub journal_mode: JournalMode,
pub freelist_count: usize,
pub checkpoint_active: bool,
pub page_set_size: usize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
struct ParallelWalPublicationAuthorization {
durability_receipt: ParallelWalDurabilityReceipt,
batch_id: u64,
assigned_commit_seq: CommitSeq,
}
fn parallel_wal_publication_intent(
authorization: &ParallelWalPublicationAuthorization,
db_size: u32,
journal_mode: JournalMode,
freelist_count: usize,
checkpoint_active: bool,
) -> Result<ParallelWalPublicationIntent> {
let certificate = &authorization.durability_receipt.certificate;
if !certificate.checksum_is_valid() {
return Err(FrankenError::internal(
"parallel WAL publication rejected a damaged commit certificate",
));
}
if authorization
.durability_receipt
.commit_seq_for_batch(authorization.batch_id)
!= Some(authorization.assigned_commit_seq)
{
return Err(FrankenError::internal(format!(
"parallel WAL publication certificate does not authorize batch {} at {}",
authorization.batch_id, authorization.assigned_commit_seq
)));
}
if authorization.assigned_commit_seq < certificate.commit_seq_lo
|| authorization.assigned_commit_seq > certificate.commit_seq_hi
{
return Err(FrankenError::internal(format!(
"parallel WAL publication sequence {} is outside certificate interval {}..={}",
authorization.assigned_commit_seq, certificate.commit_seq_lo, certificate.commit_seq_hi
)));
}
if checkpoint_active
&& authorization.durability_receipt.fallback_reason
!= Some(ParallelWalFallbackReason::CheckpointConflict)
{
return Err(FrankenError::Busy);
}
Ok(ParallelWalPublicationIntent {
certificate_epoch: certificate.certificate_epoch,
// The physical group is already durable. Publishing its high-water
// mark lets readers bind the complete authoritative WAL index even if
// individual pager Phase C callbacks arrive out of batch order.
visible_commit_seq: certificate.commit_seq_hi,
// The flusher installs the complete certificate group's page images in
// one seqlock generation before it wakes any Phase C waiter. The page
// plane therefore has the same contiguous high-water mark as the WAL
// visibility plane; an out-of-order waiter must never lower it.
page_plane_visible_commit_seq: certificate.commit_seq_hi,
db_size: db_size.max(certificate.db_size_pages),
journal_mode,
freelist_count,
checkpoint_active,
page_set_size: usize::try_from(certificate.page_set_size).unwrap_or(usize::MAX),
})
}
/// Reader-visible pager metadata classes covered by the Track E3 design
/// contract.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum PagerMetadataPublicationClass {
/// The multi-field metadata summary exposed through
/// `PublishedPagerState::snapshot()`.
SnapshotSummary,
/// The published page plane plus its lag-detecting horizon.
PagePlaneResidency,
/// The durable ordered-residue handoff that authorizes pager publication.
CertificateDerivedIntent,
}
/// Design-time publication contract for one pager metadata class.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PagerMetadataPublicationContract {
/// Concrete metadata class on the hot path.
pub class: PagerMetadataPublicationClass,
/// Current implementation touchpoint.
pub touchpoint: &'static str,
/// Primitive in the current code.
pub current_primitive: &'static str,
/// Primitive selected by the E3 design contract.
pub selected_primitive: &'static str,
/// Read-side retry / staleness contract.
pub retry_contract: &'static str,
/// Fallback boundary when the optimistic publication surface is not usable.
pub fallback_contract: &'static str,
}
/// Concrete pager metadata-publication mapping for Track E3.
pub const PAGER_METADATA_PUBLICATION_CONTRACTS: [PagerMetadataPublicationContract; 3] = [
PagerMetadataPublicationContract {
class: PagerMetadataPublicationClass::SnapshotSummary,
touchpoint: "pager.rs::PublishedPagerState::{snapshot,finalize_publish}",
current_primitive: "writer-serialized seqlock summary over visible_commit_seq/db_size/journal_mode/freelist/checkpoint/page_set_size",
selected_primitive: "keep seqlock-style summary publication",
retry_contract: "readers retry while the sequence is odd or unstable and never take the publish lock",
fallback_contract: "targeted sequence waits may park briefly before re-reading the summary",
},
PagerMetadataPublicationContract {
class: PagerMetadataPublicationClass::PagePlaneResidency,
touchpoint: "pager.rs::PublishedPages plus page_plane_visible_commit_seq",
current_primitive: "lock-free published-page plane gated by a monotone lag-detecting horizon",
selected_primitive: "keep monotone horizon with explicit cache/inner fallback",
retry_contract: "published pages are readable only when the page-plane horizon covers the bound snapshot; otherwise the read must fall back",
fallback_contract: "a lagging page plane is intentional and forces cache/inner reads instead of serving stale published pages",
},
PagerMetadataPublicationContract {
class: PagerMetadataPublicationClass::CertificateDerivedIntent,
touchpoint: "pager.rs::ParallelWalPublicationIntent",
current_primitive: "immutable certificate-derived intent handed from ordered residue into publish",
selected_primitive: "keep immutable certificate-derived handoff; never publish beyond durable certificate scope",
retry_contract: "not directly polled by readers; it constrains what the pager may expose as visible",
fallback_contract: "certificate mismatch or gap forces conservative publication and blocks newer visibility",
},
];
/// Point-in-time view of the pager metadata publication plane.
///
/// When the D1 parallel WAL path is active, this snapshot is the post-publish
/// image produced from a durable [`ParallelWalPublicationIntent`].
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct PagerPublishedSnapshot {
/// Even-numbered generation identifying the current published snapshot.
pub snapshot_gen: u64,
/// Commit sequence visible through the publication plane.
pub visible_commit_seq: CommitSeq,
/// Published database size in pages.
pub db_size: u32,
/// Published journal mode.
pub journal_mode: JournalMode,
/// Published freelist length.
pub freelist_count: usize,
/// Whether a checkpoint is currently active.
pub checkpoint_active: bool,
/// Number of committed pages currently served through the publication plane.
pub page_set_size: usize,
}
#[cfg(test)]
mod metadata_publication_contract_tests {
use super::{
PAGER_METADATA_PUBLICATION_CONTRACTS, PagerMetadataPublicationClass,
PagerMetadataPublicationContract,
};
fn contract(class: PagerMetadataPublicationClass) -> PagerMetadataPublicationContract {
PAGER_METADATA_PUBLICATION_CONTRACTS
.iter()
.find(|contract| contract.class == class)
.copied()
.expect("pager metadata contract must exist")
}
#[test]
fn test_pager_metadata_publication_contract_keeps_seqlock_summary() {
let summary = contract(PagerMetadataPublicationClass::SnapshotSummary);
assert_eq!(
summary.selected_primitive,
"keep seqlock-style summary publication"
);
assert!(
summary.retry_contract.contains("retry"),
"summary publication must preserve explicit seqlock retry semantics"
);
}
#[test]
fn test_pager_metadata_publication_contract_keeps_page_plane_fallback_boundary() {
let page_plane = contract(PagerMetadataPublicationClass::PagePlaneResidency);
assert!(
page_plane.fallback_contract.contains("cache/inner"),
"page-plane contract must preserve the explicit fallback boundary"
);
}
}
#[derive(Debug, Clone, Copy)]
struct PublishedPagerUpdate {
visible_commit_seq: CommitSeq,
db_size: u32,
journal_mode: JournalMode,
freelist_count: usize,
checkpoint_active: bool,
}
#[derive(Debug)]
struct PublishedPagerState {
publish_lock: Mutex<()>,
sequence_gate: Mutex<()>,
sequence_cv: Condvar,
sequence_waiters: KeyedWaitRegistry,
sequence: AtomicU64,
visible_commit_seq: AtomicU64,
// Latest commit sequence for which the published page plane itself is in
// sync. Metadata-only fast paths intentionally leave this behind so page
// reads fall back to cache/inner rather than serving stale published pages.
page_plane_visible_commit_seq: AtomicU64,
db_size: AtomicU32,
journal_mode: AtomicU8,
freelist_count: AtomicUsize,
checkpoint_active: AtomicBool,
page_set_size: AtomicUsize,
publication_write_count: AtomicU64,
read_retry_count: StripedCounter64,
published_page_hits: StripedCounter64,
// F2: Hybrid published pages - atomic direct slots for the hot low-page
// range, with sharded overflow for large page numbers.
pages: PublishedPages,
}
impl PublishedPagerState {
fn new(
db_size: u32,
visible_commit_seq: CommitSeq,
journal_mode: JournalMode,
freelist_count: usize,
) -> Self {
Self {
publish_lock: Mutex::new(()),
sequence_gate: Mutex::new(()),
sequence_cv: Condvar::new(),
sequence_waiters: KeyedWaitRegistry::new(),
sequence: AtomicU64::new(2),
visible_commit_seq: AtomicU64::new(visible_commit_seq.get()),
page_plane_visible_commit_seq: AtomicU64::new(visible_commit_seq.get()),
db_size: AtomicU32::new(db_size),
journal_mode: AtomicU8::new(encode_journal_mode(journal_mode)),
freelist_count: AtomicUsize::new(freelist_count),
checkpoint_active: AtomicBool::new(false),
page_set_size: AtomicUsize::new(0),
publication_write_count: AtomicU64::new(0),
read_retry_count: StripedCounter64::new(),
published_page_hits: StripedCounter64::new(),
pages: PublishedPages::new(db_size),
}
}
fn signal_sequence_waiters(&self, observed_sequence: u64, stage: &'static str) {
match PUBLISHED_SEQUENCE_WAIT_PATH_MODE {
WaitPathMode::KeyedEventcount => {
let woke_waiters = self.sequence_waiters.signal(observed_sequence);
tracing::trace!(
target: "fsqlite.snapshot_publication",
run_id = "pager-publication",
scenario_id = "sequence_wait_signal",
observed_sequence,
stage,
wait_strategy = PUBLISHED_SEQUENCE_WAIT_PATH_MODE.as_str(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
woke_waiters,
"signaled targeted publication waiters"
);
}
WaitPathMode::LegacyCondvarTimeout if stage == "publish_complete" => {
self.sequence_cv.notify_all();
}
WaitPathMode::LegacyCondvarTimeout => {}
}
}
fn snapshot(&self) -> PagerPublishedSnapshot {
loop {
let snapshot_gen = self.sequence.load(AtomicOrdering::Acquire);
if snapshot_gen % 2 == 1 {
self.record_retry();
self.wait_for_sequence_change(snapshot_gen, PUBLISHED_SNAPSHOT_WAIT_SLICE);
continue;
}
let visible_commit_seq =
CommitSeq::new(self.visible_commit_seq.load(AtomicOrdering::Acquire));
let db_size = self.db_size.load(AtomicOrdering::Acquire);
let journal_mode = decode_journal_mode(self.journal_mode.load(AtomicOrdering::Acquire));
let freelist_count = self.freelist_count.load(AtomicOrdering::Acquire);
let checkpoint_active = self.checkpoint_active.load(AtomicOrdering::Acquire);
let page_set_size = self.page_set_size.load(AtomicOrdering::Acquire);
if self.sequence.load(AtomicOrdering::Acquire) == snapshot_gen {
return PagerPublishedSnapshot {
snapshot_gen,
visible_commit_seq,
db_size,
journal_mode,
freelist_count,
checkpoint_active,
page_set_size,
};
}
self.record_retry();
self.wait_for_sequence_change(snapshot_gen, PUBLISHED_SNAPSHOT_WAIT_SLICE);
}
}
fn try_get_page(&self, page_no: PageNumber) -> Option<PageData> {
// F2: use the hybrid published-page store. Reads in the low-page hot
// range avoid shard hashing entirely.
self.pages.get(page_no)
}
/// Read just the publish-cycle generation counter without touching the
/// metadata fields published alongside it.
///
/// The seqlock recheck in [`SimpleTransaction::get_page`] only consumes
/// the snapshot generation, but `snapshot()` reads six other atomics on
/// every confirmation. At 1t a single page read goes through this path
/// twice — once to take a snapshot and once to confirm — so a gen-only
/// recheck collapses the second reading from seven atomic loads down to
/// one.
fn current_sequence_gen(&self) -> u64 {
self.sequence.load(AtomicOrdering::Acquire)
}
fn prefetch_page_hint(&self, page_no: PageNumber) {
self.pages.prefetch_hint(page_no);
}
fn page_plane_visible_commit_seq(&self) -> CommitSeq {
CommitSeq::new(
self.page_plane_visible_commit_seq
.load(AtomicOrdering::Acquire),
)
}
/// Capture a publication snapshot only when both the metadata summary and
/// resident page plane still belong to `expected_commit_seq`.
///
/// Callers must still recheck [`Self::current_sequence_gen`] after reading
/// a page. The checks here close the earlier window between a standalone
/// page-plane comparison and `snapshot()`, where a concurrent publication
/// could advance both planes before the snapshot was captured.
fn snapshot_for_page_plane(
&self,
expected_commit_seq: CommitSeq,
) -> Option<PagerPublishedSnapshot> {
if self.page_plane_visible_commit_seq() != expected_commit_seq {
return None;
}
let snapshot = self.snapshot();
if snapshot.visible_commit_seq != expected_commit_seq
|| self.page_plane_visible_commit_seq() != expected_commit_seq
{
return None;
}
Some(snapshot)
}
fn should_skip_stale_publish(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
scenario_id: &'static str,
) -> bool {
let current_visible_commit_seq =
CommitSeq::new(self.visible_commit_seq.load(AtomicOrdering::Acquire));
if current_visible_commit_seq > update.visible_commit_seq {
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id,
observed_commit_seq = update.visible_commit_seq.get(),
visible_commit_seq = current_visible_commit_seq.get(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
"skipping stale pager publication"
);
return true;
}
false
}
// D1-CRITICAL Change 3: Operation-specific publish methods using sharded pages.
// Replaces the closure-based publish API with type-safe operations.
/// Publish metadata and insert a single observed page.
fn publish_observed_page(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
page_no: PageNumber,
page: PageData,
) -> bool {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_read_publish_skip") {
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id = "stale_read_publish_page_skip",
page_no = page_no.get(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
"skipping stale published page observation"
);
return false;
}
self.publish_insert_page_locked(cx, update, page_no, page);
true
}
/// Publish metadata and optionally clear all pages.
fn publish_clear_if(&self, cx: &Cx, update: PublishedPagerUpdate, should_clear: bool) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_clear_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
if should_clear {
self.pages.clear();
}
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, should_clear);
}
/// Publish metadata and remove a single page.
fn publish_remove_page(&self, cx: &Cx, update: PublishedPagerUpdate, page_no: PageNumber) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_remove_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
self.pages.remove(page_no);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
/// Publish metadata only (no page changes).
fn publish_metadata_only(&self, cx: &Cx, update: PublishedPagerUpdate) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_metadata_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, false);
}
/// Advance published metadata for a single-connection fast path without
/// republishing the shared page plane. The page plane remains stale on
/// purpose; page reads detect that and fall back to cache/inner state.
fn publish_single_connection_metadata_update(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
clear_pages: bool,
) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_single_connection_metadata_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
if clear_pages {
// Metadata-only single-connection commits intentionally do not
// republish the page plane, but any previously published pages are
// now stale relative to the new visible commit horizon. Clear them
// under the publish lock so a later page-plane publish cannot
// accidentally combine fresh metadata with old page bytes.
self.pages.clear();
}
self.sync_metadata_without_page_publish(update);
let previous_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
let snapshot_gen = previous_sequence.saturating_add(1);
self.signal_sequence_waiters(previous_sequence, "publish_complete");
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id = if clear_pages {
"metadata_only_plane_clear"
} else {
"metadata_only_summary_only"
},
snapshot_gen,
visible_commit_seq = self.visible_commit_seq.load(AtomicOrdering::Acquire),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
freelist_count = self.freelist_count.load(AtomicOrdering::Acquire),
checkpoint_active = self.checkpoint_active.load(AtomicOrdering::Acquire),
read_retry_count = self.read_retry_count(),
page_set_size = self.page_set_size.load(AtomicOrdering::Acquire),
elapsed_ns = u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX),
"published metadata-only single-connection summary"
);
}
/// Publish truncate during checkpoint: retain pages up to max_page, then remove page one.
fn publish_truncate_checkpoint(&self, cx: &Cx, update: PublishedPagerUpdate, max_page: u32) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_truncate_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
self.pages.retain(|page_no| page_no.get() <= max_page);
self.pages.remove(PageNumber::ONE);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
// finalize_publish keeps db_size strictly monotonic to guard against
// cross-process stale-publish regressions (#70). Truncate-checkpoint
// is the only publisher permitted to shrink the authoritative db_size:
// the checkpoint lock guarantees no concurrent writer will clobber
// this store with a stale larger value.
self.db_size.store(update.db_size, AtomicOrdering::Release);
}
/// Publish a wholesale, already-durable replacement of the database
/// image. Unlike ordinary commits, VACUUM is authoritative for shrink as
/// well as growth, and every previously published page is stale.
fn publish_replaced_image(&self, cx: &Cx, update: PublishedPagerUpdate) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
self.pages.clear();
self.finalize_publish(cx, update, 0, start, true);
// The exclusive maintenance fence makes this the one publication path
// allowed to authoritatively shrink the visible database image.
self.visible_commit_seq
.store(update.visible_commit_seq.get(), AtomicOrdering::Release);
self.page_plane_visible_commit_seq
.store(update.visible_commit_seq.get(), AtomicOrdering::Release);
self.db_size.store(update.db_size, AtomicOrdering::Release);
}
/// Publish commit: retain pages up to db_size, then bulk insert from write_set.
fn publish_commit<S: std::hash::BuildHasher>(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
write_set: &HashMap<PageNumber, StagedPage, S>,
) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_commit_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
// Only sweep published pages when this publication actually shrinks
// the visible database size and is not stale relative to the current
// published commit horizon. The common retained autocommit path does
// not shrink; sweeping there is pure O(n) overhead inside the commit
// roundtrip. It is also unsafe for an older smaller publication to
// evict pages from a newer larger published snapshot.
let previous_db_size = self.db_size.load(AtomicOrdering::Acquire);
let previous_visible_commit_seq = self.visible_commit_seq.load(AtomicOrdering::Acquire);
if update.db_size < previous_db_size
&& update.visible_commit_seq.get() >= previous_visible_commit_seq
{
self.pages.retain(|page_no| page_no.get() <= update.db_size);
}
// Bulk insert committed pages
self.pages.insert_batch(
write_set
.iter()
.map(|(&page_no, staged)| (page_no, staged.published_page())),
);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
fn prepare_parallel_wal_group_pages(
batches: &[TransactionFrameBatch],
) -> Result<HashMap<PageNumber, PageData>> {
let mut complete_group_pages = HashMap::with_capacity(
batches
.iter()
.map(|batch| batch.frames.len())
.sum::<usize>(),
);
for batch in batches {
for frame in &batch.frames {
let page_no = PageNumber::new(frame.page_number).ok_or_else(|| {
FrankenError::internal(format!(
"parallel WAL group certificate contains page number {}",
frame.page_number
))
})?;
// Later frames in certificate order replace earlier images of
// the same page, matching WAL replay semantics.
complete_group_pages.insert(page_no, PageData::from_vec(frame.page_data.clone()));
}
}
Ok(complete_group_pages)
}
/// Publish already-validated page images covered by one durable group
/// certificate in a single seqlock generation.
///
/// This function is infallible by construction. Recovery prepares the map
/// before finalizing its exact combiner handle, so consuming that handle
/// cannot be followed by a retry-requiring publication error.
fn publish_prepared_parallel_wal_group(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
complete_group_pages: HashMap<PageNumber, PageData>,
certificate_commit_seq_lo: CommitSeq,
) {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PublishedPagerState);
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_parallel_wal_group_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
// When this certificate starts after the resident page plane's next
// sequence, an intervening peer commit is absent from
// `complete_group_pages`. The existing pages cannot be relabeled at
// the certificate high-water: evict them inside this same seqlock
// generation so misses fall through to the pinned WAL image.
if certificate_commit_seq_lo > self.page_plane_visible_commit_seq().next() {
self.pages.clear();
}
let previous_db_size = self.db_size.load(AtomicOrdering::Acquire);
let previous_visible_commit_seq = self.visible_commit_seq.load(AtomicOrdering::Acquire);
if update.db_size < previous_db_size
&& update.visible_commit_seq.get() >= previous_visible_commit_seq
{
self.pages.retain(|page_no| page_no.get() <= update.db_size);
}
self.pages.insert_batch(complete_group_pages);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
/// Validate and publish every page image covered by one durable group
/// certificate. Direct callers use this convenience wrapper; retained
/// recovery splits the two phases around exact-handle finalization.
#[cfg(test)]
fn publish_parallel_wal_group(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
batches: &[TransactionFrameBatch],
) -> Result<()> {
let complete_group_pages = Self::prepare_parallel_wal_group_pages(batches)?;
let certificate_commit_seq_lo = self.page_plane_visible_commit_seq().next();
self.publish_prepared_parallel_wal_group(
cx,
update,
complete_group_pages,
certificate_commit_seq_lo,
);
Ok(())
}
fn publish_commit_consuming_pages<I>(&self, cx: &Cx, update: PublishedPagerUpdate, pages: I)
where
I: IntoIterator<Item = (PageNumber, StagedPage)>,
{
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_commit_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
let previous_db_size = self.db_size.load(AtomicOrdering::Acquire);
let previous_visible_commit_seq = self.visible_commit_seq.load(AtomicOrdering::Acquire);
if update.db_size < previous_db_size
&& update.visible_commit_seq.get() >= previous_visible_commit_seq
{
self.pages.retain(|page_no| page_no.get() <= update.db_size);
}
self.pages.insert_batch(
pages
.into_iter()
.map(|(page_no, staged)| (page_no, staged.into_published_page())),
);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
fn publish_commit_single_page(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
page_no: PageNumber,
staged: StagedPage,
) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_commit_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
let previous_db_size = self.db_size.load(AtomicOrdering::Acquire);
let previous_visible_commit_seq = self.visible_commit_seq.load(AtomicOrdering::Acquire);
if update.db_size < previous_db_size
&& update.visible_commit_seq.get() >= previous_visible_commit_seq
{
self.pages
.retain(|published_page_no| published_page_no.get() <= update.db_size);
}
let _ = self.pages.insert(page_no, staged.into_published_page());
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
/// Publish commit by draining staged pages when the caller no longer needs
/// the write set after publication.
fn publish_commit_draining_write_set<S: std::hash::BuildHasher>(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
write_set: &mut HashMap<PageNumber, StagedPage, S>,
) {
if write_set.len() == 1
&& let Some((&page_no, _)) = write_set.iter().next()
&& let Some(staged) = write_set.remove(&page_no)
{
self.publish_commit_single_page(cx, update, page_no, staged);
return;
}
self.publish_commit_consuming_pages(cx, update, write_set.drain());
}
#[cfg(test)]
fn publish_commit_staged_pages(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
staged_pages: Vec<(PageNumber, StagedPage)>,
) {
self.publish_commit_consuming_pages(cx, update, staged_pages);
}
/// Insert a single page (for testing and internal use).
#[cfg(test)]
fn publish_insert_single(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
page_no: PageNumber,
page: PageData,
) {
let _publish_guard = self
.publish_lock
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if self.should_skip_stale_publish(cx, update, "stale_insert_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
self.pages.insert(page_no, page);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
/// Internal: Insert a single page (called with publish_lock held).
fn publish_insert_page_locked(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
page_no: PageNumber,
page: PageData,
) {
if self.should_skip_stale_publish(cx, update, "stale_insert_publish_skip") {
return;
}
let start = Instant::now();
let publish_start_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
self.signal_sequence_waiters(publish_start_sequence, "publish_begin");
// A fallback read may be the first page observed after metadata moved
// past the resident page plane. The observed page belongs to the new
// horizon, but every retained page still belongs to the old one. Drop
// those stale pages before marking the plane current; later fallback
// reads at this same horizon may then populate it incrementally.
if self.page_plane_visible_commit_seq() != update.visible_commit_seq {
self.pages.clear();
}
self.pages.insert(page_no, page);
let page_set_size = self.pages.len();
self.finalize_publish(cx, update, page_set_size, start, true);
}
/// Internal: Finalize publish by updating metadata and notifying waiters.
fn finalize_publish(
&self,
cx: &Cx,
update: PublishedPagerUpdate,
page_set_size: usize,
start: Instant,
page_plane_synced: bool,
) {
self.publication_write_count
.fetch_add(1, AtomicOrdering::Relaxed);
self.visible_commit_seq
.fetch_max(update.visible_commit_seq.get(), AtomicOrdering::Release);
// Cross-process correctness (#70): a publisher whose visible_commit_seq
// advanced past the previous one may still carry a stale local db_size
// (e.g. a non-extending commit following a peer's EOF extension whose
// page 1 header we have not yet observed). Using `store` here clobbers
// the peer's larger published db_size, leaving readers with
// `page N > snapshot db_size (N-1)` BusySnapshot errors. Keep this
// atomic strictly monotonic and let the explicit truncate-checkpoint
// path (`publish_truncate_checkpoint`) restore the authoritative
// shrunken db_size after finalize_publish returns.
self.db_size
.fetch_max(update.db_size, AtomicOrdering::Release);
self.journal_mode.store(
encode_journal_mode(update.journal_mode),
AtomicOrdering::Release,
);
self.freelist_count
.store(update.freelist_count, AtomicOrdering::Release);
self.checkpoint_active
.store(update.checkpoint_active, AtomicOrdering::Release);
self.page_set_size
.store(page_set_size, AtomicOrdering::Release);
if page_plane_synced {
self.page_plane_visible_commit_seq
.store(update.visible_commit_seq.get(), AtomicOrdering::Release);
}
let previous_sequence = self.sequence.fetch_add(1, AtomicOrdering::AcqRel);
let snapshot_gen = previous_sequence.saturating_add(1);
self.signal_sequence_waiters(previous_sequence, "publish_complete");
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id = "metadata_publish",
snapshot_gen,
visible_commit_seq = update.visible_commit_seq.get(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
freelist_count = update.freelist_count,
checkpoint_active = update.checkpoint_active,
read_retry_count = self.read_retry_count(),
page_set_size,
elapsed_ns = u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX),
"published pager snapshot"
);
}
/// Bind the pager plane to the certificate-derived handoff after the
/// ordinary page/metadata publish finishes.
fn bind_parallel_wal_publication(&self, intent: ParallelWalPublicationIntent) {
self.visible_commit_seq
.fetch_max(intent.visible_commit_seq.get(), AtomicOrdering::Release);
self.page_plane_visible_commit_seq.fetch_max(
intent.page_plane_visible_commit_seq.get(),
AtomicOrdering::Release,
);
self.db_size
.fetch_max(intent.db_size, AtomicOrdering::Release);
}
fn sync_metadata_without_page_publish(&self, update: PublishedPagerUpdate) {
// Single-connection fast path: keep publication metadata monotonic for
// callers that inspect the latest commit horizon, but do not mutate the
// published page plane. Any subsequent page read will skip the stale
// plane because `page_plane_visible_commit_seq` intentionally remains
// behind `visible_commit_seq`.
self.visible_commit_seq
.fetch_max(update.visible_commit_seq.get(), AtomicOrdering::Release);
// See finalize_publish for why db_size is always monotonic here — the
// same cross-process shrinking bug applies on the single-connection
// metadata-only path. publish_truncate_checkpoint is the only place
// permitted to shrink the published db_size.
self.db_size
.fetch_max(update.db_size, AtomicOrdering::Release);
self.journal_mode.store(
encode_journal_mode(update.journal_mode),
AtomicOrdering::Release,
);
self.freelist_count
.store(update.freelist_count, AtomicOrdering::Release);
self.checkpoint_active
.store(update.checkpoint_active, AtomicOrdering::Release);
self.page_set_size.store(0, AtomicOrdering::Release);
}
fn note_published_hit(&self) {
self.published_page_hits.increment();
}
fn record_retry(&self) {
self.read_retry_count.increment();
}
fn wait_for_sequence_change(&self, observed_sequence: u64, timeout: Duration) {
match PUBLISHED_SEQUENCE_WAIT_PATH_MODE {
WaitPathMode::KeyedEventcount => {
if self.sequence.load(AtomicOrdering::Acquire) != observed_sequence {
return;
}
let slot = self.sequence_waiters.slot(observed_sequence);
let observed_generation = slot.generation();
if self.sequence.load(AtomicOrdering::Acquire) != observed_sequence {
return;
}
let wait_result = slot.wait_for_change(observed_generation, timeout);
tracing::trace!(
target: "fsqlite.snapshot_publication",
run_id = "pager-publication",
scenario_id = match wait_result {
KeyedWaitResult::Signaled => "sequence_wait_woke",
KeyedWaitResult::RecoveredAfterTimeout => {
"sequence_wait_timeout_recheck"
}
KeyedWaitResult::TimedOut => "sequence_wait_timeout_fallback",
},
observed_sequence,
wait_strategy = PUBLISHED_SEQUENCE_WAIT_PATH_MODE.as_str(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
"publication wait finished"
);
}
WaitPathMode::LegacyCondvarTimeout => {
let guard = self
.sequence_gate
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let (_guard, _timeout) = self
.sequence_cv
.wait_timeout_while(guard, timeout, |()| {
self.sequence.load(AtomicOrdering::Acquire) == observed_sequence
})
.unwrap_or_else(std::sync::PoisonError::into_inner);
}
}
}
fn read_retry_count(&self) -> u64 {
self.read_retry_count.load()
}
fn published_page_hits(&self) -> u64 {
self.published_page_hits.load()
}
fn publication_write_count(&self) -> u64 {
self.publication_write_count.load(AtomicOrdering::Relaxed)
}
}
const fn encode_journal_mode(mode: JournalMode) -> u8 {
match mode {
JournalMode::Delete => 0,
JournalMode::Wal => 1,
}
}
const fn decode_journal_mode(raw: u8) -> JournalMode {
match raw {
1 => JournalMode::Wal,
_ => JournalMode::Delete,
}
}
#[inline]
const fn transaction_mode_is_eager_writer(mode: TransactionMode) -> bool {
matches!(
mode,
TransactionMode::Immediate | TransactionMode::Exclusive
)
}
fn wait_for_single_writer_baton<'a, F: VfsFile>(
inner_mutex: &'a Mutex<PagerInner<F>>,
writer_idle: &Condvar,
mut inner: MutexGuard<'a, PagerInner<F>>,
) -> Result<MutexGuard<'a, PagerInner<F>>> {
if !inner.writer_active {
return Ok(inner);
}
drop(inner);
for _ in 0..SINGLE_WRITER_BATON_SPINS {
std::hint::spin_loop();
}
inner = inner_mutex
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if !inner.writer_active {
return Ok(inner);
}
let deadline = Instant::now() + SINGLE_WRITER_BATON_PARK;
loop {
let Some(remaining) = deadline.checked_duration_since(Instant::now()) else {
return Err(FrankenError::Busy);
};
if remaining.is_zero() {
return Err(FrankenError::Busy);
}
let wait_result = writer_idle
.wait_timeout(inner, remaining)
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
inner = wait_result.0;
if !inner.writer_active {
return Ok(inner);
}
if wait_result.1.timed_out() {
return Err(FrankenError::Busy);
}
}
}
fn release_single_writer_baton<F: VfsFile>(inner: &mut PagerInner<F>) -> bool {
let was_active = inner.writer_active;
inner.writer_active = false;
was_active
}
/// A concrete single-writer pager backed by a VFS file.
pub struct SimplePager<V: Vfs> {
/// VFS used to open journal/WAL companion files.
vfs: Arc<V>,
/// Path to the database file.
db_path: PathBuf,
/// Native namespace generation retained for the pager lifetime.
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: Option<Arc<DatabaseNamespaceBinding>>,
/// Same-path in-process fence for bootstrap, transactions, and exclusive
/// maintenance publication.
maintenance_gate: Arc<PagerMaintenanceGate>,
/// Identity-bound in-process recovery serialization. Unlike the
/// maintenance gate, this bounded fence lets competing recovery attempts
/// queue without ever waiting while holding a main-file lock.
recovery_fence: Arc<RecoveryFence>,
/// Retains the opener lease through SQL-layer namespace bootstrap.
maintenance_open_lease: Mutex<Option<PagerMaintenanceLease>>,
/// Shared mutable state used by transactions.
inner: Arc<Mutex<PagerInner<V::File>>>,
/// Parks single-writer waiters for bounded baton handoff.
writer_idle: Arc<Condvar>,
/// Sharded page cache for high-concurrency workloads (bd-3wop3.2).
/// Each shard has its own mutex, eliminating global lock contention.
cache: Arc<ShardedPageCache>,
/// Shared page buffer pool cloned into transactions for write staging.
pool: PageBufPool,
/// Published metadata/page plane for lock-light steady-state reads.
published: Arc<PublishedPagerState>,
/// WAL backend for WAL-mode operation (D1-CRITICAL: separate lock for split-lock commit).
/// This enables Thread B to start its prepare phase while Thread A does WAL I/O.
wal_backend: SharedWalBackend,
/// Immutable committed-state snapshot (bd-db300.5.3.3.1 / Card 1: M6).
/// Readers clone the Arc via a brief RwLock-read to inspect committed state
/// without taking the PagerInner Mutex. Published on every commit.
committed_snapshot: Arc<RwLock<Arc<PagerCommittedSnapshot>>>,
/// Same-path connection counter injected by the SQL connection layer.
/// Unset pagers keep the legacy publication path.
shared_connection_count: OnceLock<Arc<AtomicUsize>>,
/// Identity-bound WAL publication queue shared by every pager that opened
/// the same underlying file, including cloned MemoryVfs handles.
group_commit_queue: GroupCommitQueueRef,
}
/// Identity-bound digest of a complete SQLite main-database image.
///
/// The digest covers the page size, exact file length, page numbers, and all
/// logical database pages. SQLite's reserved lock-byte page is deliberately
/// excluded because it is process-lock state rather than database content.
#[derive(Clone, PartialEq, Eq)]
pub struct DatabaseImageReceipt {
identity: FileIdentity,
file_size: u64,
header: DatabaseHeader,
logical_hash: [u8; 32],
}
impl DatabaseImageReceipt {
#[must_use]
pub const fn file_size(&self) -> u64 {
self.file_size
}
#[must_use]
pub const fn identity(&self) -> FileIdentity {
self.identity
}
#[must_use]
pub const fn header(&self) -> &DatabaseHeader {
&self.header
}
#[must_use]
pub const fn logical_hash(&self) -> [u8; 32] {
self.logical_hash
}
}
fn exact_database_page_count(file_size: u64, page_size: PageSize) -> Result<u32> {
let page_size_bytes = u64::from(page_size.get());
if file_size == 0 || !file_size.is_multiple_of(page_size_bytes) {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database image length {file_size} is not a positive multiple of page size {page_size_bytes}"
),
});
}
let page_count = file_size / page_size_bytes;
u32::try_from(page_count).map_err(|_| FrankenError::OutOfRange {
what: "database image page count".to_owned(),
value: page_count.to_string(),
})
}
/// Whole-page count of a database image that may carry a partial-page tail
/// (GH#334 trailing slack): the tail is excluded from the count, matching
/// stock SQLite's header-authoritative extent model.
fn floored_database_page_count(file_size: u64, page_size: PageSize) -> Result<u32> {
let page_size_bytes = u64::from(page_size.get());
if file_size == 0 {
return Err(FrankenError::DatabaseCorrupt {
detail: "database image length 0 holds no pages".to_owned(),
});
}
let page_count = file_size / page_size_bytes;
u32::try_from(page_count).map_err(|_| FrankenError::OutOfRange {
what: "database image page count".to_owned(),
value: page_count.to_string(),
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum DatabaseImageExtentPolicy {
Exact,
AllowTrailingPages,
}
async fn database_image_receipt_for_open_file_with_extent<F: VfsFile>(
cx: &Cx,
file: &F,
expected_page_size: Option<PageSize>,
extent_policy: DatabaseImageExtentPolicy,
) -> Result<DatabaseImageReceipt> {
let identity = file.file_identity()?.ok_or_else(|| {
FrankenError::internal("database image VFS did not provide a stable file identity")
})?;
let file_size = file.file_size(cx)?;
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let header_read = file.read(cx, &mut header_bytes, 0).await?;
if header_read != DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short database header while capturing image receipt: got {header_read} of {DATABASE_HEADER_SIZE}"
),
});
}
let header =
DatabaseHeader::from_bytes(&header_bytes).map_err(|err| FrankenError::DatabaseCorrupt {
detail: format!("invalid database image header: {err}"),
})?;
if let Some(expected_page_size) = expected_page_size
&& header.page_size != expected_page_size
{
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database image page size mismatch: image={} expected={}",
header.page_size.get(),
expected_page_size.get()
),
});
}
// GH#334 / bd-e26jr sig-2: stock SQLite treats the header page count as
// authoritative and ignores trailing bytes beyond the last whole page, so
// a VACUUM source may carry an unaligned partial-page tail. The tail is
// excluded from the page loop but still covered by the receipt: its bytes
// are hashed below and `file_size` is bound into the digest, so the
// publish-time CAS detects any slack mutation. Candidate/self-contained
// images stay on the exact policy.
let page_count = match extent_policy {
DatabaseImageExtentPolicy::Exact => exact_database_page_count(file_size, header.page_size)?,
DatabaseImageExtentPolicy::AllowTrailingPages => {
floored_database_page_count(file_size, header.page_size)?
}
};
let invalid_page_count = match extent_policy {
DatabaseImageExtentPolicy::Exact => header.page_count != page_count,
DatabaseImageExtentPolicy::AllowTrailingPages => header.page_count > page_count,
};
if invalid_page_count {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database image header page count {} does not match file length page count {page_count}",
header.page_count
),
});
}
let mut hasher = blake3::Hasher::new();
hasher.update(b"FrankenSQLite logical database image v1\0");
hasher.update(&header.page_size.get().to_be_bytes());
hasher.update(&file_size.to_be_bytes());
let page_size = header.page_size.as_usize();
let lock_byte_page = crate::journal::lock_byte_page(header.page_size);
let mut page = vec![0_u8; page_size];
for page_no in 1..=page_count {
if page_no == lock_byte_page {
continue;
}
let offset = u64::from(page_no - 1) * u64::from(header.page_size.get());
let bytes_read = file.read(cx, &mut page, offset).await?;
if bytes_read != page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short database page {page_no} while hashing image: got {bytes_read} of {page_size}"
),
});
}
hasher.update(&page_no.to_be_bytes());
hasher.update(&page);
}
let whole_pages_len = u64::from(page_count) * u64::from(header.page_size.get());
if file_size > whole_pages_len {
// Unaligned trailing slack (AllowTrailingPages only — the exact
// policy has already rejected it). Hash the tail so publish-time
// receipt recomputation detects content mutation, not just length.
let tail_len = usize::try_from(file_size - whole_pages_len)
.map_err(|_| FrankenError::internal("trailing slack length exceeds usize"))?;
let mut tail = vec![0_u8; tail_len];
let bytes_read = file.read(cx, &mut tail, whole_pages_len).await?;
if bytes_read != tail_len {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short trailing slack while hashing image: got {bytes_read} of {tail_len}"
),
});
}
hasher.update(b"tail\0");
hasher.update(&tail);
}
Ok(DatabaseImageReceipt {
identity,
file_size,
header,
logical_hash: *hasher.finalize().as_bytes(),
})
}
async fn database_image_receipt_for_open_file<F: VfsFile>(
cx: &Cx,
file: &F,
expected_page_size: Option<PageSize>,
) -> Result<DatabaseImageReceipt> {
database_image_receipt_for_open_file_with_extent(
cx,
file,
expected_page_size,
DatabaseImageExtentPolicy::Exact,
)
.await
}
async fn vacuum_source_receipt_for_open_file<F: VfsFile>(
cx: &Cx,
file: &F,
expected_page_size: PageSize,
) -> Result<DatabaseImageReceipt> {
database_image_receipt_for_open_file_with_extent(
cx,
file,
Some(expected_page_size),
DatabaseImageExtentPolicy::AllowTrailingPages,
)
.await
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ReadWriteOpenDisposition {
CreateIfMissing,
ExistingOnly,
ReservedEmpty,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition,
expected_identity: Option<FileIdentity>,
finish_namespace_bootstrap: bool,
}
/// Pager-open intent used by the SQL connection layer while it retains the
/// native namespace bootstrap lease through its higher-level initialization.
///
/// This is an internal cross-crate contract. Ordinary pager callers should use
/// the existing `open_*` constructors, which complete the namespace transition
/// before returning.
#[doc(hidden)]
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ConnectionPagerOpenMode {
/// Open or create the main database.
CreateIfMissing,
/// Open an existing read-write database, optionally identity-bound.
ExistingOnly(Option<FileIdentity>),
/// Initialize an identity-bound caller-reserved empty database.
ReservedEmpty(FileIdentity),
/// Open an existing database read-only, optionally identity-bound.
ReadOnly(Option<FileIdentity>),
}
impl<V: Vfs> traits::sealed::Sealed for SimplePager<V> {}
fn page_size_from_header_bytes(header_bytes: &[u8; DATABASE_HEADER_SIZE]) -> Option<PageSize> {
if &header_bytes[..DATABASE_HEADER_MAGIC.len()] != DATABASE_HEADER_MAGIC {
return None;
}
let raw = u16::from_be_bytes([header_bytes[16], header_bytes[17]]);
let page_size = match raw {
1 => 65_536,
0 => return None,
value => u32::from(value),
};
PageSize::new(page_size)
}
fn stale_main_header_is_wal_recoverable_error(error: &DatabaseHeaderError) -> bool {
matches!(
error,
DatabaseHeaderError::InvalidSchemaFormat { raw: 0 }
| DatabaseHeaderError::InvalidTextEncoding { raw: 0 }
)
}
fn change_counter_from_header_bytes(header_bytes: &[u8; DATABASE_HEADER_SIZE]) -> u32 {
u32::from_be_bytes([
header_bytes[24],
header_bytes[25],
header_bytes[26],
header_bytes[27],
])
}
fn stale_main_header_change_counter_under_wal(
header_bytes: &[u8; DATABASE_HEADER_SIZE],
error: &DatabaseHeaderError,
) -> Option<u64> {
if !stale_main_header_is_wal_recoverable_error(error) {
return None;
}
page_size_from_header_bytes(header_bytes)?;
Some(u64::from(change_counter_from_header_bytes(header_bytes)))
}
async fn wal_contains_valid_database_page1<F: VfsFile>(
cx: &Cx,
wal: &mut WalFile<F>,
expected_page_size: PageSize,
) -> bool {
let Ok(expected_page_size_usize) = usize::try_from(expected_page_size.get()) else {
return false;
};
if wal.page_size() != expected_page_size_usize {
return false;
}
// Trust only the committed WAL prefix. Valid trailing frames after the
// last commit are not visible to readers and must not affect recovery.
let Some(last_commit_frame) = wal.last_commit_frame(cx).ok().flatten() else {
return false;
};
for frame_index in (0..=last_commit_frame).rev() {
let Ok((frame_header, page_data)) = wal.read_frame(cx, frame_index).await else {
return false;
};
if frame_header.page_number != PageNumber::ONE.get() {
continue;
}
if page_data.len() < DATABASE_HEADER_SIZE {
return false;
}
let mut page1_header_bytes = [0_u8; DATABASE_HEADER_SIZE];
page1_header_bytes.copy_from_slice(&page_data[..DATABASE_HEADER_SIZE]);
return DatabaseHeader::from_bytes(&page1_header_bytes).is_ok();
}
false
}
async fn stale_main_header_can_be_recovered_from_live_wal<V: Vfs>(
cx: &Cx,
vfs: &V,
path: &Path,
header_bytes: &[u8; DATABASE_HEADER_SIZE],
error: &DatabaseHeaderError,
allow_readonly_wal_probe: bool,
) -> Result<bool> {
if !stale_main_header_is_wal_recoverable_error(error) {
return Ok(false);
}
let Some(expected_page_size) = page_size_from_header_bytes(header_bytes) else {
return Ok(false);
};
let mut wal_path = path.to_owned().into_os_string();
wal_path.push("-wal");
let wal_path = PathBuf::from(wal_path);
if !vfs.access(cx, &wal_path, AccessFlags::EXISTS)? {
return Ok(false);
}
// Probe WAL content with a write-capable handle first. On the Unix VFS,
// a READONLY-opened WAL can become the canonical inode-table fd, poisoning
// later writer paths with EBADF if they clone that descriptor.
let (wal_file, _) = match vfs.open(
cx,
Some(&wal_path),
VfsOpenFlags::READWRITE | VfsOpenFlags::WAL,
) {
Ok(opened) => opened,
Err(_) if allow_readonly_wal_probe => {
match vfs.open(
cx,
Some(&wal_path),
VfsOpenFlags::READONLY | VfsOpenFlags::WAL,
) {
Ok(opened) => opened,
Err(_) => return Ok(false),
}
}
Err(_) => return Ok(false),
};
let Ok(mut wal) = WalFile::open(cx, wal_file).await else {
return Ok(false);
};
let wal_contains_valid_page1 =
wal_contains_valid_database_page1(cx, &mut wal, expected_page_size).await;
let _ = wal.close(cx);
Ok(wal_contains_valid_page1)
}
fn bootstrap_header_from_stale_main_file(
header_bytes: &[u8; DATABASE_HEADER_SIZE],
page_size: PageSize,
) -> DatabaseHeader {
DatabaseHeader {
page_size,
write_version: header_bytes[18],
read_version: header_bytes[19],
reserved_per_page: header_bytes[20],
change_counter: u32::from_be_bytes([
header_bytes[24],
header_bytes[25],
header_bytes[26],
header_bytes[27],
]),
page_count: u32::from_be_bytes([
header_bytes[28],
header_bytes[29],
header_bytes[30],
header_bytes[31],
]),
freelist_trunk: u32::from_be_bytes([
header_bytes[32],
header_bytes[33],
header_bytes[34],
header_bytes[35],
]),
freelist_count: u32::from_be_bytes([
header_bytes[36],
header_bytes[37],
header_bytes[38],
header_bytes[39],
]),
schema_cookie: u32::from_be_bytes([
header_bytes[40],
header_bytes[41],
header_bytes[42],
header_bytes[43],
]),
schema_format: u32::from_be_bytes([
header_bytes[44],
header_bytes[45],
header_bytes[46],
header_bytes[47],
]),
default_cache_size: i32::from_be_bytes([
header_bytes[48],
header_bytes[49],
header_bytes[50],
header_bytes[51],
]),
largest_root_page: u32::from_be_bytes([
header_bytes[52],
header_bytes[53],
header_bytes[54],
header_bytes[55],
]),
// The authoritative encoding/schema header will be re-read from the
// WAL-backed page-1 snapshot on the first transaction begin.
text_encoding: fsqlite_types::TextEncoding::Utf8,
user_version: u32::from_be_bytes([
header_bytes[60],
header_bytes[61],
header_bytes[62],
header_bytes[63],
]),
incremental_vacuum: u32::from_be_bytes([
header_bytes[64],
header_bytes[65],
header_bytes[66],
header_bytes[67],
]),
application_id: u32::from_be_bytes([
header_bytes[68],
header_bytes[69],
header_bytes[70],
header_bytes[71],
]),
version_valid_for: u32::from_be_bytes([
header_bytes[92],
header_bytes[93],
header_bytes[94],
header_bytes[95],
]),
sqlite_version: u32::from_be_bytes([
header_bytes[96],
header_bytes[97],
header_bytes[98],
header_bytes[99],
]),
}
}
impl<V> MvccPager for SimplePager<V>
where
V: Vfs + Send + Sync,
V::File: Send + Sync + 'static,
{
type Txn = SimpleTransaction<V>;
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
fn begin<'a>(
&'a self,
cx: &'a Cx,
mode: TransactionMode,
) -> impl Future<Output = Result<Self::Txn>> + 'a {
async move {
let begin_handle_key = {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
shared_db_file_key(&inner.db_file)
};
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
begin_handle_key,
)
.await?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if shared_db_file_key(&inner.db_file) != begin_handle_key {
return Err(FrankenError::internal(
"pager database-file handle changed during begin admission",
));
}
if self
.group_commit_queue
.has_relevant_process_root(begin_handle_key)
{
return Err(FrankenError::BusyRecovery);
}
// A stale lexical namespace must not consume an identity-owned
// orphan receipt that another valid alias needs to recover.
self.validate_namespace_binding()?;
inner.adopt_orphaned_rollback_journal_recovery()?;
// Derive and admit the exact takeover owner while PagerInner is
// still locked. No await or local-state ABA can occur between this
// sample and the identity-gate comparison.
let expected_recovery_owner = if inner.active_transactions == 0
&& inner
.rollback_journal_recovery_state
.is_pager_takeover_eligible()
{
inner.rollback_journal_recovery_owner
} else {
None
};
let mut maintenance_lease = Some(
self.maintenance_gate
.enter_transaction_with_recovery_owner(expected_recovery_owner)?,
);
if inner.checkpoint_active {
let active_transactions = inner.active_transactions;
let checkpoint_active = inner.checkpoint_active;
drop(inner);
log_checkpoint_coordination(
cx,
&pager_group_commit_queue(self),
"active_gate",
"begin",
transaction_mode_name(mode),
"checkpoint_excludes_new_transactions",
true,
active_transactions,
checkpoint_active,
);
return Err(FrankenError::Busy);
}
let eager_writer = transaction_mode_is_eager_writer(mode);
if eager_writer && inner.access_mode.is_readonly() {
return Err(FrankenError::ReadOnly);
}
if eager_writer {
inner = wait_for_single_writer_baton(&self.inner, &self.writer_idle, inner)?;
}
let active_transactions_before_begin = inner.active_transactions;
if active_transactions_before_begin != 0
&& inner.rollback_journal_recovery_state.is_pending()
{
return Err(FrankenError::BusyRecovery);
}
// ── In-memory fast path ─────────────────────────────────────
// For in-memory VFS, skip persistent shared-lock ownership between
// local transactions. We still need to recover any externally-created
// hot journal and refresh connection-local metadata/publication state
// when the first local transaction starts, because another pager
// sharing the same `MemoryVfs` can mutate the durable image or the
// shared WAL backend between transactions.
if self.vfs.is_memory() {
// Declared after `inner`: on cancellation the admission guard
// publishes its rooted cleanup before the pager mutex becomes
// observable to another task. The pending restorer never
// blocks on that mutex while holding the shared file handle.
let mut admission = BeginAdmission::new(
&self.group_commit_queue,
Arc::clone(&self.inner),
Arc::clone(&inner.db_file),
Arc::clone(&self.writer_idle),
maintenance_lease
.take()
.expect("memory begin must own its transaction maintenance lease"),
None,
cx,
);
let commit_seq_before_refresh = inner.commit_seq;
let (committed_refresh, journal_visibility_invalidation) =
if active_transactions_before_begin == 0 {
let maintenance_lease =
admission.maintenance_lease.as_mut().ok_or_else(|| {
FrankenError::internal(
"memory begin admission lost its maintenance lease before refresh",
)
})?;
self.refresh_runtime_committed_state(
cx,
maintenance_lease,
&mut inner,
Some(&mut admission.external_lock),
)
.await?
} else {
(
CommittedStateRefresh {
wal_snapshot_initialized: false,
page_cache_invalidated: false,
},
false,
)
};
if active_transactions_before_begin == 0 {
let clear_published_pages = journal_visibility_invalidation
|| committed_refresh.page_cache_invalidated
|| inner.commit_seq != commit_seq_before_refresh;
// D1-CRITICAL Change 3: Use sharded publish_clear_if.
self.published.publish_clear_if(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
clear_published_pages,
);
}
if eager_writer && inner.writer_active {
return Err(FrankenError::Busy);
}
let wal_conflict_snapshot = if inner.journal_mode == JournalMode::Wal {
capture_wal_conflict_snapshot_at_begin(
&self.wal_backend,
cx,
committed_refresh.wal_snapshot_initialized,
active_transactions_before_begin != 0,
)
.await?
} else {
None
};
let active_transactions_after_begin =
inner.active_transactions.checked_add(1).ok_or_else(|| {
FrankenError::internal("active transaction count overflow during begin")
})?;
if eager_writer {
inner.writer_active = true;
admission.mark_writer_baton_owned();
}
inner.active_transactions = active_transactions_after_begin;
let original_db_size = inner.db_size;
let journal_mode = inner.journal_mode;
let published_snapshot = self.published.snapshot();
// Honor the "inner is at least as fresh as published" invariant by
// taking the per-field max of the two views; this guards against
// a transaction starting with stale visibility when a recent
// commit already advanced inner.* but the publication plane has
// not yet been re-advertised. (Regression-covered by
// self_alloc_extension_not_conflict; see 18faea82.)
let bound_visible_commit_seq =
std::cmp::max(published_snapshot.visible_commit_seq, inner.commit_seq);
let bound_db_size = published_snapshot.db_size.max(original_db_size);
let pool = self.pool.clone();
let cleanup_cx = cx.clone();
let memory_db_bump_alloc =
self.vfs.is_memory() && self.db_path == Path::new("/:memory:");
let maintenance_lease = admission.complete()?;
return Ok(SimpleTransaction {
vfs: Arc::clone(&self.vfs),
journal_path: Self::journal_path(&self.db_path),
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: self.namespace_binding.clone(),
group_commit_queue: Arc::clone(&self.group_commit_queue),
inner: Arc::clone(&self.inner),
db_file: Arc::clone(&inner.db_file),
writer_idle: Arc::clone(&self.writer_idle),
cache: Arc::clone(&self.cache),
published: Arc::clone(&self.published),
wal_backend: Arc::clone(&self.wal_backend),
committed_snapshot: Arc::clone(&self.committed_snapshot),
shared_connection_count: self.shared_connection_count.get().cloned(),
maintenance_lease: Some(maintenance_lease),
pending_group_commit_attempt: None,
owned_rollback_recovery: None,
rollback_commit_finalization_pending: false,
rollback_recovery_pending: Arc::clone(&inner.rollback_recovery_pending),
recovery_fence: Arc::clone(&self.recovery_fence),
read_only_pager: inner.access_mode.is_readonly(),
wal_conflict_snapshot,
published_visible_commit_seq: Cell::new(bound_visible_commit_seq),
published_db_size: Cell::new(bound_db_size),
write_set: PagePageMap::default(),
write_pages_sorted: Vec::new(),
freed_pages: Vec::new(),
freed_page_bounds: None,
allocated_from_freelist: Vec::new(),
allocated_from_eof: Vec::new(),
writes_observed: false,
mode,
is_writer: eager_writer,
committed: false,
finished: false,
original_db_size,
savepoint_stack: Vec::new(),
journal_mode,
pool,
cleanup_cx,
page_lease: Vec::new(),
memory_db_bump_alloc,
rolled_back_pages: HashSet::new(),
txn_read_cache: RefCell::new(PagePageMap::default()),
retained_memory_overlay_dirty_pages: BTreeSet::new(),
scratch_arena: bumpalo::Bump::new(),
});
}
// ── File-backed path (full locking + recovery) ──────────────
let logical_transition_claim = if active_transactions_before_begin == 0 || eager_writer
{
Some(
GroupCommitLogicalExitClaim::try_register(
&self.group_commit_queue,
shared_db_file_key(&inner.db_file),
)
.ok_or(FrankenError::BusyRecovery)?,
)
} else {
None
};
// Declared after `inner` for the same publication-before-visibility
// rule as the memory-backed path above.
let mut admission = BeginAdmission::new(
&self.group_commit_queue,
Arc::clone(&self.inner),
Arc::clone(&inner.db_file),
Arc::clone(&self.writer_idle),
maintenance_lease
.take()
.expect("file begin must own its transaction maintenance lease"),
logical_transition_claim,
cx,
);
let commit_seq_before_refresh = inner.commit_seq;
let (committed_refresh, journal_visibility_invalidation) =
if active_transactions_before_begin == 0 {
let maintenance_lease =
admission.maintenance_lease.as_mut().ok_or_else(|| {
FrankenError::internal(
"file begin admission lost its maintenance lease before refresh",
)
})?;
self.refresh_runtime_committed_state(
cx,
maintenance_lease,
&mut inner,
Some(&mut admission.external_lock),
)
.await?
} else {
(
CommittedStateRefresh {
wal_snapshot_initialized: false,
page_cache_invalidated: false,
},
false,
)
};
if active_transactions_before_begin == 0 {
// Retain one stock-visible SHARED snapshot fence for the lifetime
// of the first local transaction. The refresh above must hand
// off the exact fence under which it verified journal and
// metadata state; reacquiring here would open an ABA window to
// a cross-process writer. Later local transactions share this
// file handle and the last one releases it.
if !admission.external_lock.is_armed() {
return Err(FrankenError::internal(
"file begin refresh did not retain its verified external snapshot",
));
}
}
if active_transactions_before_begin == 0 {
let clear_published_pages = journal_visibility_invalidation
|| committed_refresh.page_cache_invalidated
|| inner.commit_seq != commit_seq_before_refresh;
// D1-CRITICAL Change 3: Use sharded publish_clear_if.
self.published.publish_clear_if(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
clear_published_pages,
);
}
let published_snapshot = self.published.snapshot();
let commit_seq_lagged = published_snapshot.visible_commit_seq < inner.commit_seq;
let db_size_lagged = published_snapshot.db_size < inner.db_size;
let journal_mode_lagged = published_snapshot.journal_mode != inner.journal_mode;
let freelist_lagged = published_snapshot.freelist_count != inner.freelist.len();
let checkpoint_lagged = published_snapshot.checkpoint_active != inner.checkpoint_active;
let publication_lagged = commit_seq_lagged
|| db_size_lagged
|| journal_mode_lagged
|| freelist_lagged
|| checkpoint_lagged;
if publication_lagged {
let publication_update = PublishedPagerUpdate {
visible_commit_seq: std::cmp::max(
published_snapshot.visible_commit_seq,
inner.commit_seq,
),
db_size: published_snapshot.db_size.max(inner.db_size),
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
};
let clear_published_pages = published_snapshot.visible_commit_seq
!= publication_update.visible_commit_seq
|| published_snapshot.db_size != publication_update.db_size
|| published_snapshot.journal_mode != publication_update.journal_mode;
self.published
.publish_clear_if(cx, publication_update, clear_published_pages);
}
if eager_writer && inner.writer_active {
return Err(FrankenError::Busy);
}
// For write transactions, escalate to RESERVED to signal write intent
// to other processes. This is a non-blocking advisory lock that
// prevents multiple processes from writing simultaneously.
if eager_writer {
if active_transactions_before_begin != 0 {
admission.external_lock.arm_lock_level(LockLevel::Shared);
}
{
let mut db_file = shared_db_file_write(&inner.db_file, cx).await?;
db_file.lock(cx, LockLevel::Reserved)?;
}
if active_transactions_before_begin != 0 {
admission.external_lock.mark_lock_level_acquired();
}
inner.writer_active = true;
admission.mark_writer_baton_owned();
}
let wal_conflict_snapshot = if inner.journal_mode == JournalMode::Wal {
capture_wal_conflict_snapshot_at_begin(
&self.wal_backend,
cx,
committed_refresh.wal_snapshot_initialized,
active_transactions_before_begin != 0,
)
.await?
} else {
None
};
inner.active_transactions =
inner.active_transactions.checked_add(1).ok_or_else(|| {
FrankenError::internal("active transaction count overflow during begin")
})?;
let original_db_size = inner.db_size;
let journal_mode = inner.journal_mode;
let pool = self.pool.clone();
// The WAL backend pinned this exact refreshed image above. Shared
// publication may advance after that pin, so binding the
// transaction to a later global snapshot would mix two images.
let bound_visible_commit_seq = inner.commit_seq;
let bound_db_size = inner.db_size;
let cleanup_cx = cleanup_child_cx(cx);
let memory_db_bump_alloc =
self.vfs.is_memory() && self.db_path == Path::new("/:memory:");
let read_only_pager = inner.access_mode.is_readonly();
let db_file = Arc::clone(&inner.db_file);
let rollback_recovery_pending = Arc::clone(&inner.rollback_recovery_pending);
let maintenance_lease = admission.complete()?;
drop(inner);
Ok(SimpleTransaction {
vfs: Arc::clone(&self.vfs),
journal_path: Self::journal_path(&self.db_path),
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: self.namespace_binding.clone(),
group_commit_queue: Arc::clone(&self.group_commit_queue),
inner: Arc::clone(&self.inner),
db_file,
writer_idle: Arc::clone(&self.writer_idle),
cache: Arc::clone(&self.cache),
published: Arc::clone(&self.published),
wal_backend: Arc::clone(&self.wal_backend),
committed_snapshot: Arc::clone(&self.committed_snapshot),
shared_connection_count: self.shared_connection_count.get().cloned(),
maintenance_lease: Some(maintenance_lease),
pending_group_commit_attempt: None,
owned_rollback_recovery: None,
rollback_commit_finalization_pending: false,
rollback_recovery_pending,
recovery_fence: Arc::clone(&self.recovery_fence),
read_only_pager,
wal_conflict_snapshot,
published_visible_commit_seq: Cell::new(bound_visible_commit_seq),
published_db_size: Cell::new(bound_db_size),
write_set: PagePageMap::default(),
write_pages_sorted: Vec::new(),
freed_pages: Vec::new(),
freed_page_bounds: None,
allocated_from_freelist: Vec::new(),
allocated_from_eof: Vec::new(),
writes_observed: false,
mode,
is_writer: eager_writer,
committed: false,
finished: false,
original_db_size,
savepoint_stack: Vec::new(),
journal_mode,
pool,
cleanup_cx,
page_lease: Vec::new(),
memory_db_bump_alloc,
rolled_back_pages: HashSet::new(),
txn_read_cache: RefCell::new(PagePageMap::default()),
retained_memory_overlay_dirty_pages: BTreeSet::new(),
scratch_arena: bumpalo::Bump::new(),
})
}
}
fn journal_mode(&self) -> JournalMode {
self.published.snapshot().journal_mode
}
fn is_readonly(&self) -> bool {
let inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.access_mode.is_readonly()
}
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
fn set_journal_mode<'a>(
&'a self,
cx: &'a Cx,
mode: JournalMode,
) -> impl Future<Output = Result<JournalMode>> + 'a {
async move {
settle_pending_group_commit_finalization(&self.group_commit_queue).await?;
let _maintenance_lease = self.maintenance_gate.enter_transaction()?;
self.validate_namespace_binding()?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if inner.journal_mode == mode {
if mode == JournalMode::Wal && !has_wal_backend(&self.wal_backend)? {
return Err(FrankenError::Unsupported);
}
if mode == JournalMode::Wal {
self.cache.evict(PageNumber::ONE);
self.published.publish_remove_page(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
PageNumber::ONE,
);
}
return Ok(mode);
}
if inner.checkpoint_active {
return Err(FrankenError::Busy);
}
if inner.active_transactions > 0 {
// Cannot switch journal mode while any transaction is active.
return Err(FrankenError::Busy);
}
if mode == JournalMode::Wal && !has_wal_backend(&self.wal_backend)? {
return Err(FrankenError::Unsupported);
}
// Update the file format version in the database header (bytes 18-19).
// WAL mode uses version 2; all rollback journal modes use version 1.
// Without this, standard SQLite tools cannot detect WAL mode from the
// on-disk header and will fail to look for the WAL file.
let version_byte: u8 = if mode == JournalMode::Wal { 2 } else { 1 };
if inner.db_size > 0 && !inner.access_mode.is_readonly() {
let page_size = inner.page_size.as_usize();
let mut page1 = vec![0u8; page_size];
let db_file = shared_db_file_read(&inner.db_file, cx).await?;
let bytes_read = db_file.read(cx, &mut page1, 0).await?;
if bytes_read >= DATABASE_HEADER_SIZE {
page1[18] = version_byte;
page1[19] = version_byte;
db_file.write(cx, &page1, 0).await?;
self.cache.evict(PageNumber::ONE);
}
}
inner.journal_mode = mode;
// D1-CRITICAL Change 3: Use sharded publish_remove_page.
self.published.publish_remove_page(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
PageNumber::ONE,
);
drop(inner);
Ok(mode)
}
}
fn set_wal_backend(&self, backend: Box<dyn WalBackend>) -> Result<()> {
if self
.group_commit_queue
.has_process_root_finalization_attempt()
{
return Err(FrankenError::BusyRecovery);
}
let replacing_backend = has_wal_backend(&self.wal_backend)?;
let maintenance_lease = replacing_backend
.then(|| self.maintenance_gate.enter_exclusive_maintenance())
.transpose()?;
if self
.group_commit_queue
.has_process_root_finalization_attempt()
{
return Err(FrankenError::BusyRecovery);
}
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if inner.checkpoint_active {
return Err(FrankenError::Busy);
}
drop(inner);
// D1-CRITICAL: Store in shared lock, NOT inner.wal_backend
let mut wal_guard = self
.wal_backend
.write()
.map_err(|_| FrankenError::internal("SharedWalBackend lock poisoned"))?;
if wal_guard.is_some() && maintenance_lease.is_none() {
return Err(FrankenError::Busy);
}
*wal_guard = Some(Arc::new(AsyncRwLock::with_name("wal_backend", backend)));
drop(wal_guard);
Ok(())
}
}
impl<V: Vfs> SimplePager<V>
where
V::File: Send + Sync + 'static,
{
const EXPORT_COPY_CHUNK_SIZE: usize = 64 * 1024;
async fn prepare_fresh_journal_for_maintenance(&self, cx: &Cx) -> Result<()> {
let journal_path = Self::journal_path(&self.db_path);
if !self.vfs.access(cx, &journal_path, AccessFlags::EXISTS)? {
return Ok(());
}
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal, _) = self.vfs.open(cx, Some(&journal_path), flags)?;
let result = classify_rollback_journal_prefix(cx, &journal)
.await
.and_then(|(state, _)| {
if state == RollbackJournalPrefixState::NonHot {
Ok(())
} else {
Err(FrankenError::Busy)
}
});
let close_result = journal.close(cx);
result?;
close_result?;
// Remove an accepted non-hot leftover before the publisher creates a
// new EXCLUSIVE journal. This prevents following a pre-existing
// symlink or truncating an attacker-controlled hard link.
self.vfs.delete(cx, &journal_path, true)?;
if self.vfs.access(cx, &journal_path, AccessFlags::EXISTS)? {
return Err(FrankenError::CannotOpen { path: journal_path });
}
Ok(())
}
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
async fn with_exclusive_maintenance<S, T>(
&self,
cx: &Cx,
state: &mut S,
operation: impl for<'a> FnOnce(
&'a Self,
&'a Cx,
&'a mut PagerInner<V::File>,
&'a mut S,
) -> LocalPagerFuture<'a, T>,
) -> Result<T> {
struct MaintenanceActivityGuard<'a, F: VfsFile> {
inner: &'a Mutex<PagerInner<F>>,
published: &'a PublishedPagerState,
cleanup_cx: Cx,
active: bool,
}
impl<F: VfsFile> MaintenanceActivityGuard<'_, F> {
fn disarm(&mut self) {
self.active = false;
}
}
impl<F: VfsFile> Drop for MaintenanceActivityGuard<'_, F> {
fn drop(&mut self) {
if !self.active {
return;
}
let mut inner = match self.inner.lock() {
Ok(inner) => inner,
Err(error) => {
tracing::error!(
"maintenance activity guard recovered a poisoned PagerInner for fail-closed cleanup"
);
error.into_inner()
}
};
let _mask = self.cleanup_cx.masked();
inner.checkpoint_active = false;
self.published.publish_metadata_only(
&self.cleanup_cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: false,
},
);
}
}
settle_pending_group_commit_finalization(&self.group_commit_queue).await?;
let _maintenance_lease = self.maintenance_gate.enter_exclusive_maintenance()?;
let mut activity_guard = MaintenanceActivityGuard {
inner: &self.inner,
published: self.published.as_ref(),
cleanup_cx: cleanup_child_cx(cx),
active: false,
};
self.validate_namespace_binding()?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if inner.access_mode.is_readonly()
|| inner.active_transactions != 0
|| inner.writer_active
|| inner.checkpoint_active
|| inner.rollback_journal_recovery_state.is_pending()
{
return Err(FrankenError::Busy);
}
inner.checkpoint_active = true;
self.published.publish_metadata_only(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: true,
},
);
activity_guard.active = true;
let wal_handle = if inner.journal_mode == JournalMode::Wal {
match wal_backend_handle(&self.wal_backend) {
Ok(handle) => Some(handle),
Err(error) => return Err(error),
}
} else {
None
};
// Acquire one VFS-defined fence over every lock surface relevant to a
// whole-image replacement. Each native backend composes its main-file
// and shared-memory lock surfaces according to its platform protocol.
let wal_mode = inner.journal_mode == JournalMode::Wal;
let mut external_lock =
BeginExternalLockState::new(&self.group_commit_queue, Arc::clone(&inner.db_file), cx);
let maintenance_lock_result = external_lock.acquire_maintenance(cx, wal_mode).await;
if let Err(err) = maintenance_lock_result {
// Terminalize synchronously or publish the process-root retry
// before advertising that maintenance is inactive.
drop(external_lock);
return Err(err);
}
let preflight = async {
self.prepare_fresh_journal_for_maintenance(cx).await?;
if let Some(wal_handle) = wal_handle.as_ref() {
let mut wal = async_rwlock_write(wal_handle, cx, "WAL backend").await?;
// Refresh from the durable WAL while the main-file EXCLUSIVE
// lock prevents external SQLite readers/writers from entering.
wal.begin_transaction(cx).await?;
if wal.frame_count() != 0 {
return Err(FrankenError::Busy);
}
}
Ok(())
}
.await;
let operation_result = match preflight {
Ok(()) => operation(self, cx, &mut inner, state).await,
Err(err) => Err(err),
};
// Releasing the cross-process maintenance fence is cleanup, not
// cancellable application work. In particular, VACUUM may have had
// to synchronously replay a hot rollback journal after its caller was
// cancelled. Keep the final main/SHM unlocks in an independently
// masked child so inherited cancellation cannot strand either lock.
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let unlock_result = external_lock.restore().await;
// `restore()` deliberately leaves the attempt armed on error. Its
// Drop either finishes synchronously or queues a process-root owner;
// only after that handoff may observers see maintenance as inactive.
drop(external_lock);
inner.checkpoint_active = false;
self.published.publish_metadata_only(
&cleanup_cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: false,
},
);
activity_guard.disarm();
self.publish_committed_snapshot_from_inner(&inner);
drop(wal_handle);
match (operation_result, unlock_result) {
(Ok(value), Ok(())) => Ok(value),
(Err(err), Ok(())) | (Ok(_), Err(err)) => Err(err),
(operation_result, unlock_result) => {
let operation_detail = operation_result
.err()
.map_or_else(|| "ok".to_owned(), |err| err.to_string());
let unlock_detail = unlock_result
.err()
.map_or_else(|| "ok".to_owned(), |err| err.to_string());
Err(FrankenError::internal(format!(
"exclusive maintenance cleanup failed: operation={operation_detail}; external_unlock={unlock_detail}"
)))
}
}
}
/// Return the database path used by this pager.
#[must_use]
pub fn db_path(&self) -> &Path {
&self.db_path
}
/// Clone the native lifetime binding for components that derive companion
/// paths (notably the path-refreshing WAL backend).
#[cfg(all(feature = "native", any(unix, windows)))]
pub fn namespace_binding(&self) -> Option<Arc<DatabaseNamespaceBinding>> {
self.namespace_binding.clone()
}
#[cfg(all(feature = "native", any(unix, windows)))]
pub fn validate_namespace_binding(&self) -> Result<()> {
if let Some(binding) = &self.namespace_binding {
binding.validate_path_identity()?;
}
Ok(())
}
#[cfg(not(all(feature = "native", any(unix, windows))))]
pub fn validate_namespace_binding(&self) -> Result<()> {
Ok(())
}
fn release_maintenance_open_lease(&self) {
self.maintenance_open_lease
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.take();
}
#[cfg(all(feature = "native", any(unix, windows)))]
#[doc(hidden)]
pub fn finish_namespace_bootstrap(&self) -> Result<()> {
if let Some(binding) = &self.namespace_binding {
binding.finish_bootstrap()?;
}
self.release_maintenance_open_lease();
Ok(())
}
#[cfg(not(all(feature = "native", any(unix, windows))))]
#[doc(hidden)]
pub fn finish_namespace_bootstrap(&self) -> Result<()> {
self.release_maintenance_open_lease();
Ok(())
}
/// Clone the pager's VFS handle for companion-file operations.
pub fn vfs_handle(&self) -> Arc<V> {
Arc::clone(&self.vfs)
}
/// Return the identity of the already-open main database file.
///
/// The VFS implementation determines whether a stable descriptor identity
/// is available. The pager never re-resolves [`Self::db_path`] here.
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
pub async fn file_identity(&self, cx: &Cx) -> Result<Option<FileIdentity>> {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
let db_file = shared_db_file_read(&inner.db_file, cx).await?;
db_file.file_identity()
}
/// Propagate the connection's busy-timeout to the underlying VFS file so
/// that `posix_lock` retries with backoff instead of returning BUSY
/// immediately on cross-process contention.
pub async fn set_vfs_busy_timeout_ms(&self, cx: &Cx, ms: u64) {
// bd-b4mwn rework: publish the connection's busy budget to the shared
// group-commit queue so settle's claim-contention retry envelope
// scales with the caller's patience instead of a fixed ceiling. Keep
// the largest published value: settle serves every connection on the
// path and must not be starved by one impatient peer.
self.group_commit_queue
.settle_budget_ms
.fetch_max(ms.max(1), AtomicOrdering::AcqRel);
let db_file = self
.inner
.lock()
.ok()
.map(|inner| Arc::clone(&inner.db_file));
if let Some(db_file) = db_file
&& let Ok(mut db_file) = shared_db_file_write(&db_file, cx).await
{
db_file.set_busy_timeout_ms(ms);
}
}
/// Resolve any group-commit finalization a prior transaction admission on
/// this pager rooted but deferred — for example the SHARED-snapshot
/// restore that `BeginAdmission::drop` queues when a read admission (such
/// as the schema/header probe during a read-only open) is torn down
/// without a synchronous unlock. Read-only WAL installs call this first so
/// a benign self-inflicted pending external unlock does not make
/// `set_wal_backend_owned` refuse with `BusyRecovery`, mirroring the
/// settle `set_journal_mode` already performs before mutating journal
/// state.
pub async fn quiesce_pending_group_commit_finalization(&self) -> Result<()> {
settle_pending_group_commit_finalization(&self.group_commit_queue).await
}
/// Install a concrete WAL backend while preserving ownership on failure.
///
/// This lets callers explicitly close any underlying VFS resources when
/// the pager rejects installation.
pub fn set_wal_backend_owned<B>(&self, backend: B) -> std::result::Result<(), (FrankenError, B)>
where
B: WalBackend + 'static,
{
if self
.group_commit_queue
.has_process_root_finalization_attempt()
{
return Err((FrankenError::BusyRecovery, backend));
}
let replacing_backend = match has_wal_backend(&self.wal_backend) {
Ok(replacing) => replacing,
Err(err) => return Err((err, backend)),
};
let maintenance_lease = if replacing_backend {
match self.maintenance_gate.enter_exclusive_maintenance() {
Ok(lease) => Some(lease),
Err(err) => return Err((err, backend)),
}
} else {
None
};
if self
.group_commit_queue
.has_process_root_finalization_attempt()
{
return Err((FrankenError::BusyRecovery, backend));
}
let inner = match self.inner.lock() {
Ok(inner) => inner,
Err(_) => {
return Err((FrankenError::internal("SimplePager lock poisoned"), backend));
}
};
if inner.checkpoint_active {
return Err((FrankenError::Busy, backend));
}
drop(inner);
let mut wal_guard = match self.wal_backend.write() {
Ok(guard) => guard,
Err(_) => {
return Err((
FrankenError::internal("SharedWalBackend lock poisoned"),
backend,
));
}
};
if wal_guard.is_some() && maintenance_lease.is_none() {
return Err((FrankenError::Busy, backend));
}
*wal_guard = Some(Arc::new(AsyncRwLock::with_name(
"wal_backend",
Box::new(backend),
)));
drop(wal_guard);
Ok(())
}
/// Return the current WAL commit sync policy.
#[must_use]
pub fn wal_commit_sync_policy(&self) -> WalCommitSyncPolicy {
self.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.wal_commit_sync_policy
}
/// Configure whether WAL-mode commits sync the WAL file immediately.
pub fn set_wal_commit_sync_policy(&self, policy: WalCommitSyncPolicy) -> Result<()> {
let _maintenance_lease = self.maintenance_gate.enter_transaction()?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
inner.wal_commit_sync_policy = policy;
Ok(())
}
/// Inspect a fully-written candidate database image without creating any
/// SQLite sidecars. Callers should perform semantic validation first, then
/// retain this receipt for identity/hash verification at publication.
pub async fn inspect_database_image(
&self,
cx: &Cx,
image_path: &Path,
) -> Result<DatabaseImageReceipt> {
let full_path = self.vfs.full_pathname(cx, image_path)?;
let flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_DB;
let (mut file, _) = self.vfs.open(cx, Some(&full_path), flags)?;
let result = database_image_receipt_for_open_file(cx, &file, Some(self.page_size())).await;
let close_result = file.close(cx);
let receipt = result?;
close_result?;
Ok(receipt)
}
/// Inspect a private database image and prove that no recovery sidecar is
/// part of its current state.
///
/// `VACUUM INTO` uses this before semantic validation and again at each
/// success boundary. Checking companions on both sides of the full-image
/// digest closes the window in which a cooperating opener could switch a
/// nominally complete main file into a WAL- or journal-backed generation
/// without changing the main-file bytes themselves.
pub async fn inspect_self_contained_database_image(
&self,
cx: &Cx,
image_path: &Path,
) -> Result<DatabaseImageReceipt> {
let full_path = self.vfs.full_pathname(cx, image_path)?;
self.ensure_vacuum_candidate_is_self_contained(cx, &full_path)?;
let receipt = self.inspect_database_image(cx, &full_path).await?;
self.ensure_vacuum_candidate_is_self_contained(cx, &full_path)?;
Ok(receipt)
}
/// Install source-derived change-counter provenance on a private VACUUM
/// candidate without trusting its pathname between inspection and write.
///
/// The full provisional receipt is recomputed from the identity-bound open
/// handle while an exclusive file lock is held. The method writes one
/// complete page-1 image, durably syncs it, verifies that every byte outside
/// offsets 24..28 and 92..96 is unchanged, and returns the final full-image
/// receipt for later semantic validation and publication CAS.
pub async fn restore_vacuum_candidate_change_counter(
&self,
cx: &Cx,
image_path: &Path,
provisional: &DatabaseImageReceipt,
change_counter: u32,
) -> Result<DatabaseImageReceipt> {
let full_path = self.vfs.full_pathname(cx, image_path)?;
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut file, _) =
self.vfs
.open_with_expected_identity(cx, &full_path, flags, provisional.identity)?;
let mut lock_held = false;
let update_result = async {
file.lock(cx, LockLevel::Exclusive)?;
lock_held = true;
let current = database_image_receipt_for_open_file(
cx,
&file,
Some(provisional.header.page_size),
)
.await?;
if current != *provisional {
return Err(FrankenError::BusySnapshot {
conflicting_pages:
"VACUUM candidate changed before change-counter provenance repair"
.to_owned(),
});
}
let page_size = provisional.header.page_size.as_usize();
let mut page_one = vec![0_u8; page_size];
let bytes_read = file.read(cx, &mut page_one, 0).await?;
if bytes_read != page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"VACUUM candidate page 1 is truncated: read {bytes_read} of {page_size} bytes"
),
});
}
let mut expected_page_one = page_one.clone();
let counter_bytes = change_counter.to_be_bytes();
expected_page_one[24..28].copy_from_slice(&counter_bytes);
expected_page_one[92..96].copy_from_slice(&counter_bytes);
file.write(cx, &expected_page_one, 0).await?;
file.durable_sync(cx, SyncKind::FullDurable)?;
let mut verified_page_one = vec![0_u8; page_size];
let verified_read = file.read(cx, &mut verified_page_one, 0).await?;
if verified_read != page_size || verified_page_one != expected_page_one {
return Err(FrankenError::DatabaseCorrupt {
detail: "VACUUM candidate page 1 changed outside the intended counter fields"
.to_owned(),
});
}
let final_receipt = database_image_receipt_for_open_file(
cx,
&file,
Some(provisional.header.page_size),
)
.await?;
let mut expected_header = provisional.header.clone();
expected_header.change_counter = change_counter;
expected_header.version_valid_for = change_counter;
if final_receipt.identity != provisional.identity
|| final_receipt.file_size != provisional.file_size
|| final_receipt.header != expected_header
{
return Err(FrankenError::DatabaseCorrupt {
detail: "VACUUM candidate provenance repair changed its identity, length, or unrelated header fields"
.to_owned(),
});
}
Ok(final_receipt)
}
.await;
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let unlock_result = if lock_held {
file.unlock(&cleanup_cx, LockLevel::None)
} else {
Ok(())
};
let close_result = file.close(&cleanup_cx);
match (update_result, unlock_result, close_result) {
(Ok(receipt), Ok(()), Ok(())) => Ok(receipt),
(Err(error), Ok(()), Ok(()))
| (Ok(_), Err(error), Ok(()))
| (Ok(_), Ok(()), Err(error)) => Err(error),
(update_result, unlock_result, close_result) => Err(FrankenError::internal(format!(
"VACUUM candidate provenance cleanup failed: update={}; unlock={}; close={}",
update_result
.err()
.map_or_else(|| "ok".to_owned(), |error| error.to_string()),
unlock_result
.err()
.map_or_else(|| "ok".to_owned(), |error| error.to_string()),
close_result
.err()
.map_or_else(|| "ok".to_owned(), |error| error.to_string()),
))),
}
}
/// Capture the post-checkpoint source image used to build VACUUM's
/// candidate. The header defines the logical page extent, while any
/// whole-page trailing slack remains covered by the receipt digest and
/// file size. Publication later recomputes the same receipt under the
/// exclusive maintenance protocol and aborts if any byte changed.
pub async fn capture_vacuum_source_image(&self, cx: &Cx) -> Result<DatabaseImageReceipt> {
self.with_exclusive_maintenance(cx, &mut (), |_, cx, inner, ()| {
Box::pin(async move {
let db_file = shared_db_file_read(&inner.db_file, cx).await?;
vacuum_source_receipt_for_open_file(cx, &*db_file, inner.page_size).await
})
})
.await
}
fn ensure_vacuum_candidate_is_self_contained(&self, cx: &Cx, image_path: &Path) -> Result<()> {
for suffix in ["-journal", "-wal", "-shm", "-wal-fec"] {
let mut sidecar = image_path.as_os_str().to_owned();
sidecar.push(suffix);
let sidecar = PathBuf::from(sidecar);
if self.vfs.access(cx, &sidecar, AccessFlags::EXISTS)? {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"VACUUM candidate is not self-contained: companion {} exists",
sidecar.display()
),
});
}
}
Ok(())
}
/// Publish a semantically-validated VACUUM image over the already-open
/// database inode using a durable rollback journal.
///
/// `source` is a full-image compare-and-swap token captured before the
/// rebuild began. `candidate` is captured only after schema reload,
/// `quick_check`, and `integrity_check` succeeded. Both receipts are
/// recomputed from their original open handles inside the exclusive
/// maintenance epoch before any durable source byte is changed.
#[allow(clippy::too_many_lines)]
pub async fn publish_validated_database_image(
&self,
cx: &Cx,
image_path: &Path,
source: &DatabaseImageReceipt,
candidate: &DatabaseImageReceipt,
) -> Result<()> {
if self.vfs.is_memory() {
return Err(FrankenError::Unsupported);
}
let image_full_path = self.vfs.full_pathname(cx, image_path)?;
self.ensure_vacuum_candidate_is_self_contained(cx, &image_full_path)?;
let journal_path = self
.vfs
.full_pathname(cx, &Self::journal_path(&self.db_path))?;
if image_full_path == journal_path || candidate.identity == source.identity {
return Err(FrankenError::CannotOpen {
path: image_full_path,
});
}
let candidate_flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_DB;
let (candidate_file, _) = self.vfs.open_with_expected_identity(
cx,
&image_full_path,
candidate_flags,
candidate.identity,
)?;
let mut publication_state = (
candidate_file,
image_full_path,
source.clone(),
candidate.clone(),
None::<RollbackRecoveryOwnerId>,
);
let mut publication_result = self.with_exclusive_maintenance(
cx,
&mut publication_state,
|pager, cx, inner, state| {
let (candidate_file, image_full_path, source, candidate, recovery_owner_receipt) = state;
Box::pin(async move {
pager.ensure_vacuum_candidate_is_self_contained(cx, image_full_path)?;
let shared_db_file = Arc::clone(&inner.db_file);
let mut db_file = shared_db_file_write(&shared_db_file, cx).await?;
let current_source =
vacuum_source_receipt_for_open_file(cx, &*db_file, inner.page_size).await?;
if current_source != *source {
return Err(FrankenError::BusySnapshot {
conflicting_pages: "VACUUM source image changed while rebuilding".to_owned(),
});
}
let current_candidate = database_image_receipt_for_open_file(
cx,
&*candidate_file,
Some(inner.page_size),
)
.await?;
if current_candidate != *candidate {
return Err(FrankenError::DatabaseCorrupt {
detail: "VACUUM candidate identity or content changed after validation"
.to_owned(),
});
}
let expected_format_version = if inner.journal_mode == JournalMode::Wal {
2
} else {
1
};
if candidate.header.write_version != expected_format_version
|| candidate.header.read_version != expected_format_version
{
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"VACUUM candidate file-format versions ({}, {}) do not match {:?} mode",
candidate.header.write_version,
candidate.header.read_version,
inner.journal_mode
),
});
}
let expected_change_counter = source.header.change_counter.wrapping_add(1).max(1);
if candidate.header.change_counter != expected_change_counter
|| candidate.header.version_valid_for != expected_change_counter
{
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"VACUUM candidate change-counter provenance mismatch: source={}, candidate={}, version_valid_for={}, expected={expected_change_counter}",
source.header.change_counter,
candidate.header.change_counter,
candidate.header.version_valid_for
),
});
}
let old_page_count = source.header.page_count;
let new_page_count = candidate.header.page_count;
let page_size = inner.page_size;
let page_size_bytes = u64::from(page_size.get());
let page_size_usize = page_size.as_usize();
let lock_byte_page = crate::journal::lock_byte_page(page_size);
let journal_record_count = old_page_count
.checked_sub(u32::from(lock_byte_page <= old_page_count))
.ok_or_else(|| FrankenError::internal("VACUUM journal page count underflow"))?;
// SQLite's -1 sentinel derives the record count from the exact
// journal length and supports databases whose page count exceeds
// the positive i32 range.
let journal_page_count = i32::try_from(journal_record_count).unwrap_or(-1);
let mut nonce_bytes = [0_u8; 4];
pager.vfs.randomness(cx, &mut nonce_bytes);
let nonce = u32::from_be_bytes(nonce_bytes);
let journal_path = Self::journal_path(&pager.db_path);
let recovery_owner = inner.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::JournalConstructionPending,
&journal_path,
)?;
*recovery_owner_receipt = Some(recovery_owner);
let journal_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_JOURNAL;
let (mut journal_file, _) = pager.vfs.open(cx, Some(&journal_path), journal_flags)?;
let journal_identity = journal_file.file_identity()?.ok_or_else(|| {
FrankenError::internal(
"rollback-journal VFS did not provide a stable file identity",
)
})?;
if journal_identity == source.identity || journal_identity == candidate.identity {
return Err(FrankenError::CannotOpen { path: journal_path });
}
let mut journal_is_hot = false;
let mut journal_is_recoverable = true;
let cleanup_cx = cleanup_child_cx(cx);
// This guard deliberately outlives the inner publication closure.
// Once the hot header is durable, failures unwind into the replay
// path below; dropping the mask at the closure boundary would let
// a cancelled parent abort that replay and expose a partial main
// image while the maintenance fence is still held.
let _cleanup_mask = cleanup_cx.masked();
let mut publish_result = async {
journal_file.truncate(cx, 0)?;
let requested_sector_size = db_file.sector_size().max(journal_file.sector_size());
let sector_size = if (512..=65_536).contains(&requested_sector_size)
&& requested_sector_size.is_power_of_two()
{
requested_sector_size
} else {
4096
};
let initial_header = JournalHeader {
page_count: 0,
nonce,
initial_db_size: old_page_count,
sector_size,
page_size: page_size.get(),
};
let mut initial_header_bytes = initial_header.encode_padded();
mark_local_journal_header(&mut initial_header_bytes);
// A rollback journal becomes hot only after every pre-image
// record is durable. Keep the magic zero through construction
// so a crash or I/O failure cannot advertise an incomplete
// journal as recoverable while the source database is still
// untouched. The complete, magic-bearing header is installed
// and synced immediately before the first database write.
initial_header_bytes[..JOURNAL_MAGIC.len()].fill(0);
journal_file.write(cx, &initial_header_bytes, 0).await?;
let mut journal_offset = u64::try_from(initial_header_bytes.len()).map_err(|_| {
FrankenError::OutOfRange {
what: "VACUUM rollback-journal header length".to_owned(),
value: initial_header_bytes.len().to_string(),
}
})?;
let mut page = vec![0_u8; page_size_usize];
let mut records_written = 0_u32;
for raw_page_no in 1..=old_page_count {
if raw_page_no == lock_byte_page {
continue;
}
let offset = u64::from(raw_page_no - 1)
.checked_mul(page_size_bytes)
.ok_or_else(|| FrankenError::OutOfRange {
what: "VACUUM source page offset".to_owned(),
value: raw_page_no.to_string(),
})?;
let bytes_read = db_file.read(cx, &mut page, offset).await?;
if bytes_read != page_size_usize {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short source read while journaling VACUUM page {raw_page_no}: got {bytes_read} of {page_size_usize}"
),
});
}
let record = JournalPageRecord::new(raw_page_no, page.clone(), nonce);
let encoded = record.encode();
journal_file.write(cx, &encoded, journal_offset).await?;
journal_offset = journal_offset
.checked_add(u64::try_from(encoded.len()).map_err(|_| {
FrankenError::OutOfRange {
what: "VACUUM rollback-journal record length".to_owned(),
value: encoded.len().to_string(),
}
})?)
.ok_or_else(|| FrankenError::OutOfRange {
what: "VACUUM rollback-journal file length".to_owned(),
value: raw_page_no.to_string(),
})?;
records_written = records_written.checked_add(1).ok_or_else(|| {
FrankenError::OutOfRange {
what: "VACUUM rollback-journal records written".to_owned(),
value: raw_page_no.to_string(),
}
})?;
}
if records_written != journal_record_count {
return Err(FrankenError::internal(format!(
"VACUUM rollback-journal record mismatch: wrote {records_written}, expected {journal_record_count}"
)));
}
journal_file.truncate(cx, journal_offset)?;
let durable_journal_size = journal_file.file_size(cx)?;
if durable_journal_size != journal_offset {
return Err(FrankenError::internal(format!(
"VACUUM rollback-journal length mismatch: got {durable_journal_size}, expected {journal_offset}"
)));
}
journal_file.durable_sync(cx, SyncKind::FullDurable)?;
pager.vfs.sync_parent_directory(cx, &journal_path)?;
// The final record count is the commit point for the journal,
// not for the database. Once this barrier succeeds, recovery
// can restore every original logical page and exact length.
let final_header = JournalHeader {
page_count: journal_page_count,
..initial_header
};
let mut final_header_bytes = final_header.encode_padded();
mark_local_journal_header(&mut final_header_bytes);
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::JournalActivationPending,
)?;
durable_write_and_verify_journal_header(
cx,
&mut journal_file,
&final_header_bytes,
)
.await?;
journal_is_hot = true;
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::ReplayPending,
)?;
// Once the main image starts changing, inherited cancellation
// must not strand a half-published image. The outer mask stays
// live through this closure and any synchronous replay below.
let mut candidate_page = vec![0_u8; page_size_usize];
for raw_page_no in 1..=new_page_count {
if raw_page_no == lock_byte_page {
continue;
}
let offset = u64::from(raw_page_no - 1)
.checked_mul(page_size_bytes)
.ok_or_else(|| FrankenError::OutOfRange {
what: "VACUUM candidate page offset".to_owned(),
value: raw_page_no.to_string(),
})?;
let bytes_read = candidate_file
.read(&cleanup_cx, &mut candidate_page, offset)
.await?;
if bytes_read != page_size_usize {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short candidate read while publishing VACUUM page {raw_page_no}: got {bytes_read} of {page_size_usize}"
),
});
}
db_file
.write(&cleanup_cx, &candidate_page, offset)
.await?;
#[cfg(any(test, feature = "fault-injection"))]
crate::fault_hooks::maybe_inject_vacuum_after_target_page(raw_page_no)?;
}
db_file.truncate(&cleanup_cx, candidate.file_size)?;
db_file.durable_sync(&cleanup_cx, SyncKind::FullDurable)?;
let published_receipt = database_image_receipt_for_open_file(
&cleanup_cx,
&*db_file,
Some(inner.page_size),
)
.await?;
if published_receipt.identity != source.identity
|| published_receipt.file_size != candidate.file_size
|| published_receipt.header != candidate.header
|| published_receipt.logical_hash != candidate.logical_hash
{
return Err(FrankenError::DatabaseCorrupt {
detail: "VACUUM target verification did not match the validated candidate"
.to_owned(),
});
}
let rebuilt_freelist = load_freelist_from_disk(
&cleanup_cx,
&*db_file,
page_size,
new_page_count,
candidate.header.freelist_trunk,
candidate.header.freelist_count,
)
.await?;
if rebuilt_freelist.len()
!= usize::try_from(candidate.header.freelist_count).map_err(|_| {
FrankenError::OutOfRange {
what: "VACUUM candidate freelist count".to_owned(),
value: candidate.header.freelist_count.to_string(),
}
})?
{
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"VACUUM candidate freelist contains {} valid unique pages but header declares {}",
rebuilt_freelist.len(),
candidate.header.freelist_count
),
});
}
// Persist mode: make the hot journal non-hot by durably
// clearing its magic. If that barrier fails, restore the full
// header and roll back. A truncate+sync fallback is used only
// when restoring the hot header itself fails; that fallback
// durably selects the already-synced candidate as committed.
#[cfg(any(test, feature = "fault-injection"))]
crate::fault_hooks::maybe_inject_vacuum_before_commit_marker()?;
let invalidate_result = durable_invalidate_journal(
&cleanup_cx,
&mut journal_file,
JournalInvalidation::ZeroMagic,
)
.await;
if let Err(invalidate_err) = invalidate_result {
let restore_result = durable_write_and_verify_journal_header(
&cleanup_cx,
&mut journal_file,
&final_header_bytes,
)
.await;
if let Err(restore_err) = restore_result {
let truncate_commit_result = durable_invalidate_journal(
&cleanup_cx,
&mut journal_file,
JournalInvalidation::Truncate,
)
.await;
if let Err(truncate_err) = truncate_commit_result {
journal_is_recoverable = false;
return Err(FrankenError::internal(format!(
"VACUUM publication outcome is indeterminate after rollback-journal invalidation failure: invalidate={invalidate_err}; restore={restore_err}; truncate_commit={truncate_err}"
)));
}
} else {
return Err(FrankenError::internal(format!(
"VACUUM could not commit its rollback journal; restored the hot journal for rollback: {invalidate_err}"
)));
}
}
// The zero-magic barrier above is the durable database commit
// point. Shrinking the now-non-hot journal is cleanup only.
let _ = journal_file.truncate(&cleanup_cx, 0);
let _ = journal_file.durable_sync(&cleanup_cx, SyncKind::FullDurable);
Ok(rebuilt_freelist)
}
.await;
let mut pre_hot_artifact_proven_non_hot = false;
if publish_result.is_err() && !journal_is_hot {
match durable_invalidate_journal(
&cleanup_cx,
&mut journal_file,
JournalInvalidation::ZeroMagic,
)
.await
{
Ok(()) => pre_hot_artifact_proven_non_hot = true,
Err(cleanup_err) => {
let publication_err = publish_result
.expect_err("pre-hot cleanup only runs after publication failure");
publish_result = Err(FrankenError::internal(format!(
"VACUUM publication failed before the journal became hot and its zero-magic cleanup also failed: publication={publication_err}; cleanup={cleanup_err}"
)));
}
}
}
let close_result = journal_file.close(&cleanup_cx);
let rebuilt_freelist = match publish_result {
Ok(freelist) => {
match close_result {
Ok(()) => {
if let Err(delete_err) =
pager.vfs.delete(&cleanup_cx, &journal_path, true)
{
tracing::warn!(
error = %delete_err,
journal = %journal_path.display(),
"VACUUM committed with a non-hot rollback-journal leftover"
);
}
}
Err(close_err) => {
tracing::error!(
error = %close_err,
journal = %journal_path.display(),
"VACUUM committed but rollback-journal close failed"
);
}
}
freelist
}
Err(publication_err) if journal_is_hot && journal_is_recoverable => {
let publication_error_detail = publication_err.to_string();
let expected_source = source.clone();
let publication_error_detail = publication_error_detail.clone();
let rollback_result = Self::replay_journal_with_validator(
&cleanup_cx,
pager.vfs.as_ref(),
&mut *db_file,
&journal_path,
page_size,
move |validator_cx, restored_file, _journal_page_size| Box::pin(async move {
let restored = database_image_receipt_for_open_file(
validator_cx,
restored_file,
Some(page_size),
)
.await?;
if restored != expected_source {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"VACUUM rollback completed but the source receipt did not match: publication={publication_error_detail}"
),
});
}
Ok(())
}),
)
.await;
match rollback_result {
Ok(()) => {
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::ExternalFinalizationPending,
)?;
if let Err(delete_error) =
pager.vfs.delete(&cleanup_cx, &journal_path, true)
{
tracing::warn!(
%delete_error,
journal = %journal_path.display(),
"VACUUM rollback left a durable non-hot journal"
);
}
return Err(publication_err);
}
Err(rollback_err) => {
return Err(FrankenError::internal(format!(
"VACUUM publication failed and rollback did not restore the source image: publication={publication_err}; rollback={rollback_err}; close={}",
close_result
.err()
.map_or_else(|| "ok".to_owned(), |err| err.to_string())
)));
}
}
}
Err(publication_err) => {
if !journal_is_hot && pre_hot_artifact_proven_non_hot {
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::ExternalFinalizationPending,
)?;
if close_result.is_ok() {
let _ = pager.vfs.delete(&cleanup_cx, &journal_path, true);
}
}
return Err(match close_result {
Ok(()) => publication_err,
Err(close_err) => FrankenError::internal(format!(
"VACUUM publication failed and rollback-journal close also failed: publication={publication_err}; close={close_err}"
)),
});
}
};
let next_commit_seq = inner.commit_seq.next();
pager.cache.clear();
inner.page_size = candidate.header.page_size;
inner.db_size = new_page_count;
inner.next_page = if new_page_count >= 2 {
new_page_count.saturating_add(1)
} else {
2
};
inner.freelist = rebuilt_freelist;
// The durable SQLite header counter is a wrapping u32, whereas
// the pager visibility clock is monotonic u64. VACUUM is exactly
// one logical commit, so advance the latter rather than replacing
// it with a potentially smaller wrapped header value.
inner.commit_seq = next_commit_seq;
inner.committed_db_file_size_bytes = candidate.file_size;
inner.committed_db_change_counter = u64::from(candidate.header.change_counter);
inner.committed_wal_generation = None;
inner.committed_wal_visible_commit_count = 0;
pager.published.publish_replaced_image(
&cleanup_cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: true,
},
);
pager.publish_committed_snapshot_from_inner(inner);
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::ExternalFinalizationPending,
)?;
remove_group_commit_queue(&pager.db_path);
Ok(())
})
})
.await;
if publication_result.is_ok() {
let finish_result = publication_state
.4
.ok_or_else(|| {
FrankenError::internal(
"successful VACUUM publication omitted its exact recovery owner",
)
})
.and_then(|recovery_owner| {
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
inner.finish_rollback_journal_recovery(recovery_owner)
});
if let Err(error) = finish_result {
publication_result = Err(error);
}
} else if publication_state.4.is_some() {
// A failed publication may already have restored the source (or
// proven that no main-file write began) while retaining its exact
// owner until the outer maintenance fence is terminal. Eagerly run
// the canonical pager recovery/finalization path so an expected
// VACUUM error does not leave unrelated work at BusyRecovery. If
// the external unlock or journal state is genuinely unresolved,
// this attempt fails closed and preserves the exact owner.
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
if let Err(finalization_error) = self.refresh_published_snapshot(&cleanup_cx).await {
let publication_error = publication_result
.expect_err("failed VACUUM cleanup requires an original publication error");
publication_result = Err(FrankenError::internal(format!(
"VACUUM publication failed and exact recovery finalization also failed: publication={publication_error}; finalization={finalization_error}"
)));
}
}
let (mut candidate_file, image_full_path, _, _, _) = publication_state;
let close_result = candidate_file.close(cx);
match publication_result {
Ok(()) => {
if let Err(close_err) = close_result {
tracing::error!(
error = %close_err,
candidate = %image_full_path.display(),
"VACUUM committed but candidate close failed"
);
}
Ok(())
}
Err(publication_err) => match close_result {
Ok(()) => Err(publication_err),
Err(close_err) => Err(FrankenError::internal(format!(
"VACUUM publication failed and candidate close also failed: publication={publication_err}; close={close_err}"
))),
},
}
}
/// Export the pager's main database image as a self-contained SQLite file.
///
/// The pager must be quiescent. In WAL mode we first checkpoint and
/// truncate the WAL so the returned bytes contain the durable main image.
pub async fn export_database_bytes(&self, cx: &Cx) -> Result<Vec<u8>> {
settle_pending_group_commit_finalization(&self.group_commit_queue).await?;
let _maintenance_lease = self.maintenance_gate.enter_transaction()?;
self.validate_namespace_binding()?;
let source_full = self.vfs.full_pathname(cx, &self.db_path)?;
let journal_mode = {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if inner.active_transactions > 0 || inner.checkpoint_active {
return Err(FrankenError::Busy);
}
inner.journal_mode
};
if journal_mode == JournalMode::Wal {
self.checkpoint(cx, traits::CheckpointMode::Truncate)
.await?;
}
let source_flags = VfsOpenFlags::MAIN_DB | VfsOpenFlags::READWRITE;
let (mut source_file, _) = self.vfs.open(cx, Some(&source_full), source_flags)?;
let export_result = async {
let file_size = source_file.file_size(cx)?;
let output_len = usize::try_from(file_size).map_err(|_| FrankenError::OutOfRange {
what: "database export size".to_owned(),
value: file_size.to_string(),
})?;
let mut bytes = vec![0_u8; output_len];
let mut copied = 0_usize;
while copied < output_len {
let chunk_len = (output_len - copied).min(Self::EXPORT_COPY_CHUNK_SIZE);
let bytes_read = source_file
.read(cx, &mut bytes[copied..copied + chunk_len], copied as u64)
.await?;
if bytes_read == 0 {
return Err(FrankenError::internal(
"unexpected EOF while exporting database image",
));
}
copied = copied
.checked_add(bytes_read)
.ok_or_else(|| FrankenError::internal("export size overflow"))?;
}
Ok(bytes)
}
.await;
let source_close = source_file.close(cx);
let bytes = export_result?;
source_close?;
Ok(bytes)
}
/// Copy the pager's main database file to `target_path` via the active VFS.
///
/// This is the pager-side export primitive used by higher-level features
/// like `VACUUM INTO` and backup/canonicalization flows. The copy is only
/// allowed when the pager is quiescent. In WAL mode we first checkpoint and
/// truncate the WAL so the destination contains a self-contained main DB.
pub async fn copy_database_to(&self, cx: &Cx, target_path: &Path) -> Result<()> {
settle_pending_group_commit_finalization(&self.group_commit_queue).await?;
let _maintenance_lease = self.maintenance_gate.enter_transaction()?;
self.validate_namespace_binding()?;
let source_path = self.db_path.clone();
let source_full = self.vfs.full_pathname(cx, &source_path)?;
let target_full = self.vfs.full_pathname(cx, target_path)?;
if source_full == target_full {
return Err(FrankenError::CannotOpen { path: target_full });
}
if self.vfs.access(cx, &target_full, AccessFlags::EXISTS)? {
return Err(FrankenError::CannotOpen { path: target_full });
}
let journal_mode = {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
if inner.active_transactions > 0 || inner.checkpoint_active {
return Err(FrankenError::Busy);
}
inner.journal_mode
};
if journal_mode == JournalMode::Wal {
self.checkpoint(cx, traits::CheckpointMode::Truncate)
.await?;
}
let source_flags = VfsOpenFlags::MAIN_DB | VfsOpenFlags::READWRITE;
let target_flags = VfsOpenFlags::MAIN_DB
| VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE;
let (mut source_file, _) = self.vfs.open(cx, Some(&source_full), source_flags)?;
let (mut target_file, _) = self.vfs.open(cx, Some(&target_full), target_flags)?;
let copy_result = async {
let file_size = source_file.file_size(cx)?;
let mut copied = 0_u64;
let mut buffer = vec![0_u8; Self::EXPORT_COPY_CHUNK_SIZE];
while copied < file_size {
let remaining = file_size - copied;
let chunk_len = usize::try_from(remaining.min(buffer.len() as u64))
.map_err(|_| FrankenError::internal("copy chunk length overflow"))?;
let bytes_read = source_file
.read(cx, &mut buffer[..chunk_len], copied)
.await?;
if bytes_read == 0 {
return Err(FrankenError::internal(
"unexpected EOF while copying database image",
));
}
target_file.write(cx, &buffer[..bytes_read], copied).await?;
copied = copied
.checked_add(
u64::try_from(bytes_read)
.map_err(|_| FrankenError::internal("copy size overflow"))?,
)
.ok_or_else(|| FrankenError::internal("copy offset overflow"))?;
}
target_file.truncate(cx, file_size)?;
target_file.sync(cx, SyncFlags::FULL)?;
Ok(())
}
.await;
let source_close = source_file.close(cx);
let target_close = target_file.close(cx);
copy_result?;
source_close?;
target_close?;
Ok(())
}
/// Capture point-in-time page-cache counters.
pub fn cache_metrics_snapshot(&self) -> Result<PageCacheMetricsSnapshot> {
Ok(self.cache.metrics_snapshot())
}
/// Capture cheap cache counters for hot-path statistical sampling.
///
/// Skips per-slot iteration and the eviction-policy mutex. Suitable for
/// callers that only need hit/miss/admit/evict counts and resident-page
/// totals (e.g. the e-process oracle refresh loop).
pub fn cache_metrics_lightweight_snapshot(
&self,
) -> Result<crate::page_cache::PageCacheLightweightSnapshot> {
Ok(self.cache.metrics_lightweight_snapshot())
}
/// Capture a read-only snapshot of the resident page-cache entries.
pub fn cache_page_snapshots(&self) -> Result<Vec<PageCachePageSnapshot>> {
Ok(self.cache.page_snapshots())
}
/// Capture page-cache efficiency metrics via the shared observability API.
pub fn cache_efficiency_snapshot(&self) -> Result<PageCacheEfficiencySnapshot> {
Ok(self.cache.metrics_snapshot().efficiency_snapshot())
}
/// Reset page-cache counters without altering resident pages.
pub fn reset_cache_metrics(&self) -> Result<()> {
self.cache.reset_metrics();
Ok(())
}
/// Capture the current published pager metadata snapshot.
#[must_use]
pub fn published_snapshot(&self) -> PagerPublishedSnapshot {
self.published.snapshot()
}
/// Refresh the publication plane from the latest committed pager state.
///
/// This is used by upper layers that need a coherent published visibility
/// snapshot before starting a new transaction or deciding whether a
/// connection-local execution image is stale.
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
pub async fn refresh_published_snapshot(&self, cx: &Cx) -> Result<PagerPublishedSnapshot> {
settle_pending_group_commit_finalization(&self.group_commit_queue).await?;
// Validate before adoption so a stale namespace cannot pin an orphaned
// exact-owner receipt inside a pager that cannot safely recover it.
self.validate_namespace_binding()?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
inner.adopt_orphaned_rollback_journal_recovery()?;
let expected_recovery_owner = if inner.active_transactions == 0
&& inner
.rollback_journal_recovery_state
.is_pager_takeover_eligible()
{
inner.rollback_journal_recovery_owner
} else {
None
};
let mut maintenance_lease = self
.maintenance_gate
.enter_transaction_with_recovery_owner(expected_recovery_owner)?;
if inner.active_transactions > 0 || inner.checkpoint_active {
if inner.rollback_journal_recovery_state.is_pending() {
return Err(FrankenError::BusyRecovery);
}
return Ok(self.published.snapshot());
}
let had_recovery_pending = inner.rollback_journal_recovery_state.is_pending();
let commit_seq_before_refresh = inner.commit_seq;
let (refresh, journal_visibility_invalidation) = self
.refresh_runtime_committed_state(cx, &mut maintenance_lease, &mut inner, None)
.await?;
let clear_published_pages = had_recovery_pending
|| journal_visibility_invalidation
|| refresh.page_cache_invalidated
|| inner.commit_seq != commit_seq_before_refresh;
// D1-CRITICAL Change 3: Use sharded publish_clear_if.
self.published.publish_clear_if(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
clear_published_pages,
);
Ok(self.published.snapshot())
}
/// Refresh the publication plane for a clean WAL-mode read boundary.
///
/// Clean prepared-read fast paths only need to learn whether the visible
/// WAL/header horizon changed. When the pager is already in WAL mode and
/// rollback-journal recovery is clean, the WAL refresh itself supplies the
/// required cross-process visibility probe; taking the database-file shared
/// lock and probing the rollback journal only adds work to the hot read
/// path.
pub async fn refresh_published_snapshot_for_clean_wal_read(
&self,
cx: &Cx,
) -> Result<PagerPublishedSnapshot> {
self.refresh_published_snapshot(cx).await
}
/// Number of snapshot retries steady-state readers have taken.
#[must_use]
pub fn published_read_retry_count(&self) -> u64 {
self.published.read_retry_count()
}
/// Returns the database page size.
#[must_use]
pub fn page_size(&self) -> PageSize {
let inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.page_size
}
/// Read the committed-state snapshot without taking the PagerInner Mutex.
///
/// Returns a cheap `Arc` clone — the RwLock read-hold is ~nanoseconds.
/// Use this for staleness checks, visibility probes, and begin-path
/// fast-path gating instead of locking `PagerInner`.
#[must_use]
pub fn committed_snapshot(&self) -> Arc<PagerCommittedSnapshot> {
if self
.shared_connection_count
.get()
.is_some_and(|counter| counter.load(AtomicOrdering::Acquire) == 1)
{
let published = self.published.snapshot();
let inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
return Arc::new(PagerCommittedSnapshot {
commit_seq: published.visible_commit_seq,
db_size: published.db_size,
journal_mode: published.journal_mode,
freelist_count: published.freelist_count,
checkpoint_active: published.checkpoint_active,
writer_active: inner.writer_active,
db_file_size_bytes: inner.committed_db_file_size_bytes,
});
}
Arc::clone(
&self
.committed_snapshot
.read()
.unwrap_or_else(std::sync::PoisonError::into_inner),
)
}
/// Publish a new committed-state snapshot while the pager inner lock is still held.
///
/// This mirrors the transaction commit helper so pager-level tests can
/// exercise publication/reclamation invariants without manufacturing a
/// full commit path.
fn publish_committed_snapshot_from_inner(&self, inner: &PagerInner<V::File>) {
let snapshot = Arc::new(PagerCommittedSnapshot::from_inner(inner));
let mut guard = self
.committed_snapshot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = snapshot;
}
/// Bind a same-path connection counter owned by the SQL connection layer.
pub fn bind_shared_connection_count(&self, counter: Arc<AtomicUsize>) {
let _ = self.shared_connection_count.set(counter);
}
/// Number of page reads satisfied directly from the publication plane.
#[must_use]
pub fn published_page_hits(&self) -> u64 {
self.published.published_page_hits()
}
/// Number of publish-plane writes applied to this pager.
#[must_use]
pub fn publication_write_count(&self) -> u64 {
self.published.publication_write_count()
}
/// Number of frames currently in the WAL for this pager.
///
/// Read-only probe: `WalBackend::frame_count(&self) -> usize` is a
/// plain field read, so we take the `SharedWalBackend` RwLock in
/// `read()` mode — shared with other readers and, critically,
/// non-blocking against concurrent WAL writers that only take
/// `read()` themselves on their own read paths. The previous
/// `with_wal_backend` (write-lock) acquisition here made every
/// `maybe_run_adaptive_autocheckpoint` probe serialize against
/// every other WAL operation — the checkpoint advisor samples this
/// on every commit, so under MT-writer workloads it turned each
/// post-commit poll into a global WAL RwLock write-contention
/// point.
pub async fn wal_frame_count(&self, cx: &Cx) -> usize {
if self
.group_commit_queue
.has_process_root_finalization_attempt()
{
return 0;
}
with_wal_backend_read(&self.wal_backend, cx, |wal, _| {
Box::pin(async move { Ok(wal.frame_count()) })
})
.await
.unwrap_or(0)
}
/// Compute the journal path from the database path.
fn journal_path(db_path: &Path) -> PathBuf {
let mut jp = db_path.as_os_str().to_owned();
jp.push("-journal");
PathBuf::from(jp)
}
fn ensure_reserved_recovery_artifacts_absent(cx: &Cx, vfs: &V, db_path: &Path) -> Result<()> {
for suffix in ["-journal", "-wal", "-wal-fec", "-shm"] {
let mut artifact_path = db_path.as_os_str().to_owned();
artifact_path.push(suffix);
if vfs.path_entry_exists(cx, Path::new(&artifact_path))? {
return Err(FrankenError::CannotOpen {
path: db_path.to_owned(),
});
}
}
Ok(())
}
fn validate_rollback_recovery_namespace(
cx: &Cx,
vfs: &V,
current_db_path: &Path,
current_db_identity: Option<FileIdentity>,
namespace: &RollbackRecoveryNamespace,
) -> Result<()> {
let cannot_open = || FrankenError::CannotOpen {
path: namespace.db_path.clone(),
};
if let (Some(origin), Some(current)) = (namespace.db_identity, current_db_identity)
&& origin != current
{
return Err(cannot_open());
}
#[cfg(all(feature = "native", any(unix, windows)))]
if let Some(binding) = &namespace.namespace_binding {
if binding.stable_path() != namespace.db_path
|| namespace.db_identity != Some(binding.identity())
{
return Err(cannot_open());
}
return binding.validate_path_identity();
}
let Some(expected_identity) = namespace.db_identity else {
return if namespace.db_path == current_db_path {
Ok(())
} else {
Err(cannot_open())
};
};
let flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_DB;
let (mut origin_file, _) =
vfs.open_with_expected_identity(cx, &namespace.db_path, flags, expected_identity)?;
let identity_result = origin_file.file_identity().and_then(|observed| {
if observed == Some(expected_identity) {
Ok(())
} else {
Err(cannot_open())
}
});
let close_result = origin_file.close(cx);
match (identity_result, close_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(identity_error), Err(close_error)) => Err(FrankenError::internal(format!(
"rollback-recovery namespace validation failed and its identity probe could not close: validation={identity_error}; close={close_error}"
))),
}
}
fn journal_mode_from_database_header(header: &DatabaseHeader) -> Result<JournalMode> {
match (header.write_version, header.read_version) {
(1, 1) => Ok(JournalMode::Delete),
(2, 2) => Ok(JournalMode::Wal),
(write_version, read_version) => Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database header has incompatible file-format versions: write={write_version}, read={read_version}"
),
}),
}
}
async fn journal_mode_from_database_file(cx: &Cx, db_file: &V::File) -> Result<JournalMode> {
let file_size = db_file.file_size(cx)?;
if file_size < DATABASE_HEADER_SIZE as u64 {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database file too small for recovery-mode detection: {file_size} bytes (< {DATABASE_HEADER_SIZE})"
),
});
}
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let bytes_read = db_file.read(cx, &mut header_bytes, 0).await?;
if bytes_read != DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short database-header read during recovery-mode detection: got {bytes_read} of {DATABASE_HEADER_SIZE} bytes"
),
});
}
let header = DatabaseHeader::from_bytes(&header_bytes).map_err(|error| {
FrankenError::DatabaseCorrupt {
detail: format!("invalid database header during recovery-mode detection: {error}"),
}
})?;
Self::journal_mode_from_database_header(&header)
}
/// Recover one runtime pager while its identity-bound maintenance lease is
/// exclusive and no VFS lock is held by the caller.
///
/// The cross-process order is always WAL write/checkpoint slots followed
/// by main-file EXCLUSIVE. Replay and the exact metadata rebuild occur in
/// the same epoch. If the rebuild fails after durable replay, the distinct
/// `MetadataRefreshPending` state makes a later attempt retry metadata
/// without requiring the already-invalidated journal.
#[allow(clippy::too_many_arguments)]
async fn recover_runtime_rollback_journal(
cx: &Cx,
vfs: &V,
inner: &mut PagerInner<V::File>,
journal_path: &Path,
cache: &ShardedPageCache,
wal_backend: &SharedWalBackend,
group_commit_queue: &Arc<GroupCommitQueue>,
begin_external_lock: Option<&mut BeginExternalLockState<V::File>>,
) -> Result<bool> {
if matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
) {
return Err(FrankenError::BusyRecovery);
}
let recovery_is_pending = inner.rollback_journal_recovery_state.is_pending();
let recovery_has_owner = inner.rollback_journal_recovery_owner.is_some();
let recovery_has_namespace = inner.rollback_journal_recovery_namespace.is_some();
if recovery_is_pending != recovery_has_owner || recovery_has_owner != recovery_has_namespace
{
return Err(FrankenError::internal(
"rollback-recovery state, exact owner, and origin namespace disagree",
));
}
let recovery_namespace = inner.rollback_journal_recovery_namespace.clone();
let recovery_journal_path = recovery_namespace.as_ref().map_or_else(
|| journal_path.to_owned(),
|namespace| namespace.journal_path.clone(),
);
let journal_path = recovery_journal_path.as_path();
let pre_main_write_pending = matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::JournalConstructionPending
| RollbackJournalRecoveryState::JournalActivationPending
);
let mut standalone_external_lock = begin_external_lock.is_none().then(|| {
BeginExternalLockState::new(group_commit_queue, Arc::clone(&inner.db_file), cx)
});
let external_lock = begin_external_lock
.or(standalone_external_lock.as_mut())
.expect("runtime recovery must own one external-lock attempt guard");
// Recovery may be adopted by a pager whose cached journal mode lags
// the origin owner. Conservatively exclude WAL writers/checkpointers
// as well as main-file users for every rollback-journal recovery.
external_lock.acquire_maintenance(cx, true).await?;
let recovery_result = async {
if let Some(namespace) = &recovery_namespace {
Self::validate_rollback_recovery_namespace(
cx,
vfs,
&inner.database_path,
inner.database_identity,
namespace,
)?;
}
let journal_exists = vfs.access(cx, journal_path, AccessFlags::EXISTS)?;
if inner.rollback_journal_recovery_state.needs_replay() && !journal_exists {
return Err(FrankenError::internal(
"rollback journal missing while failed commit replay was pending",
));
}
let mut journal_observed = false;
if journal_exists {
journal_observed = true;
// Construction/activation provenance belongs to the original
// local artifact, not to whatever later occupies this path.
// Once a native maintenance epoch has ended, an external
// writer may replace it with a different valid hot journal.
// Therefore every current hot journal is validated and
// replayed; only absent or proven non-hot debris is discarded.
let shared_db_file = Arc::clone(&inner.db_file);
let mut db_file = shared_db_file_write(&shared_db_file, cx).await?;
let outcome = Self::replay_journal_with_optional_page_size_validator(
cx,
vfs,
&mut *db_file,
journal_path,
Some(inner.page_size),
|_, _, _| Box::pin(async { Ok(()) }),
)
.await?;
match outcome {
RollbackJournalReplayOutcome::Replayed(_) => {
if let Some(owner) = inner.rollback_journal_recovery_owner {
inner.transition_rollback_journal_recovery(
owner,
RollbackJournalRecoveryState::MetadataRefreshPending,
)?;
} else {
inner.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::MetadataRefreshPending,
journal_path,
)?;
}
}
RollbackJournalReplayOutcome::NonHot
if inner.rollback_journal_recovery_state.needs_replay() =>
{
return Err(FrankenError::internal(
"rollback journal became non-hot before pending replay completed",
));
}
RollbackJournalReplayOutcome::NonHot => {}
}
}
// A valid hot journal at the receipt's path may have been created
// after the original owner released its external locks. Even an
// ExternalFinalizationPending retry crosses such an unlocked
// interval. Derive the authoritative mode for every generic
// recovery before this attempt or its caller publishes metadata.
let recovered_journal_mode = {
let db_file = shared_db_file_read(&inner.db_file, cx).await?;
Self::journal_mode_from_database_file(cx, &*db_file).await?
};
inner.journal_mode = recovered_journal_mode;
if let Some(namespace) = inner.rollback_journal_recovery_namespace.as_mut() {
namespace.journal_mode = recovered_journal_mode;
}
// A takeover receipt can move to a different PagerInner whose
// cache has no provenance tying it to the recovered image. This
// includes ExternalFinalizationPending after another opener
// already replayed the journal. Invalidate for every generic
// recovery before either this attempt or its caller publishes a
// post-recovery snapshot.
cache.clear();
if pre_main_write_pending
|| matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::MetadataRefreshPending
)
{
inner
.refresh_committed_state_after_recovery(cx, cache, wal_backend)
.await?;
}
// Persist the caller-owned lock/lease/snapshot obligation before
// either deleting a harmless non-hot leftover or awaiting the
// external maintenance restoration.
if let Some(owner) = inner.rollback_journal_recovery_owner {
inner.transition_rollback_journal_recovery(
owner,
RollbackJournalRecoveryState::ExternalFinalizationPending,
)?;
} else {
inner.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::ExternalFinalizationPending,
journal_path,
)?;
}
if journal_observed
&& matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::ExternalFinalizationPending
)
&& let Err(error) = vfs.delete(cx, journal_path, true)
{
tracing::warn!(
%error,
journal = %journal_path.display(),
"runtime recovery left a durable non-hot rollback journal"
);
}
Ok(journal_observed)
}
.await;
let unlock_result = external_lock.restore().await;
match (recovery_result, unlock_result) {
(Ok(recovered), Ok(())) => Ok(recovered),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(recovery_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"runtime rollback recovery failed and could not release the maintenance lock: recovery={recovery_error}; unlock={unlock_error}"
))),
}
}
async fn recover_rollback_journal_for_open(
cx: &Cx,
vfs: &V,
context: RollbackJournalOpenRecoveryContext<'_, V::File>,
mut orphaned_recovery: Option<&mut OrphanedRollbackRecovery>,
) -> Result<Option<RollbackJournalReplayOutcome>> {
let RollbackJournalOpenRecoveryContext {
group_commit_queue,
db_file,
current_db_path,
current_db_identity,
journal_path,
} = context;
let exact_journal_path = orphaned_recovery.as_deref().map_or_else(
|| journal_path.to_owned(),
|recovery| recovery.namespace.journal_path.clone(),
);
let exact_journal_path = exact_journal_path.as_path();
if orphaned_recovery.is_none()
&& !vfs.access(cx, exact_journal_path, AccessFlags::EXISTS)?
{
return Ok(None);
}
// Open-time recovery has not yet established a trustworthy journal
// mode. Conservatively exclude WAL writers/checkpointers as well as
// main-file users; this is safe for rollback-mode files and prevents a
// crashed WAL-mode whole-image publication racing recovery.
let mut maintenance_attempt =
BeginExternalLockState::new(group_commit_queue, Arc::clone(db_file), cx);
maintenance_attempt.acquire_maintenance(cx, true).await?;
let mut file = shared_db_file_write(db_file, cx).await?;
let recovery_result = async {
if let Some(recovery) = orphaned_recovery.as_deref() {
Self::validate_rollback_recovery_namespace(
cx,
vfs,
current_db_path,
current_db_identity,
&recovery.namespace,
)?;
}
if !vfs.access(cx, exact_journal_path, AccessFlags::EXISTS)? {
if orphaned_recovery.as_deref().is_some_and(|recovery| {
recovery.recovery_state.needs_replay()
}) {
return Err(FrankenError::internal(
"rollback journal missing while orphaned replay was pending",
));
}
if let Some(recovery) = orphaned_recovery.as_deref_mut() {
recovery.recovery_state =
RollbackJournalRecoveryState::ExternalFinalizationPending;
}
return Ok(None);
}
let outcome = Self::replay_journal_with_optional_page_size_validator(
cx,
vfs,
&mut *file,
exact_journal_path,
None,
|validator_cx, restored_file, journal_page_size| Box::pin(async move {
let restored_size = restored_file.file_size(validator_cx)?;
if restored_size == 0 {
return Ok(());
}
if restored_size < DATABASE_HEADER_SIZE as u64 {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback recovery restored a {restored_size}-byte main file, too short for its database header"
),
});
}
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let bytes_read = restored_file
.read(validator_cx, &mut header_bytes, 0)
.await?;
if bytes_read != DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short database-header read after rollback recovery: got {bytes_read} of {DATABASE_HEADER_SIZE} bytes"
),
});
}
let restored_header =
DatabaseHeader::from_bytes(&header_bytes).map_err(|error| {
FrankenError::DatabaseCorrupt {
detail: format!(
"rollback recovery restored an invalid database header: {error}"
),
}
})?;
if restored_header.page_size != journal_page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback recovery restored page size {}, but the hot journal used {}",
restored_header.page_size.get(),
journal_page_size.get()
),
});
}
Ok(())
}),
)
.await?;
match outcome {
RollbackJournalReplayOutcome::Replayed(_) => {
if let Some(recovery) = orphaned_recovery.as_deref_mut() {
// Replay has already durably invalidated the journal.
// Persist that fact in the claimed receipt before the
// next fallible/cancellable operation.
recovery.recovery_state =
RollbackJournalRecoveryState::MetadataRefreshPending;
}
}
RollbackJournalReplayOutcome::NonHot
if orphaned_recovery
.as_deref()
.is_some_and(|recovery| recovery.recovery_state.needs_replay()) =>
{
return Err(FrankenError::internal(
"rollback journal became non-hot before orphaned replay completed",
));
}
RollbackJournalReplayOutcome::NonHot => {
if let Some(recovery) = orphaned_recovery.as_deref_mut()
&& matches!(
recovery.recovery_state,
RollbackJournalRecoveryState::JournalConstructionPending
| RollbackJournalRecoveryState::JournalActivationPending
)
{
recovery.recovery_state =
RollbackJournalRecoveryState::MetadataRefreshPending;
}
}
}
if let Some(recovery) = orphaned_recovery.as_deref_mut() {
recovery.namespace.journal_mode =
Self::journal_mode_from_database_file(cx, &*file).await?;
}
if let Err(error) = vfs.delete(cx, exact_journal_path, true) {
tracing::warn!(
%error,
journal = %exact_journal_path.display(),
"open-time recovery left a durable non-hot rollback journal"
);
} else if vfs.access(cx, exact_journal_path, AccessFlags::EXISTS)? {
return Err(FrankenError::CannotOpen {
path: exact_journal_path.to_owned(),
});
}
if let Some(recovery) = orphaned_recovery.as_deref_mut() {
recovery.recovery_state =
RollbackJournalRecoveryState::ExternalFinalizationPending;
}
Ok(Some(outcome))
}
.await;
drop(file);
let unlock_result = maintenance_attempt.restore().await;
drop(maintenance_attempt);
match (recovery_result, unlock_result) {
(Ok(outcome), Ok(())) => Ok(outcome),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(recovery_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"open-time rollback recovery failed and could not release the maintenance lock: recovery={recovery_error}; unlock={unlock_error}"
))),
}
}
async fn verify_readonly_rollback_journal_state(
cx: &Cx,
vfs: &V,
journal_path: &Path,
) -> Result<()> {
if !vfs.access(cx, journal_path, AccessFlags::EXISTS)? {
return Ok(());
}
let flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal_file, _) = vfs.open(cx, Some(journal_path), flags)?;
let classification = classify_rollback_journal_prefix(cx, &journal_file).await;
let close_result = journal_file.close(cx);
match (classification, close_result) {
(Ok((RollbackJournalPrefixState::NonHot, _)), Ok(())) => Ok(()),
(Ok((RollbackJournalPrefixState::Hot, _)), Ok(())) => Err(FrankenError::BusyRecovery),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(classification_error), Err(close_error)) => Err(FrankenError::internal(format!(
"read-only rollback-journal classification failed and the journal handle could not close: classification={classification_error}; close={close_error}"
))),
}
}
/// Bind a transaction/read boundary to one coherent durable state.
///
/// Every caller enters with exactly one transaction maintenance lease and
/// no active transaction in this pager. A hot journal is first discovered
/// under main-file SHARED, then SHARED is released before the lease is
/// upgraded and the canonical WAL-slots -> main-EXCLUSIVE recovery epoch
/// begins. Read-only pagers only classify and fail closed.
async fn refresh_runtime_committed_state(
&self,
cx: &Cx,
maintenance_lease: &mut PagerMaintenanceLease,
inner: &mut PagerInner<V::File>,
begin_external_lock: Option<&mut BeginExternalLockState<V::File>>,
) -> Result<(CommittedStateRefresh, bool)> {
let journal_path = Self::journal_path(&self.db_path);
// File-backed begin admission passes its persistent external-lock
// guard here. Retain the final verified SHARED snapshot in that guard
// so no cross-process writer can change the image between recovery
// verification and transaction admission. Standalone refresh callers
// still release their temporary snapshot before returning.
let retain_snapshot_for_caller = begin_external_lock.is_some() && !self.vfs.is_memory();
let mut standalone_external_lock = begin_external_lock.is_none().then(|| {
BeginExternalLockState::new(&self.group_commit_queue, Arc::clone(&inner.db_file), cx)
});
let external_lock = begin_external_lock
.or(standalone_external_lock.as_mut())
.expect("runtime refresh must own one external-lock attempt guard");
external_lock.acquire_snapshot(cx).await?;
let journal_exists = match self.vfs.access(cx, &journal_path, AccessFlags::EXISTS) {
Ok(exists) => exists,
Err(error) => {
return match external_lock.restore().await {
Ok(()) => Err(error),
Err(unlock_error) => Err(FrankenError::internal(format!(
"rollback-journal probe failed and could not release SHARED: probe={error}; unlock={unlock_error}"
))),
};
}
};
let had_pending = inner.rollback_journal_recovery_state.is_pending();
if inner.access_mode.is_readonly() {
let operation_result = if had_pending {
Err(FrankenError::BusyRecovery)
} else {
match Self::verify_readonly_rollback_journal_state(cx, &*self.vfs, &journal_path)
.await
{
Ok(()) => {
inner
.refresh_committed_state(cx, &self.cache, &self.wal_backend)
.await
}
Err(error) => Err(error),
}
};
if retain_snapshot_for_caller {
let refresh = operation_result?;
return Ok((refresh, journal_exists));
}
let unlock_result = external_lock.restore().await;
return match (operation_result, unlock_result) {
(Ok(refresh), Ok(())) => Ok((refresh, journal_exists)),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(operation_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"read-only durable snapshot failed and could not release SHARED: operation={operation_error}; unlock={unlock_error}"
))),
};
}
if had_pending || journal_exists {
// Never wait for the recovery fence or acquire WAL slots while
// retaining main SHARED: maintenance publishers use the opposite,
// canonical order.
external_lock.restore().await?;
let _recovery_guard = self.recovery_fence.acquire_for_recovery()?;
let prior_kind =
maintenance_lease.upgrade_to_exclusive(inner.rollback_journal_recovery_owner)?;
let recovery_journal_path = inner
.rollback_journal_recovery_namespace
.as_ref()
.map_or_else(
|| journal_path.clone(),
|namespace| namespace.journal_path.clone(),
);
let recovery_result = Self::recover_runtime_rollback_journal(
cx,
&*self.vfs,
inner,
&recovery_journal_path,
&self.cache,
&self.wal_backend,
&self.group_commit_queue,
Some(&mut *external_lock),
)
.await;
let external_restore_pending = external_lock.is_armed();
if external_restore_pending {
return match recovery_result {
Err(error) => Err(error),
Ok(_) => Err(FrankenError::internal(
"runtime recovery reported success while its external maintenance lock remained armed",
)),
};
}
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let downgrade_result = maintenance_lease.downgrade_from_exclusive(prior_kind);
match (recovery_result, downgrade_result) {
(Ok(_), Ok(())) => {}
(Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error),
(Err(recovery_error), Err(downgrade_error)) => {
return Err(FrankenError::internal(format!(
"runtime recovery failed and could not restore its maintenance lease: recovery={recovery_error}; downgrade={downgrade_error}"
)));
}
}
external_lock.acquire_snapshot(cx).await?;
// A non-hot leftover is harmless if deletion failed. A new hot
// record here means another process won a publication race after
// our recovery epoch; fail closed instead of reading through it.
if let Err(error) =
Self::verify_readonly_rollback_journal_state(cx, &*self.vfs, &recovery_journal_path)
.await
{
return match external_lock.restore().await {
Ok(()) => Err(error),
Err(unlock_error) => Err(FrankenError::internal(format!(
"post-recovery journal verification failed and could not release SHARED: verification={error}; unlock={unlock_error}"
))),
};
}
let refresh_result = inner
.refresh_committed_state_after_recovery(cx, &self.cache, &self.wal_backend)
.await;
let finalization_result = refresh_result.and_then(|refresh| {
let owner = inner.rollback_journal_recovery_owner.ok_or_else(|| {
FrankenError::internal(
"runtime rollback recovery completed without an exact owner receipt",
)
})?;
inner.finish_rollback_journal_recovery(owner)?;
Ok(refresh)
});
if retain_snapshot_for_caller {
return finalization_result.map(|refresh| (refresh, true));
}
// Clear the exact recovery owner while the verified SHARED fence
// is still held. If releasing that final fence fails, the armed
// external-lock guard publishes a process-root cleanup owner; a
// cross-process writer never gets an unlock-to-owner-clear gap.
let unlock_result = external_lock.restore().await;
return match (finalization_result, unlock_result) {
(Ok(refresh), Ok(())) => Ok((refresh, true)),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(finalization_error), Err(unlock_error)) => {
Err(FrankenError::internal(format!(
"durable pager refresh finalization failed and could not release SHARED: finalization={finalization_error}; unlock={unlock_error}"
)))
}
};
}
let refresh_result = inner
.refresh_committed_state(cx, &self.cache, &self.wal_backend)
.await;
if retain_snapshot_for_caller {
return refresh_result.map(|refresh| (refresh, false));
}
let unlock_result = external_lock.restore().await;
match (refresh_result, unlock_result) {
(Ok(refresh), Ok(())) => Ok((refresh, false)),
(Err(error), Ok(())) | (Ok(_), Err(error)) => Err(error),
(Err(refresh_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"durable pager refresh failed and could not release SHARED: refresh={refresh_error}; unlock={unlock_error}"
))),
}
}
/// Open a pager for the SQL connection layer while deliberately retaining
/// the native namespace bootstrap lease. The caller must invoke
/// [`Self::finish_namespace_bootstrap`] immediately before returning a
/// successfully initialized connection.
#[doc(hidden)]
#[allow(clippy::too_many_lines)]
pub async fn open_for_connection_with_cx_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
requested_page_size: PageSize,
page_buffer_max: Option<usize>,
mode: ConnectionPagerOpenMode,
) -> Result<Self> {
match mode {
ConnectionPagerOpenMode::CreateIfMissing => {
Self::open_readwrite_with_cx_and_page_buffer_max(
cx,
vfs,
path,
requested_page_size,
page_buffer_max,
ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition::CreateIfMissing,
expected_identity: None,
finish_namespace_bootstrap: false,
},
)
.await
}
ConnectionPagerOpenMode::ExistingOnly(expected_identity) => {
Self::open_readwrite_with_cx_and_page_buffer_max(
cx,
vfs,
path,
requested_page_size,
page_buffer_max,
ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition::ExistingOnly,
expected_identity,
finish_namespace_bootstrap: false,
},
)
.await
}
ConnectionPagerOpenMode::ReservedEmpty(expected_identity) => {
Self::open_readwrite_with_cx_and_page_buffer_max(
cx,
vfs,
path,
requested_page_size,
page_buffer_max,
ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition::ReservedEmpty,
expected_identity: Some(expected_identity),
finish_namespace_bootstrap: false,
},
)
.await
}
ConnectionPagerOpenMode::ReadOnly(expected_identity) => {
Self::open_readonly_with_optional_expected_identity(
cx,
vfs,
path,
requested_page_size,
expected_identity,
page_buffer_max,
false,
)
.await
}
}
}
/// Open (or create) a database and return a pager using a caller-owned
/// capability context.
///
/// Existing databases adopt the page size encoded in their on-disk header;
/// `requested_page_size` is used when creating a new database or when the
/// header is unavailable/corrupt and recovery must fall back to a caller
/// default.
///
/// If a hot journal is detected (leftover from a crash), it is replayed
/// to restore the database to a consistent state before returning.
#[allow(clippy::too_many_lines)]
pub async fn open_with_cx(
cx: &Cx,
vfs: V,
path: &Path,
requested_page_size: PageSize,
) -> Result<Self> {
Self::open_with_cx_and_page_buffer_max(cx, vfs, path, requested_page_size, None).await
}
/// Like [`open_with_cx`](Self::open_with_cx) but allows overriding the
/// page-buffer-pool ceiling.
///
/// `page_buffer_max` is resolved via [`crate::resolve_page_buffer_max`]: `Some(n)`
/// uses that value directly, `None` checks the `FSQLITE_PAGE_BUFFER_MAX`
/// env var, then falls back to [`crate::DEFAULT_PAGE_BUFFER_MAX`] (262 144).
#[allow(clippy::too_many_lines)]
pub async fn open_with_cx_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
requested_page_size: PageSize,
page_buffer_max: Option<usize>,
) -> Result<Self> {
Self::open_readwrite_with_cx_and_page_buffer_max(
cx,
vfs,
path,
requested_page_size,
page_buffer_max,
ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition::CreateIfMissing,
expected_identity: None,
finish_namespace_bootstrap: true,
},
)
.await
}
/// Initialize a caller-reserved empty file only if the opened VFS handle
/// has `expected_identity`.
///
/// This keeps create-new workflows bound to the descriptor that reserved
/// the pathname. A missing or non-empty file is refused, as is a
/// pre-existing rollback journal, WAL, WAL-FEC, or shared-memory sidecar.
/// The identity and sidecar checks precede database initialization, and
/// this path never performs recovery.
pub async fn open_reserved_with_cx_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
requested_page_size: PageSize,
expected_identity: FileIdentity,
page_buffer_max: Option<usize>,
) -> Result<Self> {
Self::open_readwrite_with_cx_and_page_buffer_max(
cx,
vfs,
path,
requested_page_size,
page_buffer_max,
ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition::ReservedEmpty,
expected_identity: Some(expected_identity),
finish_namespace_bootstrap: true,
},
)
.await
}
/// Open an existing database for reading and writing without creating or
/// initializing the main database file.
///
/// The VFS open omits [`VfsOpenFlags::CREATE`]. Empty and malformed files
/// are rejected before rollback-journal recovery, ensuring that failed
/// opens cannot bootstrap or otherwise mutate their main database image.
/// When `expected_identity` is present, it is compared with the identity
/// of the already-open VFS handle before any file read or recovery action.
#[allow(clippy::too_many_lines)]
pub async fn open_existing_with_cx_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
requested_page_size: PageSize,
expected_identity: Option<FileIdentity>,
page_buffer_max: Option<usize>,
) -> Result<Self> {
Self::open_readwrite_with_cx_and_page_buffer_max(
cx,
vfs,
path,
requested_page_size,
page_buffer_max,
ReadWriteOpenPolicy {
disposition: ReadWriteOpenDisposition::ExistingOnly,
expected_identity,
finish_namespace_bootstrap: true,
},
)
.await
}
#[allow(clippy::too_many_lines)]
async fn open_readwrite_with_cx_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
requested_page_size: PageSize,
page_buffer_max: Option<usize>,
policy: ReadWriteOpenPolicy,
) -> Result<Self> {
let ReadWriteOpenPolicy {
disposition,
expected_identity,
finish_namespace_bootstrap,
} = policy;
let vfs = Arc::new(vfs);
let db_path = vfs.full_pathname(cx, path)?;
// A lexical path gate runs before namespace admission or file open so
// a prior caller cannot leave process-root finalization work while a
// replacement opener begins inspecting the same name.
//
// Private memory VFS instances may legitimately reuse the same
// synthetic path for unrelated storage identities. Their identity
// gate below provides same-database coordination without cross-wiring
// independent databases through a process-global path.
if !vfs.is_memory() {
settle_process_root_finalizations_for_path(&db_path).await?;
}
let path_maintenance_gate = maintenance_gate_for_backend(&*vfs, &db_path);
let (path_maintenance_lease, path_orphaned_recovery_claim) =
path_maintenance_gate.enter_readwrite_open_for_orphan_recovery()?;
#[cfg(all(feature = "native", any(unix, windows)))]
let (pending_namespace, replace_quiescent_namespace_record) = if vfs.is_memory() {
(None, false)
} else {
let intent = if disposition == ReadWriteOpenDisposition::ReservedEmpty {
NamespaceOpenIntent::ReservedExclusive
} else {
NamespaceOpenIntent::Shared
};
let pending = PendingNamespaceOpen::begin(&db_path, intent)?;
// A Shared admission without an expected identity owns both
// namespace locks exclusively. Its bind may therefore repair a
// plain copied/corrupt base record after validating the opened
// main-file identity; joined and transition-bearing generations
// remain fail-closed in the namespace layer.
let replace_record =
intent == NamespaceOpenIntent::Shared && pending.has_quiescent_record_bytes()?;
(Some(pending), replace_record)
};
#[cfg(all(feature = "native", any(unix, windows)))]
if disposition == ReadWriteOpenDisposition::ReservedEmpty && pending_namespace.is_some() {
validate_reserved_database_artifacts(&db_path, WindowsLockSidecarPolicy::RejectAll)?;
}
#[cfg(all(feature = "native", any(unix, windows)))]
let namespace_expected_identity = pending_namespace
.as_ref()
.and_then(PendingNamespaceOpen::expected_identity);
#[cfg(all(feature = "native", any(unix, windows)))]
let effective_expected_identity = match (expected_identity, namespace_expected_identity) {
(Some(caller), Some(generation)) if caller != generation => {
return Err(FrankenError::CannotOpen {
path: db_path.clone(),
});
}
(Some(caller), _) => Some(caller),
(None, generation) => generation,
};
#[cfg(not(all(feature = "native", any(unix, windows))))]
let effective_expected_identity = expected_identity;
let mut flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
if disposition == ReadWriteOpenDisposition::CreateIfMissing {
flags |= VfsOpenFlags::CREATE;
}
#[cfg(all(feature = "native", any(unix, windows)))]
if replace_quiescent_namespace_record {
// A stale nonempty namespace record must never turn a missing
// main database into a silently created empty replacement.
flags.remove(VfsOpenFlags::CREATE);
}
let (db_file, _actual_flags) = match (disposition, effective_expected_identity) {
(ReadWriteOpenDisposition::ReservedEmpty, Some(expected_identity)) => {
vfs.open_reserved_with_expected_identity(cx, &db_path, flags, expected_identity)?
}
(_, Some(expected_identity)) => {
vfs.open_with_expected_identity(cx, &db_path, flags, expected_identity)?
}
(ReadWriteOpenDisposition::ReservedEmpty, None) => {
return Err(FrankenError::CannotOpen { path: db_path });
}
(_, None) => vfs.open(cx, Some(&db_path), flags)?,
};
if let Some(identity) = db_file.file_identity()? {
settle_process_root_finalizations_for_identity(identity).await?;
}
let maintenance_gate =
identity_bound_maintenance_gate(&*vfs, &path_maintenance_gate, &db_file)?;
let recovery_fence = identity_bound_recovery_fence(&*vfs, &db_path, &db_file)?;
let group_commit_queue = identity_bound_group_commit_queue(&*vfs, &db_path, &db_file)?;
// A prior caller may have disappeared after physical WAL mutation
// started. Settle that identity-bound obligation before this opener
// inspects the main file, journal, WAL, or cached header state.
settle_pending_group_commit_finalization(&group_commit_queue).await?;
let (mut maintenance_open_lease, mut orphaned_recovery_claim) =
if Arc::ptr_eq(&maintenance_gate, &path_maintenance_gate) {
(path_maintenance_lease, path_orphaned_recovery_claim)
} else {
let (identity_lease, identity_claim) =
maintenance_gate.enter_readwrite_open_for_orphan_recovery()?;
drop(path_orphaned_recovery_claim);
drop(path_maintenance_lease);
(identity_lease, identity_claim)
};
#[cfg(all(feature = "native", any(unix, windows)))]
let namespace_binding = if let Some(pending) = pending_namespace {
let identity = db_file
.file_identity()?
.ok_or_else(|| FrankenError::CannotOpen {
path: db_path.clone(),
})?;
let binding = if replace_quiescent_namespace_record {
pending.bind_replacing_quiescent_record(identity)?
} else {
pending.bind(identity)?
};
binding.validate_path_identity()?;
Some(binding)
} else {
None
};
let database_identity = db_file.file_identity()?;
let inner_database_path = db_path.clone();
#[cfg(all(feature = "native", any(unix, windows)))]
let inner_namespace_binding = namespace_binding.clone();
let db_file = Arc::new(AsyncRwLock::with_name("pager_db_file", db_file));
let journal_path = Self::journal_path(&db_path);
if disposition == ReadWriteOpenDisposition::ReservedEmpty
&& orphaned_recovery_claim.is_some()
{
return Err(FrankenError::BusyRecovery);
}
if let Some(claim) = orphaned_recovery_claim.as_mut() {
let recovery_owner = claim.recovery().owner;
let _recovery_guard = recovery_fence.acquire_for_recovery()?;
let prior_kind = maintenance_open_lease.upgrade_to_exclusive(Some(recovery_owner))?;
let recovery_result = Self::recover_rollback_journal_for_open(
cx,
&*vfs,
RollbackJournalOpenRecoveryContext {
group_commit_queue: &group_commit_queue,
db_file: &db_file,
current_db_path: &db_path,
current_db_identity: database_identity,
journal_path: &journal_path,
},
Some(claim.recovery_mut()),
)
.await;
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let downgrade_result = maintenance_open_lease.downgrade_from_exclusive(prior_kind);
match (recovery_result, downgrade_result) {
(Ok(_), Ok(())) => {}
(Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error),
(Err(recovery_error), Err(downgrade_error)) => {
return Err(FrankenError::internal(format!(
"orphaned open-time recovery failed and could not restore its maintenance lease: recovery={recovery_error}; downgrade={downgrade_error}"
)));
}
}
// ExternalFinalizationPending is not terminal merely because the
// maintenance lock was restored. Reacquire a verified SHARED
// snapshot, recheck the receipt's exact sidecar path for a writer
// that won the intervening gap, and clear the exact owner while
// that final fence is still held.
let recovery_journal_path = claim.recovery().namespace.journal_path.clone();
let mut final_snapshot =
BeginExternalLockState::new(&group_commit_queue, Arc::clone(&db_file), &cleanup_cx);
final_snapshot.acquire_snapshot(&cleanup_cx).await?;
let finalization_result = Self::verify_readonly_rollback_journal_state(
&cleanup_cx,
&*vfs,
&recovery_journal_path,
)
.await
.and_then(|()| {
maintenance_gate.release_rollback_recovery_owner(recovery_owner)?;
Ok(claim.finish())
});
let unlock_result = final_snapshot.restore().await;
let completed = match (finalization_result, unlock_result) {
(Ok(completed), Ok(())) => completed,
(Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error),
(Err(finalization_error), Err(unlock_error)) => {
return Err(FrankenError::internal(format!(
"orphaned open-time finalization failed and could not release its verified snapshot: finalization={finalization_error}; unlock={unlock_error}"
)));
}
};
debug_assert_eq!(completed.owner, recovery_owner);
}
if disposition == ReadWriteOpenDisposition::ExistingOnly
&& with_main_shared_lock(
cx,
&group_commit_queue,
&db_file,
&mut (),
|cx, db_file, ()| Box::pin(async move { db_file.file_size(cx) }),
)
.await?
== 0
{
// Existing-only open never lets a sidecar bootstrap an empty main
// file, even if that sidecar looks like a valid hot journal.
return Err(FrankenError::CannotOpen { path: db_path });
}
let (mut file_size, coherent_header_bytes, accept_proven_non_hot_leftover) = if disposition
== ReadWriteOpenDisposition::ReservedEmpty
{
(
shared_db_file_read(&db_file, cx).await?.file_size(cx)?,
None,
false,
)
} else {
let mut accept_proven_non_hot_leftover = false;
loop {
// Never wait for the recovery fence while holding main
// SHARED: another opener may own the fence while trying to
// upgrade to EXCLUSIVE. An unlocked existence probe is only
// a routing hint; every decision is rechecked under a lock.
if vfs.access(cx, &journal_path, AccessFlags::EXISTS)?
&& !accept_proven_non_hot_leftover
{
let _recovery_guard = recovery_fence.acquire_for_recovery()?;
let prior_kind = maintenance_open_lease.upgrade_to_exclusive(None)?;
let recovery_result = Self::recover_rollback_journal_for_open(
cx,
&*vfs,
RollbackJournalOpenRecoveryContext {
group_commit_queue: &group_commit_queue,
db_file: &db_file,
current_db_path: &db_path,
current_db_identity: database_identity,
journal_path: &journal_path,
},
None,
)
.await;
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let downgrade_result =
maintenance_open_lease.downgrade_from_exclusive(prior_kind);
match (recovery_result, downgrade_result) {
(Ok(outcome), Ok(())) => {
accept_proven_non_hot_leftover = outcome.is_some();
}
(Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error),
(Err(recovery_error), Err(downgrade_error)) => {
return Err(FrankenError::internal(format!(
"open-time rollback recovery failed and could not restore its maintenance lease: recovery={recovery_error}; downgrade={downgrade_error}"
)));
}
}
continue;
}
let mut snapshot_state = (
Arc::clone(&vfs),
journal_path.clone(),
accept_proven_non_hot_leftover,
disposition,
db_path.clone(),
);
let snapshot = with_main_shared_lock(
cx,
&group_commit_queue,
&db_file,
&mut snapshot_state,
|cx, db_file, state| {
let (vfs, journal_path, accept_non_hot, disposition, db_path) = state;
Box::pin(async move {
if vfs.access(cx, journal_path, AccessFlags::EXISTS)? {
if !*accept_non_hot {
return Ok(None);
}
match Self::verify_readonly_rollback_journal_state(
cx,
&*vfs,
journal_path,
)
.await {
Ok(()) => {}
Err(FrankenError::BusyRecovery) => return Ok(None),
Err(error) => return Err(error),
}
}
let file_size = db_file.file_size(cx)?;
if *disposition == ReadWriteOpenDisposition::ExistingOnly && file_size == 0 {
return Err(FrankenError::CannotOpen {
path: db_path.clone(),
});
}
let header_bytes = if file_size >= DATABASE_HEADER_SIZE as u64 {
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let bytes_read = db_file.read(cx, &mut header_bytes, 0).await?;
if bytes_read != DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read fetching database header: got {bytes_read} of {DATABASE_HEADER_SIZE}"
),
});
}
Some(header_bytes)
} else {
None
};
Ok(Some((file_size, header_bytes)))
})
})
.await?;
if let Some((file_size, header_bytes)) = snapshot {
break (file_size, header_bytes, accept_proven_non_hot_leftover);
}
accept_proven_non_hot_leftover = false;
}
};
let page_size = if let Some(header_bytes) = coherent_header_bytes.as_ref() {
match DatabaseHeader::from_bytes(header_bytes) {
Ok(header) => header.page_size,
Err(error)
if stale_main_header_can_be_recovered_from_live_wal(
cx,
&*vfs,
&db_path,
header_bytes,
&error,
false,
)
.await? =>
{
page_size_from_header_bytes(header_bytes).unwrap_or(requested_page_size)
}
Err(error) if disposition == ReadWriteOpenDisposition::ExistingOnly => {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("invalid database header: {error}"),
});
}
Err(_) => requested_page_size,
}
} else if disposition == ReadWriteOpenDisposition::ExistingOnly {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database file too small for header: {file_size} bytes (< {DATABASE_HEADER_SIZE})"
),
});
} else {
requested_page_size
};
let (header, bootstrapped_from_live_wal_stub) = if file_size == 0 {
if disposition == ReadWriteOpenDisposition::ExistingOnly {
return Err(FrankenError::CannotOpen { path: db_path });
}
let reserved_bootstrap = disposition == ReadWriteOpenDisposition::ReservedEmpty;
let mut bootstrap_lock = reserved_bootstrap.then(|| {
BeginExternalLockState::new(&group_commit_queue, Arc::clone(&db_file), cx)
});
if let Some(lock) = bootstrap_lock.as_mut() {
lock.arm_lock_level(LockLevel::None);
shared_db_lock(&db_file, cx, LockLevel::Exclusive).await?;
lock.mark_lock_level_acquired();
}
let bootstrap_result = async {
let mut file = shared_db_file_write(&db_file, cx).await?;
if reserved_bootstrap {
if file.file_identity()? != expected_identity || file.file_size(cx)? != 0 {
return Err(FrankenError::CannotOpen {
path: db_path.clone(),
});
}
#[cfg(all(feature = "native", any(unix, windows)))]
if let Some(binding) = &namespace_binding {
binding.validate_path_identity()?;
validate_reserved_database_artifacts(
&db_path,
WindowsLockSidecarPolicy::AllowExpected,
)?;
}
Self::ensure_reserved_recovery_artifacts_absent(cx, &*vfs, &db_path)?;
}
// SQLite databases are never truly empty: page 1 contains the
// 100-byte database header followed by the sqlite_master root page.
//
// This makes newly-created databases valid for downstream layers
// (B-tree, schema) and avoids surprising "empty file" semantics.
let page_len = page_size.as_usize();
let mut page1 = vec![0u8; page_len];
let header = DatabaseHeader {
page_size,
page_count: 1,
sqlite_version: FRANKENSQLITE_SQLITE_VERSION_NUMBER,
..DatabaseHeader::default()
};
let hdr_bytes = header.to_bytes().map_err(|err| {
FrankenError::internal(format!("failed to encode new database header: {err}"))
})?;
page1[..DATABASE_HEADER_SIZE].copy_from_slice(&hdr_bytes);
// Initialize sqlite_master root page as an empty leaf table B-tree
// page (type 0x0D) with zero cells.
let usable = page_size.usable(header.reserved_per_page);
BTreePageHeader::write_empty_leaf_table(&mut page1, DATABASE_HEADER_SIZE, usable);
file.write(cx, &page1, 0).await?;
file.sync(cx, SyncFlags::NORMAL)?;
Ok((header, file.file_size(cx)?))
}
.await;
let unlock_result = if let Some(lock) = bootstrap_lock.as_mut() {
lock.restore().await
} else {
Ok(())
};
let (header, initialized_size) = match (bootstrap_result, unlock_result) {
(Ok(value), Ok(())) => value,
(Err(error), Ok(())) | (Ok(_), Err(error)) => return Err(error),
(Err(bootstrap_error), Err(unlock_error)) => {
return Err(FrankenError::internal(format!(
"reserved database bootstrap failed and could not release the main-file lock: bootstrap={bootstrap_error}; unlock={unlock_error}"
)));
}
};
file_size = initialized_size;
(header, false)
} else {
if file_size < DATABASE_HEADER_SIZE as u64 {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database file too small for header: {file_size} bytes (< {DATABASE_HEADER_SIZE})"
),
});
}
let header_bytes = coherent_header_bytes.ok_or_else(|| FrankenError::CannotOpen {
path: db_path.clone(),
})?;
let (header, bootstrapped_from_live_wal_stub) =
match DatabaseHeader::from_bytes(&header_bytes) {
Ok(header) => (header, false),
Err(error)
if stale_main_header_can_be_recovered_from_live_wal(
cx,
&*vfs,
&db_path,
&header_bytes,
&error,
false,
)
.await? =>
{
// A live SQLite WAL can carry the authoritative page-1
// header while the main file still contains the stale
// bootstrap stub. Accept the file here and let the
// first WAL-backed refresh validate and load the real
// header from page 1 in the committed snapshot.
(
bootstrap_header_from_stale_main_file(&header_bytes, page_size),
true,
)
}
Err(error) => {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("invalid database header: {error}"),
});
}
};
if header.page_size != page_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database page size mismatch: header={} requested={}",
header.page_size.get(),
page_size.get()
),
});
}
(header, bootstrapped_from_live_wal_stub)
};
let page_size_u64 = page_size.as_usize() as u64;
// GH#334 / bd-e26jr sig-2: stock SQLite ignores trailing bytes beyond
// the last whole page (header page count is authoritative), and
// doctor-repair flows depend on opening such slack-bearing files. A
// partial-page tail is therefore slack, not corruption: floor to
// whole pages exactly like the aligned-slack case already does.
let db_pages = file_size
.checked_div(page_size_u64)
.ok_or_else(|| FrankenError::internal("page size must be non-zero"))?;
let db_size = u32::try_from(db_pages).map_err(|_| FrankenError::OutOfRange {
what: "database page count".to_owned(),
value: db_pages.to_string(),
})?;
let next_page = if db_size >= 2 {
db_size.saturating_add(1)
} else {
2
};
let freelist = if bootstrapped_from_live_wal_stub {
Vec::new()
} else {
let expected_header_bytes = header.to_bytes().map_err(|error| {
FrankenError::internal(format!(
"validated database header could not be re-encoded: {error}"
))
})?;
let mut freelist_state = (
Arc::clone(&vfs),
journal_path.clone(),
accept_proven_non_hot_leftover,
file_size,
expected_header_bytes,
page_size,
db_size,
header.freelist_trunk,
header.freelist_count,
);
with_main_shared_lock(
cx,
&group_commit_queue,
&db_file,
&mut freelist_state,
|cx, db_file, state| {
let (
vfs,
journal_path,
accept_non_hot,
file_size,
expected_header_bytes,
page_size,
db_size,
freelist_trunk,
freelist_count,
) = state;
Box::pin(async move {
if vfs.access(cx, journal_path, AccessFlags::EXISTS)? {
if !*accept_non_hot {
return Err(FrankenError::BusyRecovery);
}
Self::verify_readonly_rollback_journal_state(cx, &*vfs, journal_path)
.await?;
}
if db_file.file_size(cx)? != *file_size {
return Err(FrankenError::BusyRecovery);
}
let mut observed_header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let bytes_read = db_file.read(cx, &mut observed_header_bytes, 0).await?;
if bytes_read != DATABASE_HEADER_SIZE
|| observed_header_bytes != *expected_header_bytes
{
return Err(FrankenError::BusyRecovery);
}
load_freelist_from_disk(
cx,
db_file,
*page_size,
*db_size,
*freelist_trunk,
*freelist_count,
)
.await
})
},
)
.await?
};
let initial_commit_seq = CommitSeq::new(u64::from(header.change_counter));
let initial_journal_mode = Self::journal_mode_from_database_header(&header)?;
let freelist_count = freelist.len();
let resolved_max = crate::page_cache::resolve_page_buffer_max(page_buffer_max);
let cache =
ShardedPageCache::with_max_buffers_for_initial_pages(page_size, resolved_max, db_size);
cache.set_eviction_policy(PageCacheEvictionPolicy::S3Fifo(S3FifoConfig::new(
resolved_max,
)));
let pool = cache.pool().clone();
let rollback_recovery_pending = Arc::clone(&maintenance_gate.rollback_recovery_pending);
let pager = Self {
vfs,
db_path,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding,
maintenance_gate: Arc::clone(&maintenance_gate),
recovery_fence,
maintenance_open_lease: Mutex::new(Some(maintenance_open_lease)),
inner: Arc::new(Mutex::new(PagerInner {
db_file,
database_path: inner_database_path,
database_identity,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: inner_namespace_binding,
page_size,
db_size,
next_page,
writer_active: false,
active_transactions: 0,
checkpoint_active: false,
access_mode: PagerAccessMode::ReadWrite,
freelist,
journal_mode: initial_journal_mode,
wal_commit_sync_policy: WalCommitSyncPolicy::PerCommit,
rollback_journal_recovery_state: RollbackJournalRecoveryState::Clean,
rollback_journal_recovery_owner: None,
rollback_journal_recovery_namespace: None,
maintenance_gate,
rollback_recovery_pending,
commit_seq: initial_commit_seq,
committed_db_file_size_bytes: file_size,
committed_db_change_counter: u64::from(header.change_counter),
committed_wal_generation: None,
committed_wal_visible_commit_count: 0,
})),
writer_idle: Arc::new(Condvar::new()),
cache: Arc::new(cache),
pool,
published: Arc::new(PublishedPagerState::new(
db_size,
initial_commit_seq,
initial_journal_mode,
freelist_count,
)),
wal_backend: new_shared_wal_backend(),
committed_snapshot: Arc::new(RwLock::new(Arc::new(PagerCommittedSnapshot {
commit_seq: initial_commit_seq,
db_size,
journal_mode: initial_journal_mode,
freelist_count,
checkpoint_active: false,
writer_active: false,
db_file_size_bytes: file_size,
}))),
shared_connection_count: OnceLock::new(),
group_commit_queue,
};
if finish_namespace_bootstrap {
pager.finish_namespace_bootstrap()?;
}
Ok(pager)
}
/// Enable the single-connection cache fast path when this pager is still
/// uniquely owned.
///
/// Returns `true` when the cache could be mutated in place and `false`
/// when the cache `Arc` was already shared.
pub fn enable_single_connection_cache_fast_path(&mut self) -> bool {
let Some(cache) = Arc::get_mut(&mut self.cache) else {
return false;
};
cache.enable_fast_path();
true
}
/// Report whether the page cache fast path has been enabled.
#[must_use]
pub fn is_single_connection_cache_fast_path_enabled(&self) -> bool {
self.cache.is_fast_path_enabled()
}
/// Open a database in true read-only mode for fast analytical queries.
///
/// Unlike [`Self::open_with_cx`], this:
/// - Opens the file with `READONLY` VFS flags (no write lock acquisition)
/// - Skips journal recovery (read-only connections cannot replay journals)
/// - Skips freelist traversal (not needed for read-only queries)
/// - Does NOT create the file if it doesn't exist
///
/// This makes opening a 22GB database nearly instant instead of taking
/// minutes, because it avoids the expensive freelist scan and journal
/// recovery that the read-write path performs.
#[allow(clippy::too_many_lines)]
pub async fn open_readonly_with_cx(
cx: &Cx,
vfs: V,
path: &Path,
_requested_page_size: PageSize,
) -> Result<Self> {
Self::open_readonly_with_cx_and_page_buffer_max(cx, vfs, path, _requested_page_size, None)
.await
}
/// Like [`open_readonly_with_cx`](Self::open_readonly_with_cx) but allows
/// overriding the page-buffer-pool ceiling.
///
/// See [`open_with_cx_and_page_buffer_max`](Self::open_with_cx_and_page_buffer_max)
/// for parameter semantics.
pub async fn open_readonly_with_cx_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
_requested_page_size: PageSize,
page_buffer_max: Option<usize>,
) -> Result<Self> {
Self::open_readonly_with_optional_expected_identity(
cx,
vfs,
path,
_requested_page_size,
None,
page_buffer_max,
true,
)
.await
}
/// Open an existing database read-only only if its VFS handle has
/// `expected_identity`.
///
/// The identity-bound VFS open occurs before the header or any live-WAL
/// sidecar is inspected.
pub async fn open_readonly_with_expected_identity_and_page_buffer_max(
cx: &Cx,
vfs: V,
path: &Path,
_requested_page_size: PageSize,
expected_identity: FileIdentity,
page_buffer_max: Option<usize>,
) -> Result<Self> {
Self::open_readonly_with_optional_expected_identity(
cx,
vfs,
path,
_requested_page_size,
Some(expected_identity),
page_buffer_max,
true,
)
.await
}
#[allow(clippy::too_many_lines)]
async fn open_readonly_with_optional_expected_identity(
cx: &Cx,
vfs: V,
path: &Path,
_requested_page_size: PageSize,
expected_identity: Option<FileIdentity>,
page_buffer_max: Option<usize>,
finish_namespace_bootstrap: bool,
) -> Result<Self> {
let vfs = Arc::new(vfs);
let db_path = vfs.full_pathname(cx, path)?;
if !vfs.is_memory() {
settle_process_root_finalizations_for_path(&db_path).await?;
}
let path_maintenance_gate = maintenance_gate_for_backend(&*vfs, &db_path);
let path_maintenance_lease = path_maintenance_gate.enter_open()?;
#[cfg(all(feature = "native", any(unix, windows)))]
let (mut pending_namespace, mut replace_quiescent_namespace_record) = if vfs.is_memory() {
(None, false)
} else {
// GH #140: prefer strictly read-only admission — join the
// existing namespace generation without creating or rewriting
// any sidecar record. Only when no admissible records exist
// (a database never opened by FrankenSQLite, e.g. a stock
// SQLite file) fall back to the writable Shared admission so
// the database still opens read-only on writable media; the
// zero-mutation clean-database case remains tracked in #140.
// Busy is NOT a fallback trigger: it means a live generation
// transition is in flight and must stay retryable.
match PendingNamespaceOpen::begin(&db_path, NamespaceOpenIntent::ReadOnlyExisting) {
Ok(pending) => (Some(pending), false),
Err(FrankenError::CannotOpen { .. }) => {
let pending =
PendingNamespaceOpen::begin(&db_path, NamespaceOpenIntent::Shared)?;
let replace_record = pending.has_quiescent_record_bytes()?;
(Some(pending), replace_record)
}
Err(error) => return Err(error),
}
};
#[cfg(all(feature = "native", any(unix, windows)))]
let namespace_expected_identity = pending_namespace
.as_ref()
.and_then(PendingNamespaceOpen::expected_identity);
#[cfg(all(feature = "native", any(unix, windows)))]
let effective_expected_identity = match (expected_identity, namespace_expected_identity) {
(Some(caller), Some(generation)) if caller != generation => {
return Err(FrankenError::CannotOpen {
path: db_path.clone(),
});
}
(Some(caller), _) => Some(caller),
(None, generation) => generation,
};
#[cfg(not(all(feature = "native", any(unix, windows))))]
let effective_expected_identity = expected_identity;
let flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_DB;
#[cfg(all(feature = "native", any(unix, windows)))]
let (db_file, _actual_flags) = {
let initial_open = if let Some(identity) = effective_expected_identity {
vfs.open_with_expected_identity(cx, &db_path, flags, identity)
} else {
vfs.open(cx, Some(&db_path), flags)
};
match initial_open {
Ok(opened) => opened,
Err(FrankenError::CannotOpen { .. })
if expected_identity.is_none() && namespace_expected_identity.is_some() =>
{
// bd-g5rdj: namespace sidecars are machine-local runtime
// state and are routinely copied alongside the database.
// A valid record from the source inode must not make the
// copied database permanently unopenable. Release the
// read-only join and retry through Shared admission: when
// the copied namespace is quiescent this takes `use`
// exclusively, opens the current database identity, and
// republishes that identity during `bind`. If a live peer
// owns the namespace, Shared admission joins its identity
// instead and the exact-identity open remains fail-closed.
drop(pending_namespace.take());
let replacement_pending =
PendingNamespaceOpen::begin(&db_path, NamespaceOpenIntent::Shared)?;
let replacement_expected = replacement_pending.expected_identity();
replace_quiescent_namespace_record =
replacement_pending.has_quiescent_record_bytes()?;
let opened = if let Some(identity) = replacement_expected {
vfs.open_with_expected_identity(cx, &db_path, flags, identity)?
} else {
vfs.open(cx, Some(&db_path), flags)?
};
pending_namespace = Some(replacement_pending);
opened
}
Err(error) => return Err(error),
}
};
#[cfg(not(all(feature = "native", any(unix, windows))))]
let (db_file, _actual_flags) = if let Some(expected_identity) = effective_expected_identity
{
vfs.open_with_expected_identity(cx, &db_path, flags, expected_identity)?
} else {
vfs.open(cx, Some(&db_path), flags)?
};
if let Some(identity) = db_file.file_identity()? {
settle_process_root_finalizations_for_identity(identity).await?;
}
let maintenance_gate =
identity_bound_maintenance_gate(&*vfs, &path_maintenance_gate, &db_file)?;
let recovery_fence = identity_bound_recovery_fence(&*vfs, &db_path, &db_file)?;
let group_commit_queue = identity_bound_group_commit_queue(&*vfs, &db_path, &db_file)?;
// Read-only open is still an observer of the physical database
// generation, so it must not inspect storage while a prior admitted
// WAL finalization remains unresolved.
settle_pending_group_commit_finalization(&group_commit_queue).await?;
let maintenance_open_lease = if Arc::ptr_eq(&maintenance_gate, &path_maintenance_gate) {
path_maintenance_lease
} else {
let identity_lease = maintenance_gate.enter_open()?;
drop(path_maintenance_lease);
identity_lease
};
#[cfg(all(feature = "native", any(unix, windows)))]
let namespace_binding = if let Some(pending) = pending_namespace {
let identity = db_file
.file_identity()?
.ok_or_else(|| FrankenError::CannotOpen {
path: db_path.clone(),
})?;
let binding = if replace_quiescent_namespace_record {
pending.bind_replacing_quiescent_record(identity)?
} else {
pending.bind(identity)?
};
binding.validate_path_identity()?;
Some(binding)
} else {
None
};
let database_identity = db_file.file_identity()?;
let inner_database_path = db_path.clone();
#[cfg(all(feature = "native", any(unix, windows)))]
let inner_namespace_binding = namespace_binding.clone();
let db_file = Arc::new(AsyncRwLock::with_name("pager_db_file", db_file));
let journal_path = Self::journal_path(&db_path);
let mut header_state = (Arc::clone(&vfs), journal_path.clone(), db_path.clone());
let (file_size, header_bytes) = with_main_shared_lock(
cx,
&group_commit_queue,
&db_file,
&mut header_state,
|cx, db_file, state| {
let (vfs, journal_path, db_path) = state;
Box::pin(async move {
// Read-only and schema-only opens cannot replay. They may accept a
// proven non-hot construction leftover, but they must never cache
// a database image for which recovery is required or ambiguous.
Self::verify_readonly_rollback_journal_state(cx, &*vfs, journal_path).await?;
let file_size = db_file.file_size(cx)?;
if file_size == 0 {
return Err(FrankenError::CannotOpen {
path: db_path.clone(),
});
}
if file_size < DATABASE_HEADER_SIZE as u64 {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"database file too small for header: {file_size} bytes (< {DATABASE_HEADER_SIZE})"
),
});
}
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let bytes_read = db_file.read(cx, &mut header_bytes, 0).await?;
if bytes_read != DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read fetching database header: got {bytes_read} of {DATABASE_HEADER_SIZE}"
),
});
}
Ok((file_size, header_bytes))
})
})
.await?;
let (header, page_size) = match DatabaseHeader::from_bytes(&header_bytes) {
Ok(header) => {
let page_size = header.page_size;
(Some(header), page_size)
}
Err(error)
if stale_main_header_can_be_recovered_from_live_wal(
cx,
&*vfs,
&db_path,
&header_bytes,
&error,
true,
)
.await? =>
{
let page_size = page_size_from_header_bytes(&header_bytes).ok_or_else(|| {
FrankenError::DatabaseCorrupt {
detail: "live WAL bootstrap could not recover database page size"
.to_owned(),
}
})?;
(None, page_size)
}
Err(error) => {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("invalid database header: {error}"),
});
}
};
let page_size_u64 = page_size.as_usize() as u64;
// GH#334 / bd-e26jr sig-2: a partial-page tail is slack, not
// corruption — floor to whole pages (see the read-write open above).
let db_pages = file_size
.checked_div(page_size_u64)
.ok_or_else(|| FrankenError::internal("page size must be non-zero"))?;
let db_size = u32::try_from(db_pages).map_err(|_| FrankenError::OutOfRange {
what: "database page count".to_owned(),
value: db_pages.to_string(),
})?;
let next_page = if db_size >= 2 {
db_size.saturating_add(1)
} else {
2
};
// Skip freelist traversal for read-only — use empty freelist.
let freelist = Vec::new();
let initial_commit_seq = CommitSeq::new(u64::from(
header.as_ref().map_or(0, |header| header.change_counter),
));
let resolved_max = crate::page_cache::resolve_page_buffer_max(page_buffer_max);
let cache =
ShardedPageCache::with_max_buffers_for_initial_pages(page_size, resolved_max, db_size);
cache.set_eviction_policy(PageCacheEvictionPolicy::S3Fifo(S3FifoConfig::new(
resolved_max,
)));
let pool = cache.pool().clone();
let rollback_recovery_pending = Arc::clone(&maintenance_gate.rollback_recovery_pending);
let pager = Self {
vfs,
db_path,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding,
maintenance_gate: Arc::clone(&maintenance_gate),
recovery_fence,
maintenance_open_lease: Mutex::new(Some(maintenance_open_lease)),
inner: Arc::new(Mutex::new(PagerInner {
db_file,
database_path: inner_database_path,
database_identity,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: inner_namespace_binding,
page_size,
db_size,
next_page,
writer_active: false,
active_transactions: 0,
checkpoint_active: false,
freelist,
journal_mode: JournalMode::Delete,
wal_commit_sync_policy: WalCommitSyncPolicy::PerCommit,
access_mode: PagerAccessMode::ReadOnly,
rollback_journal_recovery_state: RollbackJournalRecoveryState::Clean,
rollback_journal_recovery_owner: None,
rollback_journal_recovery_namespace: None,
maintenance_gate,
rollback_recovery_pending,
commit_seq: initial_commit_seq,
committed_db_file_size_bytes: file_size,
committed_db_change_counter: header
.as_ref()
.map_or(0, |header| u64::from(header.change_counter)),
committed_wal_generation: None,
committed_wal_visible_commit_count: 0,
})),
writer_idle: Arc::new(Condvar::new()),
cache: Arc::new(cache),
pool,
published: Arc::new(PublishedPagerState::new(
db_size,
initial_commit_seq,
JournalMode::Delete,
0, // freelist_count = 0 for read-only
)),
wal_backend: new_shared_wal_backend(),
committed_snapshot: Arc::new(RwLock::new(Arc::new(PagerCommittedSnapshot {
commit_seq: initial_commit_seq,
db_size,
journal_mode: JournalMode::Delete,
freelist_count: 0,
checkpoint_active: false,
writer_active: false,
db_file_size_bytes: file_size,
}))),
shared_connection_count: OnceLock::new(),
group_commit_queue,
};
if finish_namespace_bootstrap {
pager.finish_namespace_bootstrap()?;
}
Ok(pager)
}
/// Open (or create) a database and return a pager using a detached test context.
#[cfg(test)]
#[allow(clippy::too_many_lines)]
pub async fn open(vfs: V, path: &Path, page_size: PageSize) -> Result<Self> {
let cx = Cx::new();
Self::open_with_cx(&cx, vfs, path, page_size).await
}
/// Replay a hot journal by writing original pages back to the database.
#[cfg(test)]
async fn replay_journal(
cx: &Cx,
vfs: &V,
db_file: &mut V::File,
journal_path: &Path,
page_size: PageSize,
) -> Result<()> {
Self::replay_journal_with_validator(cx, vfs, db_file, journal_path, page_size, |_, _, _| {
Box::pin(async { Ok(()) })
})
.await
}
/// Replay a hot journal, verify the durable restored image, and only then
/// invalidate the recovery record.
///
/// The page-by-page verifier protects every recovery caller against a VFS
/// that reports a successful but misdirected or silently corrupted write.
/// `validate_restored` adds any caller-specific whole-image invariant while
/// the same hot-journal handle is still open. A validation error closes the
/// handle without invalidating it, leaving recovery retryable and fail-closed.
async fn replay_journal_with_validator<Validate>(
cx: &Cx,
vfs: &V,
db_file: &mut V::File,
journal_path: &Path,
page_size: PageSize,
validate_restored: Validate,
) -> Result<()>
where
Validate: for<'a> FnOnce(&'a Cx, &'a V::File, PageSize) -> WalFuture<'a, ()>,
{
Self::replay_journal_with_optional_page_size_validator(
cx,
vfs,
db_file,
journal_path,
Some(page_size),
validate_restored,
)
.await
.map(|_| ())
}
async fn replay_journal_with_optional_page_size_validator<Validate>(
cx: &Cx,
vfs: &V,
db_file: &mut V::File,
journal_path: &Path,
expected_page_size: Option<PageSize>,
validate_restored: Validate,
) -> Result<RollbackJournalReplayOutcome>
where
Validate: for<'a> FnOnce(&'a Cx, &'a V::File, PageSize) -> WalFuture<'a, ()>,
{
let jrnl_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut jrnl_file, _) = vfs.open(cx, Some(journal_path), jrnl_flags)?;
let (prefix_state, jrnl_size) = classify_rollback_journal_prefix(cx, &jrnl_file).await?;
if prefix_state == RollbackJournalPrefixState::NonHot {
jrnl_file.close(cx)?;
return Ok(RollbackJournalReplayOutcome::NonHot);
}
if jrnl_size < crate::journal::JOURNAL_HEADER_SIZE as u64 {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback journal is truncated: {jrnl_size} bytes is shorter than its header"
),
});
}
// Read and parse the journal header.
let mut hdr_buf = vec![0u8; crate::journal::JOURNAL_HEADER_SIZE];
let header_read = jrnl_file.read(cx, &mut hdr_buf, 0).await?;
if header_read != hdr_buf.len() {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short rollback-journal header read: got {header_read} of {} bytes",
hdr_buf.len()
),
});
}
let header =
JournalHeader::decode(&hdr_buf).map_err(|error| FrankenError::DatabaseCorrupt {
detail: format!("invalid rollback-journal header: {error}"),
})?;
if !(512..=65_536).contains(&header.sector_size) || !header.sector_size.is_power_of_two() {
return Err(FrankenError::DatabaseCorrupt {
detail: format!("hot journal has invalid sector size {}", header.sector_size),
});
}
let page_size =
PageSize::new(header.page_size).ok_or_else(|| FrankenError::DatabaseCorrupt {
detail: format!("hot journal has invalid page size {}", header.page_size),
})?;
if expected_page_size.is_some_and(|expected| expected != page_size) {
let expected_page_size = expected_page_size.expect("checked as present");
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"hot journal page size mismatch: header={} expected={}",
header.page_size,
expected_page_size.get()
),
});
}
let header_size = u64::try_from(crate::journal::JOURNAL_HEADER_SIZE)
.expect("journal header size should fit in u64");
let hdr_padded = u64::from(header.sector_size).max(header_size);
let ps = page_size.as_usize();
let record_size = 4 + ps + 4;
let mut offset = hdr_padded;
if jrnl_size < hdr_padded {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"hot journal is shorter than its {}-byte padded header: {jrnl_size} bytes",
hdr_padded
),
});
}
let is_local_journal = local_journal_marker_present(cx, &jrnl_file, hdr_padded).await?;
let record_size_u64 = u64::try_from(record_size).expect("journal record size fits u64");
if header.page_count < 0 && (jrnl_size - hdr_padded) % record_size_u64 != 0 {
if is_local_journal {
return Err(FrankenError::DatabaseCorrupt {
detail: "local hot journal has a partial trailing page record".to_owned(),
});
}
tracing::warn!(
journal = %journal_path.display(),
"refusing unsupported external rollback journal with a trailing section or master-journal payload"
);
return Err(FrankenError::Unsupported);
}
let page_count = if header.page_count < 0 {
header.compute_page_count_from_file_size(jrnl_size)
} else {
#[allow(clippy::cast_sign_loss)]
let c = header.page_count as u32;
c
};
let required_size = hdr_padded
.checked_add(
u64::from(page_count)
.checked_mul(record_size_u64)
.ok_or_else(|| FrankenError::OutOfRange {
what: "rollback-journal record span".to_owned(),
value: page_count.to_string(),
})?,
)
.ok_or_else(|| FrankenError::OutOfRange {
what: "rollback-journal required length".to_owned(),
value: page_count.to_string(),
})?;
if jrnl_size < required_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"hot journal is truncated at {jrnl_size} bytes, expected {required_size}"
),
});
}
if jrnl_size > required_size {
if is_local_journal {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"local hot journal length is {jrnl_size} bytes, expected exactly {required_size}"
),
});
}
tracing::warn!(
journal = %journal_path.display(),
trailing_bytes = jrnl_size - required_size,
"refusing unsupported external rollback journal with additional sections or a master-journal payload"
);
return Err(FrankenError::Unsupported);
}
// Validate the complete recovery surface before changing the first
// database byte. A checksum or page-number failure in a later record
// must not leave a prefix of pre-images applied to the live file.
// The caller holds the database EXCLUSIVE lock throughout recovery,
// so no legitimate SQLite writer can replace this journal between the
// validation and application passes.
let mut validation_offset = hdr_padded;
// Do not reserve from attacker-controlled nRec. Capacity grows only
// after each record has been read, decoded, and checksum-validated.
let mut validated_pages = HashSet::new();
for _ in 0..page_count {
let mut rec_buf = vec![0u8; record_size];
let bytes_read = jrnl_file.read(cx, &mut rec_buf, validation_offset).await?;
if bytes_read != record_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short rollback-journal record during validation: got {bytes_read} of {record_size} bytes"
),
});
}
#[allow(clippy::cast_possible_truncation)]
let record = JournalPageRecord::decode(&rec_buf, ps as u32).map_err(|err| {
FrankenError::DatabaseCorrupt {
detail: format!("invalid rollback-journal record: {err}"),
}
})?;
record
.verify_checksum(header.nonce)
.map_err(|err| FrankenError::DatabaseCorrupt {
detail: format!("rollback-journal checksum mismatch: {err}"),
})?;
let page_no = PageNumber::new(record.page_number).ok_or_else(|| {
FrankenError::DatabaseCorrupt {
detail: format!(
"hot journal contains invalid page number {}",
record.page_number
),
}
})?;
if page_no.get() == crate::journal::lock_byte_page(page_size) {
return Err(FrankenError::DatabaseCorrupt {
detail: "rollback journal contains the reserved lock-byte page".to_owned(),
});
}
if page_no.get() > header.initial_db_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback journal page {} exceeds its initial database size {}",
page_no.get(),
header.initial_db_size
),
});
}
if !validated_pages.insert(page_no) {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback journal contains duplicate page record {}",
page_no.get()
),
});
}
validation_offset = validation_offset
.checked_add(u64::try_from(record_size).expect("journal record size fits u64"))
.ok_or_else(|| FrankenError::OutOfRange {
what: "rollback-journal validation offset".to_owned(),
value: validation_offset.to_string(),
})?;
}
for _ in 0..page_count {
let mut rec_buf = vec![0u8; record_size];
let bytes_read = jrnl_file.read(cx, &mut rec_buf, offset).await?;
if bytes_read < record_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short rollback-journal record: got {bytes_read} of {record_size} bytes"
),
});
}
#[allow(clippy::cast_possible_truncation)]
let record = JournalPageRecord::decode(&rec_buf, ps as u32).map_err(|err| {
FrankenError::DatabaseCorrupt {
detail: format!("invalid rollback-journal record: {err}"),
}
})?;
// Verify checksum before applying.
record
.verify_checksum(header.nonce)
.map_err(|err| FrankenError::DatabaseCorrupt {
detail: format!("rollback-journal checksum mismatch: {err}"),
})?;
// Write the pre-image back to the database file.
let Some(page_no) = PageNumber::new(record.page_number) else {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"hot journal contains invalid page number {}",
record.page_number
),
});
};
if page_no.get() == crate::journal::lock_byte_page(page_size) {
// SQLite reserves this page number as the super-journal
// sentinel. It is never database content and must never be
// replayed over the process-lock byte range.
return Err(FrankenError::DatabaseCorrupt {
detail: "rollback journal contains the reserved lock-byte page".to_owned(),
});
}
let page_offset = u64::from(page_no.get() - 1) * ps as u64;
db_file.write(cx, &record.content, page_offset).await?;
offset += record_size as u64;
}
// Sync the database after replaying.
db_file.durable_sync(cx, SyncKind::FullDurable)?;
// Truncate the database to the original size from the journal header.
let target_size = u64::from(header.initial_db_size) * ps as u64;
let current_size = db_file.file_size(cx)?;
if current_size != target_size {
db_file.truncate(cx, target_size)?;
// Exact-size restoration is required after both growth and
// shrink publication failures, including rollback to an empty
// source image.
db_file.durable_sync(cx, SyncKind::FullDurable)?;
}
// Do not destroy the only recovery record until the durable database
// has been re-read and proven to equal every pre-image in this journal.
// Growth pages deliberately have no records; exact file-size
// verification proves they were removed by rollback.
let verification_result = async {
let restored_size = db_file.file_size(cx)?;
if restored_size != target_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback recovery restored file length {restored_size}, expected {target_size}"
),
});
}
let mut verification_offset = hdr_padded;
let mut restored_page = vec![0_u8; ps];
for _ in 0..page_count {
let mut rec_buf = vec![0_u8; record_size];
let bytes_read = jrnl_file
.read(cx, &mut rec_buf, verification_offset)
.await?;
if bytes_read != record_size {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short rollback-journal record during restored-image verification: got {bytes_read} of {record_size} bytes"
),
});
}
#[allow(clippy::cast_possible_truncation)]
let record = JournalPageRecord::decode(&rec_buf, ps as u32).map_err(|error| {
FrankenError::DatabaseCorrupt {
detail: format!(
"invalid rollback-journal record during restored-image verification: {error}"
),
}
})?;
record
.verify_checksum(header.nonce)
.map_err(|error| FrankenError::DatabaseCorrupt {
detail: format!(
"rollback-journal checksum mismatch during restored-image verification: {error}"
),
})?;
let page_no = PageNumber::new(record.page_number).ok_or_else(|| {
FrankenError::DatabaseCorrupt {
detail: format!(
"rollback journal contains invalid page number {} during restored-image verification",
record.page_number
),
}
})?;
if page_no.get() == crate::journal::lock_byte_page(page_size) {
return Err(FrankenError::DatabaseCorrupt {
detail: "rollback journal contains the reserved lock-byte page".to_owned(),
});
}
let page_offset = u64::from(page_no.get() - 1)
.checked_mul(u64::try_from(ps).map_err(|_| FrankenError::OutOfRange {
what: "rollback recovery page size".to_owned(),
value: ps.to_string(),
})?)
.ok_or_else(|| FrankenError::OutOfRange {
what: "rollback recovery verification page offset".to_owned(),
value: page_no.get().to_string(),
})?;
let restored_read = db_file
.read(cx, &mut restored_page, page_offset)
.await?;
if restored_read != ps || restored_page != record.content {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"rollback recovery verification mismatch on page {}",
page_no.get()
),
});
}
verification_offset = verification_offset
.checked_add(u64::try_from(record_size).expect("journal record size fits u64"))
.ok_or_else(|| FrankenError::OutOfRange {
what: "rollback recovery verification record offset".to_owned(),
value: verification_offset.to_string(),
})?;
}
validate_restored(cx, db_file, page_size).await
}
.await;
if let Err(verification_error) = verification_result {
let close_result = jrnl_file.close(cx);
return match close_result {
Ok(()) => Err(verification_error),
Err(close_error) => Err(FrankenError::internal(format!(
"rollback recovery verification failed while preserving the hot journal: verification={verification_error}; close={close_error}"
))),
};
}
// Recovery is now durably and independently verified. Invalidate the
// journal before best-effort deletion so a later delete failure does
// not replay the same pre-images on every open.
let invalidate_result =
durable_invalidate_journal(cx, &mut jrnl_file, JournalInvalidation::Truncate).await;
let close_result = jrnl_file.close(cx);
match (invalidate_result, close_result) {
(Ok(()), Ok(())) => Ok(RollbackJournalReplayOutcome::Replayed(page_size)),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(invalidate_error), Err(close_error)) => Err(FrankenError::internal(format!(
"rollback recovery was verified but journal invalidation and close failed: invalidate={invalidate_error}; close={close_error}"
))),
}
}
}
/// A snapshot of the transaction state at a savepoint boundary.
struct SavepointEntry {
/// The user-supplied savepoint name.
name: String,
/// Snapshot of the write-set at the time the savepoint was created.
/// Stores published page data so savepoint capture can reuse the staged
/// page's shared `Arc<Vec<u8>>` when available instead of cloning bytes.
write_set_snapshot: HashMap<PageNumber, PageData>,
/// Sorted unique page ids in the write-set snapshot.
write_pages_sorted_snapshot: Vec<PageNumber>,
/// Snapshot of freed pages at the time the savepoint was created.
freed_pages_snapshot: Vec<PageNumber>,
/// Snapshot of the pager's next_page counter.
/// Used to restore allocation state on rollback.
next_page_snapshot: u32,
/// Snapshot of the pager's freelist.
/// Used to restore allocation state on rollback.
freelist_snapshot: Vec<PageNumber>,
/// Snapshot of pages allocated from freelist by this transaction.
allocated_from_freelist_snapshot: Vec<PageNumber>,
/// Snapshot of pages allocated from EOF by this transaction.
allocated_from_eof_snapshot: Vec<PageNumber>,
}
#[derive(Debug)]
enum StagedPageBacking {
Buffered(PageBuf),
Owned(PageData),
}
#[derive(Debug)]
struct StagedPage {
backing: StagedPageBacking,
published: OnceLock<PageData>,
}
impl StagedPage {
fn from_buf(buf: PageBuf) -> Self {
Self {
backing: StagedPageBacking::Buffered(buf),
// Keep the staged page single-copy until a read/publication path
// actually asks for a shared snapshot. Eager publication turned
// every write into an unconditional full-page clone.
published: OnceLock::new(),
}
}
#[cfg(test)]
fn from_bytes(pool: &PageBufPool, data: &[u8]) -> Result<Self> {
let mut buf = pool.acquire()?;
let len = buf.len().min(data.len());
buf[..len].copy_from_slice(&data[..len]);
if len < buf.len() {
buf[len..].fill(0);
}
Ok(Self::from_buf(buf))
}
fn from_page_data(data: PageData) -> Self {
Self {
backing: StagedPageBacking::Owned(data),
published: OnceLock::new(),
}
}
fn from_page_data_with_cache_recovery(
pool: &PageBufPool,
cache: &ShardedPageCache,
data: PageData,
operation: &'static str,
) -> Result<Self> {
if data.len() == pool.page_size() {
return Ok(Self::from_page_data(data));
}
let mut buffer = acquire_page_buf_with_clean_cache_recovery(pool, cache, operation)?;
let len = buffer.len().min(data.len());
buffer[..len].copy_from_slice(&data.as_bytes()[..len]);
if len < buffer.len() {
buffer[len..].fill(0);
}
Ok(Self::from_buf(buffer))
}
/// Convert a committed staged image into a cache buffer without escaping
/// the configured pool bound. A foreign/standalone backing is copied into
/// a pool-owned buffer; if no clean resident can be reclaimed, the caller
/// may safely omit this post-commit cache admission.
fn into_cache_buf(self, pool: &PageBufPool, cache: &ShardedPageCache) -> Result<PageBuf> {
match self.backing {
StagedPageBacking::Buffered(buffer) if buffer.returns_to_pool(pool) => Ok(buffer),
StagedPageBacking::Buffered(source) => {
let mut buffer = acquire_page_buf_with_clean_cache_recovery(
pool,
cache,
"committed_page_cache_admission",
)?;
let len = buffer.len().min(source.len());
buffer[..len].copy_from_slice(&source[..len]);
if len < buffer.len() {
buffer[len..].fill(0);
}
Ok(buffer)
}
StagedPageBacking::Owned(source) => {
let mut buffer = acquire_page_buf_with_clean_cache_recovery(
pool,
cache,
"committed_page_cache_admission",
)?;
let len = buffer.len().min(source.len());
buffer[..len].copy_from_slice(&source.as_bytes()[..len]);
if len < buffer.len() {
buffer[len..].fill(0);
}
Ok(buffer)
}
}
}
fn as_page_bytes(&self) -> &[u8] {
match &self.backing {
StagedPageBacking::Buffered(buf) => buf.as_slice(),
StagedPageBacking::Owned(data) => data.as_bytes(),
}
}
fn as_page_bytes_mut(&mut self) -> &mut [u8] {
debug_assert!(
self.published.get().is_none(),
"staged pages must be unpublished before in-place mutation"
);
match &mut self.backing {
StagedPageBacking::Buffered(buf) => buf.as_mut_slice(),
StagedPageBacking::Owned(data) => data.as_bytes_mut(),
}
}
fn make_unpublished_for_mutation(
&mut self,
pool: &PageBufPool,
cache: &ShardedPageCache,
operation: &'static str,
) -> Result<()> {
if self.published.get().is_none() {
return Ok(());
}
if matches!(self.backing, StagedPageBacking::Buffered(_)) {
// Buffered publication copies into an independent Arc<[u8]>, so
// discarding our publication handle is enough to make the unique
// PageBuf mutable again without allocating.
let _ = self.published.take();
return Ok(());
}
// Owned publication clones the same PageData backing. Acquire the
// replacement first, so a capacity error leaves both backing and
// published snapshot completely unchanged.
let mut buffer = acquire_page_buf_with_clean_cache_recovery(pool, cache, operation)?;
let source = self.as_page_bytes();
let len = buffer.len().min(source.len());
buffer[..len].copy_from_slice(&source[..len]);
if len < buffer.len() {
buffer[len..].fill(0);
}
self.backing = StagedPageBacking::Buffered(buffer);
self.published = OnceLock::new();
Ok(())
}
fn published_page(&self) -> PageData {
self.published
.get_or_init(|| match &self.backing {
StagedPageBacking::Buffered(buf) => {
PageData::from_shared(Arc::<[u8]>::from(buf.as_slice()))
}
StagedPageBacking::Owned(data) => data.clone(),
})
.clone()
}
fn into_published_page(self) -> PageData {
let Self { backing, published } = self;
if let Some(page) = published.into_inner() {
return page;
}
match backing {
StagedPageBacking::Buffered(buf) => {
PageData::from_shared(Arc::<[u8]>::from(buf.as_slice()))
}
StagedPageBacking::Owned(data) => data,
}
}
fn try_into_unpublished_owned_page_data(self) -> std::result::Result<PageData, Self> {
let Self { backing, published } = self;
if let Some(page) = published.into_inner() {
let published = OnceLock::new();
let _ = published.set(page);
return Err(Self { backing, published });
}
match backing {
StagedPageBacking::Owned(data) => Ok(data),
other => Err(Self {
backing: other,
published: OnceLock::new(),
}),
}
}
/// Attempt to overwrite this staged page's bytes in place with `data`.
///
/// Returns `true` when the existing backing buffer was reused (no
/// allocation, no pool round-trip). Returns `false` when the backing
/// either has an outstanding shared snapshot (`published` already set) or
/// cannot otherwise be safely mutated, in which case the caller must fall
/// back to inserting a fresh [`StagedPage`].
///
/// Safety / correctness contract:
/// * If `published` is populated, an external reader (MVCC snapshot,
/// cache probe, etc.) already took a shared view of these bytes, so we
/// must never mutate them in place.
/// * For `Buffered(PageBuf)` the backing buffer is single-owner by
/// construction, so overwriting its bytes is safe when `published`
/// is empty.
/// * For `Owned(PageData)` we only reuse when the internal shared-snapshot
/// cache has not yet been materialised (i.e. the bytes are still
/// single-owner). `PageData::as_bytes_mut` would otherwise perform a
/// copy-on-write via `Arc::make_mut`, which defeats the whole point of
/// the fast path.
fn try_overwrite_bytes_in_place(&mut self, data: &[u8]) -> bool {
if self.published.get().is_some() {
return false;
}
match &mut self.backing {
StagedPageBacking::Buffered(buf) => {
if buf.len() != data.len() {
return false;
}
buf.as_mut_slice().copy_from_slice(data);
true
}
StagedPageBacking::Owned(page_data) => {
if page_data.len() != data.len() {
return false;
}
// Only mutate when the `Owned` variant still holds
// single-owner bytes. `as_bytes_mut` would otherwise CoW
// through `Arc::make_mut`, which allocates — defeating the
// fast path.
if !page_data.is_single_owner_owned() {
return false;
}
page_data.as_bytes_mut().copy_from_slice(data);
true
}
}
}
/// Attempt to overwrite this staged page with an owned [`PageData`].
///
/// Same contract as [`Self::try_overwrite_bytes_in_place`], but avoids
/// the intermediate byte-slice copy when the caller already owns a
/// `PageData` whose bytes can be moved in.
fn try_overwrite_page_data_in_place(&mut self, data: &PageData) -> bool {
self.try_overwrite_bytes_in_place(data.as_bytes())
}
}
/// Transaction handle produced by [`SimplePager`].
/// Number of EOF pages to pre-allocate in a single lock acquisition.
/// Reduces `inner` mutex contention when concurrent writers cause
/// frequent B-tree splits that each need a new page. 8 is a reasonable
/// balance — larger batches waste pages on small transactions, smaller
/// batches increase lock contention on write-heavy workloads.
const PAGE_LEASE_BATCH_SIZE: u32 = 8;
// Dirty shared-cache buffers must never be reclaimed by raw writeback here,
// because that would bypass rollback-journal/WAL durability and MVCC publication.
fn acquire_page_buf_with_clean_cache_recovery(
pool: &PageBufPool,
cache: &ShardedPageCache,
operation: &'static str,
) -> Result<PageBuf> {
match pool.acquire() {
Ok(buffer) => return Ok(buffer),
Err(FrankenError::OutOfMemory) => {}
Err(error) => return Err(error),
}
if let Some(buffer) = cache.take_clean_buffer() {
return Ok(buffer);
}
// A concurrent holder may have returned a buffer while the cache victim
// scan was in progress. Recheck the free list so the failure snapshot has
// a clean linearization point and never reports capacity with an idle
// buffer already available.
match pool.acquire() {
Ok(buffer) => return Ok(buffer),
Err(FrankenError::OutOfMemory) => {}
Err(error) => return Err(error),
}
let snapshots = cache.page_snapshots();
let cached_dirty = snapshots.iter().filter(|page| page.dirty).count();
let cached_clean = snapshots.len().saturating_sub(cached_dirty);
Err(FrankenError::PageBufferCapacityExhausted {
operation,
page_size: pool.page_size(),
max_buffers: pool.capacity(),
total_buffers: pool.total_buffers(),
available_buffers: pool.available(),
cached_clean,
cached_dirty,
successful_evictions: 0,
})
}
#[allow(clippy::struct_excessive_bools)]
pub struct SimpleTransaction<V>
where
V: Vfs,
V::File: 'static,
{
vfs: Arc<V>,
journal_path: PathBuf,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: Option<Arc<DatabaseNamespaceBinding>>,
group_commit_queue: GroupCommitQueueRef,
inner: Arc<Mutex<PagerInner<V::File>>>,
/// Exact stable handle used for lock-transition coordination. Distinct
/// handles for one file identity may proceed independently; transitions on
/// this handle must not overtake one another.
db_file: SharedDbFile<V::File>,
writer_idle: Arc<Condvar>,
cache: Arc<ShardedPageCache>,
published: Arc<PublishedPagerState>,
/// WAL backend for WAL-mode operation (D1-CRITICAL: separate lock for split-lock commit).
wal_backend: SharedWalBackend,
/// Shared committed-state snapshot (bd-db300.5.3.3.1 / M6).
committed_snapshot: Arc<RwLock<Arc<PagerCommittedSnapshot>>>,
/// Shared connection counter for single-connection fast path.
shared_connection_count: Option<Arc<AtomicUsize>>,
/// Same-path transaction lease. Released as soon as commit/rollback
/// finishes, before the transaction value itself is dropped.
maintenance_lease: Option<PagerMaintenanceLease>,
/// Exact logical owner for one WAL group-commit attempt. It retains
/// admission evidence plus Phase-A allocation/undo state until durable
/// authorization or rejection and logical Phase C are terminal.
pending_group_commit_attempt: Option<Arc<PendingGroupCommitTxnAttempt<V::File>>>,
/// Exact identity-wide rollback-recovery receipt owned by this
/// transaction. It is acquired before the first shared journal mutation
/// and retained across cancellation, retry, durable Phase C, and detached
/// cleanup. A sibling may observe the shared barrier, but only the holder
/// of this exact monotonic id may transition or clear it.
owned_rollback_recovery: Option<RollbackRecoveryOwnerId>,
/// A rollback-journal commit crossed its durable decision, but logical
/// Phase C did not yet reach terminal transaction exit. Retry and Drop must
/// preserve that committed outcome instead of replaying or reapplying it.
rollback_commit_finalization_pending: bool,
/// Pager-wide lock-free recovery gate shared by every live transaction.
/// This closes the sibling-handle window that a transaction-local flag
/// cannot observe.
rollback_recovery_pending: Arc<AtomicUsize>,
recovery_fence: Arc<RecoveryFence>,
/// The physical pager was opened read-only. This is stronger than a
/// read-only transaction mode and must reject every later writer upgrade.
read_only_pager: bool,
/// WAL visibility horizon captured for this exact transaction at BEGIN.
/// The backend's shared pinned snapshot can be replaced when a sibling
/// transaction begins, so commit-time FCW validation must not reread it.
wal_conflict_snapshot: Option<traits::WalPublicationSnapshot>,
/// Visible commit sequence at snapshot capture. This remains fixed during
/// reads; transaction-owned commit paths may advance it after publishing
/// their own writes.
published_visible_commit_seq: Cell<CommitSeq>,
/// Database size at snapshot capture. Used for MVCC visibility: pages
/// beyond this bound didn't exist when this snapshot was taken.
published_db_size: Cell<u32>,
write_set: PagePageMap<StagedPage>,
write_pages_sorted: Vec<PageNumber>,
freed_pages: Vec<PageNumber>,
freed_page_bounds: Option<(PageNumber, PageNumber)>,
allocated_from_freelist: Vec<PageNumber>,
allocated_from_eof: Vec<PageNumber>,
/// #70 BUG-A / ghost-commit guard: set true as soon as any write-staging
/// entry-point (`write_page`, `write_page_data`, `allocate_page`, or
/// savepoint-driven commit reshuffle) runs on this transaction. Cleared
/// only on explicit rollback. At commit entry we assert that writes_
/// observed + !has_pending_writes is never true, which would indicate a
/// silent state-loss path (the symptom behind the swarm ghost-commit
/// diagnostic: commit returns Ok but neither the INSERT nor the same-
/// txn progress UPDATE land on disk). Turns that class of bug into a
/// retryable error instead of silent data loss.
writes_observed: bool,
mode: TransactionMode,
is_writer: bool,
committed: bool,
finished: bool,
original_db_size: u32,
/// Stack of savepoints, pushed on SAVEPOINT and popped on RELEASE.
savepoint_stack: Vec<SavepointEntry>,
/// Journal mode captured at transaction start.
journal_mode: JournalMode,
/// Buffer pool for allocating write-set pages.
pool: PageBufPool,
/// Caller-rooted cleanup context used for drop-time finalization.
cleanup_cx: Cx,
/// Local page lease: pre-allocated EOF pages that can be handed out
/// without re-acquiring the global `inner` mutex. Reduces lock
/// convoy pressure during concurrent insert workloads with B-tree
/// splits. Unused pages are returned to the global next_page on
/// commit/rollback.
page_lease: Vec<PageNumber>,
/// True only for real `:memory:` databases. Those databases never need
/// durable freelist reuse mid-transaction, so allocation can stay on a
/// simple bump-only fast path without pulling page 1 into the conflict
/// surface.
memory_db_bump_alloc: bool,
/// Pages that were allocated after a savepoint but then rolled back.
/// These pages should return zeros when read, not BusySnapshot error.
rolled_back_pages: HashSet<PageNumber>,
/// Per-transaction read cache: pages read via inner.lock() are cached
/// here so subsequent reads of the same page (e.g., B-tree root during
/// repeated INSERTs) skip inner.lock entirely. This eliminates the
/// ~80,000 inner.lock acquisitions at 16 threads (reduces to ~80).
/// Only used in WAL mode where the published snapshot fast path is
/// defeated by constant commit_seq advancement.
txn_read_cache: RefCell<PagePageMap<PageData>>,
/// Committed page images whose backing-store flush was intentionally
/// deferred for the private `:memory:` retained-autocommit fast path.
/// These pages stay authoritative in `txn_read_cache` until a real
/// release boundary flushes them once.
retained_memory_overlay_dirty_pages: BTreeSet<PageNumber>,
/// Per-transaction bumpalo scratch arena (IMPL-3 / AG-4B).
///
/// Amortizes transient allocations over the transaction's lifetime.
/// Created fresh on transaction begin, reset at commit AND rollback,
/// and dropped when the transaction drops. Never carries allocations
/// across transaction boundaries.
///
/// Callers access the arena via [`SimpleTransaction::scratch_arena`]
/// (returns `&bumpalo::Bump`) — `Bump::alloc` takes `&self`, so
/// interior mutability is not required.
scratch_arena: bumpalo::Bump,
}
impl<V> traits::sealed::Sealed for SimpleTransaction<V>
where
V: Vfs,
V::File: 'static,
{
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct WalPageOneWritePlan {
max_written: u32,
page_one_dirty: bool,
freelist_metadata_dirty: bool,
db_growth: bool,
}
impl WalPageOneWritePlan {
/// In WAL mode, Page 1 rewrite is only required when Page 1 was explicitly
/// modified (schema changes, VACUUM, etc.). Synthetic Page 1 changes are
/// kept out of the MVCC conflict surface for:
///
/// 1. `db_growth` - commit preparation may inject a header frame so readers
/// can observe the new page count, but that frame is bookkeeping rather
/// than a direct Page 1 write for first-committer-wins purposes.
/// 2. `freelist_metadata_dirty` - Freelist changes from allocations/frees
/// are implicitly captured by the WAL frames (the pages that were
/// allocated/freed are in the WAL). At checkpoint time, the freelist
/// can be reconstructed from the final database state.
///
/// bd-3wop3.8 (D1-CRITICAL): This eliminates ~2000 MVCC conflicts per
/// 16-thread benchmark iteration where every thread's freelist operations
/// (batch page_lease allocation and return) triggered Page 1 writes.
#[must_use]
fn requires_page_one_rewrite(self) -> bool {
self.page_one_dirty
}
/// Whether commit preparation needs to advance Page 1's page-count header
/// for WAL readers. This is intentionally separate from
/// [`Self::requires_page_one_rewrite`] so synthetic growth frames can be
/// written without becoming cross-process conflict pages.
#[must_use]
fn requires_page_count_advance(self) -> bool {
self.db_growth
}
}
impl<V> SimpleTransaction<V>
where
V: Vfs,
V::File: 'static,
{
#[cfg(all(feature = "native", any(unix, windows)))]
fn validate_namespace_binding(&self) -> Result<()> {
if let Some(binding) = &self.namespace_binding {
binding.validate_path_identity()?;
}
Ok(())
}
#[cfg(not(all(feature = "native", any(unix, windows))))]
fn validate_namespace_binding(&self) -> Result<()> {
Ok(())
}
async fn reacquire_external_snapshot(&self, cx: &Cx) -> Result<()> {
let mut attempt =
BeginExternalLockState::new(&self.group_commit_queue, Arc::clone(&self.db_file), cx);
attempt.acquire_snapshot(cx).await?;
// The live transaction resumes ownership of the successful snapshot
// attempt. Its ordinary commit/rollback/Drop path performs the exact
// terminal restoration.
attempt.disarm();
Ok(())
}
/// Finish a locally failed rollback-journal commit through the same
/// identity-bound recovery epoch used by pager begin/refresh. The
/// transaction's retained snapshot lock is released before the recovery
/// fence and maintenance lease are acquired, then restored before return
/// so ordinary transaction finalization still owns exactly one snapshot
/// fence to release.
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
async fn recover_pending_rollback_journal(&mut self, cx: &Cx) -> Result<bool> {
if !self.is_writer || self.journal_mode == JournalMode::Wal {
return Ok(false);
}
let Some(recovery_owner) = self.owned_rollback_recovery else {
return Ok(false);
};
let inner_arc = Arc::clone(&self.inner);
let mut inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if inner.rollback_journal_recovery_owner != Some(recovery_owner)
|| inner.maintenance_gate.rollback_recovery_owner() != Some(recovery_owner)
|| !inner.rollback_journal_recovery_state.is_pending()
{
return Err(FrankenError::internal(
"transaction rollback recovery lost its exact owner receipt",
));
}
if matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
) {
return Err(FrankenError::internal(
"durable rollback commit must use its exact Phase-C finalizer",
));
}
// The exact receipt was stored before the first shared-journal await.
// Retain it while the snapshot marker and maintenance lease move
// through recovery so cancellation cannot turn this caller into an
// ownerless or sibling-owned cleanup attempt.
shared_db_restore_external_snapshot_attempt(&inner.db_file, cx).await?;
let recovery_guard = match self.recovery_fence.acquire_for_recovery() {
Ok(guard) => guard,
Err(error) => {
return match self.reacquire_external_snapshot(cx).await {
Ok(()) => Err(error),
Err(relock_error) => Err(FrankenError::internal(format!(
"could not enter rollback recovery or restore the transaction snapshot lock: recovery={error}; relock={relock_error}"
))),
};
}
};
let maintenance_lease = self.maintenance_lease.as_mut().ok_or_else(|| {
FrankenError::internal("pending rollback recovery lost its maintenance lease")
})?;
let prior_kind = match maintenance_lease.upgrade_to_exclusive(Some(recovery_owner)) {
Ok(prior_kind) => prior_kind,
Err(error) => {
return match self.reacquire_external_snapshot(cx).await {
Ok(()) => Err(error),
Err(relock_error) => Err(FrankenError::internal(format!(
"could not exclude same-process pagers for rollback recovery or restore the transaction snapshot lock: recovery={error}; relock={relock_error}"
))),
};
}
};
let recovery_result = SimplePager::<V>::recover_runtime_rollback_journal(
cx,
&*self.vfs,
&mut inner,
&self.journal_path,
&self.cache,
&self.wal_backend,
&self.group_commit_queue,
None,
)
.await;
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let downgrade_result = maintenance_lease.downgrade_from_exclusive(prior_kind);
let relock_result = self.reacquire_external_snapshot(&cleanup_cx).await;
let operation_result = match (recovery_result, downgrade_result, relock_result) {
(Ok(recovered), Ok(()), Ok(())) => Ok(recovered),
(Err(error), Ok(()), Ok(()))
| (Ok(_), Err(error), Ok(()))
| (Ok(_), Ok(()), Err(error)) => Err(error),
(recovery, downgrade, relock) => Err(FrankenError::internal(format!(
"rollback recovery cleanup failed: recovery={recovery:?}; downgrade={downgrade:?}; relock={relock:?}"
))),
};
let _journal_observed = operation_result?;
// The external maintenance lock was released before the snapshot lock
// was reacquired. Reclassify a surviving/new journal while the shared
// snapshot is held; a new hot journal is a retry boundary, never a
// state that may be published through.
SimplePager::<V>::verify_readonly_rollback_journal_state(
&cleanup_cx,
&*self.vfs,
&self.journal_path,
)
.await?;
self.cache.clear();
inner
.refresh_committed_state_after_recovery(&cleanup_cx, &self.cache, &self.wal_backend)
.await?;
self.published.publish_clear_if(
&cleanup_cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
true,
);
self.publish_committed_snapshot_from_inner(&inner);
inner.finish_rollback_journal_recovery(recovery_owner)?;
self.owned_rollback_recovery = None;
drop(recovery_guard);
Ok(true)
}
/// Whether this transaction has been upgraded to a writer.
#[must_use]
pub fn is_writer(&self) -> bool {
self.is_writer
}
/// Access the per-transaction bumpalo scratch arena (IMPL-3 / AG-4B).
///
/// Returns `&Bump` (not `&mut Bump`) — `Bump::alloc` and related
/// allocation entry points all take `&self`. The arena is reset on
/// commit and rollback, so callers must not hold arena-allocated
/// references across a transaction-boundary method call on
/// `SimpleTransaction`.
///
/// First wired (OPT-5) by
/// [`SimpleTransaction::materialize_retained_memory_overlay_into_write_set`],
/// which routes its transient `Vec<PageNumber>` scratch through this
/// arena. Additional callers can opt in by allocating purely transient
/// buffers (nothing stored into `self` fields that survive the current
/// method call) via `bumpalo::collections::Vec::new_in(&self.scratch_arena)`.
#[must_use]
pub(crate) fn scratch_arena(&self) -> &bumpalo::Bump {
&self.scratch_arena
}
#[inline]
fn has_pending_recovery_barrier(&self) -> bool {
self.pending_group_commit_attempt.is_some()
|| self.owned_rollback_recovery.is_some()
|| self.rollback_commit_finalization_pending
|| self.rollback_recovery_pending.load(AtomicOrdering::Acquire) != 0
}
fn identity_rollback_recovery_pending(&self) -> bool {
self.identity_rollback_recovery_owner().is_some()
}
fn identity_rollback_recovery_owner(&self) -> Option<RollbackRecoveryOwnerId> {
RollbackRecoveryOwnerId::new(self.rollback_recovery_pending.load(AtomicOrdering::Acquire))
}
fn exact_recovery_owner_matches_inner(&self, inner: &PagerInner<V::File>) -> bool {
inner.rollback_journal_recovery_owner == self.owned_rollback_recovery
&& self.identity_rollback_recovery_owner() == self.owned_rollback_recovery
}
fn has_local_changes_to_commit(&self) -> bool {
self.writes_observed
|| !self.write_set.is_empty()
|| !self.freed_pages.is_empty()
|| !self.allocated_from_freelist.is_empty()
|| !self.allocated_from_eof.is_empty()
|| !self.page_lease.is_empty()
|| !self.retained_memory_overlay_dirty_pages.is_empty()
}
#[allow(clippy::await_holding_lock)]
async fn drain_nonowner_during_rollback_recovery(
&mut self,
cx: &Cx,
committed: bool,
) -> Result<()> {
if !self.identity_rollback_recovery_pending() || self.owned_rollback_recovery.is_some() {
return Err(FrankenError::internal(
"rollback-recovery sibling drain called outside a non-owner recovery epoch",
));
}
if !self.retained_memory_overlay_dirty_pages.is_empty() {
return Err(FrankenError::BusyRecovery);
}
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let logical_exit_claim = GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
&cleanup_cx,
)
.await?;
if self.owned_rollback_recovery.is_some()
|| self.identity_rollback_recovery_owner().is_none()
{
return Err(FrankenError::BusyRecovery);
}
// The recovery owner will rebuild committed metadata. Never merge a
// sibling's speculative allocations into that image while recovery is
// pending; discard every transaction-local staging surface instead.
self.write_set.clear();
self.write_pages_sorted.clear();
self.clear_freed_pages();
self.savepoint_stack.clear();
self.rolled_back_pages.clear();
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
self.page_lease.clear();
self.writes_observed = false;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
let notify_writer_idle = coordinated_transaction_exit(
&self.group_commit_queue,
&cleanup_cx,
&mut inner,
self.is_writer && self.mode != TransactionMode::Concurrent,
&logical_exit_claim,
)
.await?;
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
self.committed = committed;
self.maintenance_lease.take();
self.finished = true;
self.scratch_arena.reset();
Ok(())
}
#[allow(clippy::await_holding_lock)]
async fn finish_durable_rollback_commit(&mut self, cx: &Cx) -> Result<()> {
let recovery_owner = self.owned_rollback_recovery.ok_or_else(|| {
FrankenError::internal(
"durable rollback-commit finalization lost its exact recovery owner",
)
})?;
if !self.rollback_commit_finalization_pending || !self.committed {
return Err(FrankenError::internal(
"durable rollback-commit finalization lost its outcome receipt",
));
}
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
if self.maintenance_lease.is_some() {
let logical_exit_claim = GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
&cleanup_cx,
)
.await?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
) || inner.rollback_journal_recovery_owner != Some(recovery_owner)
|| inner.maintenance_gate.rollback_recovery_owner() != Some(recovery_owner)
{
return Err(FrankenError::internal(
"durable rollback-commit receipt lost its exact pager recovery owner",
));
}
let notify_writer_idle = coordinated_transaction_exit(
&self.group_commit_queue,
&cleanup_cx,
&mut inner,
self.mode != TransactionMode::Concurrent,
&logical_exit_claim,
)
.await?;
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
self.maintenance_lease.take();
}
self.finish_durable_rollback_commit_after_exit(&cleanup_cx)
}
fn finish_durable_rollback_commit_after_exit(&mut self, cx: &Cx) -> Result<()> {
let recovery_owner = self.owned_rollback_recovery.ok_or_else(|| {
FrankenError::internal(
"post-exit rollback-commit finalization lost its exact recovery owner",
)
})?;
if !self.rollback_commit_finalization_pending
|| !self.committed
|| self.maintenance_lease.is_some()
{
return Err(FrankenError::internal(
"post-exit rollback-commit finalization lost its phase receipt",
));
}
let metadata_only_single_connection_fast_path = self.single_connection_fast_path_enabled();
let publish_update = {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
) || inner.rollback_journal_recovery_owner != Some(recovery_owner)
|| inner.maintenance_gate.rollback_recovery_owner() != Some(recovery_owner)
{
return Err(FrankenError::internal(
"durable rollback-commit receipt lost its exact pager recovery owner",
));
}
let update = PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
};
self.publish_committed_snapshot_from_inner(&inner);
update
};
if metadata_only_single_connection_fast_path {
self.publish_single_connection_metadata_only(cx, publish_update);
} else {
self.publish_committed_state(cx, publish_update);
}
self.published_visible_commit_seq
.set(publish_update.visible_commit_seq);
self.published_db_size.set(publish_update.db_size);
if metadata_only_single_connection_fast_path {
self.drain_committed_cache_pages_into_cache();
} else {
let committed_cache_pages = self.drain_committed_cache_pages();
if !committed_cache_pages.is_empty() {
let inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if inner.commit_seq == publish_update.visible_commit_seq {
for (page_no, buffer) in committed_cache_pages {
self.cache.insert_buffer(page_no, buffer);
}
}
}
}
self.clear_freed_pages();
self.savepoint_stack.clear();
self.rolled_back_pages.clear();
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
self.page_lease.clear();
self.retained_memory_overlay_dirty_pages.clear();
self.writes_observed = false;
{
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
inner.finish_rollback_journal_recovery(recovery_owner)?;
}
self.owned_rollback_recovery = None;
self.rollback_commit_finalization_pending = false;
self.finished = true;
self.scratch_arena.reset();
Ok(())
}
fn take_detached_durable_rollback_commit_exit(
&mut self,
cleanup_cx: Cx,
logical_exit_completed: bool,
) -> Result<DetachedDurableRollbackCommitExit<V::File>> {
if !self.rollback_commit_finalization_pending || !self.committed {
return Err(FrankenError::internal(
"detached durable rollback-commit cleanup lost its phase receipt",
));
}
let recovery_owner = self.owned_rollback_recovery.take().ok_or_else(|| {
FrankenError::internal(
"detached durable rollback-commit cleanup lost its exact recovery owner",
)
})?;
Ok(DetachedDurableRollbackCommitExit {
queue: Arc::clone(&self.group_commit_queue),
inner: Arc::clone(&self.inner),
db_file: Arc::clone(&self.db_file),
writer_idle: Arc::clone(&self.writer_idle),
cleanup_cx,
mode: self.mode,
maintenance_lease: self.maintenance_lease.take(),
recovery_owner,
logical_exit_completed,
cache: Arc::clone(&self.cache),
published: Arc::clone(&self.published),
committed_snapshot: Arc::clone(&self.committed_snapshot),
})
}
fn ensure_no_pending_group_commit_attempt(&self) -> Result<()> {
if self.has_pending_recovery_barrier() {
return Err(FrankenError::BusyRecovery);
}
Ok(())
}
fn ensure_writer_upgrade_recovery_clean(&self, inner: &PagerInner<V::File>) -> Result<()> {
if self.has_pending_recovery_barrier()
|| inner.rollback_journal_recovery_state.is_pending()
|| inner.rollback_journal_recovery_owner.is_some()
|| inner.maintenance_gate.rollback_recovery_owner().is_some()
{
return Err(FrankenError::BusyRecovery);
}
Ok(())
}
/// Allocate a staged write buffer, reclaiming only clean shared-cache
/// entries if the common pool has reached its configured ceiling.
fn stage_page_bytes(&self, data: &[u8]) -> Result<StagedPage> {
let mut buffer = acquire_page_buf_with_clean_cache_recovery(
&self.pool,
&self.cache,
"transaction_write_stage",
)?;
let len = buffer.len().min(data.len());
buffer[..len].copy_from_slice(&data[..len]);
if len < buffer.len() {
buffer[len..].fill(0);
}
Ok(StagedPage::from_buf(buffer))
}
/// Admit a snapshot page to the per-transaction read cache without
/// letting a large scan bypass the configured page-buffer ceiling.
///
/// Once the cache reaches the pool capacity, first reads continue through
/// the normal pager/WAL snapshot path but are not pinned for the remaining
/// transaction lifetime. Existing entries may still be refreshed without
/// increasing retained memory.
fn cache_transaction_read_page(&self, page_no: PageNumber, page: &PageData) {
let mut txn_read_cache = self.txn_read_cache.borrow_mut();
if txn_read_cache.contains_key(&page_no) || txn_read_cache.len() < self.pool.capacity() {
txn_read_cache.insert(page_no, page.clone());
}
}
/// Fail closed instead of serving a potentially newer image after this
/// transaction has exhausted its bounded snapshot cache.
fn ensure_uncached_snapshot_sequence(
&self,
page_no: PageNumber,
observed_commit_seq: CommitSeq,
) -> Result<()> {
if self.txn_read_cache.borrow().len() < self.pool.capacity()
|| observed_commit_seq == self.published_visible_commit_seq.get()
{
return Ok(());
}
Err(FrankenError::BusySnapshot {
conflicting_pages: format!(
"page {} is not retained after the transaction read cache reached {} pages; \
snapshot commit_seq {} != observed {}",
page_no.get(),
self.pool.capacity(),
self.published_visible_commit_seq.get().get(),
observed_commit_seq.get()
),
})
}
/// Submodular greedy prefetch selector (IMPL-8 / AG-O3 + AAC-P4).
///
/// Given a set of prefetch candidates and a budget `B`, selects up to `B`
/// pages using Nemhauser-Wolsey-Fisher greedy maximization of a monotone
/// submodular objective (see [`crate::submodular_prefetch`] for the math
/// and the `(1 - 1/e) ≈ 0.6321` approximation bound).
///
/// For each selected page, issues a prefetch hint through the same
/// fast-path ladder as [`TransactionHandle::prefetch_page_hint`]:
/// write-set → per-txn read cache → published snapshot → shared cache.
///
/// Existing `prefetch_page_hint` callers are unaffected; this method is
/// offered in parallel so callers that have a *set* of plausible pages
/// (e.g. B-tree scan look-ahead, CTE materialization) can pay the
/// bandwidth budget on the submodular-optimal subset instead of a
/// round-robin or top-k heuristic.
pub fn prefetch_page_hints_greedy(
&self,
candidates: &[crate::submodular_prefetch::Candidate],
budget: usize,
penalty: f64,
) {
if self.has_pending_recovery_barrier() {
return;
}
let selected = crate::submodular_prefetch::greedy_select(candidates, budget, penalty);
if selected.is_empty() {
return;
}
for page_no in selected {
if let Some(staged) = self.write_set.get(&page_no) {
prefetch_l1_read(staged.as_page_bytes().as_ptr());
continue;
}
if let Ok(txn_read_cache) = self.txn_read_cache.try_borrow()
&& let Some(page) = txn_read_cache.get(&page_no)
{
prefetch_l1_read(page.as_bytes().as_ptr());
continue;
}
if self.published.page_plane_visible_commit_seq()
== self.published_visible_commit_seq.get()
{
self.published.prefetch_page_hint(page_no);
}
self.cache.prefetch_page_hint(page_no);
}
}
/// Check if single-connection fast path is enabled.
fn single_connection_fast_path_enabled(&self) -> bool {
self.shared_connection_count
.as_ref()
.is_some_and(|counter| counter.load(AtomicOrdering::Acquire) == 1)
}
#[must_use]
fn durable_freelist_pages_with_inner(
inner: &PagerInner<V::File>,
db_size: u32,
restored_pages: &[PageNumber],
) -> Vec<PageNumber> {
if db_size == 0 {
return Vec::new();
}
let upper_bound = inner.next_page.saturating_sub(1).max(db_size);
let mut freelist = inner.freelist.clone();
return_pages_to_freelist(&mut freelist, restored_pages.iter().copied());
normalize_freelist(&freelist, upper_bound)
.into_iter()
.filter(|page| page.get() <= db_size)
.collect()
}
#[must_use]
fn committed_durable_freelist_pages_with_inner(
&self,
inner: &PagerInner<V::File>,
) -> Vec<PageNumber> {
Self::durable_freelist_pages_with_inner(inner, inner.db_size, &self.allocated_from_freelist)
}
#[must_use]
fn predicted_durable_freelist_pages_with_inner(
&self,
inner: &PagerInner<V::File>,
committed_db_size: u32,
) -> Vec<PageNumber> {
Self::durable_freelist_pages_with_inner(inner, committed_db_size, &self.freed_pages)
}
/// The set of pages that are free in this transaction's *live* view:
/// `inner.freelist` (the committed freelist with pages allocated this
/// transaction already popped) plus the pages freed this transaction.
///
/// During an open write transaction the on-disk freelist trunk pages and
/// the page-1 header (offsets 32/36) are a deferred, commit-time
/// projection of this set — `serialize_freelist_to_write_set` rewrites
/// them only at COMMIT (so concurrent readers never observe uncommitted
/// freelist changes; see beads_rust#138). This method therefore returns
/// the authoritative freelist mid-transaction, where the on-disk trunk is
/// intentionally stale. For a read-only transaction the deltas are empty,
/// so it returns the committed freelist. Used by `PRAGMA integrity_check`
/// (GH#113) to cross-reference page ownership against the live freelist
/// instead of the stale on-disk trunk.
#[must_use]
pub fn live_freelist_pages(&self) -> Vec<PageNumber> {
if let Some(attempt) = &self.pending_group_commit_attempt {
return attempt.projected_live_freelist();
}
self.inner.lock().map_or_else(
|_| Vec::new(),
|inner| {
let committed_db_size = self.committed_db_size_with_inner(&inner);
self.predicted_durable_freelist_pages_with_inner(&inner, committed_db_size)
},
)
}
/// The page high-water mark visible to this transaction: the largest page
/// number that has actually been handed to the transaction, including
/// uncommitted growth, floored at the committed `db_size`.
///
/// The pager's *published* snapshot db_size and the committed `db_size`
/// only reflect the last committed state, but a write transaction can
/// allocate new btree pages beyond it (e.g. leaf/interior pages created by
/// inserts/splits). `PRAGMA integrity_check` running inside this write
/// transaction must use this high-water mark as its page-extent bound —
/// otherwise the btree walk reaches legitimately in-transaction-allocated
/// pages and falsely reports them as lying "past the end of the database"
/// (GH#113). Pages still sitting in `page_lease` are deliberately excluded:
/// they are only reservations, not reachable database pages, and counting
/// them makes `integrity_check` scan for orphan ownership that cannot exist
/// yet. Returns 0 only if the inner lock is poisoned, in which case the
/// caller falls back to the published size.
#[must_use]
pub fn live_db_size(&self) -> u32 {
if let Some(attempt) = &self.pending_group_commit_attempt {
return attempt.projected_db_size();
}
self.inner.lock().map_or(0, |inner| {
self.allocated_from_eof
.iter()
.chain(self.allocated_from_freelist.iter())
.chain(self.write_set.keys())
.map(|page| page.get())
.max()
.unwrap_or(inner.db_size)
.max(inner.db_size)
})
}
/// Database size captured by this transaction's currently published
/// snapshot.
///
/// Unlike [`Self::live_db_size`], this does not consult mutable pager
/// state and therefore cannot widen a read transaction's page-visibility
/// bound after a concurrent commit.
#[must_use]
pub fn snapshot_db_size(&self) -> u32 {
self.published_db_size.get()
}
fn staged_page_high_water(&self, floor: u32) -> u32 {
self.write_set
.keys()
.map(|page| page.get())
.max()
.unwrap_or(floor)
.max(floor)
}
/// Largest page that can be visible through this transaction.
///
/// This starts at the fixed snapshot bound and includes only pages issued
/// or staged by this transaction. It deliberately excludes the pager's
/// mutable global database size, which may advance after an unrelated
/// concurrent commit, and excludes unused page-lease reservations.
#[must_use]
pub fn visible_db_size_bound(&self) -> u32 {
if let Some(attempt) = &self.pending_group_commit_attempt {
return attempt.transaction_visible_db_size_bound(self.snapshot_db_size());
}
let snapshot_db_size = self.snapshot_db_size();
self.allocated_from_eof
.iter()
.chain(self.allocated_from_freelist.iter())
.chain(self.write_set.keys())
.filter(|page| !self.contains_freed_page(**page))
.map(|page| page.get())
.max()
.unwrap_or(snapshot_db_size)
.max(snapshot_db_size)
}
#[must_use]
fn freelist_metadata_dirty_with_inner(
&self,
inner: &PagerInner<V::File>,
committed_db_size: u32,
) -> bool {
self.committed_durable_freelist_pages_with_inner(inner)
!= self.predicted_durable_freelist_pages_with_inner(inner, committed_db_size)
}
#[must_use]
fn freelist_metadata_dirty_with_pending_free_pages(
&self,
inner: &PagerInner<V::File>,
committed_db_size: u32,
pending_free_pages: &[PageNumber],
) -> bool {
if !pending_free_pages
.iter()
.any(|page| page.get() <= committed_db_size)
&& self.allocated_from_freelist.is_empty()
{
return false;
}
self.committed_durable_freelist_pages_with_inner(inner)
!= Self::durable_freelist_pages_with_inner(inner, committed_db_size, pending_free_pages)
}
#[must_use]
fn freelist_metadata_dirty(&self) -> bool {
self.inner.lock().map_or(true, |inner| {
let committed_db_size = self.committed_db_size_with_inner(&inner);
self.freelist_metadata_dirty_with_inner(&inner, committed_db_size)
})
}
#[must_use]
fn max_live_written_page(&self) -> Option<PageNumber> {
self.write_pages_sorted
.iter()
.rev()
.copied()
.find(|page| self.write_set.contains_key(page))
}
fn note_freed_page_bound(&mut self, page_no: PageNumber) {
match self.freed_page_bounds {
Some((low, high)) => {
self.freed_page_bounds = Some((low.min(page_no), high.max(page_no)));
}
None => self.freed_page_bounds = Some((page_no, page_no)),
}
}
fn refresh_freed_page_bounds(&mut self) {
let mut pages = self.freed_pages.iter().copied();
let Some(first) = pages.next() else {
self.freed_page_bounds = None;
return;
};
let (low, high) = pages.fold((first, first), |(low, high), page| {
(low.min(page), high.max(page))
});
self.freed_page_bounds = Some((low, high));
}
fn clear_freed_pages(&mut self) {
self.freed_pages.clear();
self.freed_page_bounds = None;
}
fn restore_pending_freed_pages(&mut self, pending_freed: Vec<PageNumber>) {
self.freed_pages.extend(pending_freed);
self.refresh_freed_page_bounds();
}
fn might_have_freed_page(&self, page_no: PageNumber) -> bool {
self.freed_page_bounds
.is_some_and(|(low, high)| low <= page_no && page_no <= high)
}
fn contains_freed_page(&self, page_no: PageNumber) -> bool {
self.might_have_freed_page(page_no) && self.freed_pages.contains(&page_no)
}
fn remove_freed_page_if_present(&mut self, page_no: PageNumber) {
if !self.might_have_freed_page(page_no) {
return;
}
if let Some(pos) = self.freed_pages.iter().position(|&p| p == page_no) {
self.freed_pages.swap_remove(pos);
if self.freed_pages.is_empty() {
self.freed_page_bounds = None;
}
}
}
#[must_use]
fn committed_db_size_with_inner(&self, inner: &PagerInner<V::File>) -> u32 {
self.max_live_written_page()
.map_or(inner.db_size, |page| inner.db_size.max(page.get()))
}
#[must_use]
fn classify_wal_page_one_write(
&self,
current_db_size: u32,
freelist_dirty: bool,
) -> WalPageOneWritePlan {
let max_written = self.max_live_written_page().map_or(0, |page| page.get());
WalPageOneWritePlan {
max_written,
page_one_dirty: self.write_set.contains_key(&PageNumber::ONE),
freelist_metadata_dirty: freelist_dirty,
db_growth: max_written > current_db_size,
}
}
#[must_use]
fn current_page_one_conflict_tracking_required_with_inner(
&self,
inner: &PagerInner<V::File>,
) -> bool {
let committed_db_size = self.committed_db_size_with_inner(inner);
let freelist_dirty = self.freelist_metadata_dirty_with_inner(inner, committed_db_size);
let wal_page1_plan = self.classify_wal_page_one_write(inner.db_size, freelist_dirty);
if self.journal_mode == JournalMode::Wal {
wal_page1_plan.requires_page_one_rewrite()
} else {
!self.write_set.is_empty() || freelist_dirty
}
}
#[must_use]
fn allocate_page_requires_page_one_conflict_tracking_with_inner(
&self,
inner: &PagerInner<V::File>,
) -> bool {
if self.memory_db_bump_alloc {
return false;
}
if self.mode == TransactionMode::Concurrent {
// Concurrent-mode allocator/header/page-count reconciliation is a
// commit-planning concern. Ordinary page growth stays on the local
// leased fast path and does not need to pull page 1 into the live
// MVCC conflict surface up front.
return false;
}
if self.current_page_one_conflict_tracking_required_with_inner(inner) {
return true;
}
let committed_db_size = self.committed_db_size_with_inner(inner);
let committed_freelist_is_snapshot_pinned = inner.active_transactions > 1;
if committed_freelist_is_snapshot_pinned {
return false;
}
match inner.freelist.last().copied() {
Some(page) => page.get() <= committed_db_size,
None => false,
}
}
#[must_use]
fn free_page_requires_page_one_conflict_tracking_with_inner(
&self,
inner: &PagerInner<V::File>,
page_no: PageNumber,
) -> bool {
if self.mode == TransactionMode::Concurrent {
// Free-list/page-one reconciliation for concurrent transactions is
// likewise deferred to the commit-time pending surface. Per-op free
// should not synthesize page 1 into the hot path.
return false;
}
if self.current_page_one_conflict_tracking_required_with_inner(inner) {
return true;
}
page_no.get() <= self.committed_db_size_with_inner(inner)
}
#[must_use]
fn write_page_requires_page_one_conflict_tracking_with_inner(
&self,
inner: &PagerInner<V::File>,
page_no: PageNumber,
) -> bool {
if page_no == PageNumber::ONE {
return true;
}
if self.mode == TransactionMode::Concurrent {
// Concurrent growth rewrites page 1 only at commit publication
// time. Do not drag synthetic page-one conflict tracking through
// every ordinary high-page write.
return false;
}
if self.current_page_one_conflict_tracking_required_with_inner(inner) {
return true;
}
page_no.get() > self.committed_db_size_with_inner(inner)
}
#[must_use]
fn page_one_in_pending_commit_surface_with_inner(&self, inner: &PagerInner<V::File>) -> bool {
let committed_db_size = self.committed_db_size_with_inner(inner);
let durable_freelist =
self.predicted_durable_freelist_pages_with_inner(inner, committed_db_size);
let freelist_dirty =
self.committed_durable_freelist_pages_with_inner(inner) != durable_freelist;
let wal_page1_plan = self.classify_wal_page_one_write(inner.db_size, freelist_dirty);
if self.journal_mode == JournalMode::Wal {
wal_page1_plan.requires_page_one_rewrite()
} else {
!self.write_set.is_empty() || freelist_dirty
}
}
fn predicted_commit_pages_with_inner(&self, inner: &PagerInner<V::File>) -> Vec<PageNumber> {
let mut pages = self.write_pages_sorted.clone();
let committed_db_size = self.committed_db_size_with_inner(inner);
let durable_freelist =
self.predicted_durable_freelist_pages_with_inner(inner, committed_db_size);
let freelist_dirty =
self.committed_durable_freelist_pages_with_inner(inner) != durable_freelist;
if freelist_dirty && !durable_freelist.is_empty() {
let max_leaf_entries = (inner.page_size.as_usize() / 4).saturating_sub(2).max(1);
let trunk_count = durable_freelist.len().div_ceil(max_leaf_entries + 1);
pages.extend(durable_freelist.into_iter().take(trunk_count));
// The freelist serializer (see the commit-time serialize path) takes
// `take(trunk_count)` of the *predicted* freelist as the new trunk
// page(s) and rewrites page 1's head to point at the first of them.
// Any page that was a trunk in the *committed* freelist but is no
// longer a predicted trunk gets repurposed as a leaf, so its
// role/contents change on commit. Those previously-committed trunk
// pages must therefore enter the predicted conflict surface for MVCC
// correctness. Union them in here; the final sort/dedup below removes
// any overlap with the predicted trunks.
let committed_freelist = self.committed_durable_freelist_pages_with_inner(inner);
if !committed_freelist.is_empty() {
let committed_trunk_count = committed_freelist.len().div_ceil(max_leaf_entries + 1);
pages.extend(committed_freelist.into_iter().take(committed_trunk_count));
}
}
if self.page_one_in_pending_commit_surface_with_inner(inner) {
pages.push(PageNumber::ONE);
}
pages.sort_unstable();
pages.dedup();
pages
}
fn predicted_conflict_pages_with_inner(&self, inner: &PagerInner<V::File>) -> Vec<PageNumber> {
let committed_db_size = self.committed_db_size_with_inner(inner);
let freelist_dirty = self.freelist_metadata_dirty_with_inner(inner, committed_db_size);
let wal_page1_plan = self.classify_wal_page_one_write(inner.db_size, freelist_dirty);
self.predicted_conflict_pages_for_wal_commit_with_inner(
inner,
wal_page1_plan,
&self.freed_pages,
)
}
fn predicted_conflict_pages_for_wal_commit_with_inner(
&self,
inner: &PagerInner<V::File>,
wal_page1_plan: WalPageOneWritePlan,
freed_conflict_pages: &[PageNumber],
) -> Vec<PageNumber> {
let mut pages = self.predicted_commit_pages_with_inner(inner);
pages.extend(freed_conflict_pages.iter().copied());
if self.mode == TransactionMode::Concurrent && self.journal_mode == JournalMode::Wal {
// bd-3wop3.8 (D1-CRITICAL): In WAL mode, synthetic page 1 changes
// (change counter, page count, freelist metadata) are safely
// serialized by the pager inner.lock() during Phase A commit.
// The commit protocol ensures that concurrent freelist/page-count
// updates are merged correctly (last committer includes all prior
// state). Only track page 1 as a conflict when directly modified
// by schema operations (CREATE TABLE, DROP TABLE, etc.).
//
// Use the caller's pre-synthetic Page 1 plan. Commit preparation
// may inject Page 1 later for WAL bookkeeping, and checking
// write_set at that point would incorrectly treat synthetic
// page-count/header frames as direct Page 1 writes.
//
// This eliminates ~2000 spurious MVCC conflicts on page 1 that
// occurred when concurrent INSERTs (db_growth) and DELETEs
// (freelist_dirty) both touched page 1 header metadata.
if !wal_page1_plan.requires_page_one_rewrite() {
pages.retain(|page| *page != PageNumber::ONE);
}
}
pages.sort_unstable();
pages.dedup();
pages
}
/// Capture exact, snapshot-bound full-page hashes for conflict candidates.
///
/// The transaction read cache is the only safe source here: it is pinned
/// to this handle's begin-time visibility and is never refreshed in place.
/// A candidate absent from that cache deliberately has no baseline; the
/// cross-generation validator must then fail closed with `BusySnapshot`.
fn conflict_page_baselines(
&self,
conflict_pages: &[PageNumber],
) -> Vec<TransactionConflictPageBaseline> {
let txn_read_cache = self.txn_read_cache.borrow();
conflict_pages
.iter()
.filter_map(|page_no| {
txn_read_cache.get(page_no).map(|page| {
let page_hash = *blake3::hash(page.as_bytes()).as_bytes();
TransactionConflictPageBaseline {
page_number: page_no.get(),
page_hash,
}
})
})
.collect()
}
fn publish_committed_state(&self, cx: &Cx, update: PublishedPagerUpdate) {
// D1-CRITICAL Change 3: Use sharded publish_commit.
self.published.publish_commit(cx, update, &self.write_set);
}
fn publish_committed_state_draining_write_set(
&mut self,
cx: &Cx,
update: PublishedPagerUpdate,
) {
self.published
.publish_commit_draining_write_set(cx, update, &mut self.write_set);
}
fn publish_single_connection_metadata_only(&self, cx: &Cx, update: PublishedPagerUpdate) {
let clear_pages = self.published.page_set_size.load(AtomicOrdering::Acquire) != 0;
self.published
.publish_single_connection_metadata_update(cx, update, clear_pages);
}
fn note_retained_memory_overlay_from_write_set(&mut self) {
self.retained_memory_overlay_dirty_pages
.extend(self.write_pages_sorted.iter().copied());
}
fn materialize_retained_memory_overlay_into_write_set(&mut self) -> Result<()> {
if self.retained_memory_overlay_dirty_pages.is_empty() {
return Ok(());
}
// OPT-5: route the transient page-number scratch through the
// per-transaction bumpalo arena (IMPL-3 / AG-4B). Lifetime analysis:
//
// * `overlay_page_nos` is born here, consumed by `iter().copied()`
// in the next block, and dropped before method return.
// * No reference to the arena-allocated storage escapes this
// method frame — `PageNumber` is `Copy` and the page numbers
// are materialized into fresh heap-owned `PageData` entries
// inserted into `self.write_set`.
// * `self.scratch_arena()` aliases `self` immutably only for the
// lifetime of this local vector. `&mut self` recovers before we
// call `insert_staged_page`, which requires `&mut self.write_set`.
//
// This converts one glibc `malloc(capacity * sizeof(PageNumber))` per
// retained-autocommit commit into a bump-pointer reservation that is
// reset for free at commit/rollback.
let mut overlay_page_nos: bumpalo::collections::Vec<'_, PageNumber> =
bumpalo::collections::Vec::new_in(self.scratch_arena());
overlay_page_nos.extend(
self.retained_memory_overlay_dirty_pages
.iter()
.copied()
.filter(|page_no| !self.write_set.contains_key(page_no)),
);
if overlay_page_nos.is_empty() {
return Ok(());
}
let overlay_pages = {
let txn_read_cache = self.txn_read_cache.borrow();
overlay_page_nos
.iter()
.copied()
.map(|page_no| {
let page = txn_read_cache.get(&page_no).cloned().ok_or_else(|| {
FrankenError::internal(format!(
"retained memory overlay missing authoritative page {}",
page_no.get()
))
})?;
Ok((page_no, page))
})
.collect::<Result<Vec<_>>>()?
};
// Drop the arena-allocated scratch before we re-borrow `&mut self`.
drop(overlay_page_nos);
for (page_no, page) in overlay_pages {
insert_staged_page(
&mut self.write_set,
&mut self.write_pages_sorted,
page_no,
StagedPage::from_page_data_with_cache_recovery(
&self.pool,
&self.cache,
page,
"retained_memory_overlay_stage",
)?,
);
}
Ok(())
}
fn collect_retained_memory_overlay_pages(&self) -> Result<Vec<(PageNumber, PageData)>> {
if self.retained_memory_overlay_dirty_pages.is_empty() {
return Ok(Vec::new());
}
let txn_read_cache = self.txn_read_cache.borrow();
self.retained_memory_overlay_dirty_pages
.iter()
.copied()
.map(|page_no| {
let page = txn_read_cache.get(&page_no).cloned().ok_or_else(|| {
FrankenError::internal(format!(
"retained memory overlay missing authoritative page {}",
page_no.get()
))
})?;
Ok((page_no, page))
})
.collect::<Result<Vec<_>>>()
}
async fn flush_retained_memory_overlay_pages_to_db_file(
cx: &Cx,
inner: &mut PagerInner<V::File>,
original_db_size: u32,
overlay_pages: &[(PageNumber, PageData)],
) -> Result<()> {
if overlay_pages.is_empty() {
return Ok(());
}
let page_size_bytes = u64::from(inner.page_size.get());
let mut batched_writes: SmallVec<[(u64, &[u8]); 8]> =
SmallVec::with_capacity(overlay_pages.len());
for (page_no, page) in overlay_pages {
let offset = u64::from(page_no.get() - 1) * page_size_bytes;
batched_writes.push((offset, page.as_bytes()));
}
let db_file = shared_db_file_read(&inner.db_file, cx).await?;
db_file
.write_page_batch(cx, batched_writes.as_slice())
.await?;
inner.committed_db_file_size_bytes =
u64::from(original_db_size) * u64::from(inner.page_size.get());
Ok(())
}
fn retain_committed_pages_in_txn_read_cache(&mut self, invalidate_prior_snapshot: bool) {
let mut txn_read_cache = self.txn_read_cache.borrow_mut();
if invalidate_prior_snapshot {
txn_read_cache.clear();
}
for (page_no, staged) in self.write_set.drain() {
txn_read_cache.insert(page_no, staged.into_published_page());
}
self.write_pages_sorted.clear();
}
/// Publish a new committed-state snapshot while the pager inner lock is still held.
///
/// The write lock is held only long enough to swap the immutable snapshot Arc.
fn publish_committed_snapshot_from_inner(&self, inner: &PagerInner<V::File>) {
let snapshot = Arc::new(PagerCommittedSnapshot::from_inner(inner));
let mut guard = self
.committed_snapshot
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner);
*guard = snapshot;
}
fn drain_committed_cache_pages(&mut self) -> Vec<(PageNumber, PageBuf)> {
let mut committed_pages = Vec::with_capacity(self.write_set.len());
for (page_no, staged) in self.write_set.drain() {
match staged.into_cache_buf(&self.pool, &self.cache) {
Ok(buffer) => committed_pages.push((page_no, buffer)),
Err(error) => tracing::debug!(
page_no = page_no.get(),
error = %error,
"skipping optional committed-page cache admission at pool capacity"
),
}
}
self.write_pages_sorted.clear();
committed_pages
}
fn drain_committed_cache_pages_into_cache(&mut self) {
for (page_no, staged) in self.write_set.drain() {
match staged.into_cache_buf(&self.pool, &self.cache) {
Ok(buffer) => self.cache.insert_buffer(page_no, buffer),
Err(error) => tracing::debug!(
page_no = page_no.get(),
error = %error,
"skipping optional committed-page cache admission at pool capacity"
),
}
}
self.write_pages_sorted.clear();
}
fn discard_committed_pages(&mut self) {
self.write_set.clear();
self.write_pages_sorted.clear();
}
fn collect_unstaged_allocated_pages(&self) -> Vec<PageNumber> {
self.allocated_from_eof
.iter()
.chain(self.allocated_from_freelist.iter())
.copied()
.filter(|page| !self.write_set.contains_key(page) && !self.contains_freed_page(*page))
.collect()
}
fn pending_free_pages_for_commit(&self) -> Vec<PageNumber> {
let mut pending = self.collect_unstaged_allocated_pages();
pending.extend(self.page_lease.iter().copied());
pending.extend(self.freed_pages.iter().copied());
pending
}
fn drain_unstaged_allocated_pages(&mut self) -> PendingReturnedAllocations {
let mut unstaged = PendingReturnedAllocations::default();
let write_set = &self.write_set;
let freed_pages = &self.freed_pages;
let freed_page_bounds = self.freed_page_bounds;
self.allocated_from_eof.retain(|page| {
let keep = write_set.contains_key(page)
|| freed_page_bounds.is_some_and(|(low, high)| low <= *page && *page <= high)
&& freed_pages.contains(page);
if !keep {
unstaged.from_eof.push(*page);
}
keep
});
self.allocated_from_freelist.retain(|page| {
let keep = write_set.contains_key(page)
|| freed_page_bounds.is_some_and(|(low, high)| low <= *page && *page <= high)
&& freed_pages.contains(page);
if !keep {
unstaged.from_freelist.push(*page);
}
keep
});
unstaged
}
fn restore_uncommitted_allocations_for_clean_commit(
&mut self,
inner: &mut PagerInner<V::File>,
) {
return_pages_to_freelist(&mut inner.freelist, self.allocated_from_freelist.drain(..));
if self.mode == TransactionMode::Concurrent {
return_pages_to_freelist(&mut inner.freelist, self.page_lease.drain(..));
return_pages_to_freelist(&mut inner.freelist, self.allocated_from_eof.drain(..));
} else {
self.page_lease.clear();
self.allocated_from_eof.clear();
inner.db_size = self.original_db_size;
inner.next_page = if inner.db_size >= 2 {
inner.db_size.saturating_add(1)
} else {
2
};
}
}
}
fn cleanup_child_cx(cx: &Cx) -> Cx {
cx.create_child()
}
struct RollbackJournalCommitInput<'a, S> {
journal_path: &'a Path,
write_set: &'a HashMap<PageNumber, StagedPage, S>,
original_db_size: u32,
allocated_from_freelist: &'a [PageNumber],
}
impl<V> SimpleTransaction<V>
where
V: Vfs + Send,
V::File: Send + Sync + 'static,
{
async fn invalidate_journal_after_commit(cx: &Cx, journal_file: &mut V::File) -> Result<()> {
durable_invalidate_journal(cx, journal_file, JournalInvalidation::ZeroMagic).await
}
/// Commit using the rollback journal protocol.
///
/// `allocated_from_freelist` is the committing transaction's freelist
/// allocations (see `SimpleTransaction::allocated_from_freelist`); it is
/// needed to reconstruct the transaction's begin-time view of the
/// committed freelist for the cross-connection aliasing check below.
#[allow(clippy::too_many_lines)]
async fn commit_journal<S: std::hash::BuildHasher>(
cx: &Cx,
vfs: &Arc<V>,
inner: &mut PagerInner<V::File>,
input: RollbackJournalCommitInput<'_, S>,
owned_recovery: &mut Option<RollbackRecoveryOwnerId>,
) -> Result<()> {
let RollbackJournalCommitInput {
journal_path,
write_set,
original_db_size,
allocated_from_freelist,
} = input;
if owned_recovery.is_some() {
return Err(FrankenError::internal(
"rollback-journal commit started while retaining an earlier recovery owner",
));
}
if inner
.rollback_recovery_pending
.load(AtomicOrdering::Acquire)
!= 0
{
return Err(FrankenError::BusyRecovery);
}
if !write_set.is_empty() {
// Escalate to EXCLUSIVE before writing to the database file.
// This prevents concurrent processes from reading partially
// written pages during the commit.
let shared_db_file = Arc::clone(&inner.db_file);
let mut db_file = shared_db_file_write(&shared_db_file, cx).await?;
db_file.lock(cx, LockLevel::Exclusive)?;
if inner
.rollback_recovery_pending
.load(AtomicOrdering::Acquire)
!= 0
{
return Err(FrankenError::BusyRecovery);
}
let mut nonce_bytes = [0_u8; 4];
vfs.randomness(cx, &mut nonce_bytes);
let nonce = u32::from_be_bytes(nonce_bytes);
let page_size = inner.page_size;
let ps = page_size.as_usize();
let lock_byte_page = crate::journal::lock_byte_page(page_size);
if write_set
.keys()
.any(|page_no| page_no.get() == lock_byte_page)
{
return Err(FrankenError::DatabaseCorrupt {
detail:
"rollback-journal commit attempted to write the reserved lock-byte page"
.to_owned(),
});
}
let mut journal_pages: Vec<PageNumber> = write_set
.keys()
.copied()
.filter(|page_no| page_no.get() <= original_db_size)
.collect();
journal_pages.sort_unstable_by_key(|page_no| page_no.get());
// bd-9inpb / am#152: cross-connection page-allocation conflict
// detection for rollback-journal mode.
//
// The WAL commit path guards against two connections committing the
// same physical page for different b-trees via
// `cross_process_conflict_pages` + `conflicting_pages_since_snapshot`.
// Rollback-journal commits had NO such check (that vector is empty
// for non-WAL commits), so two concurrent connections — each with its
// own `PagerInner` and therefore its own `next_page` snapshot — could
// both allocate the same EOF page number for different trees and both
// commit, aliasing the page on disk ("page N referenced multiple
// times" / lost rows).
//
// The EXCLUSIVE lock serializes committers across connections, so the
// most-recently-committed peer's database size is already durable in
// the on-disk page-1 header (page count at offset 28..32). If the
// committed db has grown past our snapshot (`original_db_size`), then
// any page we are about to write that falls in the peer-claimed range
// `(original_db_size, committed]` aliases a page that peer now owns
// (concurrent `allocate_page` only hands out pages above the snapshot
// db_size, and every committed page <= db_size is owned by either a
// b-tree or the freelist, so such a page is provably an alias rather
// than a coincidental overlap). Abort first-committer-wins style with
// `BusySnapshot` so the caller retries against the refreshed db size;
// the retry re-snapshots the grown db_size and allocates a
// non-conflicting EOF range. Page 1 and any modified existing page are
// <= original_db_size, so they never trip this check.
//
// The page-1 image is read exactly once here and REUSED as the
// journal pre-image below, so the check adds no extra counted I/O
// (page 1 is always staged in the write_set for a journal commit and
// the pre-image loop would read it regardless).
let mut page_one_preimage: Option<Vec<u8>> = if write_set.contains_key(&PageNumber::ONE)
&& original_db_size >= PageNumber::ONE.get()
{
let mut image = vec![0u8; ps];
let bytes_read = db_file.read(cx, &mut image, 0).await?;
if bytes_read < ps {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read while journaling pre-image for page 1: got {bytes_read} of {ps}"
),
});
}
let committed_db_size =
u32::from_be_bytes([image[28], image[29], image[30], image[31]]);
if committed_db_size > original_db_size {
let mut conflicts: Vec<u32> = write_set
.keys()
.map(|page| page.get())
.filter(|&page| page > original_db_size && page <= committed_db_size)
.collect();
if !conflicts.is_empty() {
conflicts.sort_unstable();
return Err(FrankenError::BusySnapshot {
conflicting_pages: conflicts
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(","),
});
}
}
// A peer can also consume (or repopulate) a *committed
// freelist* page without growing the file, which the
// db-size check above cannot see. Reconstruct this
// transaction's begin-time view of the committed freelist
// (`inner.freelist` still excludes the pages this
// transaction popped, so restore them) and compare it
// against the on-disk committed freelist under the
// EXCLUSIVE lock. Any divergence within the snapshot range
// means a peer commit changed freelist ownership since our
// snapshot: a page we popped may now be owned by the
// peer's b-tree (writing it would alias), and a page the
// peer popped would be resurrected onto the freelist by
// our commit's stale serialization (aliasing a later
// allocation). Abort first-committer-wins style with
// `BusySnapshot`; the caller retries against refreshed
// committed state.
let snapshot_freelist = Self::durable_freelist_pages_with_inner(
inner,
original_db_size,
allocated_from_freelist,
);
let first_trunk = u32::from_be_bytes([image[32], image[33], image[34], image[35]]);
let freelist_count =
u32::from_be_bytes([image[36], image[37], image[38], image[39]]);
if !(snapshot_freelist.is_empty() && (first_trunk == 0 || freelist_count == 0)) {
let committed_freelist: HashSet<u32> = load_freelist_from_disk(
cx,
&*db_file,
page_size,
committed_db_size,
first_trunk,
freelist_count,
)
.await?
.into_iter()
.map(|page| page.get())
.filter(|&page| page <= original_db_size)
.collect();
let snapshot_freelist: HashSet<u32> =
snapshot_freelist.iter().map(|page| page.get()).collect();
if snapshot_freelist != committed_freelist {
let mut conflicts: Vec<u32> = snapshot_freelist
.symmetric_difference(&committed_freelist)
.copied()
.collect();
conflicts.sort_unstable();
return Err(FrankenError::BusySnapshot {
conflicting_pages: conflicts
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(","),
});
}
}
Some(image)
} else {
None
};
// Phase 1: Write rollback journal with pre-images. Reserve the
// exact identity owner before CREATE/open can mutate the shared
// journal path, and store it transaction-locally before the first
// subsequent await. Every later transition and cleanup must prove
// possession of this same receipt.
let recovery_owner = inner.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::JournalConstructionPending,
journal_path,
)?;
*owned_recovery = Some(recovery_owner);
let jrnl_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_JOURNAL;
let (mut jrnl_file, _) = vfs.open(cx, Some(journal_path), jrnl_flags)?;
let requested_sector_size = db_file.sector_size().max(jrnl_file.sector_size());
let sector_size = if (512..=65_536).contains(&requested_sector_size)
&& requested_sector_size.is_power_of_two()
{
requested_sector_size
} else {
4096
};
let header = JournalHeader {
// The -1 sentinel derives the count from the exact file
// length when the positive i32 field cannot represent it.
page_count: i32::try_from(journal_pages.len()).unwrap_or(-1),
nonce,
initial_db_size: original_db_size,
sector_size,
page_size: page_size.get(),
};
let mut hdr_bytes = header.encode_padded();
mark_local_journal_header(&mut hdr_bytes);
hdr_bytes[..JOURNAL_MAGIC.len()].fill(0);
jrnl_file.truncate(cx, 0)?;
jrnl_file.write(cx, &hdr_bytes, 0).await?;
let mut jrnl_offset = hdr_bytes.len() as u64;
for &page_no in &journal_pages {
// Read current on-disk content as the pre-image. Rollback only
// needs images for pages that existed when this transaction
// began; pages allocated later may not exist on disk yet even
// if in-memory db_size/page-count metadata has already advanced.
// Page 1's image was already read above for the conflict check;
// reuse it instead of reading it a second time.
let pre_image = if page_no == PageNumber::ONE
&& let Some(image) = page_one_preimage.take()
{
image
} else {
let mut pre_image = vec![0u8; ps];
let disk_offset = u64::from(page_no.get() - 1) * ps as u64;
let bytes_read = db_file.read(cx, &mut pre_image, disk_offset).await?;
if bytes_read < ps {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read while journaling pre-image for page {}: got {bytes_read} of {ps}",
page_no.get()
),
});
}
pre_image
};
let record = JournalPageRecord::new(page_no.get(), pre_image, nonce);
let rec_bytes = record.encode();
jrnl_file.write(cx, &rec_bytes, jrnl_offset).await?;
jrnl_offset += rec_bytes.len() as u64;
}
// First make the complete pre-image payload durable while the
// zero magic keeps the journal non-hot. Then durably install the
// final header and directory entry before touching the database.
jrnl_file.durable_sync(cx, SyncKind::FullDurable)?;
vfs.sync_parent_directory(cx, journal_path)?;
let mut final_hdr_bytes = header.encode_padded();
mark_local_journal_header(&mut final_hdr_bytes);
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::JournalActivationPending,
)?;
durable_write_and_verify_journal_header(cx, &mut jrnl_file, &final_hdr_bytes).await?;
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::ReplayPending,
)?;
// Phase 2: Write dirty pages to database. Once the journal is hot,
// cancellation must not interrupt the write, rollback selection,
// or durable commit decision.
let cleanup_cx = cleanup_child_cx(cx);
let _mask = cleanup_cx.masked();
let saved_db_size = inner.db_size;
// bd-trfah/bd-bjm5d: one batched VFS call instead of one write
// (and one blocking-pool hop on Unix) per page. The batch covers
// the ENTIRE write_set — not `journal_pages`, which deliberately
// excludes newly-allocated pages (they need no pre-image but
// must still be written) — sorted so the batch issues
// ascending-offset pwrites. Failure semantics are unchanged:
// `write_page_batch` short-circuits leaving earlier writes
// applied — exactly what the per-page loop did — and the hot
// journal already guarantees rollback, so `db_size` is simply
// restored on error and advanced only on success.
let page_size_bytes = u64::from(inner.page_size.get());
let mut db_write_pages: Vec<(PageNumber, &StagedPage)> =
write_set.iter().map(|(p, s)| (*p, s)).collect();
db_write_pages.sort_unstable_by_key(|(page_no, _)| page_no.get());
let mut batched_writes: SmallVec<[(u64, &[u8]); 8]> =
SmallVec::with_capacity(db_write_pages.len());
for &(page_no, staged) in &db_write_pages {
let offset = u64::from(page_no.get() - 1) * page_size_bytes;
batched_writes.push((offset, staged.as_page_bytes()));
}
if let Err(e) = db_file
.write_page_batch(&cleanup_cx, batched_writes.as_slice())
.await
{
inner.db_size = saved_db_size;
return Err(e);
}
for &(page_no, _) in &db_write_pages {
inner.db_size = inner.db_size.max(page_no.get());
}
db_file.durable_sync(&cleanup_cx, SyncKind::FullDurable)?;
// Phase 3: Make the journal non-hot before best-effort deletion.
// If invalidation fails, restore and sync the complete hot header
// so rollback is definite. Only if restoration itself fails may a
// truncate+sync choose the already-durable database as committed.
if let Err(invalidate_err) =
Self::invalidate_journal_after_commit(&cleanup_cx, &mut jrnl_file).await
{
let restore_result = durable_write_and_verify_journal_header(
&cleanup_cx,
&mut jrnl_file,
&final_hdr_bytes,
)
.await;
if let Err(restore_err) = restore_result {
let truncate_commit_result = durable_invalidate_journal(
&cleanup_cx,
&mut jrnl_file,
JournalInvalidation::Truncate,
)
.await;
if let Err(truncate_err) = truncate_commit_result {
return Err(FrankenError::internal(format!(
"rollback-journal commit outcome is indeterminate: invalidate={invalidate_err}; restore={restore_err}; truncate_commit={truncate_err}"
)));
}
} else {
return Err(FrankenError::internal(format!(
"could not commit rollback journal; restored the hot journal for rollback: {invalidate_err}"
)));
}
}
if let Ok(file_size) = db_file.file_size(&cleanup_cx) {
inner.committed_db_file_size_bytes = file_size;
}
inner.transition_rollback_journal_recovery(
recovery_owner,
RollbackJournalRecoveryState::DurableCommitFinalizationPending,
)?;
match jrnl_file.close(&cleanup_cx) {
Ok(()) => {
if let Err(delete_err) = vfs.delete(&cleanup_cx, journal_path, true) {
tracing::warn!(
error = %delete_err,
journal = %journal_path.display(),
"rollback-journal commit left a durable non-hot journal"
);
}
}
Err(close_err) => {
tracing::warn!(
error = %close_err,
journal = %journal_path.display(),
"rollback-journal commit succeeded but journal close failed"
);
}
}
}
Ok(())
}
async fn flush_write_set_to_db_file_batch<S: std::hash::BuildHasher>(
cx: &Cx,
inner: &PagerInner<V::File>,
write_set: &HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &[PageNumber],
) -> Result<()> {
if write_pages_sorted.is_empty() {
return Ok(());
}
let page_size_bytes = u64::from(inner.page_size.get());
let mut batched_writes: SmallVec<[(u64, &[u8]); 8]> =
SmallVec::with_capacity(write_pages_sorted.len());
for &page_no in write_pages_sorted {
let staged = write_set.get(&page_no).ok_or_else(|| {
FrankenError::internal(format!(
"write_set missing staged page {} referenced by write_pages_sorted",
page_no.get()
))
})?;
let offset = u64::from(page_no.get() - 1) * page_size_bytes;
batched_writes.push((offset, staged.as_page_bytes()));
}
let db_file = shared_db_file_read(&inner.db_file, cx).await?;
db_file
.write_page_batch(cx, batched_writes.as_slice())
.await
}
/// Commit using the WAL protocol with group commit batching.
///
/// This method implements the group commit pattern (D1: bd-3wop3.1) which
/// replaces the old `WAL_APPEND_GATES` global mutex with a cooperative
/// batching protocol.
///
/// **D1-CRITICAL (bd-3wop3.8): Real flusher/waiter cooperative batching**
///
/// Protocol:
/// 1. Each thread builds a `TransactionFrameBatch` with OWNED frame data
/// 2. Thread submits batch to consolidator, receives `Flusher` or `Waiter` role
/// 3. **Flusher**: Uses a tail-safe arrival wait. Fresh epochs fall back to
/// the legacy 20μs spin, but epochs that already spent that budget
/// gathering peers flush immediately. The flusher then writes ALL
/// batched frames from all transactions in ONE consolidated I/O + fsync
/// 4. **Waiter**: Parks on condvar until flusher signals completion
///
/// Benefits over immediate-flush:
/// - N commits × fsync → 1 group × fsync (major latency reduction under load)
/// - Consolidated I/O: one large write instead of N small writes
/// - Reduced lock contention: waiters don't serialize through WAL I/O
///
/// This function takes `Arc<Mutex<PagerInner>>` instead of `&mut PagerInner`.
/// The CALLER drops their inner.lock() before calling this function, allowing
/// other transactions to start their prepare phase while we wait/batch.
#[cfg(test)]
async fn commit_wal_group_commit<S: std::hash::BuildHasher>(
cx: &Cx,
wal_backend: &SharedWalBackend,
inner_arc: &Arc<Mutex<PagerInner<V::File>>>,
write_set: &HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &[PageNumber],
conflict_pages: &[PageNumber],
queue: &GroupCommitQueueRef,
) -> Result<()> {
let (current_db_size, sync_policy) = {
let inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
(inner.db_size, inner.wal_commit_sync_policy)
};
let mut publication_authorization = None;
let conflict_snapshot = with_wal_backend_read(wal_backend, cx, |wal, _| {
Box::pin(async move { Ok(wal.pinned_read_snapshot()) })
})
.await?;
Self::commit_wal_group_commit_with_snapshot(
cx,
wal_backend,
inner_arc,
None,
current_db_size,
sync_policy,
write_set,
write_pages_sorted,
conflict_pages,
conflict_snapshot,
&[],
queue,
&mut publication_authorization,
None,
)
.await
}
#[allow(clippy::too_many_lines)]
#[allow(clippy::too_many_arguments)]
async fn commit_wal_group_commit_with_snapshot<S: std::hash::BuildHasher>(
cx: &Cx,
wal_backend: &SharedWalBackend,
inner_arc: &Arc<Mutex<PagerInner<V::File>>>,
published: Option<Arc<PublishedPagerState>>,
current_db_size: u32,
sync_policy: WalCommitSyncPolicy,
write_set: &HashMap<PageNumber, StagedPage, S>,
write_pages_sorted: &[PageNumber],
conflict_pages: &[PageNumber],
conflict_snapshot: Option<traits::WalPublicationSnapshot>,
conflict_page_baselines: &[TransactionConflictPageBaseline],
queue: &GroupCommitQueueRef,
publication_authorization: &mut Option<ParallelWalPublicationAuthorization>,
txn_attempt: Option<&Arc<PendingGroupCommitTxnAttempt<V::File>>>,
) -> Result<()> {
// A prior flusher on this exact handle may have been dropped while
// restoration was contended. Settle the global lane and this handle,
// but do not convoy submission behind unrelated exact-handle exits.
let current_handle_key = if let Some(attempt) = txn_attempt {
shared_db_file_key(&attempt.db_file)
} else {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
shared_db_file_key(&inner.db_file)
};
settle_pending_group_commit_finalization_for_handle(queue, current_handle_key).await?;
let detailed_metrics = detailed_consolidation_metrics_enabled();
let lane_staging_debug_enabled =
tracing::enabled!(target: "fsqlite::wal::lane_staging", tracing::Level::DEBUG);
let lock_scope_debug_enabled =
tracing::enabled!(target: "fsqlite::wal::lock_scope", tracing::Level::DEBUG);
let phase_timing =
commit_phase_timing_enabled() || lane_staging_debug_enabled || lock_scope_debug_enabled;
// ── Phase timing instrumentation ──
let t_start = phase_timing.then(Instant::now);
// Step 1: Build our batch with OWNED frame data.
// The caller supplies the Phase A snapshot so production commits do not
// re-acquire the pager mutex before entering the group-commit queue.
let t_batch_build_start = detailed_metrics.then(Instant::now);
let (batch, _our_new_db_size) =
match build_group_commit_batch(current_db_size, write_set, write_pages_sorted)? {
Some(b) => b,
None => {
*publication_authorization = None;
return Ok(());
} // Nothing to commit
};
let batch_build_us = elapsed_profile_us(t_batch_build_start);
if !conflict_pages.is_empty() && conflict_snapshot.is_none() {
// bd-dk9ra / def2ed8c5 reconciliation: the fail-closed refusal
// protects bd-1fc2c's coherent-snapshot
// invariant: a commit must never skip page validation because
// its snapshot went missing ON A BACKEND THAT HAS SNAPSHOTS. On
// snapshot-less backends (raw-pager harnesses, plain adapters:
// pinned AND published snapshots both None) validation is
// impossible by construction — def2ed8c5's eager begin-side
// capture records conflict pages that nothing can ever validate,
// and refusing killed every Immediate commit there (self_alloc
// reds + the publish-window suite wedge). Tradeoff, stated:
// snapshot-less backends commit WITHOUT page-conflict validation
// (exactly their pre-D1 semantics; the write lock serializes
// their writers); snapshot-capable backends keep the full
// fail-closed refusal, preserving what def2ed8c5 protects.
let backend_has_snapshot_facility =
with_wal_backend_read(wal_backend, cx, |wal, _| {
Box::pin(async move {
Ok(wal.published_snapshot().is_some()
|| wal.pinned_read_snapshot().is_some())
})
})
.await?;
if backend_has_snapshot_facility {
if let Some(attempt) = txn_attempt {
attempt.complete_not_committed_global()?;
}
tracing::warn!(
target: "fsqlite.pager.commit",
conflict_page_count = conflict_pages.len(),
"commit refused: conflict pages present without a conflict snapshot on a snapshot-capable backend (bd-dk9ra receipt)"
);
return Err(FrankenError::Unsupported);
}
tracing::debug!(
target: "fsqlite.pager.commit",
conflict_page_count = conflict_pages.len(),
"immediate-mode commit proceeds without conflict snapshot: write lock serializes writers (bd-dk9ra ruling)"
);
}
let t_conflict_snapshot_start = detailed_metrics.then(Instant::now);
let batch = attach_group_commit_conflict_metadata(
batch,
conflict_pages,
conflict_snapshot,
conflict_page_baselines,
);
let conflict_snapshot_us = elapsed_profile_us(t_conflict_snapshot_start);
let parallel_wal_control = queue.parallel_wal_control().clone();
let batch_id = queue.next_parallel_wal_batch_id();
let lane_id = queue.current_parallel_wal_lane_id();
let mut staging_fallback_reason = if matches!(
parallel_wal_control.mode,
ParallelWalOperatingMode::Conservative
) {
Some(ParallelWalFallbackReason::OperatorForced)
} else if let Some(limit) = parallel_wal_control.max_parallel_commit_bytes {
if group_commit_batch_staged_bytes(&batch) > limit {
Some(ParallelWalFallbackReason::LaneOverflow)
} else {
None
}
} else {
None
};
let mut lane_shadow_verdict = ParallelWalShadowVerdict::NotRun;
let mut lane_backlog = if lane_staging_debug_enabled {
Some(queue.current_lane_backlog(lane_id))
} else {
None
};
let mut batch = batch.with_context(TransactionFrameBatchContext {
batch_id,
lane_id,
staged_frame_count: 0,
staging_elapsed_ns: 0,
});
let mut lane_prepare_us = 0;
if staging_fallback_reason.is_none() {
let t_lane_prepare_start = detailed_metrics.then(Instant::now);
let staged_prepared = prepare_group_commit_batch_for_lane(
cx,
wal_backend,
&batch,
batch_id,
lane_id,
¶llel_wal_control,
)
.await?;
lane_prepare_us = elapsed_profile_us(t_lane_prepare_start);
if let Some(staged_prepared) = staged_prepared {
let staged_frame_count = staged_prepared.staged_frame_count;
let staging_elapsed_ns = staged_prepared.staging_elapsed_ns;
lane_shadow_verdict = staged_prepared.shadow_verdict;
let new_lane_backlog = queue.record_prepared_batch(staged_prepared);
if lane_staging_debug_enabled {
lane_backlog = Some(new_lane_backlog);
}
batch = batch.with_context(TransactionFrameBatchContext {
batch_id,
lane_id,
staged_frame_count,
staging_elapsed_ns,
});
} else {
staging_fallback_reason = Some(ParallelWalFallbackReason::ControllerEvidenceLost);
}
}
if lane_staging_debug_enabled {
let queue_submit_max_wait = {
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator.max_group_delay()
};
let queue_submit_batch_membership = batch.context.batch_id.to_string();
let queue_submit_rollback_mode_active = physical_writer_rollback_mode_active(
parallel_wal_control.mode,
staging_fallback_reason,
);
tracing::debug!(
target: "fsqlite::wal::lane_staging",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_LANE_RUN_ID,
scenario_id = PARALLEL_WAL_STAGE_SCENARIO_ID,
batch_id = batch.context.batch_id,
batch_membership = queue_submit_batch_membership.as_str(),
queue_delay_ns = 0_u64,
target_wait_ns = 0_u64,
max_wait_ns =
u64::try_from(queue_submit_max_wait.as_nanos()).unwrap_or(u64::MAX),
fsync_boundary = physical_writer_fsync_boundary(sync_policy),
ordering_phase = "queue_submit",
rollback_mode_active = queue_submit_rollback_mode_active,
wal_lane_id = lane_id,
lane_backlog = lane_backlog.unwrap_or(0),
staged_frame_count = batch.context.staged_frame_count,
flush_trigger = "queue_submit",
control_mode = parallel_wal_mode_name(parallel_wal_control.mode),
lane_policy_version = PARALLEL_WAL_LANE_POLICY_VERSION,
shadow_verdict = parallel_wal_shadow_verdict_name(lane_shadow_verdict),
compatibility_selector = PARALLEL_WAL_COMPATIBILITY_SELECTOR,
fallback_reason = parallel_wal_fallback_reason_name(staging_fallback_reason),
elapsed_ns = batch.context.staging_elapsed_ns,
"queued lane-local WAL staging candidate"
);
}
let prepare_us = elapsed_profile_us(t_start);
let waiter_id = batch.context.batch_id;
let waiter_frames_contributed = batch.frames.len();
// Step 2: Submit batch to consolidator, get Flusher or Waiter role and
// the exact epoch that will make this batch durable.
let t_consolidator_lock_start = phase_timing.then(Instant::now);
let (
outcome,
our_epoch,
target_epoch,
consolidator_lock_wait_us,
flushing_wait_us,
_epoch_consumer,
) = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueConsolidator);
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let lock_wait_us = elapsed_profile_us(t_consolidator_lock_start);
// ── Epoch pipelining: NO waiting during FLUSHING ──
// The consolidator now accepts submissions during FLUSHING,
// queuing them for the next epoch. This eliminates the
// flushing_wait bottleneck that was 1-2.7ms at 16 threads.
let flushing_wait = 0u64; // No longer blocks
let epoch_at_queue = consolidator.epoch();
let receipt = consolidator.submit_batch(batch)?;
// Register before releasing the consolidator mutex. A flusher
// cannot publish this target epoch between admission and consumer
// ownership becoming visible.
let epoch_consumer = queue.register_epoch_consumer(receipt.target_epoch);
if let Some(attempt) = txn_attempt {
attempt.admit(Arc::clone(&epoch_consumer), waiter_id)?;
let operation: Arc<dyn PendingGroupCommitTxnAttemptOperation> = attempt.clone();
queue.register_txn_attempt(receipt.target_epoch, waiter_id, operation)?;
}
(
receipt.outcome,
epoch_at_queue,
receipt.target_epoch,
lock_wait_us,
flushing_wait,
epoch_consumer,
)
};
trace_group_commit(format_args!(
"waiter waiter_id={waiter_id} role={outcome:?} epoch_at_queue={our_epoch} target_epoch={target_epoch} frames_i_contributed={waiter_frames_contributed}"
));
let run_flusher_loop = |mut record_initial_metrics: bool,
mut needs_arrival_wait: bool,
mut filling_obligation: Option<GroupCommitFillingObligation>,
mut prefetched_flush: Option<(
Vec<TransactionFrameBatch>,
u64,
GroupCommitFlushObligation,
)>| async move {
'flusher_loop: loop {
let arrival_wait_decision = {
let (observation, max_wait) = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueConsolidator);
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let observation = if prefetched_flush.is_none() && needs_arrival_wait {
Some(ArrivalWaitObservation {
pending_batch_count: consolidator.pending_batch_count(),
should_flush_now: consolidator.should_flush_now(),
fill_age: consolidator.fill_age(),
})
} else {
None
};
(observation, consolidator.max_group_delay())
};
let fairness_budget =
commit_service_fairness_budget(queue.parallel_wal_control(), max_wait);
let queue_age_p95 = observation
.map_or(Duration::ZERO, |obs| recent_queue_age_p95(obs.fill_age));
let previous_mode = queue.current_commit_service_mode();
let control_epoch = queue.next_commit_service_control_epoch();
let decision = decide_group_commit_arrival_wait(
observation,
max_wait,
fairness_budget,
queue_age_p95,
previous_mode,
control_epoch,
);
queue.store_commit_service_mode(decision.mode);
decision
};
let arrival_wait_us = if !arrival_wait_decision.wait_budget.is_zero() {
let t_arrival_wait_start = Instant::now();
let deadline = t_arrival_wait_start + arrival_wait_decision.wait_budget;
loop {
let should_flush = {
#[cfg(test)]
record_commit_fast_path_lock(
CommitFastPathLockClass::QueueConsolidator,
);
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator.should_flush_now()
};
if should_flush || Instant::now() >= deadline {
break;
}
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
asupersync::runtime::yield_now().await;
}
Instant::now()
.duration_since(t_arrival_wait_start)
.as_micros() as u64
} else {
0
};
let actual_wait_ns =
u64::try_from(Duration::from_micros(arrival_wait_us).as_nanos())
.unwrap_or(u64::MAX);
let (mut batches, flush_epoch, mut flush_obligation) = if let Some(prefetched) =
prefetched_flush.take()
{
prefetched
} else {
let maybe_flush = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::QueueConsolidator);
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if !record_initial_metrics
&& consolidator.phase() != fsqlite_wal::ConsolidationPhase::Filling
{
None
} else {
let batches = consolidator.begin_flush()?;
let flush_epoch = consolidator.epoch();
Some((batches, flush_epoch))
}
};
let Some(flush) = maybe_flush else {
break;
};
if let Some(obligation) = filling_obligation.as_mut() {
obligation.disarm();
}
let flush_obligation = GroupCommitFlushObligation::new(queue, flush.1);
(flush.0, flush.1, flush_obligation)
};
let t_flush_frame_prep_start = detailed_metrics.then(Instant::now);
let conflicting_pages = conflicting_pages_across_group_commit_batches(&batches);
if !conflicting_pages.is_empty() {
let flush_batch_membership = physical_writer_batch_membership(&batches);
let flush_batch_id = physical_writer_primary_batch_id(&batches);
let queue_delay_ns = arrival_wait_decision.queue_delay_ns();
let target_wait_ns = arrival_wait_decision.target_wait_ns();
let max_wait_ns = arrival_wait_decision.max_wait_ns();
let fsync_boundary = physical_writer_fsync_boundary(sync_policy);
let error = FrankenError::BusySnapshot {
conflicting_pages: conflicting_pages
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(","),
};
tracing::warn!(
target: "fsqlite::wal::lock_scope",
epoch = flush_epoch,
conflicting_pages = ?conflicting_pages,
"aborting group-commit epoch with cross-batch same-page overlap"
);
tracing::warn!(
target: "fsqlite::wal::lane_staging",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_LANE_RUN_ID,
scenario_id = PARALLEL_WAL_FLUSH_SCENARIO_ID,
batch_id = flush_batch_id,
batch_membership = flush_batch_membership.as_str(),
queue_delay_ns,
target_wait_ns,
actual_wait_ns,
max_wait_ns,
control_epoch = arrival_wait_decision.control_epoch,
queue_age_p95_ns = arrival_wait_decision.queue_age_p95_ns(),
batch_size = batches.len(),
fairness_budget_ns = arrival_wait_decision.fairness_budget_ns(),
starvation_prevented = arrival_wait_decision.starvation_prevented,
mode_switch_reason = arrival_wait_decision.mode_switch_reason,
service_policy_mode = commit_service_mode_name(arrival_wait_decision.mode),
fsync_boundary,
ordering_phase = "overlap_abort",
rollback_mode_active =
physical_writer_rollback_mode_active(parallel_wal_control.mode, None),
failure_context = "cross_batch_page_overlap",
conflicting_pages = ?conflicting_pages,
"aborting physical writer flush because batch membership overlaps on the same page"
);
let discarded_prepared_batches =
queue.discard_prepared_batches_for_flush(&batches);
if discarded_prepared_batches > 0 {
tracing::trace!(
target: "fsqlite::wal::lane_staging",
discarded_prepared_batches,
"discarded stale prepared WAL lane payloads before group-commit overlap abort"
);
}
let abort_result = queue
.abort_flushing_epoch_as_failed(flush_epoch, &error)
.map(|_| ());
if abort_result.is_ok() {
flush_obligation.disarm();
}
if let Err(abort_error) = abort_result {
if flush_epoch != target_epoch {
tracing::debug!(
target: "fsqlite::wal::lock_scope",
epoch = flush_epoch,
caller_target_epoch = target_epoch,
error = %error,
abort_error = %abort_error,
"promoted group-commit epoch overlap abort failed after caller epoch completed"
);
return Ok(());
}
return Err(FrankenError::internal(format!(
"group commit overlap abort failed for epoch {flush_epoch}: overlap={error}; abort={abort_error}"
)));
}
if flush_epoch != target_epoch {
tracing::debug!(
target: "fsqlite::wal::lock_scope",
epoch = flush_epoch,
caller_target_epoch = target_epoch,
error = %error,
"promoted group-commit epoch failed after caller epoch completed"
);
return Ok(());
}
return Err(error);
}
// Compute the consolidated commit size before borrowing frame
// refs so synthetic Page 1 headers can be promoted to the same
// db_size carried by the final WAL commit marker.
let batch_count = batches.len();
let flush_base_db_size = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
inner.db_size
};
let consolidated_db_size = group_commit_final_db_size(flush_base_db_size, &batches);
let promoted_page_one_headers =
promote_group_commit_page_one_headers(&mut batches, consolidated_db_size);
let (frame_refs, final_db_size) =
flatten_group_commit_batches(flush_base_db_size, &batches);
debug_assert_eq!(final_db_size, consolidated_db_size);
let frame_count = frame_refs.len();
let wal_frame_payload_digest = {
let mut digest = ParallelWalFramePayloadDigestBuilder::new();
for frame in &frame_refs {
let page_number = PageNumber::new(frame.page_number).ok_or_else(|| {
FrankenError::internal(format!(
"group commit contains invalid WAL page number {}",
frame.page_number
))
})?;
digest.update(page_number, frame.db_size_if_commit, frame.page_data);
}
digest.finalize()
};
let mut fallback_reason = if matches!(
parallel_wal_control.mode,
ParallelWalOperatingMode::Conservative
) {
Some(ParallelWalFallbackReason::OperatorForced)
} else {
None
};
let mut shadow_verdict = ParallelWalShadowVerdict::NotRun;
let mut prepared_batch = if fallback_reason.is_none() {
match queue.take_prepared_batches_for_flush(&batches) {
Some(mut staged_by_batch_id) => {
if promoted_page_one_headers {
// Lane-local prepared batches were created
// before the flusher knew the consolidated
// group db_size. Once Page 1 header bytes are
// promoted to that final size, those prepared
// byte streams are stale. They have been
// drained from the lane queues here; discard
// them and rebuild from patched `frame_refs`.
fallback_reason =
Some(ParallelWalFallbackReason::ControllerEvidenceLost);
None
} else {
match batches
.iter()
.map(|batch| staged_by_batch_id.remove(&batch.context.batch_id))
.collect::<Option<Vec<_>>>()
{
Some(ordered_staged_batches) => {
let staged_shadow_verdict =
aggregate_shadow_verdict(&ordered_staged_batches);
match merge_prepared_group_commit_batches(
ordered_staged_batches,
final_db_size,
) {
Ok(merged) => {
if should_shadow_compare_batches(
¶llel_wal_control,
&batches,
) {
if prepared_batch_matches_frame_refs(
&merged,
&frame_refs,
) {
shadow_verdict =
ParallelWalShadowVerdict::Clean;
Some(merged)
} else {
shadow_verdict =
ParallelWalShadowVerdict::Diverged;
fallback_reason = Some(
ParallelWalFallbackReason::PublicationMismatch,
);
None
}
} else {
shadow_verdict = staged_shadow_verdict;
Some(merged)
}
}
Err(_) => {
fallback_reason = Some(
ParallelWalFallbackReason::ControllerEvidenceLost,
);
None
}
}
}
None => {
fallback_reason =
Some(ParallelWalFallbackReason::ControllerEvidenceLost);
None
}
}
}
}
None => {
let discarded_prepared_batches =
queue.discard_prepared_batches_for_flush(&batches);
if discarded_prepared_batches > 0 {
tracing::trace!(
target: "fsqlite::wal::lane_staging",
discarded_prepared_batches,
"discarded stale prepared WAL lane payloads after raw fallback"
);
}
fallback_reason =
Some(ParallelWalFallbackReason::ControllerEvidenceLost);
None
}
}
} else {
None
};
// Operator-forced conservative mode is the production
// comparator for D1.c: it retains the former centralized
// frame serialization/checksum preparation inside the
// durability combiner. Every other mode prepares before the
// ordered residue, either from a lane-staged batch or through
// this safe raw fallback.
if prepared_batch.is_none()
&& !matches!(
parallel_wal_control.mode,
ParallelWalOperatingMode::Conservative
)
{
let backend = wal_backend_handle(wal_backend)?;
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::WalBackendRead);
let wal = async_rwlock_read(&backend, cx, "WAL backend").await?;
let mut prepared = wal.prepare_append_frames(&frame_refs)?;
if let Some(prepared) = prepared.as_mut() {
wal.finalize_prepared_frames(cx, prepared)?;
}
drop(wal);
prepared_batch = prepared;
}
let flush_frame_prep_us = elapsed_profile_us(t_flush_frame_prep_start);
GLOBAL_CONSOLIDATION_METRICS.transactions_batched.fetch_add(
u64::try_from(batch_count).unwrap_or(u64::MAX),
AtomicOrdering::Relaxed,
);
const MAX_FLUSH_RETRIES: u32 = 10;
let mut flush_result: Result<()> = Ok(());
let mut inner_lock_wait_us: u64 = 0;
let mut exclusive_lock_us: u64 = 0;
let mut wal_append_us: u64 = 0;
let mut append_conflict_check_us: u64 = 0;
let mut append_frames_us: u64 = 0;
let mut wal_sync_us: u64 = 0;
let mut frames_written_start: u64 = 0;
let mut frames_written_end: u64 = 0;
let mut fsync_seq: u64 = 0;
for attempt in 0..MAX_FLUSH_RETRIES {
let t_inner_lock_start = phase_timing.then(Instant::now);
let db_file = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let inner = inner_arc.lock().map_err(|_| {
FrankenError::internal("SimpleTransaction lock poisoned")
})?;
Arc::clone(&inner.db_file)
};
// Never wait for exact-handle lock coordination while
// retaining a PagerInner. A logical exit may need that
// exact inner before it can release the gate.
let physical_lock_window = GroupCommitPhysicalLockWindow::acquire(
queue,
shared_db_file_key(&db_file),
cx,
)
.await?;
let (restore_lock_level, initial_visible_commit_seq, checkpoint_active) = {
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let inner = inner_arc.lock().map_err(|_| {
FrankenError::internal("SimpleTransaction lock poisoned")
})?;
inner_lock_wait_us = elapsed_profile_us(t_inner_lock_start);
(
if inner.writer_active {
// Immediate/exclusive transactions enter commit
// already owning RESERVED. If WAL append fails,
// the caller must keep that writer lock so a
// retry or rollback cannot be interleaved by a
// different writer.
LockLevel::Reserved
} else {
LockLevel::Shared
},
inner.commit_seq,
inner.checkpoint_active,
)
};
flush_result = async {
let t_excl_start = phase_timing.then(Instant::now);
// WAL appends need a cross-process writer gate, but
// they must not wait for every concurrent reader or
// writer transaction that already holds SHARED on the
// main database file. SQLite's RESERVED byte is the
// narrow lock for this: one appender at a time, while
// peer SHARED holders keep running.
shared_db_lock(&db_file, cx, LockLevel::Reserved).await?;
let mut db_lock_obligation = GroupCommitDbLockObligation::new(
queue,
flush_epoch,
&db_file,
cx,
restore_lock_level,
flush_obligation.durability_started_signal(),
flush_obligation.durable_io_signal(),
flush_obligation.external_lock_state(),
physical_lock_window,
);
exclusive_lock_us = elapsed_profile_us(t_excl_start);
let flush_io_result = async {
let backend = wal_backend_handle(wal_backend)?;
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::WalBackendWrite);
let mut wal_guard =
async_rwlock_write(&backend, cx, "WAL backend").await?;
let wal = wal_guard.as_mut();
// A connection-local WAL backend can lag a
// commit published through a peer backend.
// Refresh it while the cross-process append
// gate is held before deriving the certified
// frame interval. Conflict detection also
// refreshes, so compute the interval only
// after every operation that can advance the
// backend's view of the durable tail.
let _ = wal.refresh_published_snapshot(cx).await?;
let t_append_conflict_check_start =
detailed_metrics.then(Instant::now);
let stale_conflict_pages =
conflicting_pages_since_batch_snapshots(cx, wal, &batches)
.await?;
append_conflict_check_us =
elapsed_profile_us(t_append_conflict_check_start);
if !stale_conflict_pages.is_empty() {
return Err(FrankenError::BusySnapshot {
conflicting_pages: stale_conflict_pages
.iter()
.map(u32::to_string)
.collect::<Vec<_>>()
.join(","),
});
}
let authorized_seed = wal
.latest_authorized_parallel_wal_commit_certificate(cx)
.await?;
frames_written_start = u64::try_from(wal.frame_count())
.unwrap_or(u64::MAX)
.saturating_add(1);
frames_written_end = frames_written_start
.checked_add(
u64::try_from(frame_count)
.unwrap_or(u64::MAX)
.saturating_sub(1),
)
.ok_or(FrankenError::DatabaseFull)?;
if sync_policy.should_sync_on_commit() {
fsync_seq = GROUP_COMMIT_TRACE_FSYNC_SEQ
.fetch_add(1, AtomicOrdering::Relaxed)
.saturating_add(1);
}
let durability_started =
flush_obligation.durability_started_signal();
let durable_io_signal = flush_obligation.durable_io_signal();
let publication = queue
.prepare_persisted_epoch(
cx,
PersistedGroupCommitInput {
trace_id: cx.trace_id(),
epoch: flush_epoch,
batches: &batches,
frames_start: frames_written_start,
frames_end: frames_written_end,
fsync_seq,
initial_visible_commit_seq,
db_size_pages: final_db_size,
page_set_size: frame_count,
checkpoint_active,
fallback_reason,
authorized_seed,
wal_frame_payload_digest,
},
)
.await;
let publication = match publication {
Ok(publication) => publication,
Err(error) => return Err(error),
};
let sidecar_completion =
Arc::new(Mutex::new(None::<VfsWriteCompletion>));
let wal_completion =
Arc::new(Mutex::new(None::<VfsWriteCompletion>));
let recovery_epoch_consumer =
queue.register_epoch_consumer(flush_epoch);
let recovery =
Arc::new(PendingGroupCommitRecovery::<V::File> {
queue: Arc::downgrade(queue),
epoch: flush_epoch,
_epoch_consumer: recovery_epoch_consumer,
publication: Arc::clone(&publication),
wal_backend: Arc::clone(&backend),
inner: Arc::clone(inner_arc),
published: published.clone(),
batches: batches.clone(),
final_db_size,
sync: sync_policy.should_sync_on_commit(),
sidecar_completion: Arc::clone(&sidecar_completion),
wal_completion: Arc::clone(&wal_completion),
cleanup_cx: cleanup_child_cx(cx),
durability_started: Arc::clone(&durability_started),
durable_io_completed: Arc::clone(&durable_io_signal),
resolution: Mutex::new(None),
});
let recovery_operation: Arc<
dyn PendingGroupCommitRecoveryOperation,
> = recovery.clone();
if cx.checkpoint().is_err() {
publication.abort()?;
return Err(FrankenError::Abort);
}
db_lock_obligation.set_recovery(recovery_operation);
let durability_cx = cleanup_child_cx(cx);
let _durability_mask = durability_cx.masked();
let certificate = publication.certificate()?;
let certificate_completion = VfsWriteCompletion::new();
*sidecar_completion
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(certificate_completion.clone());
durability_started.store(true, AtomicOrdering::Release);
match wal
.persist_parallel_wal_commit_certificate_tracked(
&durability_cx,
&certificate,
frames_written_start,
frames_written_end,
sync_policy.should_sync_on_commit(),
certificate_completion.clone(),
)
.await
{
Ok(()) => {}
Err(FrankenError::Unsupported) => {
// Unsupported is the backend DECLARING
// it has no certificate-sidecar
// facility (plain adapters, process-
// local harness backends) — a backend
// property, not a test-build property.
// The former #[cfg(test)] gate on this
// arm made every integration-test
// (non-cfg(test)) Immediate commit on
// such backends fail outright once the
// D1 stack landed (bd-dk9ra: self_alloc
// reds + publish-window suite wedge).
// Such backends simply operate with
// pre-D1 durability semantics — no
// parallel-WAL certificate — exactly as
// they did before the stack. Real
// production backends implement
// persist; their genuine failures
// still fail closed below.
certificate_completion.complete_success();
}
Err(error) => return Err(error),
}
let frame_completion = VfsWriteCompletion::new();
*wal_completion
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(frame_completion.clone());
let t_append_frames_start =
detailed_metrics.then(Instant::now);
if let Some(prepared) = prepared_batch.as_mut() {
wal.append_prepared_frames_tracked(
&durability_cx,
prepared,
frame_completion,
)
.await?;
} else {
wal.append_frames_tracked(
&durability_cx,
&frame_refs,
frame_completion,
)
.await?;
}
append_frames_us =
elapsed_profile_us(t_append_frames_start);
wal_append_us = append_frames_us;
let actual_frames_written_end =
u64::try_from(wal.frame_count()).unwrap_or(u64::MAX);
if actual_frames_written_end != frames_written_end {
return Err(FrankenError::internal(format!(
"parallel WAL certificate covers frames {frames_written_start}..={frames_written_end}, append ended at {actual_frames_written_end}"
)));
}
if sync_policy.should_sync_on_commit() {
let t_sync_start = phase_timing.then(Instant::now);
wal.sync(&durability_cx)?;
wal_sync_us = elapsed_profile_us(t_sync_start);
GLOBAL_CONSOLIDATION_METRICS
.fsyncs_total
.fetch_add(1, AtomicOrdering::Relaxed);
} else {
wal.publish_authorized_deferred_commit(&durability_cx)
.await?;
}
drop(wal_guard);
recovery.complete_authorized(&durability_cx)?;
#[cfg(any(test, feature = "fault-injection"))]
crate::fault_hooks::maybe_inject_after_flush_before_publish(
flush_epoch,
batch_count,
frame_count,
)?;
Ok(())
}
.await;
if flush_io_result.is_err()
&& flush_obligation.durability_state()
== GroupCommitFlushDurability::InDoubt
{
// A terminal callback error does not prove that a
// completed WAL marker was never written. Keep
// RESERVED fail-closed and let the lock obligation
// transfer ownership to the queue on Drop.
return flush_io_result;
}
// Lock restoration is mandatory even when the caller
// was cancelled. In particular, once the flush is
// durable, cancellation must not strand RESERVED or
// turn the completed epoch into Abort.
let restore_result = db_lock_obligation.restore().await;
match (flush_io_result, restore_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(flush_error), Err(restore_error)) => {
Err(FrankenError::internal(format!(
"flush failed and could not restore SHARED lock: flush={flush_error}; restore={restore_error}"
)))
}
}
}
.await;
if flush_obligation.durability_state() == GroupCommitFlushDurability::InDoubt {
// An in-doubt physical writer is never retryable: a
// retry could append a second copy while the first
// callback's bytes remain unresolved.
break;
}
match &flush_result {
Err(
FrankenError::Busy
| FrankenError::BusyRecovery
| FrankenError::BusySnapshot { .. },
) if attempt + 1 < MAX_FLUSH_RETRIES => {
perform_flush_busy_retry_handoff(flush_busy_retry_wait(attempt + 1));
// This is a physical flusher retry, not a waiter
// wake. Keep it out of the mutually exclusive wake
// reason ledger.
GLOBAL_CONSOLIDATION_METRICS.record_busy_retry();
}
_ => break,
}
}
if flush_result.is_err()
&& flush_obligation.durability_state() == GroupCommitFlushDurability::InDoubt
{
let error =
flush_result.expect_err("in-doubt group-commit branch requires an error");
tracing::error!(
epoch = flush_epoch,
%error,
"group-commit callback failed after durable mutation started; retaining RESERVED and leaving epoch FLUSHING"
);
// Do not publish a failed epoch or abort FLUSHING. Dropping
// the outer obligation observes the queued external-lock
// owner and defers resolution to durable reconciliation.
return Err(error);
}
if flush_result.is_err() && flush_obligation.is_durable() {
let error = flush_result
.expect_err("durable group-commit failure branch requires an error");
let (completed_epoch, has_promoted) = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let promoted = consolidator.complete_flush()?;
(consolidator.epoch(), promoted)
};
let caller_target_completed = completed_epoch >= target_epoch;
queue.publish_completed_epoch(
completed_epoch,
has_promoted && caller_target_completed,
);
flush_obligation.disarm();
tracing::error!(
epoch = flush_epoch,
%error,
"durable group-commit epoch completed despite local publication or lock-restoration failure"
);
if flush_epoch != target_epoch {
return Ok(());
}
return Err(error);
}
match flush_result {
Ok(()) => {
GLOBAL_CONSOLIDATION_METRICS
.groups_flushed
.fetch_add(1, AtomicOrdering::Relaxed);
GLOBAL_CONSOLIDATION_METRICS.frames_consolidated.fetch_add(
u64::try_from(frame_count).unwrap_or(u64::MAX),
AtomicOrdering::Relaxed,
);
GLOBAL_CONSOLIDATION_METRICS
.max_group_size_observed
.fetch_max(
u64::try_from(frame_count).unwrap_or(u64::MAX),
AtomicOrdering::Relaxed,
);
if detailed_metrics {
if record_initial_metrics {
GLOBAL_CONSOLIDATION_METRICS.record_prepare_breakdown(
batch_build_us,
conflict_snapshot_us,
lane_prepare_us,
);
}
GLOBAL_CONSOLIDATION_METRICS.record_flush_breakdown(
flush_frame_prep_us,
append_conflict_check_us,
append_frames_us,
);
}
if phase_timing {
GLOBAL_CONSOLIDATION_METRICS.record_phase_timing(
if record_initial_metrics {
prepare_us
} else {
0
},
if record_initial_metrics {
consolidator_lock_wait_us
} else {
0
},
flushing_wait_us,
true,
arrival_wait_us,
inner_lock_wait_us,
exclusive_lock_us,
wal_append_us,
wal_sync_us,
0,
);
}
// bd-db300.3.8.2: per-flush structured event splitting
// lock-wait time from WAL service time.
let lock_wait_total_us =
inner_lock_wait_us + exclusive_lock_us + flushing_wait_us;
let wal_service_total_us = wal_append_us + wal_sync_us;
tracing::debug!(
target: "fsqlite::wal::lock_scope",
role = "flusher",
epoch = flush_epoch,
frames = frame_count,
lock_wait_us = lock_wait_total_us,
inner_lock_wait_us,
exclusive_lock_us,
flushing_wait_us,
wal_service_us = wal_service_total_us,
wal_append_us,
wal_sync_us,
arrival_wait_us,
arrival_wait_policy = arrival_wait_decision.policy,
arrival_wait_reason = arrival_wait_decision.reason,
arrival_wait_budget_us = arrival_wait_decision.wait_budget_us(),
arrival_wait_fill_age_us = arrival_wait_decision.fill_age_us(),
service_policy_mode =
commit_service_mode_name(arrival_wait_decision.mode),
service_policy_control_epoch = arrival_wait_decision.control_epoch,
queue_age_p95_us =
arrival_wait_decision.queue_age_p95.as_micros() as u64,
fairness_budget_us =
arrival_wait_decision.fairness_budget.as_micros() as u64,
starvation_prevented = arrival_wait_decision.starvation_prevented,
arrival_wait_used_legacy_fallback =
arrival_wait_decision.used_legacy_fallback,
"WAL backend commit: lock_wait={lock_wait_total_us}us \
service={wal_service_total_us}us \
(append={wal_append_us}us sync={wal_sync_us}us) \
frames={frame_count}"
);
if tracing::enabled!(target: "fsqlite::wal::lane_staging", tracing::Level::DEBUG)
{
let flush_batch_membership = physical_writer_batch_membership(&batches);
let flush_batch_id = physical_writer_primary_batch_id(&batches);
let queue_delay_ns = arrival_wait_decision.queue_delay_ns();
let target_wait_ns = arrival_wait_decision.target_wait_ns();
let max_wait_ns = arrival_wait_decision.max_wait_ns();
let fsync_boundary = physical_writer_fsync_boundary(sync_policy);
let ordering_phase =
physical_writer_ordering_phase(arrival_wait_decision.reason);
let flush_rollback_mode_active = physical_writer_rollback_mode_active(
parallel_wal_control.mode,
fallback_reason,
);
let lane_stats = lane_flush_stats(queue, &batches);
for (lane_id, lane_backlog, staged_frame_count, elapsed_ns) in
&lane_stats
{
tracing::debug!(
target: "fsqlite::wal::lane_staging",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_LANE_RUN_ID,
scenario_id = PARALLEL_WAL_FLUSH_SCENARIO_ID,
batch_id = flush_batch_id,
batch_membership = flush_batch_membership.as_str(),
queue_delay_ns,
target_wait_ns,
actual_wait_ns,
max_wait_ns,
control_epoch = arrival_wait_decision.control_epoch,
queue_age_p95_ns = arrival_wait_decision.queue_age_p95_ns(),
batch_size = batch_count,
fairness_budget_ns = arrival_wait_decision.fairness_budget_ns(),
starvation_prevented =
arrival_wait_decision.starvation_prevented,
mode_switch_reason = arrival_wait_decision.mode_switch_reason,
service_policy_mode =
commit_service_mode_name(arrival_wait_decision.mode),
fsync_boundary,
ordering_phase,
rollback_mode_active = flush_rollback_mode_active,
wal_lane_id = *lane_id,
lane_backlog = *lane_backlog,
staged_frame_count = *staged_frame_count,
flush_trigger = arrival_wait_decision.reason,
control_mode = parallel_wal_mode_name(parallel_wal_control.mode),
lane_policy_version = PARALLEL_WAL_LANE_POLICY_VERSION,
shadow_verdict =
parallel_wal_shadow_verdict_name(shadow_verdict),
compatibility_selector = PARALLEL_WAL_COMPATIBILITY_SELECTOR,
fallback_reason =
parallel_wal_fallback_reason_name(fallback_reason),
elapsed_ns = *elapsed_ns,
"flushed lane-local WAL staging candidate"
);
}
}
let (completed_epoch, has_promoted) = {
#[cfg(test)]
record_commit_fast_path_lock(
CommitFastPathLockClass::QueueConsolidator,
);
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let promoted = consolidator.complete_flush()?;
(consolidator.epoch(), promoted)
};
let caller_target_completed = completed_epoch >= target_epoch;
queue.publish_completed_epoch(
completed_epoch,
has_promoted && caller_target_completed,
);
flush_obligation.disarm();
if has_promoted {
if caller_target_completed {
break 'flusher_loop;
}
let claimed_promoted = {
#[cfg(test)]
record_commit_fast_path_lock(
CommitFastPathLockClass::QueueConsolidator,
);
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator.claim_flusher_vacancy()
};
if !claimed_promoted {
return Err(FrankenError::internal(format!(
"group commit flusher could not reserve promoted epoch after completing epoch {flush_epoch}"
)));
}
record_initial_metrics = false;
needs_arrival_wait = false;
continue 'flusher_loop;
}
}
Err(error) => {
let flush_batch_membership = physical_writer_batch_membership(&batches);
let flush_batch_id = physical_writer_primary_batch_id(&batches);
let queue_delay_ns = arrival_wait_decision.queue_delay_ns();
let target_wait_ns = arrival_wait_decision.target_wait_ns();
let max_wait_ns = arrival_wait_decision.max_wait_ns();
let fsync_boundary = physical_writer_fsync_boundary(sync_policy);
tracing::warn!(
target: "fsqlite::wal::lane_staging",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_LANE_RUN_ID,
scenario_id = PARALLEL_WAL_FLUSH_SCENARIO_ID,
batch_id = flush_batch_id,
batch_membership = flush_batch_membership.as_str(),
queue_delay_ns,
target_wait_ns,
actual_wait_ns,
max_wait_ns,
control_epoch = arrival_wait_decision.control_epoch,
queue_age_p95_ns = arrival_wait_decision.queue_age_p95_ns(),
batch_size = batch_count,
fairness_budget_ns = arrival_wait_decision.fairness_budget_ns(),
starvation_prevented = arrival_wait_decision.starvation_prevented,
mode_switch_reason = arrival_wait_decision.mode_switch_reason,
service_policy_mode =
commit_service_mode_name(arrival_wait_decision.mode),
fsync_boundary,
ordering_phase = "flush_error",
rollback_mode_active = physical_writer_rollback_mode_active(
parallel_wal_control.mode,
fallback_reason,
),
shadow_verdict =
parallel_wal_shadow_verdict_name(shadow_verdict),
fallback_reason =
parallel_wal_fallback_reason_name(fallback_reason),
failure_context = %error,
"physical writer flush failed"
);
let abort_result = queue
.abort_flushing_epoch_as_failed(flush_epoch, &error)
.map(|_| ());
if abort_result.is_ok() {
flush_obligation.disarm();
}
if let Err(abort_error) = abort_result {
if flush_epoch != target_epoch {
tracing::debug!(
target: "fsqlite::wal::lock_scope",
epoch = flush_epoch,
caller_target_epoch = target_epoch,
error = %error,
abort_error = %abort_error,
"promoted group-commit epoch flush abort failed after caller epoch completed"
);
return Ok(());
}
return Err(FrankenError::internal(format!(
"group commit flush failed for epoch {flush_epoch} and abort_flush also failed: flush={error}; abort={abort_error}"
)));
}
if flush_epoch != target_epoch {
tracing::debug!(
target: "fsqlite::wal::lock_scope",
epoch = flush_epoch,
caller_target_epoch = target_epoch,
error = %error,
"promoted group-commit epoch flush failed after caller epoch completed"
);
return Ok(());
}
return Err(error);
}
}
break;
}
Ok(())
};
match outcome {
SubmitOutcome::Flusher => {
let filling_obligation = GroupCommitFillingObligation::new(queue, target_epoch);
run_flusher_loop(true, true, Some(filling_obligation), None).await?;
}
SubmitOutcome::Waiter => {
// Step 3b: WAITER path — wait for flusher to complete our epoch.
//
// The consolidator receipt tells us the exact epoch that owns
// this batch. A batch submitted while another epoch is already
// flushing is queued for the promoted next epoch, so deriving
// `epoch_at_queue + 1` here can wake too early.
let t_waiter_start = phase_timing.then(Instant::now);
let wait_outcome = queue.wait_for_epoch_outcome_async(cx, target_epoch).await?;
let waiter_epoch_wait_us = elapsed_profile_us(t_waiter_start);
match wait_outcome {
WaitForEpochOutcome::Completed => {
if group_commit_trace_enabled() {
let completed_epoch =
queue.completed_epoch.load(AtomicOrdering::Acquire);
let persisted = queue.persisted_epoch_for(target_epoch);
let persisted_contains_waiter = persisted
.as_ref()
.is_some_and(|record| record.members.contains(&waiter_id));
let (frames_start, frames_end, fsync_seq) =
persisted.as_ref().map_or((0, 0, 0), |record| {
(record.frames_start, record.frames_end, record.fsync_seq)
});
trace_group_commit(format_args!(
"waiter_wake waiter_id={waiter_id} epoch_at_queue={our_epoch} target_epoch={target_epoch} completed_epoch_when_woken={completed_epoch} frames_i_contributed={waiter_frames_contributed} persisted_contains_waiter={persisted_contains_waiter} frames_written_range={frames_start}..={frames_end} fsync_seq={fsync_seq}"
));
assert!(
target_epoch <= completed_epoch && persisted_contains_waiter,
"group commit waiter {waiter_id} woke for completed_epoch={completed_epoch}, target_epoch={target_epoch}, epoch_at_queue={our_epoch}, but its frames were not recorded in the persisted epoch"
);
}
// Record phase timing for waiter
if detailed_metrics {
GLOBAL_CONSOLIDATION_METRICS.record_prepare_breakdown(
batch_build_us,
conflict_snapshot_us,
lane_prepare_us,
);
}
if phase_timing {
GLOBAL_CONSOLIDATION_METRICS.record_phase_timing(
prepare_us,
consolidator_lock_wait_us,
flushing_wait_us,
false, // is_flusher
0, // arrival_wait_us (N/A for waiter)
0, // inner_lock_wait_us (N/A for waiter)
0, // exclusive_lock_us (N/A for waiter)
0, // wal_append_us (N/A for waiter)
0, // wal_sync_us (N/A for waiter)
waiter_epoch_wait_us,
);
}
// bd-db300.3.8.2: per-waiter structured event showing
// time spent waiting for the flusher (all lock-wait,
// zero WAL service time on this thread).
let lock_wait_total_us =
consolidator_lock_wait_us + flushing_wait_us + waiter_epoch_wait_us;
tracing::debug!(
target: "fsqlite::wal::lock_scope",
role = "waiter",
lock_wait_us = lock_wait_total_us,
consolidator_lock_wait_us,
flushing_wait_us,
waiter_epoch_wait_us,
wal_service_us = 0_u64,
wal_append_us = 0_u64,
wal_sync_us = 0_u64,
"WAL backend commit: lock_wait={lock_wait_total_us}us \
service=0us (waiter — flusher did I/O)"
);
// The flusher already updated inner.db_size.
// Our frames are now durable in the WAL.
}
WaitForEpochOutcome::TakeOverFlusher {
batches,
flush_epoch,
} => {
let _ = waiter_epoch_wait_us;
let flush_obligation = GroupCommitFlushObligation::new(queue, flush_epoch);
run_flusher_loop(
true,
false,
None,
Some((batches, flush_epoch, flush_obligation)),
)
.await?;
}
}
}
}
let persisted = queue.persisted_epoch_for(target_epoch).ok_or_else(|| {
FrankenError::internal(format!(
"group commit epoch {target_epoch} completed without a durability certificate"
))
})?;
if !persisted.members.contains(&waiter_id) {
return Err(FrankenError::internal(format!(
"group commit certificate for epoch {target_epoch} does not cover batch {waiter_id}"
)));
}
if persisted
.durability_receipt
.commit_seq_for_batch(waiter_id)
.is_none()
{
return Err(FrankenError::internal(format!(
"group commit certificate for epoch {target_epoch} has no sequence assignment for batch {waiter_id}"
)));
}
let assigned_commit_seq = persisted
.durability_receipt
.commit_seq_for_batch(waiter_id)
.expect("checked batch sequence assignment above");
*publication_authorization = Some(ParallelWalPublicationAuthorization {
durability_receipt: persisted.durability_receipt,
batch_id: waiter_id,
assigned_commit_seq,
});
Ok(())
}
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
async fn ensure_writer(&mut self, cx: &Cx) -> Result<()> {
if self.read_only_pager {
return Err(FrankenError::ReadOnly);
}
self.ensure_no_pending_group_commit_attempt()?;
if self.is_writer {
return Ok(());
}
if self
.group_commit_queue
.has_process_root_finalization_attempt()
{
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
self.ensure_no_pending_group_commit_attempt()?;
}
match self.mode {
TransactionMode::ReadOnly => Err(FrankenError::ReadOnly),
TransactionMode::Concurrent => {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
self.ensure_writer_upgrade_recovery_clean(&inner)?;
if self
.group_commit_queue
.has_relevant_process_root(shared_db_file_key(&self.db_file))
{
return Err(FrankenError::BusyRecovery);
}
if inner.checkpoint_active {
let active_transactions = inner.active_transactions;
let checkpoint_active = inner.checkpoint_active;
drop(inner);
log_checkpoint_coordination(
cx,
&self.group_commit_queue,
"active_gate",
"ensure_writer",
transaction_mode_name(self.mode),
"checkpoint_excludes_foreground_writer_upgrade",
true,
active_transactions,
checkpoint_active,
);
return Err(FrankenError::Busy);
}
// Concurrent writers do not acquire the pager-global writer
// baton or rollback-mode RESERVED lock. Their page-level MVCC
// conflict surface and WAL publication protocol remain the
// source of truth, including across pagers sharing MemoryVfs.
let upgrade_gate = inner.maintenance_gate.lock_clean_writer_upgrade()?;
self.ensure_writer_upgrade_recovery_clean(&inner)?;
self.is_writer = true;
drop(upgrade_gate);
drop(inner);
Ok(())
}
TransactionMode::Deferred => {
loop {
let observed_generation =
self.group_commit_queue.external_lock_waiters.generation();
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
self.ensure_writer_upgrade_recovery_clean(&inner)?;
if self
.group_commit_queue
.has_relevant_process_root(shared_db_file_key(&self.db_file))
{
return Err(FrankenError::BusyRecovery);
}
if inner.checkpoint_active {
let active_transactions = inner.active_transactions;
let checkpoint_active = inner.checkpoint_active;
drop(inner);
log_checkpoint_coordination(
cx,
&self.group_commit_queue,
"active_gate",
"ensure_writer",
transaction_mode_name(self.mode),
"checkpoint_excludes_foreground_writer_upgrade",
true,
active_transactions,
checkpoint_active,
);
return Err(FrankenError::Busy);
}
if inner.writer_active {
inner =
wait_for_single_writer_baton(&self.inner, &self.writer_idle, inner)?;
self.ensure_writer_upgrade_recovery_clean(&inner)?;
}
if inner.checkpoint_active {
let active_transactions = inner.active_transactions;
let checkpoint_active = inner.checkpoint_active;
drop(inner);
log_checkpoint_coordination(
cx,
&self.group_commit_queue,
"active_gate",
"ensure_writer",
transaction_mode_name(self.mode),
"checkpoint_excludes_foreground_writer_upgrade_after_baton_wait",
true,
active_transactions,
checkpoint_active,
);
return Err(FrankenError::Busy);
}
let Some(physical_lock_window) = GroupCommitPhysicalLockWindow::try_register(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
) else {
drop(inner);
cx.checkpoint().map_err(|_| FrankenError::Abort)?;
let _ = self
.group_commit_queue
.external_lock_waiters
.wait_for_change_async(observed_generation)
.await;
continue;
};
// Escalate to RESERVED under a counted physical window.
// Distinct admissions may overlap; exact-handle downgrade
// and release remain fenced until writer state publishes.
let mut upgrade_lock = BeginExternalLockState::new(
&self.group_commit_queue,
Arc::clone(&inner.db_file),
cx,
);
upgrade_lock.arm_lock_level(LockLevel::Shared);
if let Err(lock_error) =
shared_db_lock(&inner.db_file, cx, LockLevel::Reserved).await
{
drop(upgrade_lock);
drop(physical_lock_window);
return Err(lock_error);
}
upgrade_lock.mark_lock_level_acquired();
let writer_upgrade_gate = Arc::clone(&inner.maintenance_gate);
let recovery_check = self
.ensure_writer_upgrade_recovery_clean(&inner)
.and_then(|()| writer_upgrade_gate.lock_clean_writer_upgrade());
let upgrade_gate = match recovery_check {
Ok(upgrade_gate) => upgrade_gate,
Err(recovery_error) => {
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let unlock_result = upgrade_lock.restore().await;
drop(upgrade_lock);
drop(physical_lock_window);
return match unlock_result {
Ok(()) => Err(recovery_error),
Err(unlock_error) => Err(FrankenError::internal(format!(
"writer upgrade observed rollback recovery after RESERVED and could not restore SHARED: recovery={recovery_error}; unlock={unlock_error}"
))),
};
}
};
self.ensure_writer_upgrade_recovery_clean(&inner)?;
inner.writer_active = true;
drop(upgrade_gate);
upgrade_lock.disarm();
drop(upgrade_lock);
drop(physical_lock_window);
drop(inner);
self.is_writer = true;
return Ok(());
}
}
TransactionMode::Immediate | TransactionMode::Exclusive => Err(FrankenError::internal(
"writer transaction lost writer role",
)),
}
}
}
impl<V> SimpleTransaction<V>
where
V: Vfs + Send,
V::File: Send + Sync + 'static,
{
fn restore_not_committed_wal_attempt(&mut self) -> Result<()> {
let attempt = self.pending_group_commit_attempt.clone().ok_or_else(|| {
FrankenError::internal("cannot restore a missing group-commit transaction attempt")
})?;
attempt.complete_not_committed_global()?;
let not_committed = attempt.take_not_committed_state()?;
self.allocated_from_freelist
.extend(not_committed.returned_allocations.from_freelist);
self.allocated_from_eof
.extend(not_committed.returned_allocations.from_eof);
self.page_lease
.extend(not_committed.returned_allocations.page_lease);
self.restore_pending_freed_pages(not_committed.pending_freed_pages);
attempt.restore_phase_a_write_set(&mut self.write_set, &mut self.write_pages_sorted);
attempt.finish_terminal(false)?;
self.pending_group_commit_attempt.take();
Ok(())
}
async fn finish_authorized_wal_attempt(&mut self, cx: &Cx, release: bool) -> Result<()> {
let attempt = self.pending_group_commit_attempt.clone().ok_or_else(|| {
FrankenError::internal("cannot finalize a missing authorized group-commit transaction")
})?;
if !matches!(
attempt.resolution(),
PendingGroupCommitTxnResolution::Authorized(_)
) {
return Err(FrankenError::BusyRecovery);
}
let publication_intent = attempt.publication_intent()?;
let (db_file, memory_file_size) = {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
(
Arc::clone(&inner.db_file),
self.memory_db_bump_alloc
.then(|| u64::from(inner.db_size) * u64::from(inner.page_size.get())),
)
};
let committed_file_size = if let Some(file_size) = memory_file_size {
Some(file_size)
} else {
match shared_db_file_read(&db_file, cx).await {
Ok(db_file) => db_file.file_size(cx).ok(),
Err(_) => None,
}
};
{
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if let Some(file_size) = committed_file_size {
inner.committed_db_file_size_bytes = file_size;
}
self.publish_committed_snapshot_from_inner(&inner);
}
if release {
let logical_exit_claim = GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
cx,
)
.await?;
attempt.finish_txn_exit(&logical_exit_claim).await?;
}
#[cfg(any(test, feature = "fault-injection"))]
crate::fault_hooks::maybe_inject_during_phase_c(
publication_intent.visible_commit_seq.get(),
publication_intent.db_size,
)?;
self.published
.bind_parallel_wal_publication(publication_intent);
self.published_visible_commit_seq
.set(publication_intent.visible_commit_seq);
self.published_db_size.set(publication_intent.db_size);
if release {
if self.single_connection_fast_path_enabled() {
self.drain_committed_cache_pages_into_cache();
} else {
self.discard_committed_pages();
}
} else {
// Group consolidation may rewrite cloned Page 1 frames to the
// certificate-wide db_size. Member-local staged images are
// therefore not authoritative after an Authorized group commit.
// Let the complete published group plane repopulate the retained
// transaction instead of caching a stale local Page 1.
self.discard_committed_pages();
self.txn_read_cache.borrow_mut().clear();
}
self.retained_memory_overlay_dirty_pages.clear();
self.clear_freed_pages();
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
self.page_lease.clear();
self.savepoint_stack.clear();
self.rolled_back_pages.clear();
self.writes_observed = false;
self.scratch_arena.reset();
attempt.finish_terminal(release)?;
self.pending_group_commit_attempt.take();
if release {
self.committed = true;
self.maintenance_lease.take();
self.finished = true;
} else {
self.original_db_size = publication_intent.db_size;
}
Ok(())
}
}
const fn retained_lock_level_after_txn_exit(
remaining_active_transactions: u32,
writer_active: bool,
) -> LockLevel {
if remaining_active_transactions == 0 {
LockLevel::None
} else if writer_active {
LockLevel::Reserved
} else {
LockLevel::Shared
}
}
#[allow(clippy::await_holding_lock)]
async fn coordinated_transaction_exit<F: VfsFile>(
queue: &Arc<GroupCommitQueue>,
cx: &Cx,
inner: &mut PagerInner<F>,
releases_writer_baton: bool,
logical_exit_claim: &GroupCommitLogicalExitClaim,
) -> Result<bool> {
debug_assert!(
Arc::ptr_eq(queue, &logical_exit_claim.queue)
&& logical_exit_claim.handle_key == shared_db_file_key(&inner.db_file)
&& logical_exit_claim.active,
"transaction exit requires a live claim for the exact identity queue"
);
let remaining_active_transactions =
inner.active_transactions.checked_sub(1).ok_or_else(|| {
FrankenError::internal("transaction exit would underflow active transactions")
})?;
let writer_active_after_exit = inner.writer_active && !releases_writer_baton;
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
if remaining_active_transactions == 0 {
shared_db_restore_external_snapshot_attempt(&inner.db_file, &cleanup_cx).await?;
} else {
let preserve_level = retained_lock_level_after_txn_exit(
remaining_active_transactions,
writer_active_after_exit,
);
shared_db_unlock(&inner.db_file, &cleanup_cx, preserve_level).await?;
}
inner.active_transactions = remaining_active_transactions;
let notify_writer_idle = releases_writer_baton && release_single_writer_baton(inner);
Ok(notify_writer_idle)
}
impl<V> TransactionHandle for SimpleTransaction<V>
where
V: Vfs + Send,
V::File: Send + Sync + 'static,
{
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
fn get_page<'a>(
&'a self,
cx: &'a Cx,
page_no: PageNumber,
) -> impl Future<Output = Result<PageData>> + 'a {
async move {
self.ensure_no_pending_group_commit_attempt()?;
if self.contains_freed_page(page_no) {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"page {} was freed earlier in this transaction",
page_no.get()
),
});
}
if let Some(staged) = self.write_set.get(&page_no) {
return Ok(staged.published_page());
}
// Pages that were allocated after a savepoint and then rolled back
// should return zeros, not BusySnapshot error.
if self.rolled_back_pages.contains(&page_no) {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
return Ok(PageData::zeroed(inner.page_size));
}
// MVCC db_size guard: pages beyond the transaction's snapshot db_size
// did not exist when this snapshot was taken. A transaction handle is
// bound to one coherent snapshot for its lifetime, so a later commit
// must never expand this boundary in-place. Callers that need the
// latest state must begin a new transaction at the statement boundary.
//
// Exception: pages allocated by THIS transaction (in allocated_from_eof
// or allocated_from_freelist) are allowed even if beyond published_db_size.
if page_no.get() > self.published_db_size.get() {
let page_allocated_by_this_txn = self.allocated_from_eof.contains(&page_no)
|| self.allocated_from_freelist.contains(&page_no)
|| self.page_lease.contains(&page_no);
if !page_allocated_by_this_txn {
let latest_snapshot = self.published.snapshot();
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id = "page_beyond_fixed_snapshot_db_size",
page_no = page_no.get(),
published_db_size = self.published_db_size.get(),
latest_db_size = latest_snapshot.db_size,
published_commit_seq = self.published_visible_commit_seq.get().get(),
latest_commit_seq = latest_snapshot.visible_commit_seq.get(),
"refused to advance a transaction's fixed snapshot"
);
return Err(FrankenError::BusySnapshot {
conflicting_pages: format!(
"page {} > snapshot db_size {} (latest: {})",
page_no.get(),
self.published_db_size.get(),
latest_snapshot.db_size
),
});
}
}
// A transaction that has already observed a page owns that exact
// snapshot image for the rest of its lifetime. Consult the local
// cache before every shared publication/cache source; otherwise a
// concurrent commit can replace those global latest-image planes and
// make a re-read return different bytes even though this handle's
// visible commit sequence remains fixed (GH #129).
if let Some(cached) = self.txn_read_cache.borrow().get(&page_no) {
return Ok(cached.clone());
}
self.ensure_uncached_snapshot_sequence(
page_no,
self.published.snapshot().visible_commit_seq,
)?;
let single_connection_fast_path = self.single_connection_fast_path_enabled();
let trace_read_start =
tracing::enabled!(target: "fsqlite.snapshot_publication", tracing::Level::TRACE)
.then(Instant::now);
let mut published_retry_count = 0_usize;
while let Some(snapshot) = self
.published
.snapshot_for_page_plane(self.published_visible_commit_seq.get())
{
if page_no.get() > snapshot.db_size {
if self.published.current_sequence_gen() == snapshot.snapshot_gen {
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id = "zero_fill_read",
snapshot_gen = snapshot.snapshot_gen,
visible_commit_seq = snapshot.visible_commit_seq.get(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
read_retry_count = self.published.read_retry_count(),
page_set_size = snapshot.page_set_size,
elapsed_ns = trace_read_start
.map_or(0, |start| {
u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX)
}),
"resolved zero-filled page from published metadata"
);
let page = PageData::from_vec(vec![0_u8; self.pool.page_size()]);
self.cache_transaction_read_page(page_no, &page);
return Ok(page);
}
self.published.record_retry();
if published_retry_count >= PUBLISHED_READ_FAST_RETRY_LIMIT {
break;
}
self.published.wait_for_sequence_change(
snapshot.snapshot_gen,
PUBLISHED_SNAPSHOT_WAIT_SLICE,
);
published_retry_count = published_retry_count.saturating_add(1);
continue;
}
if let Some(page) = self.published.try_get_page(page_no) {
if self.published.current_sequence_gen() == snapshot.snapshot_gen {
self.published.note_published_hit();
tracing::trace!(
target: "fsqlite.snapshot_publication",
trace_id = cx.trace_id(),
run_id = "pager-publication",
scenario_id = "published_read_hit",
snapshot_gen = snapshot.snapshot_gen,
visible_commit_seq = snapshot.visible_commit_seq.get(),
publication_mode = SNAPSHOT_PUBLICATION_MODE,
read_retry_count = self.published.read_retry_count(),
page_set_size = snapshot.page_set_size,
elapsed_ns = trace_read_start
.map_or(0, |start| {
u64::try_from(start.elapsed().as_nanos()).unwrap_or(u64::MAX)
}),
"served page from published snapshot"
);
self.cache_transaction_read_page(page_no, &page);
return Ok(page);
}
self.published.record_retry();
if published_retry_count >= PUBLISHED_READ_FAST_RETRY_LIMIT {
break;
}
self.published.wait_for_sequence_change(
snapshot.snapshot_gen,
PUBLISHED_SNAPSHOT_WAIT_SLICE,
);
published_retry_count = published_retry_count.saturating_add(1);
continue;
}
break;
}
let committed_snapshot = self.published.snapshot();
if committed_snapshot.visible_commit_seq == self.published_visible_commit_seq.get()
&& committed_snapshot.journal_mode != JournalMode::Wal
&& page_no.get() <= committed_snapshot.db_size
{
// bd-perf (V1.2): Use get_shared to get PageData directly,
// avoiding the 4KB memcpy + separate Arc allocation of get_copy.
if let Some(page_data) = self.cache.get_shared(page_no) {
self.cache_transaction_read_page(page_no, &page_data);
return Ok(page_data);
}
}
// WAL mode fast path: try shared-lock read first (bd-db300.3.8.7).
if self.journal_mode == JournalMode::Wal
&& let Some(data) =
read_page_from_wal_backend(&self.wal_backend, cx, page_no).await?
{
let page = PageData::from_vec(data);
self.ensure_uncached_snapshot_sequence(
page_no,
self.published.snapshot().visible_commit_seq,
)?;
self.cache_transaction_read_page(page_no, &page);
return Ok(page);
}
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
self.ensure_uncached_snapshot_sequence(page_no, inner.commit_seq)?;
let data = inner
.read_page_copy(cx, &self.cache, &self.wal_backend, page_no)
.await?;
let page = PageData::from_vec(data);
let publish_update = PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
};
let publish_page = page_no.get() <= inner.db_size
&& inner.commit_seq == self.published_visible_commit_seq.get();
drop(inner);
if publish_page && !single_connection_fast_path {
self.published
.publish_observed_page(cx, publish_update, page_no, page.clone());
}
// Cache the page read from inner.lock() for future reads.
self.cache_transaction_read_page(page_no, &page);
Ok(page)
}
}
fn prefetch_page_hint(&self, _cx: &Cx, page_no: PageNumber) {
if self.has_pending_recovery_barrier() {
return;
}
if let Some(staged) = self.write_set.get(&page_no) {
prefetch_l1_read(staged.as_page_bytes().as_ptr());
return;
}
if let Ok(txn_read_cache) = self.txn_read_cache.try_borrow()
&& let Some(page) = txn_read_cache.get(&page_no)
{
prefetch_l1_read(page.as_bytes().as_ptr());
return;
}
if self.published.page_plane_visible_commit_seq() == self.published_visible_commit_seq.get()
{
self.published.prefetch_page_hint(page_no);
}
self.cache.prefetch_page_hint(page_no);
}
fn write_page<'a>(
&'a mut self,
cx: &'a Cx,
page_no: PageNumber,
data: &'a [u8],
) -> impl Future<Output = Result<()>> + 'a {
async move {
self.ensure_writer(cx).await?;
// Fast path: a second (or Nth) write to the same page within the
// same transaction reuses the already-allocated StagedPage buffer
// instead of allocating a fresh PageBuf from the pool and dropping
// the old one. This is a frequent pattern for cursor-driven
// workloads that repeatedly restamp the same B-tree leaf as rows
// accumulate.
if let Some(existing) = self.write_set.get_mut(&page_no)
&& existing.try_overwrite_bytes_in_place(data)
{
self.writes_observed = true;
self.remove_freed_page_if_present(page_no);
STAGED_PAGE_OVERWRITE_STEALS_TOTAL.fetch_add(1, AtomicOrdering::Relaxed);
return Ok(());
}
let staged = self.stage_page_bytes(data)?;
// Mutate transaction bookkeeping only after fallible staging succeeds;
// a capacity error must leave the prior free/write state untouched.
self.writes_observed = true;
self.remove_freed_page_if_present(page_no);
insert_staged_page(
&mut self.write_set,
&mut self.write_pages_sorted,
page_no,
staged,
);
Ok(())
}
}
fn write_page_data<'a>(
&'a mut self,
cx: &'a Cx,
page_no: PageNumber,
data: PageData,
) -> impl Future<Output = Result<()>> + 'a {
async move {
self.ensure_writer(cx).await?;
// Same-page steal fast path (see `write_page`). If the existing staged
// image cannot be overwritten because it has been published or is no
// longer single-owner, replace that map entry directly. Routing through
// `insert_staged_page` would hash and insert the same key again even
// though `write_pages_sorted` is already correct.
if let Some(existing) = self.write_set.get_mut(&page_no)
&& existing.try_overwrite_page_data_in_place(&data)
{
self.writes_observed = true;
self.remove_freed_page_if_present(page_no);
STAGED_PAGE_OVERWRITE_STEALS_TOTAL.fetch_add(1, AtomicOrdering::Relaxed);
return Ok(());
}
let staged = StagedPage::from_page_data_with_cache_recovery(
&self.pool,
&self.cache,
data,
"transaction_write_page_data",
)?;
self.writes_observed = true;
self.remove_freed_page_if_present(page_no);
if let Some(existing) = self.write_set.get_mut(&page_no) {
*existing = staged;
return Ok(());
}
insert_staged_page(
&mut self.write_set,
&mut self.write_pages_sorted,
page_no,
staged,
);
Ok(())
}
}
fn try_take_staged_page_data(&mut self, page_no: PageNumber) -> Option<PageData> {
if self.has_pending_recovery_barrier() {
return None;
}
let staged = self.write_set.remove(&page_no)?;
match staged.try_into_unpublished_owned_page_data() {
Ok(data) => {
remove_page_sorted(&mut self.write_pages_sorted, page_no);
Some(data)
}
Err(staged) => {
self.write_set.insert(page_no, staged);
None
}
}
}
fn try_mutate_staged_page_data(
&mut self,
page_no: PageNumber,
f: &mut dyn FnMut(&mut PageData),
) -> bool {
if self.has_pending_recovery_barrier() {
return false;
}
let Some(staged) = self.write_set.get_mut(&page_no) else {
return false;
};
if staged.published.get().is_some() {
return false;
}
let StagedPageBacking::Owned(data) = &mut staged.backing else {
return false;
};
f(data);
true
}
fn restore_staged_page_data<'a>(
&'a mut self,
cx: &'a Cx,
page_no: PageNumber,
data: PageData,
) -> impl Future<Output = Result<()>> + 'a {
async move {
self.ensure_writer(cx).await?;
let staged = StagedPage::from_page_data_with_cache_recovery(
&self.pool,
&self.cache,
data,
"restore_staged_page_data",
)?;
// #70 ghost-commit guard: mark only after fallible staging succeeds.
self.writes_observed = true;
insert_staged_page(
&mut self.write_set,
&mut self.write_pages_sorted,
page_no,
staged,
);
Ok(())
}
}
fn allocate_page<'a>(
&'a mut self,
cx: &'a Cx,
) -> impl Future<Output = Result<PageNumber>> + 'a {
async move {
self.ensure_writer(cx).await?;
// ── Local lease fast path ──────────────────────────────────────
// If we have pre-allocated pages from a previous batch, hand one
// out without touching the global `inner` mutex at all.
if let Some(page) = self.page_lease.pop() {
self.allocated_from_eof.push(page);
return Ok(page);
}
// Pages freed earlier in the same transaction stay quarantined until
// commit. Reusing them immediately lets one B-tree operation hand a
// page to another tree before the old ownership is durably retired,
// which can surface as cross-tree page aliasing on disk.
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !self.memory_db_bump_alloc {
let committed_freelist_is_snapshot_pinned =
self.mode == TransactionMode::Concurrent || inner.active_transactions > 1;
if committed_freelist_is_snapshot_pinned {
// Concurrent writers always read against a fixed snapshot. So do
// immediate/deferred writers when another local transaction is
// still active, because that older reader snapshot can still
// observe the committed image being replaced. In both cases, pages
// at or below db_size are part of some still-visible committed
// state and cannot be safely reused from the live global freelist
// without versioned freelist metadata. Pages above db_size are
// different: they only exist because an earlier transaction
// allocated EOF pages and then rolled back, so reusing them does
// not violate committed-snapshot visibility and avoids
// page-count holes.
//
// Both reuse arms below share one safety gate: this
// transaction must be the pager's ONLY live transaction
// (`active_transactions == 1`) with a current snapshot
// (`published_visible_commit_seq == inner.commit_seq`).
// While another local transaction pins an older view, the
// pager cannot refresh durable metadata, so a page in the
// local `> db_size` pool may meanwhile have been claimed
// and committed by a PEER connection growing the same
// file — handing it out again would alias the peer's
// committed page (pinned by test_concurrent_rollback_
// quarantines_peer_claimed_eof_page_until_refresh). When
// the gate holds, begin-time refresh has replaced the
// freelist from durable state, and any race with a peer
// allocating the same page after our snapshot resolves at
// commit through WAL first-committer-wins conflict
// detection, exactly as for EOF growth.
let sole_current_snapshot = inner.active_transactions == 1
&& self.published_visible_commit_seq.get() == inner.commit_seq;
if sole_current_snapshot
&& let Some(idx) = inner
.freelist
.iter()
.rposition(|page| page.get() > inner.db_size)
{
let page = inner.freelist.remove(idx);
self.allocated_from_freelist.push(page);
return Ok(page);
}
// GH#302 bounded snapshot-safe reclamation: committed
// freelist pages at or below db_size ARE reusable under
// the same gate — with only our own (current) snapshot
// live, every page on the committed freelist is already
// free *in our own snapshot* and cannot be live content
// of any tree we can read, so the pop is exactly as safe
// as the non-concurrent arm below. External readers at an
// older mark keep reading the pre-reuse frame through
// WAL/journal versioning, the same way they survive any
// ordinary in-place page rewrite, and a racing external
// writer popping the same committed free page aborts
// second-committer (test_journal_commit_detects_cross_
// connection_committed_freelist_reuse_alias).
//
// Without this arm, default (concurrent) transactions
// never reused committed free pages and every churn
// workload grew the file at EOF without bound.
if sole_current_snapshot && let Some(page) = inner.freelist.pop() {
self.allocated_from_freelist.push(page);
return Ok(page);
}
} else if let Some(page) = inner.freelist.pop() {
self.allocated_from_freelist.push(page);
return Ok(page);
}
}
// ── EOF allocation ──────────────────────────────────────────────
// For concurrent transactions that have already allocated at least
// one page, batch-allocate PAGE_LEASE_BATCH_SIZE pages in one lock
// acquisition to reduce mutex contention during B-tree splits.
// The first allocation is always single-page to avoid over-reserving
// for short transactions. Non-concurrent writers always allocate
// one page at a time since there's no lock convoy to avoid.
let pending_byte_page = (0x4000_0000 / inner.page_size.get()) + 1;
let already_allocated =
!self.allocated_from_eof.is_empty() || !self.allocated_from_freelist.is_empty();
let batch = if self.mode == TransactionMode::Concurrent && already_allocated {
PAGE_LEASE_BATCH_SIZE
} else {
1
};
let mut first_page: Option<PageNumber> = None;
for _ in 0..batch {
let mut raw = inner.next_page;
if raw == pending_byte_page {
raw = raw.saturating_add(1);
}
let next = raw.saturating_add(1);
// Stop the batch if next_page can no longer advance (u32::MAX
// saturation). Continuing would hand out duplicate page numbers.
if next == raw {
break;
}
inner.next_page = next;
if let Some(page) = PageNumber::new(raw) {
if first_page.is_none() {
first_page = Some(page);
} else {
self.page_lease.push(page);
}
}
}
drop(inner);
let page = first_page.ok_or_else(|| FrankenError::OutOfRange {
what: "allocated page number".to_owned(),
value: "0".to_owned(),
})?;
self.allocated_from_eof.push(page);
Ok(page)
}
}
fn free_page<'a>(
&'a mut self,
cx: &'a Cx,
page_no: PageNumber,
) -> impl Future<Output = Result<()>> + 'a {
async move {
self.ensure_writer(cx).await?;
if page_no == PageNumber::ONE {
return Err(FrankenError::OutOfRange {
what: "free page number".to_owned(),
value: page_no.get().to_string(),
});
}
if !self.contains_freed_page(page_no) {
self.freed_pages.push(page_no);
self.note_freed_page_bound(page_no);
}
if self.write_set.remove(&page_no).is_some() {
remove_page_sorted(&mut self.write_pages_sorted, page_no);
}
Ok(())
}
}
#[allow(clippy::too_many_lines)]
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
fn commit<'a>(&'a mut self, cx: &'a Cx) -> impl Future<Output = Result<()>> + 'a {
async move {
if self.finished {
return Ok(());
}
if self.rollback_commit_finalization_pending {
return self.finish_durable_rollback_commit(cx).await;
}
// Rollback/close is another structured owner for a lock
// restoration stranded by a dropped commit future.
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
if self.owned_rollback_recovery.is_some() {
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
self.recover_pending_rollback_journal(&cleanup_cx).await?;
}
if let Some(attempt) = self.pending_group_commit_attempt.clone() {
match attempt.reconcile_global_from_queue()? {
PendingGroupCommitTxnResolution::Pending => {
return Err(FrankenError::BusyRecovery);
}
PendingGroupCommitTxnResolution::NotCommitted => {
self.restore_not_committed_wal_attempt()?;
}
PendingGroupCommitTxnResolution::Authorized(_) => {
return self.finish_authorized_wal_attempt(cx, true).await;
}
}
}
if self.identity_rollback_recovery_pending() {
if self.has_local_changes_to_commit() {
return Err(FrankenError::BusyRecovery);
}
self.drain_nonowner_during_rollback_recovery(cx, true)
.await?;
return Ok(());
}
self.validate_namespace_binding()?;
if !self.is_writer {
let logical_exit_claim = GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
cx,
)
.await?;
if self.identity_rollback_recovery_owner() != self.owned_rollback_recovery {
drop(logical_exit_claim);
self.drain_nonowner_during_rollback_recovery(cx, true)
.await?;
return Ok(());
}
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !self.exact_recovery_owner_matches_inner(&inner) {
let foreign_pending = self.identity_rollback_recovery_pending()
&& self.owned_rollback_recovery.is_none();
drop(inner);
drop(logical_exit_claim);
if foreign_pending {
self.drain_nonowner_during_rollback_recovery(cx, true)
.await?;
return Ok(());
}
return Err(FrankenError::BusyRecovery);
}
// Return any unused lease pages to the freelist so they can
// be reused by other transactions.
return_pages_to_freelist(&mut inner.freelist, self.page_lease.drain(..));
let notify_writer_idle = coordinated_transaction_exit(
&self.group_commit_queue,
cx,
&mut inner,
false,
&logical_exit_claim,
)
.await?;
debug_assert!(!notify_writer_idle);
drop(inner);
self.committed = true;
self.maintenance_lease.take();
self.finished = true;
// IMPL-3 / AG-4B: reset scratch arena on read-only commit path.
self.scratch_arena.reset();
return Ok(());
}
if self.vfs.is_memory()
&& self.memory_db_bump_alloc
&& !self.retained_memory_overlay_dirty_pages.is_empty()
{
self.materialize_retained_memory_overlay_into_write_set()?;
}
if !self.has_pending_writes() {
// #70 BUG-A ghost-commit guard: if writes were staged earlier on
// this transaction (write_page / write_page_data / allocate_page)
// but the write_set and freelist-dirty are BOTH empty now,
// something dropped the staged state without rolling the
// transaction back. Silently returning Ok here is what produced
// the swarm's "INSERT and same-txn UPDATE both atomically
// disappear, commit returns Ok" failure mode. Surface as a
// retryable Busy instead so retry_fsqlite can reissue the
// transaction rather than claiming success for lost writes.
if self.writes_observed {
return Err(FrankenError::internal(
"transaction committed with observed writes but empty write_set; \
state was dropped between staging and commit",
));
}
let logical_exit_claim = GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
cx,
)
.await?;
if self.identity_rollback_recovery_owner() != self.owned_rollback_recovery {
drop(logical_exit_claim);
self.drain_nonowner_during_rollback_recovery(cx, true)
.await?;
return Ok(());
}
let inner_arc = Arc::clone(&self.inner);
let mut inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !self.exact_recovery_owner_matches_inner(&inner) {
let foreign_pending = self.identity_rollback_recovery_pending()
&& self.owned_rollback_recovery.is_none();
drop(inner);
drop(logical_exit_claim);
if foreign_pending {
self.drain_nonowner_during_rollback_recovery(cx, true)
.await?;
return Ok(());
}
return Err(FrankenError::BusyRecovery);
}
self.restore_uncommitted_allocations_for_clean_commit(&mut inner);
let notify_writer_idle = coordinated_transaction_exit(
&self.group_commit_queue,
cx,
&mut inner,
self.mode != TransactionMode::Concurrent,
&logical_exit_claim,
)
.await?;
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
self.committed = true;
self.maintenance_lease.take();
self.finished = true;
// IMPL-3 / AG-4B: reset scratch arena on no-writes commit path.
self.scratch_arena.reset();
return Ok(());
}
// =====================================================================
// D1-CRITICAL: Split inner lock into prepare/IO/publish phases (bd-3wop3.8)
//
// REMAINING CLIFF (bd-wee9a, 2026-04-24): even with Phase A shrunk to
// ~20 µs, the per-pager `self.inner.lock()` taken below still
// serializes all writers of this pager. At MT 2t,
// `FSQLITE_TRACE_GROUP_COMMIT=1` shows every flush with
// `members=[N]` (size-1 batches) — writers never arrive in the
// GroupCommitQueue simultaneously because peer N+1 is blocked on
// this mutex while peer N completes Phases A+B+C. Raising
// `GROUP_COMMIT_SPARSE_ARRIVAL_WAIT` to 2 ms did not change the
// batching pattern. See bd-wee9a for the full analysis and
// candidate fixes (shrink Phase A further / per-txn prep buffers /
// optimistic BEGIN CONCURRENT fast path).
//
// BEFORE: inner.lock() held for entire commit (~100us) serializing all threads
// AFTER:
// Phase A (prepare, ~20us): Hold inner.lock() briefly to snapshot state
// DROP inner.lock() <-- allows Thread B to start Phase A while Thread A does I/O
// Phase B (WAL I/O, ~50us): Acquires inner.lock() only when needed
// Phase C (publish, ~10us): Re-acquires inner.lock() for finalization
//
// This allows N threads to overlap their prepare phases, reducing
// serialization from N*100us to N*20us + 50us + 10us.
// =====================================================================
// ── Full commit path timing instrumentation ──
let phase_timing = commit_phase_timing_enabled();
let t_commit_start = phase_timing.then(Instant::now);
let pager_commit_profile_active = pager_commit_profile_enabled();
record_pager_commit_call(pager_commit_profile_active);
let t_pager_phase_a_start = pager_commit_profile_start(pager_commit_profile_active);
// Journal and private-memory Phase B can mutate the durable image
// before Phase C. Reserve their exact logical exit before any such
// mutation so terminal cleanup can never fail with BusyRecovery
// after publication has already won.
let non_wal_exit_claim = if self.journal_mode == JournalMode::Wal {
None
} else {
Some(
GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
cx,
)
.await?,
)
};
if non_wal_exit_claim.is_some()
&& self.identity_rollback_recovery_owner() != self.owned_rollback_recovery
{
drop(non_wal_exit_claim);
return Err(FrankenError::BusyRecovery);
}
// Phase A: Prepare write_set under inner lock (~20us)
// Snapshot state needed for WAL I/O, then DROP inner.lock() immediately.
let inner_arc = Arc::clone(&self.inner);
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
let mut inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if non_wal_exit_claim.is_some() && !self.exact_recovery_owner_matches_inner(&inner) {
drop(inner);
drop(non_wal_exit_claim);
return Err(FrankenError::BusyRecovery);
}
let committed_db_size = self.committed_db_size_with_inner(&inner);
let mut returned_allocations = self.drain_unstaged_allocated_pages();
returned_allocations.page_lease.append(&mut self.page_lease);
let pending_returned_pages = returned_allocations.all_pages();
let mut pending_free_pages = pending_returned_pages.clone();
pending_free_pages.extend(self.freed_pages.iter().copied());
// Declared outside the block so it survives to Phase C where freed
// pages are promoted into inner.freelist after successful WAL commit.
let mut pending_freed: Vec<PageNumber>;
let mut wal_attempt: Option<Arc<PendingGroupCommitTxnAttempt<V::File>>> = None;
let cross_process_conflict_pages: Vec<PageNumber>;
{
// ShardedPageCache uses per-shard internal locking
//
let freelist_dirty = self.freelist_metadata_dirty_with_pending_free_pages(
&inner,
committed_db_size,
&pending_free_pages,
);
// CRITICAL FIX (beads_rust#138): Do NOT push pages that become
// free into inner.freelist during Phase A. In the split-lock WAL
// commit path, inner.lock() is released between Phase A and Phase B.
// If free pages are pushed here, a concurrent transaction's Phase A
// can observe and reuse them before this commit is durable, creating
// orphaned pages ("page N is never used") if WAL ordering flips.
//
// Instead, drain them into local vectors and pass the combined
// pending_free_pages to the serializer, which builds a predicted
// freelist without mutating inner.freelist. The actual promotion is
// deferred to Phase C (after WAL success), or to the failure cleanup
// for pages that were merely unused allocations.
//
// Capture the semantic Page 1 plan before freelist serialization
// injects synthetic Page 1 metadata into the write_set. Otherwise
// pure freelist bookkeeping would masquerade as a direct Page 1
// write and become a cross-process first-committer-wins conflict.
let wal_page1_plan =
self.classify_wal_page_one_write(inner.db_size, freelist_dirty);
pending_freed = std::mem::take(&mut self.freed_pages);
self.freed_page_bounds = None;
if self.journal_mode == JournalMode::Wal {
let staged_page_high_water =
self.staged_page_high_water(self.snapshot_db_size());
let live_committed_allocations = self
.allocated_from_freelist
.iter()
.chain(&self.allocated_from_eof)
.filter(|page| self.write_set.contains_key(*page))
.copied()
.collect();
let attempt = Arc::new(PendingGroupCommitTxnAttempt::new(
&self.group_commit_queue,
Arc::clone(&self.inner),
Arc::clone(&self.db_file),
Arc::clone(&self.committed_snapshot),
Arc::clone(&self.published),
Arc::clone(&self.writer_idle),
cleanup_child_cx(cx),
committed_db_size,
self.mode,
self.is_writer,
staged_page_high_water,
std::mem::take(&mut returned_allocations),
std::mem::take(&mut pending_freed),
live_committed_allocations,
));
self.pending_group_commit_attempt = Some(Arc::clone(&attempt));
wal_attempt = Some(attempt);
}
if freelist_dirty
&& let Err(e) = serialize_freelist_to_write_set(
cx,
&mut inner,
&self.cache,
&self.wal_backend,
&self.pool,
&mut self.write_set,
&mut self.write_pages_sorted,
committed_db_size,
&pending_free_pages,
wal_attempt.as_ref().map(|attempt| &attempt.phase_a_undo),
)
.await
{
if wal_attempt.is_some() {
drop(inner);
self.restore_not_committed_wal_attempt()?;
} else {
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
}
return Err(e);
}
// D1-CRITICAL Fix: In WAL mode, page 1 must be written to WAL not
// only when it was explicitly dirty, but also when the database
// grows (new pages allocated beyond current db_size). Without this,
// other connections reading page 1 from WAL won't see the updated
// page_count header, causing BusySnapshot errors.
let must_write_page1 = if self.journal_mode == JournalMode::Wal {
wal_page1_plan.requires_page_one_rewrite()
|| wal_page1_plan.requires_page_count_advance()
} else {
true
};
if must_write_page1 {
if let Some(attempt) = wal_attempt.as_ref() {
attempt
.phase_a_undo
.capture(&self.write_set, PageNumber::ONE);
}
let mut page1 = match ensure_page_one_in_write_set(
cx,
&inner,
&self.cache,
&self.wal_backend,
&self.pool,
&mut self.write_set,
)
.await
{
Ok(p) => p,
Err(e) => {
if wal_attempt.is_some() {
drop(inner);
self.restore_not_committed_wal_attempt()?;
} else {
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(
&mut inner.freelist,
pending_returned_pages,
);
}
return Err(e);
}
};
let page1_bytes = page1.as_page_bytes_mut();
if page1_bytes.len() >= DATABASE_HEADER_SIZE {
let mut page_count_bytes = [0_u8; 4];
page_count_bytes.copy_from_slice(&page1_bytes[28..32]);
let existing_page_count = u32::from_be_bytes(page_count_bytes);
let new_change_counter = inner.commit_seq.get().wrapping_add(1) as u32;
// Offset 24..28: change counter (big-endian u32)
page1_bytes[24..28].copy_from_slice(&new_change_counter.to_be_bytes());
if self.journal_mode != JournalMode::Wal
|| wal_page1_plan.requires_page_count_advance()
{
let new_db_size = committed_db_size.max(existing_page_count);
// Offset 28..32: page count (big-endian u32)
page1_bytes[28..32].copy_from_slice(&new_db_size.to_be_bytes());
}
// Offset 92..96: version-valid-for
page1_bytes[92..96].copy_from_slice(&new_change_counter.to_be_bytes());
}
insert_staged_page(
&mut self.write_set,
&mut self.write_pages_sorted,
PageNumber::ONE,
page1,
);
}
cross_process_conflict_pages = if self.journal_mode == JournalMode::Wal {
self.predicted_conflict_pages_for_wal_commit_with_inner(
&inner,
wal_page1_plan,
&pending_free_pages,
)
} else {
Vec::new()
};
}
let cross_process_conflict_page_baselines =
self.conflict_page_baselines(&cross_process_conflict_pages);
let wal_current_db_size = inner.db_size;
let wal_sync_policy = inner.wal_commit_sync_policy;
let t_phase_a_done = phase_timing.then(Instant::now);
record_pager_commit_duration(&PAGER_COMMIT_PHASE_A_TIME_NS, t_pager_phase_a_start);
// Phase B: Commit via WAL or journal
// D1-CRITICAL: For WAL mode, we release inner.lock() here so other
// threads can start their Phase A (prepare) while we wait for
// the consolidator lock. This is the key parallelization win.
let mut wal_publication_authorization = None;
let commit_result = if self.journal_mode == JournalMode::Wal {
// Drop inner lock BEFORE acquiring consolidator lock.
// This allows other threads to run Phase A concurrently.
drop(inner);
// WAL mode: Use group commit for same-process batching.
// commit_wal_group_commit will acquire consolidator.lock() first,
// then briefly inner.lock() for the actual WAL I/O.
let t_wal_commit_start = pager_commit_profile_start(pager_commit_profile_active);
let result = Self::commit_wal_group_commit_with_snapshot(
cx,
&self.wal_backend,
&self.inner,
Some(Arc::clone(&self.published)),
wal_current_db_size,
wal_sync_policy,
&self.write_set,
&self.write_pages_sorted,
&cross_process_conflict_pages,
self.wal_conflict_snapshot,
&cross_process_conflict_page_baselines,
&self.group_commit_queue,
&mut wal_publication_authorization,
wal_attempt.as_ref(),
)
.await;
record_pager_commit_duration(&PAGER_COMMIT_WAL_TIME_NS, t_wal_commit_start);
// Re-acquire inner lock for Phase C (finalize).
#[cfg(test)]
record_commit_fast_path_lock(CommitFastPathLockClass::PagerInner);
inner = match inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))
{
Ok(guard) => guard,
Err(e) => return Err(e),
};
result
} else if self.memory_db_bump_alloc {
// Private `:memory:` commits do not need rollback-journal
// creation/sync. Page 1 has already been staged above, so flush
// the final committed image directly once at the release boundary.
let t_memory_flush_start = pager_commit_profile_start(pager_commit_profile_active);
let result = Self::flush_write_set_to_db_file_batch(
cx,
&inner,
&self.write_set,
&self.write_pages_sorted,
)
.await;
record_pager_commit_duration(
&PAGER_COMMIT_MEMORY_FLUSH_TIME_NS,
t_memory_flush_start,
);
result
} else {
// Journal mode: Direct commit (no group commit)
// Journal mode keeps inner locked throughout - no parallelization.
let t_journal_commit_start =
pager_commit_profile_start(pager_commit_profile_active);
// The freelist-alias check needs the transaction's FULL set of
// freelist pops. `drain_unstaged_allocated_pages` (Phase A) has
// already moved never-written allocations out of
// `allocated_from_freelist` into `pending_returned_pages`, so
// restore both when reconstructing the begin-time freelist view
// (EOF-origin entries are filtered out by the db-size bound).
let mut alias_check_restored = self.allocated_from_freelist.clone();
alias_check_restored.extend_from_slice(&pending_returned_pages);
let mut result = Self::commit_journal(
cx,
&self.vfs,
&mut inner,
RollbackJournalCommitInput {
journal_path: &self.journal_path,
write_set: &self.write_set,
original_db_size: self.original_db_size,
allocated_from_freelist: &alias_check_restored,
},
&mut self.owned_rollback_recovery,
)
.await;
if result.is_ok() {
match self.owned_rollback_recovery {
Some(owner)
if inner.rollback_journal_recovery_owner == Some(owner)
&& matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
) =>
{
self.rollback_commit_finalization_pending = true;
}
_ => {
result = Err(FrankenError::internal(
"successful rollback-journal commit omitted its exact durable receipt",
));
}
}
}
record_pager_commit_duration(&PAGER_COMMIT_JOURNAL_TIME_NS, t_journal_commit_start);
result
};
let t_phase_b_done = phase_timing.then(Instant::now);
if self.journal_mode == JournalMode::Wal {
drop(inner);
let attempt = wal_attempt.clone().ok_or_else(|| {
FrankenError::internal(
"WAL commit reached Phase B without a logical transaction owner",
)
})?;
let commit_error = commit_result.err();
match attempt.resolution() {
PendingGroupCommitTxnResolution::Authorized(_) => {
if commit_error.is_none() && wal_publication_authorization.is_none() {
return Err(FrankenError::internal(
"successful WAL group commit omitted publication authorization",
));
}
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
self.finish_authorized_wal_attempt(&cleanup_cx, true)
.await?;
return Ok(());
}
PendingGroupCommitTxnResolution::NotCommitted => {
self.restore_not_committed_wal_attempt()?;
return Err(commit_error.unwrap_or_else(|| {
FrankenError::internal(
"WAL group commit reported success after a NotCommitted verdict",
)
}));
}
PendingGroupCommitTxnResolution::Pending => {
return Err(commit_error.unwrap_or(FrankenError::BusyRecovery));
}
}
}
if commit_result.is_ok() {
let t_phase_c_metadata_start =
pager_commit_profile_start(pager_commit_profile_active);
// Phase C1 (FAST, under inner.lock): Update metadata only.
// Now that WAL I/O has succeeded, promote pages that became free
// into inner.freelist. This is the deferred half of the Phase A
// fix: reusable pages become visible only after the WAL commit is
// durable.
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
return_pages_to_freelist(&mut inner.freelist, pending_freed);
// Cross-process #70: the group-commit flusher already set
// inner.db_size to final_db_size in WAL mode, but with concurrent
// peer commits extending the same file, our flusher's view of
// the consolidated db_size can lag the actual max committed page.
// Use fetch_max semantics so inner.db_size never regresses below
// what we just committed, and so peer extensions we subsequently
// observe in refresh_committed_state keep monotonic visibility.
inner.db_size = inner.db_size.max(committed_db_size);
// Keep volatile EOF lease pages in the in-memory freelist even
// when they are above the durable page_count. They are deliberately
// filtered out by serialize_freelist_to_write_set(), so page 1
// stays SQLite-compatible on disk. But next_page has already
// advanced past those page numbers; dropping them here creates
// permanent in-process holes that later commits can expose as
// "Page N: never used" once page_count grows past the gap.
let wal_publication_intent = wal_publication_authorization
.as_ref()
.map(|authorization| {
parallel_wal_publication_intent(
authorization,
inner.db_size,
inner.journal_mode,
inner.freelist.len(),
inner.checkpoint_active,
)
})
.transpose()?;
if let Some(intent) = wal_publication_intent {
inner.record_local_wal_commit_at(intent.visible_commit_seq);
} else {
inner.record_local_commit();
}
if self.rollback_commit_finalization_pending {
// From this point onward the durable decision and the exact
// in-memory metadata application are both recorded. A
// dropped future must finish logical exit as a commit.
self.committed = true;
}
let t_file_size_start = pager_commit_profile_start(pager_commit_profile_active);
if self.memory_db_bump_alloc {
inner.committed_db_file_size_bytes =
u64::from(inner.db_size) * u64::from(inner.page_size.get());
}
record_pager_commit_duration(&PAGER_COMMIT_FILE_SIZE_TIME_NS, t_file_size_start);
let publish_update = PublishedPagerUpdate {
visible_commit_seq: wal_publication_intent
.map_or(inner.commit_seq, |intent| intent.visible_commit_seq),
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
};
let single_connection_fast_path = self.single_connection_fast_path_enabled();
// In an isolated single-connection commit, page-plane publication
// is unnecessary even when page 1 was staged for internal durable
// bookkeeping such as change-counter/page-count maintenance. The
// committed bytes are already authoritative in the pager/cache.
let metadata_only_single_connection_fast_path = single_connection_fast_path;
// bd-db300.5.3.3.1: publish immutable snapshot while inner is
// still held so any later multi-connection readers inherit the
// committed metadata even if this commit skipped page-plane publish.
self.publish_committed_snapshot_from_inner(&inner);
let t_unlock_start = pager_commit_profile_start(pager_commit_profile_active);
let notify_writer_idle = coordinated_transaction_exit(
&self.group_commit_queue,
cx,
&mut inner,
self.mode != TransactionMode::Concurrent,
non_wal_exit_claim
.as_ref()
.expect("non-WAL commit must retain its pre-durability logical exit claim"),
)
.await?;
drop(non_wal_exit_claim);
if self.rollback_commit_finalization_pending {
// External snapshot restoration and active-transaction
// accounting are terminal. Preserve this as a local receipt
// before any later fault hook can return an error.
self.maintenance_lease.take();
}
record_pager_commit_duration(&PAGER_COMMIT_UNLOCK_TIME_NS, t_unlock_start);
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
record_pager_commit_duration(
&PAGER_COMMIT_PHASE_C_METADATA_TIME_NS,
t_phase_c_metadata_start,
);
let t_phase_c1_done = phase_timing.then(Instant::now);
// H4 fault hook: crash during Phase C, after commit_seq update
// but before snapshot publish. WAL frames are durable, commit_seq
// incremented in-memory, but snapshot plane not yet updated.
#[cfg(any(test, feature = "fault-injection"))]
crate::fault_hooks::maybe_inject_during_phase_c(
publish_update.visible_commit_seq.get(),
publish_update.db_size,
)?;
// Phase C2 (outside inner.lock): publish to the shared snapshot
// plane. In isolated single-connection mode, only metadata needs
// to advance; page bytes stay authoritative in pager/db_file state.
let t_publish_start = pager_commit_profile_start(pager_commit_profile_active);
if let Some(intent) = wal_publication_intent {
// The physical flusher installed the complete certificate
// group before waking this waiter. Re-publishing this
// transaction's subset here could overwrite the group's
// last-frame-wins image when Phase C callbacks run out of
// order, so the waiter only binds certificate metadata.
self.published.bind_parallel_wal_publication(intent);
} else if metadata_only_single_connection_fast_path {
self.publish_single_connection_metadata_only(cx, publish_update);
} else {
self.publish_committed_state(cx, publish_update);
}
record_pager_commit_duration(&PAGER_COMMIT_PUBLISH_TIME_NS, t_publish_start);
// Keep the transaction-local published snapshot hint aligned with
// the commit we just published so post-commit callers querying the
// still-live handle see committed metadata rather than the
// pre-commit snapshot boundary.
self.published_visible_commit_seq
.set(publish_update.visible_commit_seq);
self.published_db_size.set(publish_update.db_size);
let t_phase_c2_done = phase_timing.then(Instant::now);
// Record full commit path timing.
if self.journal_mode == JournalMode::Wal
&& let (
Some(t_commit_start),
Some(t_phase_a_done),
Some(t_phase_b_done),
Some(t_phase_c1_done),
Some(t_phase_c2_done),
) = (
t_commit_start,
t_phase_a_done,
t_phase_b_done,
t_phase_c1_done,
t_phase_c2_done,
)
{
let phase_a_us =
t_phase_a_done.duration_since(t_commit_start).as_micros() as u64;
let phase_b_us =
t_phase_b_done.duration_since(t_phase_a_done).as_micros() as u64;
let phase_c1_us =
t_phase_c1_done.duration_since(t_phase_b_done).as_micros() as u64;
let phase_c2_us =
t_phase_c2_done.duration_since(t_phase_c1_done).as_micros() as u64;
GLOBAL_CONSOLIDATION_METRICS.record_commit_phases(
phase_a_us,
phase_b_us,
phase_c1_us,
phase_c2_us,
);
}
// Metadata-only single-connection commits intentionally leave the
// published page plane stale, so keep the just-committed pages in
// shared cache even under WAL mode to give the next statement a
// cheap committed read surface.
let t_cache_finish_start = pager_commit_profile_start(pager_commit_profile_active);
if publish_update.journal_mode == JournalMode::Wal
&& !metadata_only_single_connection_fast_path
{
self.discard_committed_pages();
} else if metadata_only_single_connection_fast_path {
self.drain_committed_cache_pages_into_cache();
} else {
let committed_cache_pages = self.drain_committed_cache_pages();
if !committed_cache_pages.is_empty() {
let inner = self
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if inner.commit_seq == publish_update.visible_commit_seq {
for (page_no, buf) in committed_cache_pages {
self.cache.insert_buffer(page_no, buf);
}
}
}
}
self.retained_memory_overlay_dirty_pages.clear();
if self.rollback_commit_finalization_pending {
let recovery_owner = self.owned_rollback_recovery.ok_or_else(|| {
FrankenError::internal(
"durable rollback commit lost its exact owner before Phase C completed",
)
})?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
inner.finish_rollback_journal_recovery(recovery_owner)?;
self.owned_rollback_recovery = None;
self.rollback_commit_finalization_pending = false;
}
self.committed = true;
self.maintenance_lease.take();
self.finished = true;
// IMPL-3 / AG-4B: reset scratch arena after successful commit so
// transient per-transaction allocations do not linger. The arena
// is dropped when the transaction drops; this reset is the
// amortization hook that keeps the txn-committed state compact.
self.scratch_arena.reset();
record_pager_commit_duration(
&PAGER_COMMIT_CACHE_FINISH_TIME_NS,
t_cache_finish_start,
);
} else {
// Keep the writer lock held on commit failure so no other writer
// can interleave while the caller decides to retry or roll back.
//
// CRITICAL FIX (beads_rust#138): Restore pending freed pages so
// a retry or rollback can still observe them. In the old code
// they leaked into inner.freelist regardless of commit outcome;
// now we only promote on success and restore on failure.
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
drop(inner);
}
match commit_result {
Ok(()) => Ok(()),
Err(commit_error) => {
// A failed rollback-journal commit may have changed durable
// database pages after making the pre-image journal hot.
// Finish that recovery epoch before the async commit future
// resolves so callers never observe a half-published image
// and Drop does not need to block an executor.
let cleanup_cx = self.cleanup_cx.clone();
let _cleanup_mask = cleanup_cx.masked();
if let Err(recovery_error) =
self.recover_pending_rollback_journal(&cleanup_cx).await
{
return Err(FrankenError::internal(format!(
"commit failed and rollback-journal recovery did not complete: commit={commit_error}; recovery={recovery_error}"
)));
}
Err(commit_error)
}
}
}
}
fn pager_commit_state(&self) -> crate::traits::PagerCommitState {
use crate::traits::PagerCommitState;
if self.finished {
return if self.committed {
PagerCommitState::Committed
} else {
PagerCommitState::NotCommitted
};
}
if self.rollback_commit_finalization_pending || self.committed {
return PagerCommitState::DurableNeedsPublication;
}
if let Some(attempt) = self.pending_group_commit_attempt.as_ref() {
return match attempt.resolution() {
PendingGroupCommitTxnResolution::Pending => PagerCommitState::InDoubt,
PendingGroupCommitTxnResolution::Authorized(_) => {
PagerCommitState::DurableNeedsPublication
}
PendingGroupCommitTxnResolution::NotCommitted => PagerCommitState::NotCommitted,
};
}
PagerCommitState::NotCommitted
}
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
fn commit_and_retain<'a>(&'a mut self, cx: &'a Cx) -> impl Future<Output = Result<bool>> + 'a {
async move {
if self.rollback_commit_finalization_pending {
self.finish_durable_rollback_commit(cx).await?;
return Ok(false);
}
// The WAL backend currently owns one shared read pin. Retaining a
// logical transaction would require advancing that pin and this
// transaction's FCW horizon together after durability, which the
// backend API cannot do infallibly. Finish the transaction instead;
// the caller will open a fresh, coherently pinned transaction.
if self.journal_mode == JournalMode::Wal {
self.commit(cx).await?;
return Ok(false);
}
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
if self.owned_rollback_recovery.is_some() {
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
self.recover_pending_rollback_journal(&cleanup_cx).await?;
}
if let Some(attempt) = self.pending_group_commit_attempt.clone() {
match attempt.reconcile_global_from_queue()? {
PendingGroupCommitTxnResolution::Pending => {
return Err(FrankenError::BusyRecovery);
}
PendingGroupCommitTxnResolution::NotCommitted => {
self.restore_not_committed_wal_attempt()?;
}
PendingGroupCommitTxnResolution::Authorized(_) => {
self.finish_authorized_wal_attempt(cx, false).await?;
return Ok(true);
}
}
}
if self.identity_rollback_recovery_pending() {
if self.has_local_changes_to_commit() {
return Err(FrankenError::BusyRecovery);
}
self.commit(cx).await?;
return Ok(false);
}
// Only supported for in-memory pagers where we can skip I/O.
if !self.vfs.is_memory() {
self.commit(cx).await?;
return Ok(false);
}
// If not a writer or no pending writes, just commit normally.
if !self.is_writer || !self.has_pending_writes() {
self.commit(cx).await?;
return Ok(false);
}
// Perform the full commit but don't release writer state.
// This is the same as commit() except we:
// - Don't decrement active_transactions
// - Don't set writer_active = false
// - Don't set committed/finished = true
// - Clear write_set for reuse instead
let inner_arc = Arc::clone(&self.inner);
let mut inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !self.exact_recovery_owner_matches_inner(&inner) {
return Err(FrankenError::BusyRecovery);
}
let mut committed_db_size = self.committed_db_size_with_inner(&inner);
let mut pending_free_pages_for_retain = self.pending_free_pages_for_commit();
let mut freelist_dirty_for_retain = self
.freelist_metadata_dirty_with_pending_free_pages(
&inner,
committed_db_size,
&pending_free_pages_for_retain,
);
let mut single_connection_fast_path = self.single_connection_fast_path_enabled();
let mut metadata_only_single_connection_fast_path = single_connection_fast_path
&& !freelist_dirty_for_retain
&& !self.write_set.contains_key(&PageNumber::ONE);
let mut defer_private_memory_flush =
self.memory_db_bump_alloc && metadata_only_single_connection_fast_path;
if self.vfs.is_memory()
&& self.memory_db_bump_alloc
&& !defer_private_memory_flush
&& !self.retained_memory_overlay_dirty_pages.is_empty()
{
drop(inner);
self.materialize_retained_memory_overlay_into_write_set()?;
inner = inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if !self.exact_recovery_owner_matches_inner(&inner) {
return Err(FrankenError::BusyRecovery);
}
committed_db_size = self.committed_db_size_with_inner(&inner);
pending_free_pages_for_retain = self.pending_free_pages_for_commit();
freelist_dirty_for_retain = self.freelist_metadata_dirty_with_pending_free_pages(
&inner,
committed_db_size,
&pending_free_pages_for_retain,
);
single_connection_fast_path = self.single_connection_fast_path_enabled();
metadata_only_single_connection_fast_path = single_connection_fast_path
&& !freelist_dirty_for_retain
&& !self.write_set.contains_key(&PageNumber::ONE);
defer_private_memory_flush =
self.memory_db_bump_alloc && metadata_only_single_connection_fast_path;
}
// Drain freed_pages AFTER dirty check but do NOT push into
// inner.freelist. The serializer receives pending_free_pages so
// inner.freelist remains untouched until Phase C (after successful
// commit).
let mut returned_allocations = self.drain_unstaged_allocated_pages();
returned_allocations.page_lease.append(&mut self.page_lease);
let pending_returned_pages = returned_allocations.all_pages();
let mut pending_free_pages = pending_returned_pages.clone();
pending_free_pages.extend(self.freed_pages.iter().copied());
let mut pending_freed: Vec<PageNumber> = std::mem::take(&mut self.freed_pages);
self.freed_page_bounds = None;
let mut wal_publication_authorization = None;
let mut wal_attempt: Option<Arc<PendingGroupCommitTxnAttempt<V::File>>> = None;
let commit_result = {
let freelist_dirty = freelist_dirty_for_retain;
// Match the normal commit path: capture semantic Page 1 intent
// before freelist serialization can inject bookkeeping Page 1.
let wal_page1_plan =
self.classify_wal_page_one_write(inner.db_size, freelist_dirty);
if self.journal_mode == JournalMode::Wal {
let staged_page_high_water =
self.staged_page_high_water(self.snapshot_db_size());
let live_committed_allocations = self
.allocated_from_freelist
.iter()
.chain(&self.allocated_from_eof)
.filter(|page| self.write_set.contains_key(*page))
.copied()
.collect();
let attempt = Arc::new(PendingGroupCommitTxnAttempt::new(
&self.group_commit_queue,
Arc::clone(&self.inner),
Arc::clone(&self.db_file),
Arc::clone(&self.committed_snapshot),
Arc::clone(&self.published),
Arc::clone(&self.writer_idle),
cleanup_child_cx(cx),
committed_db_size,
self.mode,
self.is_writer,
staged_page_high_water,
std::mem::take(&mut returned_allocations),
std::mem::take(&mut pending_freed),
live_committed_allocations,
));
self.pending_group_commit_attempt = Some(Arc::clone(&attempt));
wal_attempt = Some(attempt);
}
if freelist_dirty
&& let Err(e) = serialize_freelist_to_write_set(
cx,
&mut inner,
&self.cache,
&self.wal_backend,
&self.pool,
&mut self.write_set,
&mut self.write_pages_sorted,
committed_db_size,
&pending_free_pages,
wal_attempt.as_ref().map(|attempt| &attempt.phase_a_undo),
)
.await
{
if wal_attempt.is_some() {
drop(inner);
self.restore_not_committed_wal_attempt()?;
} else {
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
}
return Err(e);
}
// D1-CRITICAL Fix: In WAL mode, page 1 must be written to WAL not
// only when it was explicitly dirty, but also when the database
// grows (new pages allocated beyond current db_size). Without this,
// other connections reading page 1 from WAL won't see the updated
// page_count header, causing BusySnapshot errors.
let must_write_page1 = if self.journal_mode == JournalMode::Wal {
wal_page1_plan.requires_page_one_rewrite()
|| wal_page1_plan.requires_page_count_advance()
} else if self.vfs.is_memory() {
// B3.4: :memory: journal mode skips page 1 header update unless:
// 1. freelist_dirty (freelist count in header must match), OR
// 2. page 1 is explicitly dirty in write_set
freelist_dirty || self.write_set.contains_key(&PageNumber::ONE)
} else {
true
};
if must_write_page1 {
if let Some(attempt) = wal_attempt.as_ref() {
attempt
.phase_a_undo
.capture(&self.write_set, PageNumber::ONE);
}
let mut page1 = match ensure_page_one_in_write_set(
cx,
&inner,
&self.cache,
&self.wal_backend,
&self.pool,
&mut self.write_set,
)
.await
{
Ok(p) => p,
Err(e) => {
if wal_attempt.is_some() {
drop(inner);
self.restore_not_committed_wal_attempt()?;
} else {
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(
&mut inner.freelist,
pending_returned_pages,
);
}
return Err(e);
}
};
let page1_bytes = page1.as_page_bytes_mut();
if page1_bytes.len() >= DATABASE_HEADER_SIZE {
let mut page_count_bytes = [0_u8; 4];
page_count_bytes.copy_from_slice(&page1_bytes[28..32]);
let existing_page_count = u32::from_be_bytes(page_count_bytes);
let new_change_counter = inner.commit_seq.get().wrapping_add(1) as u32;
page1_bytes[24..28].copy_from_slice(&new_change_counter.to_be_bytes());
if self.journal_mode != JournalMode::Wal
|| wal_page1_plan.requires_page_count_advance()
{
let new_db_size = committed_db_size.max(existing_page_count);
page1_bytes[28..32].copy_from_slice(&new_db_size.to_be_bytes());
}
page1_bytes[92..96].copy_from_slice(&new_change_counter.to_be_bytes());
}
insert_staged_page(
&mut self.write_set,
&mut self.write_pages_sorted,
PageNumber::ONE,
page1,
);
}
let cross_process_conflict_pages = if self.journal_mode == JournalMode::Wal {
self.predicted_conflict_pages_for_wal_commit_with_inner(
&inner,
wal_page1_plan,
&pending_free_pages,
)
} else {
Vec::new()
};
let cross_process_conflict_page_baselines =
self.conflict_page_baselines(&cross_process_conflict_pages);
let wal_current_db_size = inner.db_size;
let wal_sync_policy = inner.wal_commit_sync_policy;
if self.journal_mode == JournalMode::Wal {
drop(inner);
let result = Self::commit_wal_group_commit_with_snapshot(
cx,
&self.wal_backend,
&self.inner,
Some(Arc::clone(&self.published)),
wal_current_db_size,
wal_sync_policy,
&self.write_set,
&self.write_pages_sorted,
&cross_process_conflict_pages,
self.wal_conflict_snapshot,
&cross_process_conflict_page_baselines,
&self.group_commit_queue,
&mut wal_publication_authorization,
wal_attempt.as_ref(),
)
.await;
inner = match inner_arc
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))
{
Ok(guard) => guard,
Err(e) => return Err(e),
};
result
} else if self.vfs.is_memory() {
if defer_private_memory_flush {
// For a real private `:memory:` database, a retained
// single-connection metadata-only commit does not need to
// rewrite the VFS backing store on every row. Keep the
// committed page image in `txn_read_cache` and flush once
// when the retained writer is actually released.
} else {
// bd-wwqen.3: :memory: retained-commit fast path.
// Skip journal creation, pre-image backup, sync, and deletion.
// Batch dirty-page flushes through the VFS so MemoryFile can
// hold its backing-storage lock once for the whole retained
// commit. Keep the staged pages in the write set for the later
// publish step so the flush path avoids an eager drain/move of
// the whole staging map on every autocommit write.
if let Err(e) = Self::flush_write_set_to_db_file_batch(
cx,
&inner,
&self.write_set,
&self.write_pages_sorted,
)
.await
{
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
return Err(e);
}
}
Ok(())
} else {
// See commit(): restore drained never-written freelist pops
// (`pending_returned_pages`) for the freelist-alias check.
let mut alias_check_restored = self.allocated_from_freelist.clone();
alias_check_restored.extend_from_slice(&pending_returned_pages);
let result = Self::commit_journal(
cx,
&self.vfs,
&mut inner,
RollbackJournalCommitInput {
journal_path: &self.journal_path,
write_set: &self.write_set,
original_db_size: self.original_db_size,
allocated_from_freelist: &alias_check_restored,
},
&mut self.owned_rollback_recovery,
)
.await;
result
}
};
if self.journal_mode == JournalMode::Wal {
drop(inner);
let attempt = wal_attempt.clone().ok_or_else(|| {
FrankenError::internal(
"retained WAL commit reached Phase B without a logical transaction owner",
)
})?;
let commit_error = commit_result.err();
match attempt.resolution() {
PendingGroupCommitTxnResolution::Authorized(_) => {
if commit_error.is_none() && wal_publication_authorization.is_none() {
return Err(FrankenError::internal(
"successful retained WAL commit omitted publication authorization",
));
}
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
self.finish_authorized_wal_attempt(&cleanup_cx, false)
.await?;
return Ok(true);
}
PendingGroupCommitTxnResolution::NotCommitted => {
self.restore_not_committed_wal_attempt()?;
return Err(commit_error.unwrap_or_else(|| {
FrankenError::internal(
"retained WAL commit reported success after a NotCommitted verdict",
)
}));
}
PendingGroupCommitTxnResolution::Pending => {
return Err(commit_error.unwrap_or(FrankenError::BusyRecovery));
}
}
}
if commit_result.is_ok() {
// For journal mode, update db_size from our computed value.
// For WAL mode with group commit, the flusher already set inner.db_size
// to the consolidated max across all batched transactions - don't revert it.
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
return_pages_to_freelist(&mut inner.freelist, pending_freed);
// See commit() Phase C1: keep inner.db_size monotonic across
// concurrent cross-process commits so we never publish a db_size
// smaller than a peer's just-committed extent.
inner.db_size = inner.db_size.max(committed_db_size);
// Keep volatile EOF lease pages in memory; see commit() Phase C1.
let wal_publication_intent = wal_publication_authorization
.as_ref()
.map(|authorization| {
parallel_wal_publication_intent(
authorization,
inner.db_size,
inner.journal_mode,
inner.freelist.len(),
inner.checkpoint_active,
)
})
.transpose()?;
if let Some(intent) = wal_publication_intent {
inner.record_local_wal_commit_at(intent.visible_commit_seq);
} else {
inner.record_local_commit();
}
// `commit_and_retain` is restricted to MemoryVfs pagers. Their
// committed extent is the monotonic logical page high-water, so
// derive it without introducing a cancellable Phase-C await.
inner.committed_db_file_size_bytes =
u64::from(inner.db_size) * u64::from(inner.page_size.get());
// NOTE: We intentionally do NOT decrement active_transactions or
// set writer_active=false — the transaction stays "active" for reuse.
let publish_update = PublishedPagerUpdate {
visible_commit_seq: wal_publication_intent
.map_or(inner.commit_seq, |intent| intent.visible_commit_seq),
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
};
// bd-db300.5.3.3.1: publish immutable snapshot while inner is still
// held — MUST happen before publish_committed_state (same order as
// `commit()`), so concurrent readers see the immutable snapshot
// before the seqlock commit_seq advances.
self.publish_committed_snapshot_from_inner(&inner);
drop(inner);
if let Some(intent) = wal_publication_intent {
// The group flusher already published every certified page in
// WAL order. Retained transactions must not replay their
// member-local subset after another member's Phase C.
self.retained_memory_overlay_dirty_pages.clear();
self.published.bind_parallel_wal_publication(intent);
} else if metadata_only_single_connection_fast_path {
if defer_private_memory_flush {
self.note_retained_memory_overlay_from_write_set();
} else {
self.retained_memory_overlay_dirty_pages.clear();
}
self.publish_single_connection_metadata_only(cx, publish_update);
self.retain_committed_pages_in_txn_read_cache(false);
} else {
self.retained_memory_overlay_dirty_pages.clear();
self.publish_committed_state_draining_write_set(cx, publish_update);
}
// Clear retained transaction state for reuse.
self.write_pages_sorted.clear();
self.clear_freed_pages();
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
self.savepoint_stack.clear();
self.rolled_back_pages.clear();
self.writes_observed = false;
if !metadata_only_single_connection_fast_path {
self.txn_read_cache.borrow_mut().clear();
}
// IMPL-3 / AG-4B: reset scratch arena on retained commit. The
// transaction stays active for reuse, but arena-allocated scratch
// from the committed logical transaction must not leak into the
// next one.
self.scratch_arena.reset();
self.original_db_size = committed_db_size;
self.published_visible_commit_seq
.set(publish_update.visible_commit_seq);
self.published_db_size.set(publish_update.db_size);
if self.journal_mode != JournalMode::Wal {
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if let Some(recovery_owner) = self.owned_rollback_recovery {
inner.finish_rollback_journal_recovery(recovery_owner)?;
self.owned_rollback_recovery = None;
}
}
// Transaction stays active — committed/finished remain false.
Ok(true)
} else {
// CRITICAL FIX (beads_rust#138): Restore pending freed pages on
// commit failure so rollback can still see them.
self.restore_pending_freed_pages(pending_freed);
return_pages_to_freelist(&mut inner.freelist, pending_returned_pages);
drop(inner);
commit_result?;
unreachable!()
}
}
}
fn is_writer(&self) -> bool {
self.is_writer
}
fn has_pending_writes(&self) -> bool {
self.pending_group_commit_attempt.is_some()
|| self.owned_rollback_recovery.is_some()
|| self.rollback_commit_finalization_pending
|| !self.write_set.is_empty()
|| self.freelist_metadata_dirty()
}
fn published_visible_commit_seq_hint(&self) -> Option<CommitSeq> {
if self.has_pending_recovery_barrier() {
return None;
}
Some(self.published_visible_commit_seq.get())
}
fn pending_commit_pages(&self) -> Result<Vec<PageNumber>> {
self.ensure_no_pending_group_commit_attempt()?;
if !self.has_pending_writes() {
return Ok(Vec::new());
}
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Ok(self.predicted_commit_pages_with_inner(&inner))
}
fn pending_conflict_pages(&self) -> Result<Vec<PageNumber>> {
self.ensure_no_pending_group_commit_attempt()?;
if !self.has_pending_writes() {
return Ok(Vec::new());
}
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Ok(self.predicted_conflict_pages_with_inner(&inner))
}
fn pending_conflict_pages_conservative(&self) -> Vec<PageNumber> {
if self.has_pending_recovery_barrier() {
return vec![PageNumber::ONE];
}
let mut pages = Vec::with_capacity(
self.write_pages_sorted
.len()
.saturating_add(self.freed_pages.len())
.saturating_add(1),
);
pages.extend(self.write_pages_sorted.iter().copied());
pages.extend(self.freed_pages.iter().copied());
// A free at or below the transaction's begin snapshot necessarily
// rewrites durable freelist metadata. Locally allocated pages can
// also become durable frees when another staged page advances this
// transaction's eventual page-count high-water mark, so include that
// lock-free local bound as well. Page 1 is the shared FCW token that
// makes disjoint free-only transactions conflict without consulting
// PagerInner to predict exact trunk-page synthesis.
let published_db_size = self.published_db_size.get();
let durable_bound = self
.max_live_written_page()
.map_or(published_db_size, |page| published_db_size.max(page.get()));
if self
.freed_pages
.iter()
.any(|page| page.get() <= durable_bound)
{
pages.push(PageNumber::ONE);
}
pages.sort_unstable();
pages.dedup();
pages
}
fn write_set_page_numbers(&self) -> Vec<PageNumber> {
if self.has_pending_recovery_barrier() {
return vec![PageNumber::ONE];
}
self.write_pages_sorted.clone()
}
fn page_size(&self) -> PageSize {
PageSize::new(u32::try_from(self.pool.page_size()).expect("pool page size fits u32"))
.expect("pool page size invariant")
}
fn page_one_in_pending_commit_surface(&self) -> Result<bool> {
self.ensure_no_pending_group_commit_attempt()?;
if !self.has_pending_writes() {
return Ok(false);
}
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Ok(self.page_one_in_pending_commit_surface_with_inner(&inner))
}
fn allocate_page_requires_page_one_conflict_tracking(&self) -> Result<bool> {
self.ensure_no_pending_group_commit_attempt()?;
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Ok(self.allocate_page_requires_page_one_conflict_tracking_with_inner(&inner))
}
fn free_page_requires_page_one_conflict_tracking(&self, page_no: PageNumber) -> Result<bool> {
self.ensure_no_pending_group_commit_attempt()?;
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Ok(self.free_page_requires_page_one_conflict_tracking_with_inner(&inner, page_no))
}
fn write_page_requires_page_one_conflict_tracking(&self, page_no: PageNumber) -> Result<bool> {
self.ensure_no_pending_group_commit_attempt()?;
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Ok(self.write_page_requires_page_one_conflict_tracking_with_inner(&inner, page_no))
}
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
fn rollback<'a>(&'a mut self, cx: &'a Cx) -> impl Future<Output = Result<()>> + 'a {
async move {
if self.finished {
return Ok(());
}
if self.rollback_commit_finalization_pending {
self.finish_durable_rollback_commit(cx).await?;
return Err(FrankenError::internal(
"rollback could not undo a rollback-journal commit that was already durable",
));
}
settle_pending_group_commit_finalization_for_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
)
.await?;
if let Some(attempt) = self.pending_group_commit_attempt.clone() {
match attempt.reconcile_global_from_queue()? {
PendingGroupCommitTxnResolution::Pending => {
return Err(FrankenError::BusyRecovery);
}
PendingGroupCommitTxnResolution::Authorized(_) => {
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
self.finish_authorized_wal_attempt(&cleanup_cx, true)
.await?;
return Err(FrankenError::internal(
"rollback could not undo a group commit that became durable during recovery",
));
}
PendingGroupCommitTxnResolution::NotCommitted => {
self.restore_not_committed_wal_attempt()?;
}
}
}
if self.identity_rollback_recovery_pending() && self.owned_rollback_recovery.is_none() {
return self
.drain_nonowner_during_rollback_recovery(cx, false)
.await;
}
self.validate_namespace_binding()?;
// Rollback is mandatory cleanup. A caller may reach it precisely
// because its parent context was cancelled during a commit, so every
// recovery, refresh, unlock, and journal cleanup below must run from a
// masked child rather than inherit that cancellation.
let cleanup_cx = cleanup_child_cx(cx);
let _cleanup_mask = cleanup_cx.masked();
let logical_exit_claim = GroupCommitLogicalExitClaim::acquire(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
&cleanup_cx,
)
.await?;
if self.identity_rollback_recovery_owner() != self.owned_rollback_recovery {
let foreign_pending = self.identity_rollback_recovery_pending()
&& self.owned_rollback_recovery.is_none();
drop(logical_exit_claim);
if foreign_pending {
return self
.drain_nonowner_during_rollback_recovery(&cleanup_cx, false)
.await;
}
return Err(FrankenError::BusyRecovery);
}
if self.vfs.is_memory()
&& self.memory_db_bump_alloc
&& !self.retained_memory_overlay_dirty_pages.is_empty()
{
let overlay_pages = self.collect_retained_memory_overlay_pages()?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
Self::flush_retained_memory_overlay_pages_to_db_file(
&cleanup_cx,
&mut inner,
self.original_db_size,
&overlay_pages,
)
.await?;
drop(inner);
self.retained_memory_overlay_dirty_pages.clear();
}
self.write_set.clear();
self.write_pages_sorted.clear();
self.clear_freed_pages();
self.savepoint_stack.clear();
self.rolled_back_pages.clear();
// #70 ghost-commit guard: after rollback, staged writes were
// explicitly discarded, so a subsequent commit entry with an empty
// write_set must not trip the defensive assertion.
self.writes_observed = false;
let restored_from_journal = self.recover_pending_rollback_journal(&cleanup_cx).await?;
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
let mut notify_writer_idle = false;
if restored_from_journal {
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
// Lease pages were EOF allocations that were never written to
// disk. After journal recovery rebuilds committed state, these
// page numbers don't exist — just drop them.
self.page_lease.clear();
} else {
// Restore pages allocated from the freelist.
return_pages_to_freelist(
&mut inner.freelist,
self.allocated_from_freelist.drain(..),
);
if self.is_writer && self.mode != TransactionMode::Concurrent {
// Non-concurrent: next_page will be reset below, so lease
// pages (which were EOF allocations) will be re-issued
// naturally by future transactions. Just drop them — putting
// them on the freelist would create sparse page holes since
// the freelist consumer could pick a high page number while
// next_page restarts from db_size+1.
self.page_lease.clear();
inner.db_size = self.original_db_size;
// Reset next_page to avoid holes if we allocated pages that are now discarded.
// Logic matches SimplePager::open.
let db_size = inner.db_size;
inner.next_page = if db_size >= 2 {
db_size.saturating_add(1)
} else {
2
};
} else if self.is_writer && self.mode == TransactionMode::Concurrent {
// Concurrent: next_page is NOT reset, so lease pages and
// aborted EOF allocations must return to the in-memory
// freelist. Otherwise next_page skips over them permanently
// and a later commit can grow page_count past those holes,
// yielding "Page N: never used" corruption.
return_pages_to_freelist(&mut inner.freelist, self.page_lease.drain(..));
return_pages_to_freelist(
&mut inner.freelist,
self.allocated_from_eof.drain(..),
);
} else {
// Read-only transaction: lease should be empty (only writers
// allocate pages), but clear defensively.
self.page_lease.clear();
}
}
notify_writer_idle |= coordinated_transaction_exit(
&self.group_commit_queue,
&cleanup_cx,
&mut inner,
self.is_writer && self.mode != TransactionMode::Concurrent,
&logical_exit_claim,
)
.await?;
drop(inner);
if notify_writer_idle {
self.writer_idle.notify_one();
}
self.committed = false;
self.maintenance_lease.take();
self.finished = true;
// IMPL-3 / AG-4B: reset scratch arena so transient allocations do not
// carry across transaction boundaries.
self.scratch_arena.reset();
Ok(())
}
}
fn record_write_witness(&mut self, _cx: &Cx, _key: fsqlite_types::WitnessKey) {}
fn savepoint(&mut self, _cx: &Cx, name: &str) -> Result<()> {
self.ensure_no_pending_group_commit_attempt()?;
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
self.savepoint_stack.push(SavepointEntry {
name: name.to_owned(),
write_set_snapshot: self
.write_set
.iter()
.map(|(&k, v)| (k, v.published_page()))
.collect(),
write_pages_sorted_snapshot: self.write_pages_sorted.clone(),
freed_pages_snapshot: self.freed_pages.clone(),
next_page_snapshot: inner.next_page,
freelist_snapshot: inner.freelist.clone(),
allocated_from_freelist_snapshot: self.allocated_from_freelist.clone(),
allocated_from_eof_snapshot: self.allocated_from_eof.clone(),
});
drop(inner);
Ok(())
}
fn release_savepoint(&mut self, _cx: &Cx, name: &str) -> Result<()> {
self.ensure_no_pending_group_commit_attempt()?;
let pos = self
.savepoint_stack
.iter()
.rposition(|sp| sp.name == name)
.ok_or_else(|| FrankenError::internal(format!("no savepoint named '{name}'")))?;
// RELEASE removes the named savepoint and all savepoints above it.
// Changes since the savepoint are kept (merged into the parent).
self.savepoint_stack.truncate(pos);
Ok(())
}
fn rollback_to_savepoint(&mut self, _cx: &Cx, name: &str) -> Result<()> {
self.ensure_no_pending_group_commit_attempt()?;
let pos = self
.savepoint_stack
.iter()
.rposition(|sp| sp.name == name)
.ok_or_else(|| FrankenError::internal(format!("no savepoint named '{name}'")))?;
let entry = &self.savepoint_stack[pos];
// Restore write-set FIRST to ensure we don't leave the transaction in an
// inconsistent state if PageBuf allocation fails (OOM).
let new_write_set = entry
.write_set_snapshot
.iter()
.map(|(&k, v)| -> Result<(PageNumber, StagedPage)> {
Ok((
k,
StagedPage::from_page_data_with_cache_recovery(
&self.pool,
&self.cache,
v.clone(),
"savepoint_rollback_stage",
)?,
))
})
.collect::<Result<PagePageMap<_>>>()?;
// Track pages that were allocated after the savepoint so that get_page
// can return zeros for them instead of BusySnapshot error.
for page_no in self
.allocated_from_eof
.iter()
.skip(entry.allocated_from_eof_snapshot.len())
{
self.rolled_back_pages.insert(*page_no);
}
for page_no in self
.allocated_from_freelist
.iter()
.skip(entry.allocated_from_freelist_snapshot.len())
{
self.rolled_back_pages.insert(*page_no);
}
{
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimpleTransaction lock poisoned"))?;
if self.mode != TransactionMode::Concurrent {
inner.next_page = entry.next_page_snapshot;
inner.freelist.clone_from(&entry.freelist_snapshot);
// Lease pages reference the rolled-back next_page range
// and will be re-allocated by future EOF allocations, so
// just drop them.
self.page_lease.clear();
} else {
// Return unused lease pages to the freelist before
// returning post-savepoint EOF/freelist allocations.
// These are valid EOF page numbers that next_page has
// already advanced past (concurrent mode doesn't roll
// back next_page).
return_pages_to_freelist(&mut inner.freelist, self.page_lease.drain(..));
return_pages_to_freelist(
&mut inner.freelist,
self.allocated_from_eof
.drain(entry.allocated_from_eof_snapshot.len()..),
);
return_pages_to_freelist(
&mut inner.freelist,
self.allocated_from_freelist
.drain(entry.allocated_from_freelist_snapshot.len()..),
);
}
}
let allocated_from_freelist_snapshot = entry.allocated_from_freelist_snapshot.clone();
let allocated_from_eof_snapshot = entry.allocated_from_eof_snapshot.clone();
let freed_pages_snapshot = entry.freed_pages_snapshot.clone();
let write_pages_sorted_snapshot = entry.write_pages_sorted_snapshot.clone();
let snapshot_was_empty = entry.write_set_snapshot.is_empty();
self.allocated_from_freelist = allocated_from_freelist_snapshot;
self.allocated_from_eof = allocated_from_eof_snapshot;
self.freed_pages = freed_pages_snapshot;
self.refresh_freed_page_bounds();
// #70 ghost-commit guard: rollback-to-savepoint replaces write_set
// with the snapshot. If the snapshot is empty, no writes are pending
// — commit will be a legitimate no-op and the guard should not flag
// it. If the snapshot is non-empty, writes are still pending and
// writes_observed should stay consistent with that.
self.write_set = new_write_set;
self.write_pages_sorted = write_pages_sorted_snapshot;
if snapshot_was_empty {
self.writes_observed = false;
}
// Discard savepoints created after the named one, but keep
// the named savepoint itself (it can be rolled back to again).
self.savepoint_stack.truncate(pos + 1);
Ok(())
}
}
impl<V> Drop for SimpleTransaction<V>
where
V: Vfs,
V::File: 'static,
{
fn drop(&mut self) {
if self.finished {
return;
}
// Publish exact-handle ownership before touching PagerInner or
// attempting a synchronous durable tail. If any later edge is not
// terminal, this root is transferred unchanged to queued cleanup.
let mut drop_root_attempt = Some(ProcessRootFinalizationAttempt::register_exact_handle(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
));
if self.rollback_commit_finalization_pending && self.maintenance_lease.is_none() {
// Logical exit is already terminal, so the remaining publication
// tail is synchronous and safe to finish in Drop. Never run
// `record_local_commit` or external exit twice from this phase.
let cleanup_cx = self.cleanup_cx.clone();
let _cleanup_mask = cleanup_cx.masked();
match self.finish_durable_rollback_commit_after_exit(&cleanup_cx) {
Ok(()) => {
if let Some(root_attempt) = drop_root_attempt.take() {
root_attempt.release_after_terminal();
}
}
Err(error) => {
tracing::error!(
%error,
"transaction Drop queued a failed durable rollback-commit publication tail"
);
match self.take_detached_durable_rollback_commit_exit(cleanup_cx.clone(), true)
{
Ok(cleanup) => {
self.group_commit_queue.enqueue_pending_logical_cleanup(
PendingGroupCommitLogicalCleanup::new(
drop_root_attempt.take(),
Box::new(cleanup),
),
);
if let Err(resolve_error) = self
.group_commit_queue
.try_resolve_one_pending_external_unlock()
{
tracing::error!(
%resolve_error,
"transaction Drop could not claim its queued durable publication tail"
);
}
}
Err(detach_error) => {
tracing::error!(
%detach_error,
"transaction Drop could not preserve the exact durable cleanup receipt"
);
}
}
}
}
self.finished = true;
return;
}
if self.pending_group_commit_attempt.is_some() {
let attempt = self
.pending_group_commit_attempt
.take()
.expect("pending group-commit attempt remained attached during Drop");
let cleanup = DetachedPendingGroupCommitTxnCleanup {
attempt,
maintenance_lease: self.maintenance_lease.take(),
allocated_from_freelist: std::mem::take(&mut self.allocated_from_freelist),
allocated_from_eof: std::mem::take(&mut self.allocated_from_eof),
page_lease: std::mem::take(&mut self.page_lease),
allocation_cleanup_applied: false,
stale_pending_observations: 0,
};
self.group_commit_queue.enqueue_pending_logical_cleanup(
PendingGroupCommitLogicalCleanup::new(drop_root_attempt.take(), Box::new(cleanup)),
);
if let Err(error) = self
.group_commit_queue
.try_resolve_one_pending_external_unlock()
{
tracing::error!(
%error,
"transaction drop could not claim a pending group-commit external unlock"
);
}
self.finished = true;
return;
}
if let Err(error) = self
.group_commit_queue
.try_resolve_one_pending_external_unlock()
{
tracing::error!(
%error,
"transaction drop could not claim a pending group-commit external unlock"
);
}
let mut notify_writer_idle = false;
let mut direct_cleanup_terminal = false;
// Drop is the last synchronous fail-safe after a caller abandons a
// transaction. Any unfinished rollback recovery/finalization stays
// marked pending for the pager's next explicit async recovery epoch;
// Drop only restores safe in-memory state and releases whatever lock
// ownership can be finalized synchronously.
let cleanup_cx = self.cleanup_cx.clone();
let _cleanup_mask = cleanup_cx.masked();
let mut inner = match self.inner.lock() {
Ok(inner) => inner,
Err(error) => {
tracing::error!(
"transaction Drop recovered a poisoned PagerInner for fail-closed cleanup"
);
error.into_inner()
}
};
let recovery_was_pending = self.is_writer
&& self.journal_mode != JournalMode::Wal
&& inner.rollback_journal_recovery_state.is_pending();
if recovery_was_pending {
tracing::warn!(
"drop left rollback recovery/finalization pending for the pager's next recovery epoch"
);
}
if recovery_was_pending {
// Never merge transaction-local allocation state into an
// image whose recovery failed. Keep Pending set so every
// subsequent begin retries or fails closed.
self.cache.clear();
self.allocated_from_freelist.clear();
self.allocated_from_eof.clear();
self.page_lease.clear();
} else {
// Ordinary uncommitted drop: restore freelist allocations.
return_pages_to_freelist(&mut inner.freelist, self.allocated_from_freelist.drain(..));
if self.is_writer && self.mode != TransactionMode::Concurrent {
// Non-concurrent: next_page will be reset, so lease pages
// are re-issued naturally. Just drop them to avoid holes.
self.page_lease.clear();
inner.db_size = self.original_db_size;
inner.next_page = if inner.db_size >= 2 {
inner.db_size.saturating_add(1)
} else {
2
};
} else if self.is_writer && self.mode == TransactionMode::Concurrent {
// Concurrent: next_page stays advanced, so return lease
// pages and EOF allocations to the freelist.
return_pages_to_freelist(&mut inner.freelist, self.page_lease.drain(..));
return_pages_to_freelist(&mut inner.freelist, self.allocated_from_eof.drain(..));
} else {
// Read-only: lease should be empty, clear defensively.
self.page_lease.clear();
}
}
let logical_exit_claim = GroupCommitLogicalExitClaim::try_register(
&self.group_commit_queue,
shared_db_file_key(&self.db_file),
);
let mut defer_transaction_exit = logical_exit_claim.is_none();
if logical_exit_claim.is_none() {
tracing::warn!(
"drop-time transaction exit was queued behind a live physical or logical external-lock owner"
);
} else if let Some(remaining_active_transactions) = inner.active_transactions.checked_sub(1)
{
let releases_writer_baton = self.is_writer && self.mode != TransactionMode::Concurrent;
let writer_active_after_exit = inner.writer_active && !releases_writer_baton;
let restore_target = if remaining_active_transactions == 0 {
PendingExternalUnlockTarget::ExternalSnapshot
} else {
PendingExternalUnlockTarget::LockLevel(retained_lock_level_after_txn_exit(
remaining_active_transactions,
writer_active_after_exit,
))
};
let db_file = Arc::clone(&inner.db_file);
match db_file.try_write() {
Ok(mut db_file) => {
if let Err(error) = restore_target.restore(&mut *db_file, &cleanup_cx) {
tracing::warn!(
%error,
"drop-time transaction exit was queued after snapshot unlock failed"
);
defer_transaction_exit = true;
} else {
inner.active_transactions = remaining_active_transactions;
notify_writer_idle =
releases_writer_baton && release_single_writer_baton(&mut inner);
}
}
Err(error) => {
tracing::warn!(
%error,
"drop-time transaction snapshot lock was busy; queued full exact-handle exit"
);
defer_transaction_exit = true;
}
}
} else {
tracing::error!(
"drop-time transaction exit would underflow active transactions; retained as fail-closed rooted work"
);
defer_transaction_exit = true;
}
drop(logical_exit_claim);
drop(inner);
if defer_transaction_exit {
if self.rollback_commit_finalization_pending && self.committed {
match self.take_detached_durable_rollback_commit_exit(cleanup_cx.clone(), false) {
Ok(cleanup) => {
self.group_commit_queue.enqueue_pending_logical_cleanup(
PendingGroupCommitLogicalCleanup::new(
drop_root_attempt.take(),
Box::new(cleanup),
),
);
}
Err(error) => {
tracing::error!(
%error,
"transaction Drop could not transfer its exact durable cleanup receipt"
);
// An ordinary detached exit cannot publish or finish a
// durable rollback-commit receipt. Keep the process
// root and exact owner fail-closed rather than release
// accounting under a false terminal result.
}
}
} else {
let cleanup = DetachedTransactionExit {
queue: Arc::clone(&self.group_commit_queue),
inner: Arc::clone(&self.inner),
db_file: Arc::clone(&self.db_file),
writer_idle: Arc::clone(&self.writer_idle),
cleanup_cx: cleanup_cx.clone(),
mode: self.mode,
is_writer: self.is_writer,
maintenance_lease: self.maintenance_lease.take(),
};
self.group_commit_queue.enqueue_pending_logical_cleanup(
PendingGroupCommitLogicalCleanup::new(
drop_root_attempt.take(),
Box::new(cleanup),
),
);
}
} else {
direct_cleanup_terminal = true;
}
if direct_cleanup_terminal {
self.maintenance_lease.take();
if self.rollback_commit_finalization_pending
&& self.committed
&& let Err(error) = self.finish_durable_rollback_commit_after_exit(&cleanup_cx)
{
tracing::error!(
%error,
"transaction Drop queued a failed direct durable rollback-commit publication tail"
);
direct_cleanup_terminal = false;
match self.take_detached_durable_rollback_commit_exit(cleanup_cx.clone(), true) {
Ok(cleanup) => {
self.group_commit_queue.enqueue_pending_logical_cleanup(
PendingGroupCommitLogicalCleanup::new(
drop_root_attempt.take(),
Box::new(cleanup),
),
);
}
Err(detach_error) => {
tracing::error!(
%detach_error,
"transaction Drop could not preserve its direct exact durable receipt"
);
}
}
}
}
if notify_writer_idle {
self.writer_idle.notify_one();
}
// Journal cleanup is performed only by an exact recovery owner. Drop
// never deletes the shared artifact on an unowned best-effort path.
self.finished = true;
if direct_cleanup_terminal && let Some(root_attempt) = drop_root_attempt.take() {
root_attempt.release_after_terminal();
}
}
}
// ---------------------------------------------------------------------------
// CheckpointPageWriter implementation for WAL checkpointing
// ---------------------------------------------------------------------------
/// A checkpoint page writer that writes pages directly to the database file.
///
/// This type implements [`crate::CheckpointPageWriter`] and is used during WAL
/// checkpointing to transfer committed pages from the WAL back to the main
/// database file.
///
/// The writer holds a reference to the pager's inner state and acquires the
/// mutex for each operation. This is acceptable because checkpoint is an
/// infrequent operation and the writes must be serialized with other pager
/// operations anyway.
pub struct SimplePagerCheckpointWriter<V: Vfs>
where
V::File: Send + Sync,
{
inner: Arc<Mutex<PagerInner<V::File>>>,
cache: Arc<ShardedPageCache>,
published: Arc<PublishedPagerState>,
/// True once this checkpoint pass has actually mutated the database file
/// (page write or truncation). GH #294: a checkpoint that finds every
/// frame already backfilled byte-for-byte must leave the main file — its
/// header change counter included — completely untouched, so `sync`
/// only re-stamps the page-1 header after a real mutation.
dirty: bool,
}
impl<V: Vfs> traits::sealed::Sealed for SimplePagerCheckpointWriter<V> where V::File: Send + Sync {}
impl<V> SimplePagerCheckpointWriter<V>
where
V: Vfs + Send + Sync,
V::File: Send + Sync,
{
/// Patch page 1 header fields that must remain globally consistent.
///
/// This ensures external SQLite readers see:
/// - a valid change counter (24..28),
/// - the true on-disk page count (28..32),
/// - matching version-valid-for (92..96).
async fn patch_page1_header(&self, cx: &Cx) -> Result<()> {
let (db_file, db_size, page_size, current_change_counter) = {
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePagerCheckpointWriter lock poisoned"))?;
// SQLite databases always keep page 1 when non-empty.
if inner.db_size == 0 {
return Ok(());
}
(
Arc::clone(&inner.db_file),
inner.db_size,
inner.page_size.as_usize(),
inner.commit_seq.get() as u32,
)
};
let mut page1 = vec![0u8; page_size];
let db_file = shared_db_file_read(&db_file, cx).await?;
let bytes_read = db_file.read(cx, &mut page1, 0).await?;
if bytes_read < DATABASE_HEADER_SIZE {
return Err(FrankenError::DatabaseCorrupt {
detail: format!(
"short read while patching page 1 header: got {bytes_read} bytes, need at least {DATABASE_HEADER_SIZE}",
),
});
}
let mut patched_fields = [0_u8; 8];
patched_fields[..4].copy_from_slice(¤t_change_counter.to_be_bytes());
patched_fields[4..].copy_from_slice(&db_size.to_be_bytes());
// GH #294: when the on-disk header already carries the exact values
// this patch would stamp, rewriting them would only bump the main
// file's mtime/ctime. Skip the redundant write.
if page1[24..32] == patched_fields && page1[92..96] == patched_fields[..4] {
return Ok(());
}
page1[24..28].copy_from_slice(¤t_change_counter.to_be_bytes());
page1[28..32].copy_from_slice(&db_size.to_be_bytes());
page1[92..96].copy_from_slice(¤t_change_counter.to_be_bytes());
db_file.write(cx, &page1, 0).await?;
self.cache.evict(PageNumber::ONE);
Ok(())
}
}
impl<V> traits::CheckpointPageWriter for SimplePagerCheckpointWriter<V>
where
V: Vfs + Send + Sync,
V::File: Send + Sync,
{
fn write_page<'a>(
&'a mut self,
cx: &'a Cx,
page_no: PageNumber,
data: &'a [u8],
) -> WalFuture<'a, ()> {
Box::pin(async move {
let (db_file, page_size) = {
let inner = self.inner.lock().map_err(|_| {
FrankenError::internal("SimplePagerCheckpointWriter lock poisoned")
})?;
(Arc::clone(&inner.db_file), inner.page_size.as_usize())
};
let offset = u64::from(page_no.get() - 1) * page_size as u64;
{
let db_file = shared_db_file_read(&db_file, cx).await?;
// GH #294: the pager-level checkpoint entry always restarts
// from frame zero, so close-time passive checkpoints revisit
// frames an earlier checkpoint already backfilled. Re-writing
// identical bytes would still bump the main file's mtime and
// ctime on every open/close cycle, so compare first and only
// touch the file when the page content genuinely differs.
let mut existing = vec![0_u8; data.len()];
let already_backfilled =
db_file
.read(cx, &mut existing, offset)
.await
.is_ok_and(|bytes_read| {
bytes_read == data.len()
&& if page_no == PageNumber::ONE
&& data.len() >= DATABASE_HEADER_SIZE
{
// Page 1 compares with the patch-owned header
// fields masked out: `patch_page1_header`
// re-stamps the change counter (24..32) and
// version-valid-for (92..96) after every real
// backfill, so the on-disk values are the
// authoritative successors of the frame's
// values. Re-writing the frame's stale copy
// would dirty the pass and re-stamp the header
// with a drifted commit clock on every
// open/close cycle (GH #294).
existing[..24] == data[..24]
&& existing[32..92] == data[32..92]
&& existing[96..] == data[96..]
} else {
existing == data
}
});
if !already_backfilled {
db_file.write(cx, data, offset).await?;
self.dirty = true;
}
}
{
let mut inner = self.inner.lock().map_err(|_| {
FrankenError::internal("SimplePagerCheckpointWriter lock poisoned")
})?;
inner.db_size = inner.db_size.max(page_no.get());
}
if page_no == PageNumber::ONE && data.len() >= DATABASE_HEADER_SIZE && self.dirty {
self.patch_page1_header(cx).await?;
}
self.cache.evict(page_no);
if page_no == PageNumber::ONE {
let inner = self.inner.lock().map_err(|_| {
FrankenError::internal("SimplePagerCheckpointWriter lock poisoned")
})?;
self.published.publish_remove_page(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
PageNumber::ONE,
);
}
Ok(())
})
}
fn truncate<'a>(&'a mut self, cx: &'a Cx, n_pages: u32) -> WalFuture<'a, ()> {
Box::pin(async move {
let (db_file, old_db_size, page_size) = {
let inner = self.inner.lock().map_err(|_| {
FrankenError::internal("SimplePagerCheckpointWriter lock poisoned")
})?;
(
Arc::clone(&inner.db_file),
inner.db_size,
inner.page_size.as_usize(),
)
};
let target_size = u64::from(n_pages) * page_size as u64;
{
let mut db_file = shared_db_file_write(&db_file, cx).await?;
// GH #294: skip the physical truncation when the file already
// has the target length so a no-op checkpoint pass never
// updates the main file's timestamps.
if db_file.file_size(cx)? != target_size {
db_file.truncate(cx, target_size)?;
self.dirty = true;
}
}
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePagerCheckpointWriter lock poisoned"))?;
inner.db_size = n_pages;
// Invalidate cached pages beyond the new size.
// ShardedPageCache is internally synchronized, so no lock needed.
for pgno in (n_pages.saturating_add(1))..=old_db_size {
if let Some(page_no) = PageNumber::new(pgno) {
self.cache.evict(page_no);
}
}
// D1-CRITICAL Change 3: Use sharded publish_truncate_checkpoint.
self.published.publish_truncate_checkpoint(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
n_pages,
);
Ok(())
})
}
fn sync<'a>(&'a mut self, cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async move {
// Ensure header page_count reflects the final db_size after all
// checkpoint writes/truncation, even if page 1 was checkpointed early.
// ShardedPageCache is internally synchronized, so no lock needed.
//
// GH #294: only re-stamp the header when this checkpoint pass
// actually mutated the database file. A pass that found every
// frame already backfilled must not rewrite the change counter:
// the pager's commit clock re-counts still-visible WAL commits on
// every open, so an unconditional stamp inflates the header
// change counter (bytes 24-27 / 92-95) and bumps the main file's
// timestamps once per open/close cycle.
if self.dirty {
self.patch_page1_header(cx).await?;
}
// Durability barrier FIRST, publication second (GH #195): the
// published pager plane must never advertise checkpoint state whose
// backing writes have not survived their sync barrier. On sync
// failure nothing is published and the WAL remains authoritative.
//
// GH #198: this sync is the checkpoint recovery fence — the WAL
// generation may be invalidated right after it, so it must be the
// strongest platform durability barrier. `durable_sync(FullDurable)`
// reaches F_FULLFSYNC on macOS (plain `sync(FULL)` stops at fsync,
// which does not flush the drive cache there) and a full fsync
// including metadata everywhere else.
let db_file = {
let inner = self.inner.lock().map_err(|_| {
FrankenError::internal("SimplePagerCheckpointWriter lock poisoned")
})?;
Arc::clone(&inner.db_file)
};
shared_db_file_write(&db_file, cx)
.await?
.durable_sync(cx, SyncKind::FullDurable)?;
let inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePagerCheckpointWriter lock poisoned"))?;
// D1-CRITICAL Change 3: Use sharded publish_remove_page.
self.published.publish_remove_page(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
PageNumber::ONE,
);
Ok(())
})
}
}
impl<V: Vfs> SimplePager<V>
where
V::File: Send + Sync + 'static,
{
/// Create a checkpoint page writer for WAL checkpointing.
///
/// The returned writer implements [`crate::CheckpointPageWriter`] and can be
/// wrapped in a `CheckpointTargetAdapter` from `fsqlite-core` to satisfy
/// the WAL executor's `CheckpointTarget` trait.
///
/// # Panics
///
/// This method does not panic, but the returned writer's methods may
/// return errors if the pager's internal mutex is poisoned.
#[must_use]
pub(crate) fn checkpoint_writer(&self) -> SimplePagerCheckpointWriter<V> {
SimplePagerCheckpointWriter {
inner: Arc::clone(&self.inner),
cache: Arc::clone(&self.cache),
published: Arc::clone(&self.published),
dirty: false,
}
}
/// Run a WAL checkpoint to transfer frames from the WAL to the database.
///
/// This is the main checkpoint entry point for WAL mode. It:
/// 1. Acquires the pager lock
/// 2. Creates a checkpoint writer for database page writes
/// 3. Delegates to the WAL backend's checkpoint implementation
///
/// # Arguments
///
/// * `cx` - Cancellation/deadline context
/// * `mode` - Checkpoint mode (Passive, Full, Restart, Truncate)
///
/// # Returns
///
/// A `CheckpointResult` describing what was accomplished, or an error if:
/// - The pager is not in WAL mode
/// - The pager lock is poisoned
/// - Any I/O error occurs during the checkpoint
///
/// # Notes
///
/// This implementation refuses to checkpoint while any transaction is active.
/// It starts from the beginning (backfilled_frames = 0) and passes
/// `oldest_reader_frame = None`. Because pager does not yet track external
/// reader end marks, `RESTART` and `TRUNCATE` are conservatively downgraded
/// to `FULL` so we never reset or truncate WAL based on incomplete reader
/// visibility. For incremental, reader-aware checkpointing, use the
/// lower-level WAL backend API.
// bd-h9o9r: a sync mutex guard is held across an await in this
// function's body; reachable-deadlock audit and lock-scope repair
// belong to the Phase-C pager reconstruction.
#[allow(clippy::await_holding_lock)]
pub async fn checkpoint(
&self,
cx: &Cx,
mode: traits::CheckpointMode,
) -> Result<traits::CheckpointResult> {
settle_pending_group_commit_finalization(&self.group_commit_queue).await?;
let _maintenance_lease = self.maintenance_gate.enter_transaction()?;
self.validate_namespace_binding()?;
let cleanup_cx = cleanup_child_cx(cx);
let checkpoint_gate_state;
// Reserve exclusive checkpoint ownership while marking checkpoint active.
// `begin()` and deferred writer upgrades are blocked while this flag is
// set so commits cannot observe "WAL mode but no backend".
let (wal, external_lock) = {
let mut inner = self
.inner
.lock()
.map_err(|_| FrankenError::internal("SimplePager lock poisoned"))?;
// Check we're in WAL mode.
if inner.journal_mode != JournalMode::Wal {
return Err(FrankenError::Unsupported);
}
if inner.checkpoint_active {
let active_transactions = inner.active_transactions;
let checkpoint_active = inner.checkpoint_active;
drop(inner);
log_checkpoint_coordination(
cx,
&pager_group_commit_queue(self),
"active_gate",
"checkpoint_begin",
"return_busy",
"checkpoint_already_owns_serialized_backend",
true,
active_transactions,
checkpoint_active,
);
return Err(FrankenError::Busy);
}
// Without reader tracking in pager, the safe policy is to refuse
// checkpoint while any transaction is active.
if inner.active_transactions > 0 {
let active_transactions = inner.active_transactions;
let checkpoint_active = inner.checkpoint_active;
drop(inner);
log_checkpoint_coordination(
cx,
&pager_group_commit_queue(self),
"writer_gate",
"checkpoint_begin",
"return_busy",
"checkpoint_waits_for_foreground_writers_to_quiesce",
true,
active_transactions,
checkpoint_active,
);
return Err(FrankenError::Busy);
}
let wal = wal_backend_handle(&self.wal_backend)?;
// Participate in the same VFS-defined whole-image fence used by
// VACUUM. On Windows this includes stock SQLite's real main-file
// and -shm byte ranges in addition to FrankenSQLite's cooperative
// sidecars; on Unix the default hook is the native lock protocol.
let mut external_lock = BeginExternalLockState::new(
&self.group_commit_queue,
Arc::clone(&inner.db_file),
cx,
);
external_lock.acquire_maintenance(cx, true).await?;
inner.checkpoint_active = true;
checkpoint_gate_state = (inner.active_transactions, inner.checkpoint_active);
// D1-CRITICAL Change 3: Use sharded publish_metadata_only.
self.published.publish_metadata_only(
cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
);
(wal, external_lock)
};
// Lock is released here.
log_checkpoint_coordination(
cx,
&pager_group_commit_queue(self),
"backend_owned",
"foreground_idle",
"checkpoint_runs",
"checkpoint_only_enters_after_group_commit_lane_quiesces",
true,
checkpoint_gate_state.0,
checkpoint_gate_state.1,
);
struct CheckpointGuard<'a, F: VfsFile + 'static> {
inner: &'a std::sync::Mutex<PagerInner<F>>,
published: &'a PublishedPagerState,
cleanup_cx: Cx,
external_lock: Option<BeginExternalLockState<F>>,
}
impl<F: VfsFile + 'static> Drop for CheckpointGuard<'_, F> {
fn drop(&mut self) {
// Terminalize the physical fence or publish its process-root
// retry before observers can see checkpoint activity clear.
drop(self.external_lock.take());
let mut inner = match self.inner.lock() {
Ok(inner) => inner,
Err(error) => {
tracing::error!(
"checkpoint guard recovered a poisoned PagerInner for fail-closed cleanup"
);
error.into_inner()
}
};
let _mask = self.cleanup_cx.masked();
inner.checkpoint_active = false;
self.published.publish_metadata_only(
&self.cleanup_cx,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq,
db_size: inner.db_size,
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
);
}
}
let mut guard = CheckpointGuard {
inner: &self.inner,
published: self.published.as_ref(),
cleanup_cx,
external_lock: Some(external_lock),
};
// Create a checkpoint writer that writes directly to the database file.
let mut writer = self.checkpoint_writer();
// Single-connection fast path (issue #66): when this pager is the only
// connection to the database (shared_connection_count == 1) and no
// transactions are active (already verified above), there are no
// readers whose WAL end-marks we need to track. Restart and Truncate
// are safe because no other connection can be reading from the WAL.
let sole_connection = self
.shared_connection_count
.get()
.is_some_and(|counter| counter.load(std::sync::atomic::Ordering::Acquire) == 1);
let effective_mode = match mode {
traits::CheckpointMode::Restart | traits::CheckpointMode::Truncate
if !sole_connection =>
{
tracing::debug!(
requested_mode = ?mode,
"downgrading checkpoint mode because pager has multiple connections or lacks reader-tracking for safe WAL reset"
);
traits::CheckpointMode::Full
}
_ => mode,
};
// Run the checkpoint from the beginning. Reader-aware incremental
// checkpointing requires exposing oldest-reader tracking from pager.
let mut wal = async_rwlock_write(&wal, cx, "WAL backend").await?;
let checkpoint_result = wal
.checkpoint(cx, effective_mode, &mut writer, 0, None)
.await;
drop(wal);
if let Err(error) = &checkpoint_result {
let queue_snapshot =
checkpoint_coordination_queue_snapshot(&pager_group_commit_queue(self));
tracing::warn!(
target: "fsqlite::wal::checkpoint_coordination",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_CHECKPOINT_RUN_ID,
scenario_id = PHYSICAL_WRITER_CHECKPOINT_SCENARIO_ID,
checkpoint_phase = "error",
foreground_phase = "foreground_idle",
foreground_action = "checkpoint_failed",
interaction_rule = "checkpoint_keeps_foreground_writer_lane_decoupled_even_on_error",
stall_avoided = true,
active_transactions = checkpoint_gate_state.0,
checkpoint_active = checkpoint_gate_state.1,
queue_phase = queue_snapshot.queue_phase,
queue_epoch = queue_snapshot.queue_epoch,
pending_batch_count = queue_snapshot.pending_batch_count,
requested_mode = ?mode,
effective_mode = ?effective_mode,
failure_context = %error,
"checkpoint failed after acquiring exclusive checkpoint ownership"
);
}
let mut result = checkpoint_result?;
// Surface any pager-level downgrade in the result so callers can
// detect that their requested mode was not honored (issue #66 fix 4).
result.requested_mode = mode;
result.effective_mode = effective_mode;
let queue_snapshot =
checkpoint_coordination_queue_snapshot(&pager_group_commit_queue(self));
tracing::debug!(
target: "fsqlite::wal::checkpoint_coordination",
trace_id = cx.trace_id(),
run_id = PHYSICAL_WRITER_CHECKPOINT_RUN_ID,
scenario_id = PHYSICAL_WRITER_CHECKPOINT_SCENARIO_ID,
checkpoint_phase = "complete",
foreground_phase = "foreground_idle",
foreground_action = "checkpoint_complete",
interaction_rule = "checkpoint_keeps_foreground_writer_lane_idle_until_release",
stall_avoided = true,
active_transactions = checkpoint_gate_state.0,
checkpoint_active = checkpoint_gate_state.1,
queue_phase = queue_snapshot.queue_phase,
queue_epoch = queue_snapshot.queue_epoch,
pending_batch_count = queue_snapshot.pending_batch_count,
requested_mode = ?result.requested_mode,
effective_mode = ?result.effective_mode,
total_frames = result.total_frames,
frames_backfilled = result.frames_backfilled,
completed = result.completed,
wal_was_reset = result.wal_was_reset,
"checkpoint completed without re-entering the foreground physical writer lane"
);
guard
.external_lock
.as_mut()
.expect("checkpoint guard must own its external maintenance attempt")
.restore()
.await?;
drop(guard);
Ok(result)
}
}
#[cfg(test)]
// bd-h9o9r: test bodies routinely hold the pager's sync mutex guards across
// awaits; under the deterministic single-task test runtimes this cannot
// deadlock, and per-site tags would add noise without audit value. The
// production sites each carry an individual audit tag instead.
#[allow(clippy::await_holding_lock)]
mod tests {
use super::*;
use crate::traits::{MvccPager, PagerCommitState, TransactionHandle, TransactionMode};
use fsqlite_types::PageSize;
use fsqlite_types::flags::{AccessFlags, SyncFlags, VfsOpenFlags};
use fsqlite_types::{BTreePageHeader, DatabaseHeader};
use fsqlite_vfs::{MemoryFile, MemoryVfs, Vfs, VfsFile};
#[cfg(all(feature = "native", unix))]
use fsqlite_vfs::{NamespaceOpenIntent, PendingNamespaceOpen, UnixVfs};
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering as AtomicOrdering};
use std::sync::{Arc, Mutex, Weak};
static FAULT_HOOK_TEST_GUARD: crate::fault_hooks::FaultInjectionSessionLock =
crate::fault_hooks::FaultInjectionSessionLock::new();
const BEAD_ID: &str = "bd-bca.1";
const DB300_E3_3_BEAD_ID: &str = "bd-db300.5.3.3";
const DB300_E3_3_A_BEAD_ID: &str = "bd-db300.5.3.3.1";
const TRACK_U_BEAD_ID: &str = "bd-c9pxw";
const CHECKPOINT_DECOUPLING_BEAD_ID: &str = "bd-1dp9.6.7.9.2";
const COMMIT_SERVICE_POLICY_BEAD_ID: &str = "bd-1dp9.6.7.9.4";
type ObservedLockLevel = Arc<Mutex<LockLevel>>;
type ObservedUnlockTraceIds = Arc<Mutex<Vec<u64>>>;
type ObservedCleanupUnlockHarness = (
SimplePager<ObservedLockVfs>,
ObservedLockLevel,
ObservedUnlockTraceIds,
);
/// Deliberate no-op (frankensqlite#299).
///
/// This helper previously installed a process-global `TRACE` subscriber via
/// `tracing_subscriber::fmt()...with_test_writer().try_init()`. Because
/// `try_init()` is process-wide and first-caller-wins, the first of the 39
/// callers changed tracing enablement — and libtest output capture — for
/// every unrelated test that ran afterwards in this binary. When any such
/// test later failed, libtest replayed the entire captured global trace
/// stream for it; one reported run produced a 9.2 GB, 29.7M-line archive
/// and terminated in `EDQUOT` without a trustworthy summary.
///
/// `fsqlite-core` fixed the identical pattern in b262b6a6; the pager helper
/// kept it. No caller asserts on emitted trace events — the trace-shaped
/// assertions in this module read `ObservedUnlockTraceIds`, a mock-VFS
/// field populated by the mock itself, not by a subscriber — so the body is
/// simply removed. The call sites are retained so the diff stays test-only
/// and reviewable.
///
/// A test that genuinely needs events should scope a target-filtered
/// subscriber with `tracing::subscriber::set_default(..)` and a guard, and
/// bound its capture. See `pager_test_tracing_helper_installs_no_global_subscriber`.
fn init_publication_test_tracing() {}
/// frankensqlite#299 regression: the publication tracing helper must not
/// install, or otherwise disturb, a process-global subscriber.
///
/// Comparing global dispatcher state across the call is the only sound
/// assertion available: it proves the helper itself is inert regardless of
/// libtest ordering or of what any other test did first, so it cannot be
/// tainted.
///
/// An absolute `!has_been_set()` assertion is deliberately NOT made, and an
/// earlier revision of this test was wrong to make one. It cannot hold in
/// this binary: `asupersync::test_utils` installs a process-global
/// dispatcher via `tracing::dispatcher::set_global_default`, and these
/// tests run under that harness, so a global subscriber already exists
/// before this body starts. That is unrelated to frankensqlite#299, whose
/// defect was this helper installing a `TRACE` subscriber with
/// `with_test_writer()` and thereby routing the global trace stream through
/// libtest capture for every later test in the binary.
#[test]
fn pager_test_tracing_helper_installs_no_global_subscriber() {
let before = tracing::dispatcher::has_been_set();
init_publication_test_tracing();
assert_eq!(
before,
tracing::dispatcher::has_been_set(),
"init_publication_test_tracing must not install or alter a global subscriber"
);
}
async fn test_pager() -> (SimplePager<MemoryVfs>, PathBuf) {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/test.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
(pager, path)
}
#[test]
fn group_commit_allocator_delta_prefers_live_allocations_over_returns() {
let page_three = PageNumber::new(3).unwrap();
let page_four = PageNumber::new(4).unwrap();
let page_five = PageNumber::new(5).unwrap();
let page_six = PageNumber::new(6).unwrap();
let returned = PendingReturnedAllocations {
from_freelist: vec![page_four, page_five],
from_eof: vec![page_six],
page_lease: vec![page_five],
};
let delta = PendingGroupCommitAllocatorDelta::new(
vec![page_five, page_five],
&returned,
&[page_three, page_five],
);
assert_eq!(delta.live_committed_allocations, vec![page_five]);
assert_eq!(
delta.returned_or_freed_pages,
vec![page_three, page_four, page_six],
"a page committed by any group member must not be returned by another member"
);
let mut freelist = vec![page_six, page_five, page_four];
delta.apply_to_freelist(&mut freelist);
assert_eq!(
freelist,
vec![page_six, page_four, page_three],
"allocator reconciliation must be idempotent and keep committed pages live"
);
delta.apply_to_freelist(&mut freelist);
assert_eq!(freelist, vec![page_six, page_four, page_three]);
}
#[test]
fn group_commit_record_local_wal_commit_at_catches_up_once_to_certificate_horizon() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.journal_mode = JournalMode::Wal;
inner.commit_seq = CommitSeq::new(7);
inner.committed_wal_visible_commit_count = 11;
inner.record_local_wal_commit_at(CommitSeq::new(10));
assert_eq!(inner.commit_seq, CommitSeq::new(10));
assert_eq!(inner.committed_wal_visible_commit_count, 14);
inner.record_local_wal_commit_at(CommitSeq::new(9));
inner.record_local_wal_commit_at(CommitSeq::new(10));
assert_eq!(inner.commit_seq, CommitSeq::new(10));
assert_eq!(
inner.committed_wal_visible_commit_count, 14,
"replayed or out-of-order Phase C callbacks must not double-count commits"
);
});
}
#[test]
fn group_commit_queue_reconciles_reused_combiner_to_durable_pager_floor() {
let queue = GroupCommitQueue::new(GroupCommitConfig::default());
let first = queue.durability_combiner(CommitSeq::new(3), 7);
assert_eq!(
first.visibility_snapshot().visible_commit_seq,
CommitSeq::new(3)
);
let reused = queue.durability_combiner(CommitSeq::new(10), 9);
assert!(Arc::ptr_eq(&first, &reused));
assert_eq!(
reused.visibility_snapshot().visible_commit_seq,
CommitSeq::new(10),
"a reused queue must not allocate below the pager's durable identity"
);
let stale = queue.durability_combiner(CommitSeq::new(8), 9);
assert_eq!(
stale.visibility_snapshot().visible_commit_seq,
CommitSeq::new(10),
"a stale pager observation must not lower the combiner"
);
}
#[test]
fn pending_group_commit_attempt_visible_bound_fences_transaction_introspection() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let snapshot_db_size = txn.snapshot_db_size();
let owned_page = PageNumber::new(snapshot_db_size + 1).unwrap();
let staged_page = PageNumber::new(snapshot_db_size + 2).unwrap();
let unrelated_global_size = snapshot_db_size + 3;
txn.write_page(&cx, staged_page, &sample_page(0x6D))
.await
.unwrap();
let staged_page_high_water = txn.staged_page_high_water(snapshot_db_size);
assert_eq!(staged_page_high_water, staged_page.get());
txn.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.db_size = unrelated_global_size;
let attempt = Arc::new(PendingGroupCommitTxnAttempt::new(
&txn.group_commit_queue,
Arc::clone(&txn.inner),
Arc::clone(&txn.db_file),
Arc::clone(&txn.committed_snapshot),
Arc::clone(&txn.published),
Arc::clone(&txn.writer_idle),
txn.cleanup_cx.clone(),
txn.original_db_size,
txn.mode,
txn.is_writer,
staged_page_high_water,
PendingReturnedAllocations::default(),
Vec::new(),
vec![owned_page],
));
txn.pending_group_commit_attempt = Some(attempt);
assert!(txn.has_pending_writes());
assert_eq!(txn.snapshot_db_size(), snapshot_db_size);
assert_eq!(txn.live_db_size(), unrelated_global_size);
assert_eq!(
txn.visible_db_size_bound(),
staged_page.get(),
"a pending attempt may widen its fixed snapshot for its own live allocation or staged page"
);
assert_eq!(txn.published_visible_commit_seq_hint(), None);
assert!(matches!(
txn.pending_commit_pages(),
Err(FrankenError::BusyRecovery)
));
assert!(matches!(
txn.pending_conflict_pages(),
Err(FrankenError::BusyRecovery)
));
assert_eq!(
txn.pending_conflict_pages_conservative(),
vec![PageNumber::ONE]
);
assert_eq!(txn.write_set_page_numbers(), vec![PageNumber::ONE]);
assert!(matches!(
txn.page_one_in_pending_commit_surface(),
Err(FrankenError::BusyRecovery)
));
assert!(matches!(
txn.allocate_page_requires_page_one_conflict_tracking(),
Err(FrankenError::BusyRecovery)
));
assert!(matches!(
txn.free_page_requires_page_one_conflict_tracking(PageNumber::new(2).unwrap()),
Err(FrankenError::BusyRecovery)
));
assert!(matches!(
txn.write_page_requires_page_one_conflict_tracking(PageNumber::new(2).unwrap()),
Err(FrankenError::BusyRecovery)
));
txn.prefetch_page_hint(&cx, PageNumber::ONE);
txn.prefetch_page_hints_greedy(&[], 0, 0.0);
txn.pending_group_commit_attempt.take();
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn group_commit_persisted_certificate_waits_for_complete_logical_phase_c_page_plane() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let queue = Arc::clone(&txn.group_commit_queue);
let attempt = Arc::new(PendingGroupCommitTxnAttempt::new(
&queue,
Arc::clone(&txn.inner),
Arc::clone(&txn.db_file),
Arc::clone(&txn.committed_snapshot),
Arc::clone(&txn.published),
Arc::clone(&txn.writer_idle),
txn.cleanup_cx.clone(),
txn.original_db_size,
txn.mode,
txn.is_writer,
txn.snapshot_db_size(),
PendingReturnedAllocations::default(),
Vec::new(),
Vec::new(),
));
let authorization = publication_authorization_for_test(false);
let epoch = authorization
.durability_receipt
.certificate
.certificate_epoch;
let batch_id = authorization.batch_id;
attempt
.admit(queue.register_epoch_consumer(epoch), batch_id)
.unwrap();
txn.pending_group_commit_attempt = Some(Arc::clone(&attempt));
queue
.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
epoch,
PersistedGroupCommitEpoch {
members: HashSet::from([batch_id]),
frames_start: 1,
frames_end: 2,
fsync_seq: 1,
durability_receipt: authorization.durability_receipt,
},
);
assert!(matches!(
attempt.reconcile_global_from_queue().unwrap(),
PendingGroupCommitTxnResolution::Pending
));
assert!(matches!(
txn.pending_commit_pages(),
Err(FrankenError::BusyRecovery)
));
txn.restore_not_committed_wal_attempt().unwrap();
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn rollback_after_authorized_group_commit_reports_commit_and_keeps_page_live() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let committed_page = PageNumber::new(2).unwrap();
let committed_bytes = sample_page(0xA7);
txn.write_page(&cx, committed_page, &committed_bytes)
.await
.unwrap();
txn.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.freelist
.push(committed_page);
let attempt = Arc::new(PendingGroupCommitTxnAttempt::new(
&txn.group_commit_queue,
Arc::clone(&txn.inner),
Arc::clone(&txn.db_file),
Arc::clone(&txn.committed_snapshot),
Arc::clone(&txn.published),
Arc::clone(&txn.writer_idle),
txn.cleanup_cx.clone(),
txn.original_db_size,
txn.mode,
txn.is_writer,
txn.snapshot_db_size(),
PendingReturnedAllocations::default(),
Vec::new(),
vec![committed_page],
));
txn.pending_group_commit_attempt = Some(Arc::clone(&attempt));
let authorization = publication_authorization_for_test(false);
let group_delta = attempt.allocator_delta();
attempt
.complete_authorized_global(
authorization,
&HashMap::from([(committed_page, PageData::from_vec(committed_bytes))]),
&group_delta,
true,
)
.unwrap();
let error = txn
.rollback(&cx)
.await
.expect_err("a durable group commit cannot be rolled back");
assert!(
error
.to_string()
.contains("could not undo a group commit that became durable"),
"rollback must distinguish committed recovery from successful rollback: {error}"
);
assert!(txn.finished);
assert!(txn.committed);
assert!(txn.pending_group_commit_attempt.is_none());
let inner = txn
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(inner.active_transactions, 0);
assert!(
!inner.freelist.contains(&committed_page),
"authorized rollback cleanup must never recycle a committed page"
);
});
}
#[test]
fn authorized_retained_group_commit_reads_certified_page_plane_not_local_page_one() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let local_page_one = sample_page(0x11);
let certified_page_one = PageData::from_vec(sample_page(0x22));
txn.write_page(&cx, PageNumber::ONE, &local_page_one)
.await
.unwrap();
let attempt = Arc::new(PendingGroupCommitTxnAttempt::new(
&txn.group_commit_queue,
Arc::clone(&txn.inner),
Arc::clone(&txn.db_file),
Arc::clone(&txn.committed_snapshot),
Arc::clone(&txn.published),
Arc::clone(&txn.writer_idle),
txn.cleanup_cx.clone(),
txn.original_db_size,
txn.mode,
txn.is_writer,
txn.snapshot_db_size(),
PendingReturnedAllocations::default(),
Vec::new(),
Vec::new(),
));
txn.pending_group_commit_attempt = Some(Arc::clone(&attempt));
let group_delta = attempt.allocator_delta();
attempt
.complete_authorized_global(
publication_authorization_for_test(false),
&HashMap::from([(PageNumber::ONE, certified_page_one.clone())]),
&group_delta,
true,
)
.unwrap();
txn.finish_authorized_wal_attempt(&cx, false).await.unwrap();
assert!(
txn.txn_read_cache.borrow().get(&PageNumber::ONE).is_none(),
"retained Phase C must not cache the member-local pre-consolidation Page 1"
);
assert_eq!(
txn.get_page(&cx, PageNumber::ONE).await.unwrap(),
certified_page_one
);
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn identity_registries_replace_expired_same_generation_state() {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let flags = VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (file, _) = vfs
.open(&cx, Some(Path::new("/identity-registry.db")), flags)
.unwrap();
let identity = file
.file_identity()
.unwrap()
.expect("MemoryVfs exposes a stable storage identity");
let old_fence = recovery_fence_for_identity(identity);
let old_gate = maintenance_gate_for_identity(identity);
let queue_path = Path::new("/identity-registry.db");
let old_queue = group_commit_queue_for_identity(identity, queue_path, true);
assert!(Arc::ptr_eq(
&old_fence,
&recovery_fence_for_identity(identity)
));
assert!(Arc::ptr_eq(
&old_gate,
&maintenance_gate_for_identity(identity)
));
assert!(Arc::ptr_eq(
&old_queue,
&group_commit_queue_for_identity(identity, queue_path, true)
));
let old_fence_weak = Arc::downgrade(&old_fence);
let old_gate_weak = Arc::downgrade(&old_gate);
let old_queue_weak = Arc::downgrade(&old_queue);
drop(old_fence);
drop(old_gate);
drop(old_queue);
assert!(old_fence_weak.upgrade().is_none());
assert!(old_gate_weak.upgrade().is_none());
assert!(old_queue_weak.upgrade().is_none());
let replacement_fence = recovery_fence_for_identity(identity);
let replacement_gate = maintenance_gate_for_identity(identity);
let replacement_queue = group_commit_queue_for_identity(identity, queue_path, true);
assert!(!Weak::ptr_eq(
&old_fence_weak,
&Arc::downgrade(&replacement_fence)
));
assert!(!Weak::ptr_eq(
&old_gate_weak,
&Arc::downgrade(&replacement_gate)
));
assert!(!Weak::ptr_eq(
&old_queue_weak,
&Arc::downgrade(&replacement_queue)
));
let mut local_registry = IdentityWeakRegistry::<usize>::default();
for index in 0..65_u32 {
let path = PathBuf::from(format!("/identity-registry-{index}.db"));
let (unique_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let unique_identity = unique_file.file_identity().unwrap().unwrap();
let value = local_registry.get_or_insert_with(unique_identity, || Arc::new(1));
drop(value);
}
assert!(
local_registry.entries.len() <= 2,
"amortized sweeps must bound expired identity keys"
);
}
#[cfg(all(feature = "native", unix))]
#[test]
fn connection_open_mode_retains_namespace_exclusivity_until_explicit_finish() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let dir = tempfile::tempdir().expect("tempdir");
let database = dir.path().join("connection-bootstrap.db");
let pager = SimplePager::open_for_connection_with_cx_and_page_buffer_max(
&cx,
UnixVfs::new(),
&database,
PageSize::DEFAULT,
None,
ConnectionPagerOpenMode::CreateIfMissing,
)
.await
.expect("open deferred connection pager");
let binding = pager
.namespace_binding()
.expect("native pager retains a namespace binding");
assert!(binding.bootstrap_is_exclusive());
assert!(matches!(
PendingNamespaceOpen::begin(&database, NamespaceOpenIntent::Shared),
Err(FrankenError::Busy)
));
let (backend, _, _, _) = MockWalBackend::new();
pager
.set_wal_backend(Box::new(backend))
.expect("connection bootstrap must permit WAL backend installation");
pager
.finish_namespace_bootstrap()
.expect("publish completed connection generation");
assert!(!binding.bootstrap_is_exclusive());
let peer = PendingNamespaceOpen::begin(&database, NamespaceOpenIntent::Shared)
.expect("peer can join after the connection success boundary");
assert_eq!(peer.expected_identity(), Some(binding.identity()));
peer.bind(binding.identity())
.expect("peer joins the published generation");
});
}
#[cfg(all(feature = "native", unix))]
#[test]
fn copied_or_corrupt_quiescent_namespace_records_rebind_safely() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let dir = tempfile::tempdir().expect("tempdir");
let source = dir.path().join("namespace-copy-source.db");
let source_pager =
SimplePager::open_with_cx(&cx, UnixVfs::new(), &source, PageSize::DEFAULT)
.await
.expect("create source database and namespace record");
drop(source_pager);
let sidecar = |path: &Path, suffix: &str| {
let mut suffixed = path.as_os_str().to_owned();
suffixed.push(suffix);
PathBuf::from(suffixed)
};
let target = dir.path().join("namespace-copy-target.db");
std::fs::copy(&source, &target).expect("copy source database");
for suffix in ["-fsqlite-ns-gate", "-fsqlite-ns-use"] {
std::fs::copy(sidecar(&source, suffix), sidecar(&target, suffix))
.expect("copy namespace sidecar");
}
let target_bytes = std::fs::read(&target).expect("snapshot target database");
let readonly =
SimplePager::open_readonly_with_cx(&cx, UnixVfs::new(), &target, PageSize::DEFAULT)
.await
.expect("open copied namespace state read-only");
drop(readonly);
std::fs::write(sidecar(&target, "-fsqlite-ns-use"), b"corrupt base record")
.expect("corrupt quiescent namespace record");
let repaired =
SimplePager::open_readonly_with_cx(&cx, UnixVfs::new(), &target, PageSize::DEFAULT)
.await
.expect("repair corrupt quiescent namespace state");
drop(repaired);
assert_eq!(
std::fs::read(&target).expect("re-read target database"),
target_bytes,
"read-only namespace repair must not modify the main database"
);
for suffix in ["-fsqlite-ns-gate", "-fsqlite-ns-use"] {
std::fs::copy(sidecar(&source, suffix), sidecar(&target, suffix))
.expect("restore namespace sidecar");
}
let writable = SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
UnixVfs::new(),
&target,
PageSize::DEFAULT,
None,
None,
)
.await
.expect("open copied namespace state read-write");
drop(writable);
let missing = dir.path().join("namespace-copy-missing-main.db");
for suffix in ["-fsqlite-ns-gate", "-fsqlite-ns-use"] {
std::fs::copy(sidecar(&source, suffix), sidecar(&missing, suffix))
.expect("copy orphan namespace sidecar");
}
assert!(matches!(
SimplePager::open_with_cx(&cx, UnixVfs::new(), &missing, PageSize::DEFAULT,).await,
Err(FrankenError::CannotOpen { .. })
));
assert!(
!missing.exists(),
"stale sidecars must not create a replacement for a missing main database"
);
});
}
#[test]
fn gh_131_write_stage_reclaims_clean_cache_page_when_pool_is_saturated() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let pager = SimplePager::open_with_cx_and_page_buffer_max(
&cx,
MemoryVfs::new(),
Path::new("/clean_cache_write_admission.db"),
PageSize::DEFAULT,
Some(2),
)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for raw_page_no in 1..=2 {
let page_no = PageNumber::new(raw_page_no).unwrap();
let mut buffer = pager.pool.acquire().unwrap();
buffer.as_mut_slice()[0] = u8::try_from(raw_page_no).unwrap();
pager.cache.insert_buffer(page_no, buffer);
}
assert_eq!(pager.pool.total_buffers(), 2);
assert_eq!(pager.pool.available(), 0);
assert_eq!(pager.cache.metrics_snapshot().cached_pages, 2);
let write_page = PageNumber::new(3).unwrap();
txn.write_page(&cx, write_page, &sample_page(0x5A))
.await
.expect("write staging should evict one clean cache page and retry");
assert_eq!(pager.cache.metrics_snapshot().cached_pages, 1);
assert!(txn.write_set.contains_key(&write_page));
assert_eq!(pager.pool.total_buffers(), 2);
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn gh_131_write_stage_never_flushes_dirty_cache_page_to_main_db() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
for (journal_mode, path) in [
(
JournalMode::Delete,
Path::new("/dirty_cache_write_admission_delete.db"),
),
(
JournalMode::Wal,
Path::new("/dirty_cache_write_admission_wal.db"),
),
] {
let vfs = DbWriteFailOnceVfs::new(path.to_path_buf());
let pager = vfs
.open_file_backed_pager_with_page_buffer_max(&cx, path, 1)
.await
.unwrap();
if journal_mode == JournalMode::Wal {
let (backend, _frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
}
assert_eq!(pager.journal_mode(), journal_mode);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let dirty_page = PageNumber::ONE;
pager
.cache
.insert_fresh(dirty_page, |bytes| bytes[0] = 0xD1)
.unwrap();
assert_eq!(pager.pool.total_buffers(), 1);
assert_eq!(pager.pool.available(), 0);
vfs.arm_after_db_writes(0);
let error = txn
.write_page(&cx, PageNumber::new(2).unwrap(), &sample_page(0x6B))
.await
.expect_err("a dirty-only saturated cache must fail closed");
match error {
FrankenError::PageBufferCapacityExhausted {
operation,
page_size,
max_buffers,
total_buffers,
available_buffers,
cached_clean,
cached_dirty,
successful_evictions,
} => {
assert_eq!(operation, "transaction_write_stage");
assert_eq!(page_size, PageSize::DEFAULT.as_usize());
assert_eq!(max_buffers, 1);
assert_eq!(total_buffers, 1);
assert_eq!(available_buffers, 0);
assert_eq!(cached_clean, 0);
assert_eq!(cached_dirty, 1);
assert_eq!(successful_evictions, 0);
}
other => panic!("expected structured capacity exhaustion, got {other:?}"),
}
assert_eq!(
vfs.db_write_fault_observation(),
(Vec::new(), None),
"failed staging must not write a dirty shared-cache page to MAIN_DB in {journal_mode:?} mode"
);
assert!(pager.cache.contains(dirty_page));
assert!(
pager
.cache
.page_snapshots()
.iter()
.any(|snapshot| snapshot.page_no == dirty_page && snapshot.dirty)
);
assert!(!txn.writes_observed);
assert!(txn.write_set.is_empty());
assert!(txn.write_pages_sorted.is_empty());
assert!(!txn.has_pending_writes());
txn.rollback(&cx).await.unwrap();
assert_eq!(vfs.db_write_fault_observation(), (Vec::new(), None));
assert!(pager.cache.contains(dirty_page));
let mut reused = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
reused.rollback(&cx).await.unwrap();
assert_eq!(vfs.db_write_fault_observation(), (Vec::new(), None));
}
});
}
#[test]
fn gh_131_repeated_owned_page_cache_admission_stays_within_pool_bound() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let pager = SimplePager::open_with_cx_and_page_buffer_max(
&cx,
MemoryVfs::new(),
Path::new("/bounded_owned_cache_admission.db"),
PageSize::DEFAULT,
Some(1),
)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
pager
.cache
.insert_buffer(PageNumber::ONE, pager.pool.acquire().unwrap());
for raw_page_no in 2..=64 {
let page_no = PageNumber::new(raw_page_no).unwrap();
txn.write_page_data(
&cx,
page_no,
PageData::from_vec(sample_page(u8::try_from(raw_page_no).unwrap())),
)
.await
.unwrap();
txn.drain_committed_cache_pages_into_cache();
assert_eq!(pager.pool.total_buffers(), 1);
assert_eq!(pager.cache.metrics_snapshot().cached_pages, 1);
assert!(pager.cache.contains(page_no));
}
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn gh_291_transaction_read_cache_respects_page_buffer_max() {
asupersync::test_utils::run_test(|| async {
const PAGE_BUFFER_MAX: usize = 3;
let cx = Cx::new();
let pager = SimplePager::open_with_cx_and_page_buffer_max(
&cx,
MemoryVfs::new(),
Path::new("/bounded_transaction_read_cache.db"),
PageSize::DEFAULT,
Some(PAGE_BUFFER_MAX),
)
.await
.unwrap();
let mut pages = Vec::new();
for marker in 1_u8..=8 {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_no, &sample_page(marker))
.await
.unwrap();
seed.commit(&cx).await.unwrap();
pages.push((page_no, marker));
}
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for &(page_no, marker) in &pages {
let page = reader.get_page(&cx, page_no).await.unwrap();
assert_eq!(page.as_ref()[0], marker);
assert!(
reader.txn_read_cache.borrow().len() <= PAGE_BUFFER_MAX,
"a sequential scan must not pin more page images than page_buffer_max"
);
}
assert_eq!(reader.txn_read_cache.borrow().len(), PAGE_BUFFER_MAX);
let (first_page, first_marker) = pages[0];
assert_eq!(
reader.get_page(&cx, first_page).await.unwrap().as_ref()[0],
first_marker,
"a retained page must remain stable after cache saturation"
);
assert_eq!(reader.txn_read_cache.borrow().len(), PAGE_BUFFER_MAX);
let (uncached_page, uncached_marker) = *pages.last().unwrap();
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
writer
.write_page(&cx, uncached_page, &sample_page(0xEE))
.await
.unwrap();
writer.commit(&cx).await.unwrap();
assert_eq!(
reader.get_page(&cx, first_page).await.unwrap().as_ref()[0],
first_marker,
"a cached baseline must survive a concurrent commit"
);
let error = reader
.get_page(&cx, uncached_page)
.await
.expect_err("an uncached page must fail closed after the snapshot advances");
assert!(
matches!(error, FrankenError::BusySnapshot { .. }),
"expected BusySnapshot for uncached page {uncached_page:?} (old marker \
{uncached_marker}), got {error:?}"
);
reader.commit(&cx).await.unwrap();
});
}
#[test]
fn gh_131_read_copy_capacity_fallback_preserves_dirty_cache_page() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let pager = SimplePager::open_with_cx_and_page_buffer_max(
&cx,
MemoryVfs::new(),
Path::new("/dirty_cache_read_fallback.db"),
PageSize::DEFAULT,
Some(1),
)
.await
.unwrap();
let expected = sample_page(0x27);
let page_no = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_no, &expected).await.unwrap();
seed.commit(&cx).await.unwrap();
page_no
};
pager.cache.clear();
pager
.cache
.insert_fresh(PageNumber::ONE, |bytes| bytes[0] = 0xD1)
.unwrap();
let data = pager
.inner
.lock()
.unwrap()
.read_page_copy(&cx, &pager.cache, &pager.wal_backend, page_no)
.await
.unwrap();
assert_eq!(data, expected);
assert!(pager.cache.contains(PageNumber::ONE));
assert!(
pager
.cache
.page_snapshots()
.iter()
.any(|snapshot| snapshot.page_no == PageNumber::ONE && snapshot.dirty)
);
});
}
#[cfg(all(feature = "native", unix))]
#[test]
fn gh_131_file_backed_saturated_cache_write_commit_and_rollback_are_correct() {
asupersync::test_utils::run_test(|| async {
async fn saturate_cache(
pager: &SimplePager<fsqlite_vfs::UnixVfs>,
cx: &Cx,
page_numbers: &[PageNumber],
) {
pager.cache.clear();
for &page_no in page_numbers {
let bytes = pager
.inner
.lock()
.unwrap()
.read_page_copy_uncached(cx, page_no)
.await
.unwrap();
let mut buffer = pager.pool.acquire().unwrap();
buffer.copy_from_slice(&bytes);
pager.cache.insert_buffer(page_no, buffer);
}
assert_eq!(pager.pool.available(), 0);
assert_eq!(
pager.cache.metrics_snapshot().cached_pages,
page_numbers.len()
);
}
let cx = Cx::new();
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("saturated-cache-transaction-cycle.db");
let pager = SimplePager::open_with_cx_and_page_buffer_max(
&cx,
fsqlite_vfs::UnixVfs::new(),
&path,
PageSize::DEFAULT,
Some(2),
)
.await
.unwrap();
let original = sample_page(0x31);
let seeded_page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_no, &original).await.unwrap();
seed.commit(&cx).await.unwrap();
page_no
};
saturate_cache(&pager, &cx, &[PageNumber::ONE, seeded_page]).await;
let committed = sample_page(0x42);
let committed_page = {
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = writer.allocate_page(&cx).await.unwrap();
writer.write_page(&cx, page_no, &committed).await.unwrap();
writer.commit(&cx).await.unwrap();
page_no
};
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader
.get_page(&cx, committed_page)
.await
.unwrap()
.as_bytes(),
committed
);
reader.rollback(&cx).await.unwrap();
saturate_cache(&pager, &cx, &[PageNumber::ONE, committed_page]).await;
let rolled_back = sample_page(0x53);
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
writer
.write_page(&cx, committed_page, &rolled_back)
.await
.unwrap();
writer.rollback(&cx).await.unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader
.get_page(&cx, committed_page)
.await
.unwrap()
.as_bytes(),
committed
);
reader.rollback(&cx).await.unwrap();
assert!(pager.pool.total_buffers() <= pager.pool.capacity());
});
}
fn sample_page(seed: u8) -> Vec<u8> {
let page_size = PageSize::DEFAULT.as_usize();
let mut page = vec![0u8; page_size];
for (i, byte) in page.iter_mut().enumerate() {
let reduced = u8::try_from(i % 251).expect("modulo fits u8");
*byte = reduced ^ seed;
}
page
}
#[test]
fn shared_file_state_key_matches_relative_and_absolute_spellings() {
let rel = Path::new("__fsqlite_nonexistent_shared_state_key_probe__.db");
let abs = std::env::current_dir().unwrap().join(rel);
assert_eq!(shared_file_state_key(rel), shared_file_state_key(&abs));
}
#[test]
fn shared_file_state_key_normalizes_fallback_dot_components() {
let probe = Path::new("__fsqlite_nonexistent_shared_state_key_probe__.db");
let dotted = Path::new(".").join(probe);
let parent = Path::new("__fsqlite_unused_probe_parent__")
.join("..")
.join(probe);
assert_eq!(shared_file_state_key(probe), shared_file_state_key(&dotted));
assert_eq!(shared_file_state_key(probe), shared_file_state_key(&parent));
}
#[test]
fn shared_file_state_key_matches_before_and_after_file_creation() {
let dir = tempfile::tempdir().unwrap();
let canonical_path = dir.path().join("coordination.db");
let dotted_path = dir.path().join(".").join("coordination.db");
let before_create_key = shared_file_state_key(&dotted_path);
std::fs::write(&canonical_path, b"").unwrap();
let after_create_key = shared_file_state_key(&dotted_path);
assert_eq!(
before_create_key, after_create_key,
"shared state keys must stay stable when the db file is created after first open"
);
}
#[cfg(all(feature = "native", unix))]
#[test]
fn open_without_rollback_journal_skips_recovery_fence_convoy() {
asupersync::test_utils::run_test(|| async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("clean-open.db");
let cx = Cx::new();
{
let _pager = SimplePager::open_with_cx(
&cx,
fsqlite_vfs::UnixVfs::new(),
&path,
PageSize::DEFAULT,
)
.await
.expect("create clean database");
}
let journal_path = SimplePager::<fsqlite_vfs::UnixVfs>::journal_path(&path);
assert!(
!fsqlite_vfs::UnixVfs::new()
.access(&cx, &journal_path, AccessFlags::EXISTS)
.expect("journal existence probe"),
"test precondition: clean database must not have a rollback journal"
);
let fence = recovery_fence_for_path(&path);
let held = fence
.try_acquire_for_recovery()
.expect("hold recovery fence to simulate unrelated opener recovery");
let started = std::time::Instant::now();
let _reopened = SimplePager::open_with_cx(
&cx,
fsqlite_vfs::UnixVfs::new(),
&path,
PageSize::DEFAULT,
)
.await
.expect("clean open should not wait behind recovery fence");
let elapsed = started.elapsed();
drop(held);
assert!(
elapsed < Duration::from_millis(100),
"clean shared-file open should skip the recovery fence when no rollback journal exists \
(elapsed {elapsed:?})"
);
});
}
fn default_commit_service_fairness_budget(max_wait: Duration) -> Duration {
commit_service_fairness_budget(&ParallelWalControlSurface::default(), max_wait)
}
fn track_u_log_counts(
case: &str,
dirty_set_count: usize,
already_dirty_skip_count: usize,
commit_flush_count: usize,
) {
eprintln!(
"INFO bead_id={TRACK_U_BEAD_ID} case={case} dirty_set_count={dirty_set_count} \
already_dirty_skip_count={already_dirty_skip_count} \
commit_flush_count={commit_flush_count}"
);
}
async fn private_memory_pager() -> SimplePager<MemoryVfs> {
SimplePager::open(MemoryVfs::new(), Path::new("/:memory:"), PageSize::DEFAULT)
.await
.unwrap()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct FlushBusyRetryScheduleSummary {
attempts: u32,
total_spin_loops: u64,
max_spin_loops: u32,
yield_count: u32,
}
fn summarize_flush_busy_retry_schedule(attempts: u32) -> FlushBusyRetryScheduleSummary {
assert!(
attempts > 0,
"flush busy retry validation needs at least one attempt"
);
let waits = (1..=attempts)
.map(flush_busy_retry_wait)
.collect::<Vec<_>>();
let total_spin_loops = waits.iter().map(|wait| u64::from(wait.spin_loops)).sum();
let max_spin_loops = waits
.iter()
.map(|wait| wait.spin_loops)
.max()
.unwrap_or_default();
let yield_count = u32::try_from(waits.iter().filter(|wait| wait.yielded).count())
.expect("flush busy retry validation should fit within u32 attempt counts");
FlushBusyRetryScheduleSummary {
attempts,
total_spin_loops,
max_spin_loops,
yield_count,
}
}
#[test]
fn test_flush_busy_retry_yield_cadence_is_bounded() {
for attempt in 1..FLUSH_BUSY_HANDOFF_YIELD_EVERY {
assert!(
!flush_busy_retry_should_yield(attempt),
"attempt {attempt} should stay on-CPU"
);
}
assert!(flush_busy_retry_should_yield(
FLUSH_BUSY_HANDOFF_YIELD_EVERY
));
assert!(!flush_busy_retry_should_yield(
FLUSH_BUSY_HANDOFF_YIELD_EVERY + 1
));
assert!(flush_busy_retry_should_yield(
FLUSH_BUSY_HANDOFF_YIELD_EVERY * 2
));
}
#[test]
fn test_flush_busy_retry_schedule_summary_bounds_tail_budget() {
let summary = summarize_flush_busy_retry_schedule(10);
assert_eq!(summary.attempts, 10);
assert_eq!(summary.total_spin_loops, 12_224);
assert_eq!(summary.max_spin_loops, FLUSH_BUSY_HANDOFF_MAX_SPINS);
assert_eq!(
summary.yield_count,
10 / FLUSH_BUSY_HANDOFF_YIELD_EVERY,
"wake amplification must stay at one scheduler yield per bounded retry window"
);
}
#[derive(Debug, Clone, Copy)]
struct ReadSurfaceSnapshot {
cache: PageCacheMetricsSnapshot,
published_hits: u64,
}
fn read_surface_snapshot<V>(pager: &SimplePager<V>) -> ReadSurfaceSnapshot
where
V: Vfs + Send + Sync,
V::File: Send + Sync + 'static,
{
ReadSurfaceSnapshot {
cache: pager.cache_metrics_snapshot().unwrap(),
published_hits: pager.published_page_hits(),
}
}
fn observed_read_total(before: ReadSurfaceSnapshot, after: ReadSurfaceSnapshot) -> u64 {
after
.cache
.total_accesses()
.saturating_sub(before.cache.total_accesses())
.saturating_add(after.published_hits.saturating_sub(before.published_hits))
}
fn observed_read_hit_rate_percent(
before: ReadSurfaceSnapshot,
after: ReadSurfaceSnapshot,
) -> f64 {
let cache_hits = after.cache.hits.saturating_sub(before.cache.hits);
let published_hits = after.published_hits.saturating_sub(before.published_hits);
let total_reads = observed_read_total(before, after);
if total_reads == 0 {
0.0
} else {
(cache_hits.saturating_add(published_hits) as f64 * 100.0) / total_reads as f64
}
}
struct DropAwareWalBackend {
dropped: Arc<Mutex<bool>>,
}
impl Drop for DropAwareWalBackend {
fn drop(&mut self) {
*self
.dropped
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) = true;
}
}
impl crate::traits::WalBackend for DropAwareWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
0
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: false,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
#[derive(Clone)]
struct JournalDeleteFailVfs {
inner: MemoryVfs,
memory_fast_path: Arc<AtomicBool>,
journal_delete_attempts: Arc<AtomicUsize>,
}
impl JournalDeleteFailVfs {
fn new() -> Self {
Self {
inner: MemoryVfs::new(),
memory_fast_path: Arc::new(AtomicBool::new(true)),
journal_delete_attempts: Arc::new(AtomicUsize::new(0)),
}
}
async fn open_file_backed_pager(&self, path: &Path) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result = SimplePager::open(self.clone(), path, PageSize::DEFAULT).await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
}
impl Vfs for JournalDeleteFailVfs {
type File = MemoryFile;
fn name(&self) -> &'static str {
self.inner.name()
}
fn open(
&self,
cx: &Cx,
path: Option<&std::path::Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
self.inner.open(cx, path, flags)
}
fn delete(&self, cx: &Cx, path: &std::path::Path, sync_dir: bool) -> Result<()> {
if path.to_string_lossy().ends_with("-journal") {
let prior_attempts = self
.journal_delete_attempts
.fetch_add(1, AtomicOrdering::Relaxed);
assert!(
prior_attempts < 8,
"open-time recovery must not spin on a proven non-hot journal whose cleanup delete fails"
);
return Err(FrankenError::internal(
"simulated journal delete failure".to_owned(),
));
}
self.inner.delete(cx, path, sync_dir)
}
fn access(&self, cx: &Cx, path: &std::path::Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &std::path::Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
fn is_memory(&self) -> bool {
self.memory_fast_path.load(AtomicOrdering::Acquire)
}
}
#[derive(Clone, Default)]
struct WalReadonlyFallbackProbeVfs {
inner: MemoryVfs,
readonly_wal_open_attempted: Arc<AtomicBool>,
}
impl WalReadonlyFallbackProbeVfs {
fn new() -> Self {
Self {
inner: MemoryVfs::new(),
readonly_wal_open_attempted: Arc::new(AtomicBool::new(false)),
}
}
fn readonly_wal_open_attempted(&self) -> bool {
self.readonly_wal_open_attempted
.load(AtomicOrdering::Relaxed)
}
}
impl Vfs for WalReadonlyFallbackProbeVfs {
type File = MemoryFile;
fn name(&self) -> &'static str {
self.inner.name()
}
fn open(
&self,
cx: &Cx,
path: Option<&std::path::Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
let is_wal = flags.contains(VfsOpenFlags::WAL);
if is_wal && flags.contains(VfsOpenFlags::READONLY) {
self.readonly_wal_open_attempted
.store(true, AtomicOrdering::Relaxed);
}
if is_wal && flags.contains(VfsOpenFlags::READWRITE) {
return Err(FrankenError::CannotOpen {
path: path
.map(std::path::Path::to_path_buf)
.unwrap_or_else(|| PathBuf::from("<wal-probe>")),
});
}
self.inner.open(cx, path, flags)
}
fn delete(&self, cx: &Cx, path: &std::path::Path, sync_dir: bool) -> Result<()> {
self.inner.delete(cx, path, sync_dir)
}
fn access(&self, cx: &Cx, path: &std::path::Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &std::path::Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
}
#[derive(Clone)]
struct ObservedLockVfs {
inner: MemoryVfs,
observed_lock_level: ObservedLockLevel,
observed_unlock_trace_ids: ObservedUnlockTraceIds,
fail_unlock_on_checkpoint_error: bool,
memory_fast_path: Arc<AtomicBool>,
external_snapshot_acquire_failures: Arc<AtomicUsize>,
external_maintenance_acquire_failures: Arc<AtomicUsize>,
external_restore_failures: Arc<AtomicUsize>,
external_restore_publication_probe: Arc<Mutex<Option<Weak<PublishedPagerState>>>>,
external_restore_checkpoint_observations: Arc<Mutex<Vec<bool>>>,
}
impl ObservedLockVfs {
fn new() -> Self {
Self {
inner: MemoryVfs::new(),
observed_lock_level: Arc::new(Mutex::new(LockLevel::None)),
observed_unlock_trace_ids: Arc::new(Mutex::new(Vec::new())),
fail_unlock_on_checkpoint_error: false,
memory_fast_path: Arc::new(AtomicBool::new(true)),
external_snapshot_acquire_failures: Arc::new(AtomicUsize::new(0)),
external_maintenance_acquire_failures: Arc::new(AtomicUsize::new(0)),
external_restore_failures: Arc::new(AtomicUsize::new(0)),
external_restore_publication_probe: Arc::new(Mutex::new(None)),
external_restore_checkpoint_observations: Arc::new(Mutex::new(Vec::new())),
}
}
async fn open_file_backed_pager(&self, path: &Path) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result = SimplePager::open(self.clone(), path, PageSize::DEFAULT).await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
async fn open_file_backed_readonly_pager(
&self,
cx: &Cx,
path: &Path,
) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result =
SimplePager::open_readonly_with_cx(cx, self.clone(), path, PageSize::DEFAULT).await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
fn observed_lock_level(&self) -> ObservedLockLevel {
Arc::clone(&self.observed_lock_level)
}
fn observed_unlock_trace_ids(&self) -> ObservedUnlockTraceIds {
Arc::clone(&self.observed_unlock_trace_ids)
}
fn with_checkpoint_enforced_unlock() -> Self {
Self {
fail_unlock_on_checkpoint_error: true,
..Self::new()
}
}
fn with_external_attempt_failures(
snapshot_acquire_failures: usize,
maintenance_acquire_failures: usize,
restore_failures: usize,
) -> Self {
Self {
external_snapshot_acquire_failures: Arc::new(AtomicUsize::new(
snapshot_acquire_failures,
)),
external_maintenance_acquire_failures: Arc::new(AtomicUsize::new(
maintenance_acquire_failures,
)),
external_restore_failures: Arc::new(AtomicUsize::new(restore_failures)),
..Self::new()
}
}
}
struct ObservedLockFile {
inner: MemoryFile,
observe_lock_state: bool,
observed_lock_level: ObservedLockLevel,
observed_unlock_trace_ids: ObservedUnlockTraceIds,
fail_unlock_on_checkpoint_error: bool,
external_snapshot_prior_level: Option<LockLevel>,
external_maintenance_prior_level: Option<LockLevel>,
external_snapshot_acquire_failures: Arc<AtomicUsize>,
external_maintenance_acquire_failures: Arc<AtomicUsize>,
external_restore_failures: Arc<AtomicUsize>,
external_restore_publication_probe: Arc<Mutex<Option<Weak<PublishedPagerState>>>>,
external_restore_checkpoint_observations: Arc<Mutex<Vec<bool>>>,
}
fn consume_observed_lock_failure(counter: &AtomicUsize) -> bool {
atomic_usize_checked_update(
counter,
AtomicOrdering::AcqRel,
AtomicOrdering::Acquire,
|remaining| remaining.checked_sub(1),
)
.is_ok()
}
fn record_observed_restore_checkpoint_activity(
probe: &Mutex<Option<Weak<PublishedPagerState>>>,
observations: &Mutex<Vec<bool>>,
) {
let published = probe
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.and_then(Weak::upgrade);
if let Some(published) = published {
observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(published.snapshot().checkpoint_active);
}
}
impl Vfs for ObservedLockVfs {
type File = ObservedLockFile;
fn name(&self) -> &'static str {
self.inner.name()
}
fn open(
&self,
cx: &Cx,
path: Option<&std::path::Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
let (inner, actual_flags) = self.inner.open(cx, path, flags)?;
Ok((
ObservedLockFile {
inner,
observe_lock_state: actual_flags.contains(VfsOpenFlags::MAIN_DB),
observed_lock_level: self.observed_lock_level(),
observed_unlock_trace_ids: self.observed_unlock_trace_ids(),
fail_unlock_on_checkpoint_error: self.fail_unlock_on_checkpoint_error,
external_snapshot_prior_level: None,
external_maintenance_prior_level: None,
external_snapshot_acquire_failures: Arc::clone(
&self.external_snapshot_acquire_failures,
),
external_maintenance_acquire_failures: Arc::clone(
&self.external_maintenance_acquire_failures,
),
external_restore_failures: Arc::clone(&self.external_restore_failures),
external_restore_publication_probe: Arc::clone(
&self.external_restore_publication_probe,
),
external_restore_checkpoint_observations: Arc::clone(
&self.external_restore_checkpoint_observations,
),
},
actual_flags,
))
}
fn delete(&self, cx: &Cx, path: &std::path::Path, sync_dir: bool) -> Result<()> {
self.inner.delete(cx, path, sync_dir)
}
fn access(&self, cx: &Cx, path: &std::path::Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &std::path::Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
fn is_memory(&self) -> bool {
self.memory_fast_path.load(AtomicOrdering::Acquire)
}
}
impl VfsFile for ObservedLockFile {
fn close(&mut self, cx: &Cx) -> Result<()> {
let result = self.inner.close(cx);
if result.is_ok() && self.observe_lock_state {
*self.observed_lock_level.lock().unwrap() = LockLevel::None;
}
result
}
fn file_identity(&self) -> Result<Option<FileIdentity>> {
self.inner.file_identity()
}
fn read<'a>(
&'a self,
cx: &'a Cx,
buf: &'a mut [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<usize>> + Send + 'a {
self.inner.read(cx, buf, offset)
}
fn write<'a>(
&'a self,
cx: &'a Cx,
buf: &'a [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<()>> + Send + 'a {
self.inner.write(cx, buf, offset)
}
fn truncate(&mut self, cx: &Cx, size: u64) -> Result<()> {
self.inner.truncate(cx, size)
}
fn sync(&mut self, cx: &Cx, flags: SyncFlags) -> Result<()> {
self.inner.sync(cx, flags)
}
fn file_size(&self, cx: &Cx) -> Result<u64> {
self.inner.file_size(cx)
}
fn lock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
self.inner.lock(cx, level)?;
if self.observe_lock_state {
*self.observed_lock_level.lock().unwrap() = level;
}
Ok(())
}
fn unlock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
if self.observe_lock_state {
self.observed_unlock_trace_ids
.lock()
.unwrap()
.push(cx.trace_id());
}
if self.fail_unlock_on_checkpoint_error {
cx.checkpoint()
.map_err(|err| FrankenError::internal(err.to_string()))?;
}
self.inner.unlock(cx, level)?;
if self.observe_lock_state {
*self.observed_lock_level.lock().unwrap() = level;
}
Ok(())
}
fn lock_external_shared_snapshot(&mut self, cx: &Cx) -> Result<()> {
if !self.observe_lock_state {
return self.inner.lock_external_shared_snapshot(cx);
}
if self.external_snapshot_prior_level.is_some()
|| self.external_maintenance_prior_level.is_some()
{
return Err(FrankenError::internal(
"observed-lock external attempt is already active",
));
}
let prior_level = *self.observed_lock_level.lock().unwrap();
self.external_snapshot_prior_level = Some(prior_level);
self.inner.lock_external_shared_snapshot(cx)?;
*self.observed_lock_level.lock().unwrap() = prior_level.max(LockLevel::Shared);
if consume_observed_lock_failure(&self.external_snapshot_acquire_failures) {
return Err(FrankenError::internal(
"injected external snapshot acquisition failure after arming",
));
}
Ok(())
}
fn restore_external_shared_snapshot_attempt(&mut self, cx: &Cx) -> Result<()> {
if !self.observe_lock_state {
return self.inner.restore_external_shared_snapshot_attempt(cx);
}
let Some(prior_level) = self.external_snapshot_prior_level else {
return self.inner.restore_external_shared_snapshot_attempt(cx);
};
record_observed_restore_checkpoint_activity(
&self.external_restore_publication_probe,
&self.external_restore_checkpoint_observations,
);
if consume_observed_lock_failure(&self.external_restore_failures) {
return Err(FrankenError::internal(
"injected external snapshot restoration failure",
));
}
self.observed_unlock_trace_ids
.lock()
.unwrap()
.push(cx.trace_id());
if self.fail_unlock_on_checkpoint_error {
cx.checkpoint()
.map_err(|err| FrankenError::internal(err.to_string()))?;
}
self.inner.restore_external_shared_snapshot_attempt(cx)?;
*self.observed_lock_level.lock().unwrap() = prior_level;
self.external_snapshot_prior_level = None;
Ok(())
}
fn lock_external_maintenance(&mut self, cx: &Cx, wal_mode: bool) -> Result<()> {
if !self.observe_lock_state {
return self.inner.lock_external_maintenance(cx, wal_mode);
}
if self.external_snapshot_prior_level.is_some()
|| self.external_maintenance_prior_level.is_some()
{
return Err(FrankenError::internal(
"observed-lock external attempt is already active",
));
}
let prior_level = *self.observed_lock_level.lock().unwrap();
self.external_maintenance_prior_level = Some(prior_level);
self.inner.lock_external_maintenance(cx, wal_mode)?;
*self.observed_lock_level.lock().unwrap() = LockLevel::Exclusive;
if consume_observed_lock_failure(&self.external_maintenance_acquire_failures) {
return Err(FrankenError::internal(
"injected external maintenance acquisition failure after arming",
));
}
Ok(())
}
fn restore_external_maintenance_attempt(&mut self, cx: &Cx) -> Result<()> {
if !self.observe_lock_state {
return self.inner.restore_external_maintenance_attempt(cx);
}
let Some(prior_level) = self.external_maintenance_prior_level else {
return self.inner.restore_external_maintenance_attempt(cx);
};
record_observed_restore_checkpoint_activity(
&self.external_restore_publication_probe,
&self.external_restore_checkpoint_observations,
);
if consume_observed_lock_failure(&self.external_restore_failures) {
return Err(FrankenError::internal(
"injected external maintenance restoration failure",
));
}
self.observed_unlock_trace_ids
.lock()
.unwrap()
.push(cx.trace_id());
if self.fail_unlock_on_checkpoint_error {
cx.checkpoint()
.map_err(|err| FrankenError::internal(err.to_string()))?;
}
self.inner.restore_external_maintenance_attempt(cx)?;
*self.observed_lock_level.lock().unwrap() = prior_level;
self.external_maintenance_prior_level = None;
Ok(())
}
fn check_reserved_lock(&self, cx: &Cx) -> Result<bool> {
self.inner.check_reserved_lock(cx)
}
fn sector_size(&self) -> u32 {
self.inner.sector_size()
}
fn device_characteristics(&self) -> u32 {
self.inner.device_characteristics()
}
fn shm_map(
&mut self,
cx: &Cx,
region: u32,
size: u32,
extend: bool,
) -> Result<fsqlite_vfs::ShmRegion> {
self.inner.shm_map(cx, region, size, extend)
}
fn shm_lock(&mut self, cx: &Cx, offset: u32, n: u32, flags: u32) -> Result<()> {
self.inner.shm_lock(cx, offset, n, flags)
}
fn shm_barrier(&self) {
self.inner.shm_barrier();
}
fn shm_unmap(&mut self, cx: &Cx, delete: bool) -> Result<()> {
self.inner.shm_unmap(cx, delete)
}
}
async fn observed_lock_pager() -> (SimplePager<ObservedLockVfs>, ObservedLockLevel) {
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let path = PathBuf::from("/observed-lock.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
(pager, observed_lock_level)
}
async fn observed_lock_pager_with_checkpoint_enforced_unlock() -> ObservedCleanupUnlockHarness {
let vfs = ObservedLockVfs::with_checkpoint_enforced_unlock();
let observed_lock_level = vfs.observed_lock_level();
let observed_unlock_trace_ids = vfs.observed_unlock_trace_ids();
let path = PathBuf::from("/observed-lock-checkpoint.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
(pager, observed_lock_level, observed_unlock_trace_ids)
}
#[derive(Debug, Default)]
struct ExclusiveLockMetrics {
owner: Option<u64>,
acquired_at: Option<Instant>,
acquisition_count: usize,
hold_samples_ns: Vec<u64>,
wait_samples_ns: Vec<u64>,
}
#[derive(Clone)]
struct BlockingObservedLockVfs {
inner: MemoryVfs,
observed_lock_level: Arc<Mutex<LockLevel>>,
next_handle_id: StdArc<AtomicU64>,
exclusive_metrics: StdArc<(StdMutex<ExclusiveLockMetrics>, StdCondvar)>,
memory_fast_path: Arc<AtomicBool>,
}
impl BlockingObservedLockVfs {
fn new() -> Self {
Self {
inner: MemoryVfs::new(),
observed_lock_level: Arc::new(Mutex::new(LockLevel::None)),
next_handle_id: StdArc::new(AtomicU64::new(1)),
exclusive_metrics: StdArc::new((
StdMutex::new(ExclusiveLockMetrics::default()),
StdCondvar::new(),
)),
memory_fast_path: Arc::new(AtomicBool::new(true)),
}
}
async fn open_file_backed_pager(&self, path: &Path) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result = SimplePager::open(self.clone(), path, PageSize::DEFAULT).await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
fn observed_lock_level(&self) -> Arc<Mutex<LockLevel>> {
Arc::clone(&self.observed_lock_level)
}
fn wait_for_exclusive_acquisitions(&self, target: usize) {
let (metrics_lock, metrics_ready) = &*self.exclusive_metrics;
let mut metrics = metrics_lock.lock().unwrap();
while metrics.acquisition_count < target {
metrics = metrics_ready.wait(metrics).unwrap();
}
}
fn exclusive_hold_samples_ns(&self) -> Vec<u64> {
let (metrics_lock, _) = &*self.exclusive_metrics;
metrics_lock.lock().unwrap().hold_samples_ns.clone()
}
fn exclusive_wait_samples_ns(&self) -> Vec<u64> {
let (metrics_lock, _) = &*self.exclusive_metrics;
metrics_lock.lock().unwrap().wait_samples_ns.clone()
}
fn clear_exclusive_metrics(&self) {
let (metrics_lock, _) = &*self.exclusive_metrics;
let mut metrics = metrics_lock.lock().unwrap();
*metrics = ExclusiveLockMetrics::default();
}
}
struct BlockingObservedLockFile {
inner: MemoryFile,
observed_lock_level: Arc<Mutex<LockLevel>>,
handle_id: u64,
lock_level: LockLevel,
exclusive_metrics: StdArc<(StdMutex<ExclusiveLockMetrics>, StdCondvar)>,
external_snapshot_prior_level: Option<LockLevel>,
external_maintenance_prior_level: Option<LockLevel>,
}
impl BlockingObservedLockFile {
fn acquire_exclusive_hold(&self) {
let wait_started = Instant::now();
let (metrics_lock, metrics_ready) = &*self.exclusive_metrics;
let mut metrics = metrics_lock.lock().unwrap();
while metrics.owner.is_some() && metrics.owner != Some(self.handle_id) {
metrics = metrics_ready.wait(metrics).unwrap();
}
metrics
.wait_samples_ns
.push(u64::try_from(wait_started.elapsed().as_nanos()).unwrap_or(u64::MAX));
metrics.owner = Some(self.handle_id);
metrics.acquired_at = Some(Instant::now());
metrics.acquisition_count = metrics.acquisition_count.saturating_add(1);
metrics_ready.notify_all();
}
fn release_exclusive_hold(&self) {
let (metrics_lock, metrics_ready) = &*self.exclusive_metrics;
let mut metrics = metrics_lock.lock().unwrap();
if metrics.owner == Some(self.handle_id) {
if let Some(acquired_at) = metrics.acquired_at.take() {
metrics
.hold_samples_ns
.push(u64::try_from(acquired_at.elapsed().as_nanos()).unwrap_or(u64::MAX));
}
metrics.owner = None;
metrics_ready.notify_all();
}
}
}
impl Vfs for BlockingObservedLockVfs {
type File = BlockingObservedLockFile;
fn name(&self) -> &'static str {
self.inner.name()
}
fn open(
&self,
cx: &Cx,
path: Option<&std::path::Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
let (inner, actual_flags) = self.inner.open(cx, path, flags)?;
Ok((
BlockingObservedLockFile {
inner,
observed_lock_level: self.observed_lock_level(),
handle_id: self.next_handle_id.fetch_add(1, AtomicOrdering::Relaxed),
lock_level: LockLevel::None,
exclusive_metrics: StdArc::clone(&self.exclusive_metrics),
external_snapshot_prior_level: None,
external_maintenance_prior_level: None,
},
actual_flags,
))
}
fn delete(&self, cx: &Cx, path: &std::path::Path, sync_dir: bool) -> Result<()> {
self.inner.delete(cx, path, sync_dir)
}
fn access(&self, cx: &Cx, path: &std::path::Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &std::path::Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
fn is_memory(&self) -> bool {
self.memory_fast_path.load(AtomicOrdering::Acquire)
}
}
impl VfsFile for BlockingObservedLockFile {
fn close(&mut self, cx: &Cx) -> Result<()> {
self.release_exclusive_hold();
let result = self.inner.close(cx);
if result.is_ok() {
self.lock_level = LockLevel::None;
*self.observed_lock_level.lock().unwrap() = LockLevel::None;
}
result
}
fn read<'a>(
&'a self,
cx: &'a Cx,
buf: &'a mut [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<usize>> + Send + 'a {
self.inner.read(cx, buf, offset)
}
fn write<'a>(
&'a self,
cx: &'a Cx,
buf: &'a [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<()>> + Send + 'a {
self.inner.write(cx, buf, offset)
}
fn truncate(&mut self, cx: &Cx, size: u64) -> Result<()> {
self.inner.truncate(cx, size)
}
fn sync(&mut self, cx: &Cx, flags: SyncFlags) -> Result<()> {
self.inner.sync(cx, flags)
}
fn file_size(&self, cx: &Cx) -> Result<u64> {
self.inner.file_size(cx)
}
fn lock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
if self.lock_level < LockLevel::Exclusive && level >= LockLevel::Exclusive {
self.acquire_exclusive_hold();
}
self.inner.lock(cx, level)?;
if self.lock_level < level {
self.lock_level = level;
}
*self.observed_lock_level.lock().unwrap() = self.lock_level;
Ok(())
}
fn unlock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
if self.lock_level >= LockLevel::Exclusive && level < LockLevel::Exclusive {
self.release_exclusive_hold();
}
self.inner.unlock(cx, level)?;
if self.lock_level > level {
self.lock_level = level;
}
*self.observed_lock_level.lock().unwrap() = self.lock_level;
Ok(())
}
fn lock_external_shared_snapshot(&mut self, cx: &Cx) -> Result<()> {
if self.external_snapshot_prior_level.is_some()
|| self.external_maintenance_prior_level.is_some()
{
return Err(FrankenError::internal(
"blocking observed-lock external attempt is already active",
));
}
let prior_level = self.lock_level;
self.external_snapshot_prior_level = Some(prior_level);
self.inner.lock_external_shared_snapshot(cx)?;
self.lock_level = prior_level.max(LockLevel::Shared);
*self.observed_lock_level.lock().unwrap() = self.lock_level;
Ok(())
}
fn restore_external_shared_snapshot_attempt(&mut self, cx: &Cx) -> Result<()> {
let Some(prior_level) = self.external_snapshot_prior_level else {
return self.inner.restore_external_shared_snapshot_attempt(cx);
};
self.inner.restore_external_shared_snapshot_attempt(cx)?;
self.lock_level = prior_level;
*self.observed_lock_level.lock().unwrap() = self.lock_level;
self.external_snapshot_prior_level = None;
Ok(())
}
fn lock_external_maintenance(&mut self, cx: &Cx, wal_mode: bool) -> Result<()> {
if self.external_snapshot_prior_level.is_some()
|| self.external_maintenance_prior_level.is_some()
{
return Err(FrankenError::internal(
"blocking observed-lock external attempt is already active",
));
}
let prior_level = self.lock_level;
self.external_maintenance_prior_level = Some(prior_level);
if prior_level < LockLevel::Exclusive {
self.acquire_exclusive_hold();
}
self.inner.lock_external_maintenance(cx, wal_mode)?;
self.lock_level = LockLevel::Exclusive;
*self.observed_lock_level.lock().unwrap() = self.lock_level;
Ok(())
}
fn restore_external_maintenance_attempt(&mut self, cx: &Cx) -> Result<()> {
let Some(prior_level) = self.external_maintenance_prior_level else {
return self.inner.restore_external_maintenance_attempt(cx);
};
self.inner.restore_external_maintenance_attempt(cx)?;
if prior_level < LockLevel::Exclusive {
self.release_exclusive_hold();
}
self.lock_level = prior_level;
*self.observed_lock_level.lock().unwrap() = self.lock_level;
self.external_maintenance_prior_level = None;
Ok(())
}
fn check_reserved_lock(&self, cx: &Cx) -> Result<bool> {
self.inner.check_reserved_lock(cx)
}
fn sector_size(&self) -> u32 {
self.inner.sector_size()
}
fn device_characteristics(&self) -> u32 {
self.inner.device_characteristics()
}
fn shm_map(
&mut self,
cx: &Cx,
region: u32,
size: u32,
extend: bool,
) -> Result<fsqlite_vfs::ShmRegion> {
self.inner.shm_map(cx, region, size, extend)
}
fn shm_lock(&mut self, cx: &Cx, offset: u32, n: u32, flags: u32) -> Result<()> {
self.inner.shm_lock(cx, offset, n, flags)
}
fn shm_barrier(&self) {
self.inner.shm_barrier();
}
fn shm_unmap(&mut self, cx: &Cx, delete: bool) -> Result<()> {
self.inner.shm_unmap(cx, delete)
}
}
#[derive(Debug)]
struct DbWriteFailState {
target_path: PathBuf,
target_journal_path: PathBuf,
armed: bool,
remaining_successful_db_writes: usize,
live_journal_bytes: Vec<u8>,
captured_journal_bytes: Option<Vec<u8>>,
successful_db_write_offsets: Vec<u64>,
failed_db_write_offset: Option<u64>,
}
#[derive(Clone)]
struct DbWriteFailOnceVfs {
inner: MemoryVfs,
state: Arc<Mutex<DbWriteFailState>>,
memory_fast_path: Arc<AtomicBool>,
}
impl DbWriteFailOnceVfs {
fn new(target_path: PathBuf) -> Self {
let mut target_journal_path = target_path.as_os_str().to_owned();
target_journal_path.push("-journal");
Self {
inner: MemoryVfs::new(),
state: Arc::new(Mutex::new(DbWriteFailState {
target_path,
target_journal_path: PathBuf::from(target_journal_path),
armed: false,
remaining_successful_db_writes: 0,
live_journal_bytes: Vec::new(),
captured_journal_bytes: None,
successful_db_write_offsets: Vec::new(),
failed_db_write_offset: None,
})),
memory_fast_path: Arc::new(AtomicBool::new(true)),
}
}
async fn open_file_backed_pager(&self, path: &Path) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result = SimplePager::open(self.clone(), path, PageSize::DEFAULT).await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
async fn open_file_backed_pager_with_page_buffer_max(
&self,
cx: &Cx,
path: &Path,
page_buffer_max: usize,
) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result = SimplePager::open_with_cx_and_page_buffer_max(
cx,
self.clone(),
path,
PageSize::DEFAULT,
Some(page_buffer_max),
)
.await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
fn arm_after_db_writes(&self, successful_db_writes_before_failure: usize) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.armed = true;
state.remaining_successful_db_writes = successful_db_writes_before_failure;
state.captured_journal_bytes = None;
state.successful_db_write_offsets.clear();
state.failed_db_write_offset = None;
}
fn captured_journal_bytes(&self) -> Option<Vec<u8>> {
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.captured_journal_bytes
.clone()
}
fn db_write_fault_observation(&self) -> (Vec<u64>, Option<u64>) {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
(
state.successful_db_write_offsets.clone(),
state.failed_db_write_offset,
)
}
}
#[derive(Debug)]
struct DbWriteFailOnceFile {
inner: MemoryFile,
state: Arc<Mutex<DbWriteFailState>>,
is_target_db: bool,
is_target_journal: bool,
}
impl Vfs for DbWriteFailOnceVfs {
type File = DbWriteFailOnceFile;
fn name(&self) -> &'static str {
self.inner.name()
}
fn open(
&self,
cx: &Cx,
path: Option<&std::path::Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
let (inner, actual_flags) = self.inner.open(cx, path, flags)?;
let (is_target_db, is_target_journal) = {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
(
path == Some(state.target_path.as_path())
&& flags.contains(VfsOpenFlags::MAIN_DB),
path == Some(state.target_journal_path.as_path())
&& flags.contains(VfsOpenFlags::MAIN_JOURNAL),
)
};
Ok((
DbWriteFailOnceFile {
inner,
state: Arc::clone(&self.state),
is_target_db,
is_target_journal,
},
actual_flags,
))
}
fn delete(&self, cx: &Cx, path: &std::path::Path, sync_dir: bool) -> Result<()> {
self.inner.delete(cx, path, sync_dir)?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if path == state.target_journal_path.as_path() {
state.live_journal_bytes.clear();
}
Ok(())
}
fn access(&self, cx: &Cx, path: &std::path::Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &std::path::Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
fn is_memory(&self) -> bool {
self.memory_fast_path.load(AtomicOrdering::Acquire)
}
}
impl VfsFile for DbWriteFailOnceFile {
fn close(&mut self, cx: &Cx) -> Result<()> {
self.inner.close(cx)
}
fn read<'a>(
&'a self,
cx: &'a Cx,
buf: &'a mut [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<usize>> + Send + 'a {
self.inner.read(cx, buf, offset)
}
fn write<'a>(
&'a self,
cx: &'a Cx,
buf: &'a [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<()>> + Send + 'a {
async move {
if self.is_target_journal {
let start = usize::try_from(offset).map_err(|_| {
FrankenError::Io(std::io::Error::other(
"journal observation offset exceeds usize",
))
})?;
let end = start.checked_add(buf.len()).ok_or_else(|| {
FrankenError::Io(std::io::Error::other(
"journal observation range overflows usize",
))
})?;
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.live_journal_bytes.len() < end {
state.live_journal_bytes.resize(end, 0);
}
state.live_journal_bytes[start..end].copy_from_slice(buf);
}
if self.is_target_db {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.armed {
if state.remaining_successful_db_writes == 0 {
state.armed = false;
state.failed_db_write_offset = Some(offset);
let captured = state.live_journal_bytes.clone();
state.captured_journal_bytes = Some(captured);
return Err(FrankenError::Io(std::io::Error::other(
"simulated main-db write failure",
)));
}
state.successful_db_write_offsets.push(offset);
state.remaining_successful_db_writes -= 1;
}
}
self.inner.write(cx, buf, offset).await
}
}
fn truncate(&mut self, cx: &Cx, size: u64) -> Result<()> {
self.inner.truncate(cx, size)?;
if self.is_target_journal {
let new_len = usize::try_from(size).map_err(|_| {
FrankenError::Io(std::io::Error::other(
"journal observation size exceeds usize",
))
})?;
self.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.live_journal_bytes
.resize(new_len, 0);
}
Ok(())
}
fn sync(&mut self, cx: &Cx, flags: SyncFlags) -> Result<()> {
self.inner.sync(cx, flags)
}
fn file_size(&self, cx: &Cx) -> Result<u64> {
self.inner.file_size(cx)
}
fn lock(&mut self, cx: &Cx, level: fsqlite_types::LockLevel) -> Result<()> {
self.inner.lock(cx, level)
}
fn unlock(&mut self, cx: &Cx, level: fsqlite_types::LockLevel) -> Result<()> {
self.inner.unlock(cx, level)
}
fn lock_external_shared_snapshot(&mut self, cx: &Cx) -> Result<()> {
self.inner.lock_external_shared_snapshot(cx)
}
fn restore_external_shared_snapshot_attempt(&mut self, cx: &Cx) -> Result<()> {
self.inner.restore_external_shared_snapshot_attempt(cx)
}
fn lock_external_maintenance(&mut self, cx: &Cx, wal_mode: bool) -> Result<()> {
self.inner.lock_external_maintenance(cx, wal_mode)
}
fn restore_external_maintenance_attempt(&mut self, cx: &Cx) -> Result<()> {
self.inner.restore_external_maintenance_attempt(cx)
}
fn check_reserved_lock(&self, cx: &Cx) -> Result<bool> {
self.inner.check_reserved_lock(cx)
}
fn sector_size(&self) -> u32 {
self.inner.sector_size()
}
fn device_characteristics(&self) -> u32 {
self.inner.device_characteristics()
}
fn shm_map(
&mut self,
cx: &Cx,
region: u32,
size: u32,
extend: bool,
) -> Result<fsqlite_vfs::ShmRegion> {
self.inner.shm_map(cx, region, size, extend)
}
fn shm_lock(&mut self, cx: &Cx, offset: u32, n: u32, flags: u32) -> Result<()> {
self.inner.shm_lock(cx, offset, n, flags)
}
fn shm_barrier(&self) {
self.inner.shm_barrier();
}
fn shm_unmap(&mut self, cx: &Cx, delete: bool) -> Result<()> {
self.inner.shm_unmap(cx, delete)
}
}
// These independent switches intentionally compose fault-injection scenarios.
#[allow(clippy::struct_excessive_bools)]
#[derive(Debug, Clone, Copy, Default)]
struct JournalDurabilityFaultPlan {
ignore_zero_magic_write: bool,
ignore_truncate: bool,
corrupt_hot_header_activation: bool,
corrupt_hot_header_restore: bool,
}
#[derive(Debug)]
struct JournalDurabilityFaultState {
target_journal: PathBuf,
plan: JournalDurabilityFaultPlan,
armed: bool,
hot_header_writes: usize,
}
#[derive(Clone)]
struct JournalDurabilityFaultVfs {
inner: MemoryVfs,
state: Arc<Mutex<JournalDurabilityFaultState>>,
memory_fast_path: Arc<AtomicBool>,
}
impl JournalDurabilityFaultVfs {
fn new(target_journal: PathBuf) -> Self {
Self {
inner: MemoryVfs::new(),
state: Arc::new(Mutex::new(JournalDurabilityFaultState {
target_journal,
plan: JournalDurabilityFaultPlan::default(),
armed: false,
hot_header_writes: 0,
})),
memory_fast_path: Arc::new(AtomicBool::new(true)),
}
}
fn enable_file_backed_protocol(&self) {
self.memory_fast_path.store(false, AtomicOrdering::Release);
}
async fn open_file_backed_pager(&self, path: &Path) -> Result<SimplePager<Self>> {
self.memory_fast_path.store(true, AtomicOrdering::Release);
let result = SimplePager::open(self.clone(), path, PageSize::DEFAULT).await;
self.memory_fast_path.store(false, AtomicOrdering::Release);
result
}
fn arm(&self, plan: JournalDurabilityFaultPlan) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.plan = plan;
state.armed = true;
state.hot_header_writes = 0;
}
fn disarm(&self) {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
state.armed = false;
state.plan = JournalDurabilityFaultPlan::default();
state.hot_header_writes = 0;
}
}
#[derive(Debug)]
struct JournalDurabilityFaultFile {
inner: MemoryFile,
state: Arc<Mutex<JournalDurabilityFaultState>>,
is_target_journal: bool,
}
impl Vfs for JournalDurabilityFaultVfs {
type File = JournalDurabilityFaultFile;
fn name(&self) -> &'static str {
"journal-durability-fault"
}
fn open(
&self,
cx: &Cx,
path: Option<&Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
let (inner, actual_flags) = self.inner.open(cx, path, flags)?;
let is_target_journal = {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
path == Some(state.target_journal.as_path())
&& flags.contains(VfsOpenFlags::MAIN_JOURNAL)
};
Ok((
JournalDurabilityFaultFile {
inner,
state: Arc::clone(&self.state),
is_target_journal,
},
actual_flags,
))
}
fn delete(&self, cx: &Cx, path: &Path, sync_dir: bool) -> Result<()> {
self.inner.delete(cx, path, sync_dir)
}
fn access(&self, cx: &Cx, path: &Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
fn is_memory(&self) -> bool {
self.memory_fast_path.load(AtomicOrdering::Acquire)
}
}
impl VfsFile for JournalDurabilityFaultFile {
fn close(&mut self, cx: &Cx) -> Result<()> {
self.inner.close(cx)
}
fn file_identity(&self) -> Result<Option<FileIdentity>> {
self.inner.file_identity()
}
fn read<'a>(
&'a self,
cx: &'a Cx,
buf: &'a mut [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<usize>> + Send + 'a {
self.inner.read(cx, buf, offset)
}
fn write<'a>(
&'a self,
cx: &'a Cx,
buf: &'a [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<()>> + Send + 'a {
async move {
let mut corrupted = None;
if self.is_target_journal && offset == 0 {
let mut state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.armed {
if state.plan.ignore_zero_magic_write
&& buf.len() == JOURNAL_MAGIC.len()
&& buf.iter().all(|byte| *byte == 0)
{
return Ok(());
}
if buf.starts_with(&JOURNAL_MAGIC)
&& buf.len() > crate::journal::JOURNAL_HEADER_SIZE
{
state.hot_header_writes = state.hot_header_writes.saturating_add(1);
if state.plan.corrupt_hot_header_activation
&& state.hot_header_writes == 1
{
let mut damaged = buf.to_vec();
damaged[crate::journal::JOURNAL_HEADER_SIZE] ^= 0x40;
corrupted = Some(damaged);
} else if state.plan.corrupt_hot_header_restore
&& state.hot_header_writes == 2
{
let mut damaged = buf.to_vec();
damaged[crate::journal::JOURNAL_HEADER_SIZE] ^= 0x80;
corrupted = Some(damaged);
}
}
}
}
self.inner
.write(cx, corrupted.as_deref().unwrap_or(buf), offset)
.await
}
}
fn truncate(&mut self, cx: &Cx, size: u64) -> Result<()> {
if self.is_target_journal && size == 0 {
let state = self
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
if state.armed && state.plan.ignore_truncate {
return Ok(());
}
}
self.inner.truncate(cx, size)
}
fn sync(&mut self, cx: &Cx, flags: SyncFlags) -> Result<()> {
self.inner.sync(cx, flags)
}
fn file_size(&self, cx: &Cx) -> Result<u64> {
self.inner.file_size(cx)
}
fn lock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
self.inner.lock(cx, level)
}
fn unlock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
self.inner.unlock(cx, level)
}
fn lock_external_shared_snapshot(&mut self, cx: &Cx) -> Result<()> {
self.inner.lock_external_shared_snapshot(cx)
}
fn restore_external_shared_snapshot_attempt(&mut self, cx: &Cx) -> Result<()> {
self.inner.restore_external_shared_snapshot_attempt(cx)
}
fn lock_external_maintenance(&mut self, cx: &Cx, wal_mode: bool) -> Result<()> {
self.inner.lock_external_maintenance(cx, wal_mode)
}
fn restore_external_maintenance_attempt(&mut self, cx: &Cx) -> Result<()> {
self.inner.restore_external_maintenance_attempt(cx)
}
fn check_reserved_lock(&self, cx: &Cx) -> Result<bool> {
self.inner.check_reserved_lock(cx)
}
fn sector_size(&self) -> u32 {
self.inner.sector_size()
}
fn device_characteristics(&self) -> u32 {
self.inner.device_characteristics()
}
fn shm_map(
&mut self,
cx: &Cx,
region: u32,
size: u32,
extend: bool,
) -> Result<fsqlite_vfs::ShmRegion> {
self.inner.shm_map(cx, region, size, extend)
}
fn shm_lock(&mut self, cx: &Cx, offset: u32, n: u32, flags: u32) -> Result<()> {
self.inner.shm_lock(cx, offset, n, flags)
}
fn shm_barrier(&self) {
self.inner.shm_barrier();
}
fn shm_unmap(&mut self, cx: &Cx, delete: bool) -> Result<()> {
self.inner.shm_unmap(cx, delete)
}
}
#[test]
fn test_open_empty_database() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let inner = pager.inner.lock().unwrap();
assert_eq!(inner.db_size, 1, "bead_id={BEAD_ID} case=empty_db_size");
assert_eq!(
inner.page_size,
PageSize::DEFAULT,
"bead_id={BEAD_ID} case=page_size_default"
);
drop(inner);
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw_page = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
let hdr: [u8; DATABASE_HEADER_SIZE] = raw_page[..DATABASE_HEADER_SIZE]
.try_into()
.expect("page 1 must contain database header");
let parsed = DatabaseHeader::from_bytes(&hdr).expect("header should parse");
assert_eq!(
parsed.page_size,
PageSize::DEFAULT,
"bead_id={BEAD_ID} case=page1_header_page_size"
);
assert_eq!(
parsed.page_count, 1,
"bead_id={BEAD_ID} case=page1_header_page_count"
);
let btree_hdr = BTreePageHeader::parse(&raw_page, PageSize::DEFAULT, 0, true)
.expect("btree header");
assert_eq!(
btree_hdr.cell_count, 0,
"bead_id={BEAD_ID} case=sqlite_master_initially_empty"
);
});
}
#[test]
fn test_open_existing_database_uses_header_page_size() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/page_size_autodetect.db");
let expected_page_size = PageSize::new(8192).unwrap();
let _pager = SimplePager::open(vfs.clone(), &path, expected_page_size)
.await
.unwrap();
let reopened = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
assert!(
reopened.page_size() == expected_page_size,
"bead_id={BEAD_ID} case=autodetect_existing_page_size"
);
});
}
#[test]
fn test_begin_refreshes_external_page_growth_before_allocation() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/pager_refresh_external_growth.db");
let pager1 = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let pager2 = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut writer1 = pager1.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page2 = writer1.allocate_page(&cx).await.unwrap();
assert_eq!(page2.get(), 2, "first writer should allocate page 2");
writer1
.write_page(&cx, page2, &vec![0xAB; ps])
.await
.unwrap();
writer1.commit(&cx).await.unwrap();
let mut writer2 = pager2.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page3 = writer2.allocate_page(&cx).await.unwrap();
assert_eq!(
page3.get(),
3,
"bead_id={BEAD_ID} case=refresh_external_growth_reissues_next_page"
);
});
}
#[test]
fn test_open_existing_database_treats_non_page_aligned_tail_as_slack() {
// GH#334 / bd-e26jr sig-2: stock SQLite treats the header page count
// as authoritative and ignores trailing bytes beyond the last whole
// page, so a partial-page tail must not fail the open. The tail is
// slack: the whole-page extent stays what it was before the append.
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/misaligned.db");
let cx = Cx::new();
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
drop(pager);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let file_size = db_file.file_size(&cx).unwrap();
db_file.write(&cx, &[0xAB], file_size).await.unwrap();
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.expect("open must tolerate a partial-page trailing tail");
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.expect(
"slack-bearing database must stay readable after open",
);
drop(txn);
});
}
#[test]
fn test_begin_readonly_transaction() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert!(!txn.is_writer, "bead_id={BEAD_ID} case=readonly_not_writer");
});
}
#[test]
fn test_begin_write_transaction() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
assert!(txn.is_writer, "bead_id={BEAD_ID} case=immediate_is_writer");
});
}
#[test]
fn test_begin_deferred_transaction_starts_reader() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::Deferred).await.unwrap();
assert!(
!txn.is_writer,
"bead_id={BEAD_ID} case=deferred_starts_readonly"
);
});
}
#[test]
fn test_begin_concurrent_transaction_starts_reader() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(
!txn.is_writer,
"bead_id={BEAD_ID} case=concurrent_starts_readonly"
);
});
}
#[test]
fn test_deferred_upgrades_on_first_write_intent() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut deferred = pager.begin(&cx, TransactionMode::Deferred).await.unwrap();
assert!(
!deferred.is_writer,
"bead_id={BEAD_ID} case=deferred_pre_upgrade"
);
let _page = deferred.allocate_page(&cx).await.unwrap();
assert!(
deferred.is_writer,
"bead_id={BEAD_ID} case=deferred_upgraded_to_writer"
);
});
}
#[test]
fn test_deferred_upgrade_busy_when_writer_active() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut deferred = pager.begin(&cx, TransactionMode::Deferred).await.unwrap();
let started = Instant::now();
let err = deferred.allocate_page(&cx).await.unwrap_err();
assert!(matches!(err, FrankenError::Busy));
assert!(
started.elapsed() < Duration::from_millis(10),
"bead_id={BEAD_ID} case=deferred_upgrade_baton_wait_is_bounded"
);
});
}
#[test]
fn test_single_writer_baton_handoff_budget_is_bounded() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let started = Instant::now();
let err = match pager.begin(&cx, TransactionMode::Immediate).await {
Ok(_) => panic!("bead_id={BEAD_ID} case=single_writer_baton_should_return_busy"),
Err(err) => err,
};
assert!(matches!(err, FrankenError::Busy));
assert!(
started.elapsed() < Duration::from_millis(10),
"bead_id={BEAD_ID} case=single_writer_baton_wait_is_bounded"
);
});
}
#[test]
fn test_concurrent_begin_bypasses_single_writer_baton() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let concurrent = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(
!concurrent.is_writer,
"bead_id={BEAD_ID} case=concurrent_begin_not_blocked_by_single_writer"
);
});
}
#[test]
fn test_concurrent_writer_blocked() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _txn1 = pager.begin(&cx, TransactionMode::Exclusive).await.unwrap();
let result = pager.begin(&cx, TransactionMode::Immediate).await;
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=concurrent_writer_busy"
);
});
}
#[test]
fn test_multiple_readers_allowed() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _r1 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let _r2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
// Both readers can coexist.
});
}
#[test]
fn test_write_page_and_read_back() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let mut data = vec![0_u8; page_size];
data[0] = 0xDE;
data[1] = 0xAD;
txn.write_page(&cx, page_no, &data).await.unwrap();
let read_back = txn.get_page(&cx, page_no).await.unwrap();
assert_eq!(
read_back.as_ref()[0],
0xDE,
"bead_id={BEAD_ID} case=read_back_byte0"
);
assert_eq!(
read_back.as_ref()[1],
0xAD,
"bead_id={BEAD_ID} case=read_back_byte1"
);
});
}
#[test]
fn test_commit_persists_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
// Write in first transaction.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let mut data = vec![0_u8; page_size];
data[0..4].copy_from_slice(&[0xCA, 0xFE, 0xBA, 0xBE]);
txn.write_page(&cx, page_no, &data).await.unwrap();
txn.commit(&cx).await.unwrap();
// Read in second transaction.
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let read_back = txn2.get_page(&cx, page_no).await.unwrap();
assert_eq!(
&read_back.as_ref()[0..4],
&[0xCA, 0xFE, 0xBA, 0xBE],
"bead_id={BEAD_ID} case=commit_persists"
);
});
}
#[test]
fn test_rollback_discards_writes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
// Allocate and write a page, then commit so it exists on disk.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let original = vec![0x11_u8; page_size];
txn.write_page(&cx, page_no, &original).await.unwrap();
txn.commit(&cx).await.unwrap();
// Overwrite in a new transaction, then rollback.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let modified = vec![0x99_u8; page_size];
txn2.write_page(&cx, page_no, &modified).await.unwrap();
txn2.rollback(&cx).await.unwrap();
// Read again — should see original data.
let txn3 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let read_back = txn3.get_page(&cx, page_no).await.unwrap();
assert_eq!(
read_back.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=rollback_restores"
);
});
}
#[test]
fn test_allocate_returns_sequential_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
assert!(
p2.get() > p1.get(),
"bead_id={BEAD_ID} case=sequential_alloc p1={} p2={}",
p1.get(),
p2.get()
);
});
}
#[test]
fn test_free_page_reuses_on_next_alloc() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// 1. Allocate a page and commit.
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn1.allocate_page(&cx).await.unwrap();
txn1.write_page(&cx, p, &vec![0_u8; ps]).await.unwrap();
txn1.commit(&cx).await.unwrap();
// 2. Free the page and commit -> moves to freelist.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p).await.unwrap();
txn2.commit(&cx).await.unwrap();
// Verify freelist has the page.
{
let inner = pager.inner.lock().unwrap();
assert_eq!(inner.freelist.len(), 1);
assert_eq!(inner.freelist[0], p);
drop(inner);
}
// 3. Allocate the page again (pops from freelist).
let mut txn3 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = txn3.allocate_page(&cx).await.unwrap();
assert_eq!(
p2,
p,
"bead_id={BEAD_ID} case=freelist_reuse p3={} p1={}",
p2.get(),
p.get()
);
});
}
#[test]
fn test_return_pages_to_freelist_keeps_unique_descending_order() {
let p2 = PageNumber::new(2).unwrap();
let p3 = PageNumber::new(3).unwrap();
let p4 = PageNumber::new(4).unwrap();
let p5 = PageNumber::new(5).unwrap();
let p6 = PageNumber::new(6).unwrap();
let mut freelist = vec![p5, p3, p2];
return_pages_to_freelist(&mut freelist, [p4, p3, p6, p4]);
assert_eq!(
freelist,
vec![p6, p5, p4, p3, p2],
"bead_id={BEAD_ID} case=return_pages_dedupes_after_bulk_extend"
);
}
#[test]
fn test_normalize_freelist_keeps_unique_descending_order() {
let p2 = PageNumber::new(2).unwrap();
let p3 = PageNumber::new(3).unwrap();
let p4 = PageNumber::new(4).unwrap();
let p5 = PageNumber::new(5).unwrap();
let p6 = PageNumber::new(6).unwrap();
let normalized = normalize_freelist(&[p3, p6, p4, p3, p5, p2], 5);
assert_eq!(
normalized,
vec![p5, p4, p3, p2],
"bead_id={BEAD_ID} case=normalize_freelist_dedupes_descending_and_filters"
);
}
#[test]
fn test_freed_pages_are_quarantined_until_commit() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p2, &vec![0xAA; ps]).await.unwrap();
txn.free_page(&cx, p2).await.unwrap();
let p3 = txn.allocate_page(&cx).await.unwrap();
assert_eq!(
p3.get(),
p2.get() + 1,
"bead_id={BEAD_ID} case=freed_pages_quarantined_until_commit"
);
txn.write_page(&cx, p3, &vec![0xBB; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let mut next_txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused = next_txn.allocate_page(&cx).await.unwrap();
assert_eq!(
reused, p2,
"bead_id={BEAD_ID} case=freed_pages_reenter_committed_freelist_after_commit"
);
next_txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_get_page_rejects_page_freed_in_same_transaction() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &vec![0xAA; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
page
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.free_page(&cx, page).await.unwrap();
let err = txn
.get_page(&cx, page)
.await
.expect_err("read-after-free must be rejected");
let detail = err.to_string();
assert!(
detail.contains("freed earlier in this transaction"),
"expected read-after-free error, got: {detail}"
);
});
}
#[test]
fn test_cannot_free_page_one() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let result = txn.free_page(&cx, PageNumber::ONE).await;
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=cannot_free_page_one"
);
});
}
#[test]
fn test_readonly_cannot_write() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let result = txn.write_page(&cx, PageNumber::ONE, &[0_u8; 4096]).await;
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=readonly_cannot_write"
);
});
}
#[test]
fn test_readonly_cannot_allocate() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let result = txn.allocate_page(&cx).await;
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=readonly_cannot_allocate"
);
});
}
#[test]
fn test_drop_uncommitted_writer_releases_lock() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
{
let _txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
// Dropped without commit or rollback.
}
// Should be able to begin a new writer.
let txn2 = pager.begin(&cx, TransactionMode::Immediate).await;
assert!(
txn2.is_ok(),
"bead_id={BEAD_ID} case=drop_releases_writer_lock"
);
});
}
#[test]
fn test_drop_cleanup_unlock_preserves_lineage_and_masks_cancellation() {
asupersync::test_utils::run_test(|| async {
let (pager, observed_lock_level, observed_unlock_trace_ids) =
observed_lock_pager_with_checkpoint_enforced_unlock().await;
let cx = Cx::new().with_trace_context(41, 0, 0);
{
let _txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
cx.cancel();
}
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"bead_id={BEAD_ID} case=drop_cleanup_unlock_releases_lock_after_parent_cancel"
);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().last().copied(),
Some(41),
"bead_id={BEAD_ID} case=drop_cleanup_unlock_uses_parent_trace_lineage"
);
});
}
#[test]
fn test_reader_exit_preserves_shared_lock_for_other_reader() {
asupersync::test_utils::run_test(|| async {
let (pager, observed_lock_level) = observed_lock_pager().await;
let cx = Cx::new();
let mut reader1 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let reader2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::Shared);
reader1.commit(&cx).await.unwrap();
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Shared,
"bead_id={BEAD_ID} case=reader_commit_keeps_shared_for_other_reader"
);
drop(reader2);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"bead_id={BEAD_ID} case=last_reader_releases_shared"
);
});
}
#[test]
fn test_reader_exit_preserves_reserved_lock_for_active_writer() {
asupersync::test_utils::run_test(|| async {
let (pager, observed_lock_level) = observed_lock_pager().await;
let cx = Cx::new();
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::Reserved);
drop(reader);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"bead_id={BEAD_ID} case=reader_drop_keeps_reserved_for_writer"
);
writer.commit(&cx).await.unwrap();
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"bead_id={BEAD_ID} case=writer_commit_releases_last_lock"
);
});
}
#[test]
fn test_commit_then_drop_no_double_release() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.commit(&cx).await.unwrap();
// committed=true, drop should skip writer_active=false
}
// Writer should already be released by commit.
let txn2 = pager.begin(&cx, TransactionMode::Immediate).await;
assert!(
txn2.is_ok(),
"bead_id={BEAD_ID} case=commit_releases_writer"
);
});
}
#[test]
fn test_double_commit_is_idempotent() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.commit(&cx).await.unwrap();
// Second commit should be a no-op.
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_multi_page_write_commit_read() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let page_size = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut allocated_pages = Vec::new();
for i in 0_u8..5 {
let p = txn.allocate_page(&cx).await.unwrap();
let data = vec![i; page_size];
txn.write_page(&cx, p, &data).await.unwrap();
allocated_pages.push(p);
}
txn.commit(&cx).await.unwrap();
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for (i, &p) in allocated_pages.iter().enumerate() {
let data = txn2.get_page(&cx, p).await.unwrap();
#[allow(clippy::cast_possible_truncation)]
let expected = i as u8;
assert_eq!(
data.as_ref()[0],
expected,
"bead_id={BEAD_ID} case=multi_page idx={i}"
);
}
});
}
#[test]
fn test_commit_journal_skips_preimage_for_pages_allocated_after_transaction_start() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let page_size = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let new_page = txn.allocate_page(&cx).await.unwrap();
assert!(
new_page.get() > txn.original_db_size,
"bead_id={BEAD_ID} case=new_page_is_after_transaction_start"
);
txn.write_page(&cx, new_page, &vec![0x77; page_size])
.await
.unwrap();
{
let mut inner = txn.inner.lock().unwrap();
// Simulate upper-layer page-count metadata advancing before the
// database file has been extended by the rollback-journal commit.
inner.db_size = new_page.get();
}
txn.commit(&cx).await.unwrap();
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn2.get_page(&cx, new_page).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x77,
"bead_id={BEAD_ID} case=post_start_page_commits_without_preimage_read"
);
});
}
#[test]
fn test_commit_journal_page_count_ignores_stale_sorted_pages_missing_from_write_set() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let page_size = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let durable_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, durable_page, &vec![0x11; page_size])
.await
.unwrap();
let stale_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, stale_page, &vec![0x22; page_size])
.await
.unwrap();
let removed = txn.write_set.remove(&stale_page);
assert!(
removed.is_some() && txn.write_pages_sorted.contains(&stale_page),
"bead_id={BEAD_ID} case=test_setup_left_stale_sorted_page"
);
txn.commit(&cx).await.unwrap();
let inner = pager.inner.lock().unwrap();
let db_file = shared_db_file_read(&inner.db_file, &cx).await.unwrap();
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
let bytes_read = db_file.read(&cx, &mut header_bytes, 0).await.unwrap();
assert_eq!(
bytes_read, DATABASE_HEADER_SIZE,
"bead_id={BEAD_ID} case=page_one_header_read"
);
let header = DatabaseHeader::from_bytes(&header_bytes).unwrap();
assert_eq!(
header.page_count,
durable_page.get(),
"bead_id={BEAD_ID} case=page_count_uses_authoritative_write_set"
);
assert_eq!(
db_file.file_size(&cx).unwrap(),
u64::from(durable_page.get()) * u64::from(PageSize::DEFAULT.get()),
"bead_id={BEAD_ID} case=file_size_matches_page_count"
);
});
}
// ── Journal crash recovery tests ────────────────────────────────────
#[test]
fn journal_invalidation_detects_vfs_operations_that_lie_about_durability() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let journal_path = PathBuf::from("/lying-journal.db-journal");
let vfs = JournalDurabilityFaultVfs::new(journal_path.clone());
let flags = VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal, _) = vfs.open(&cx, Some(&journal_path), flags).unwrap();
let mut header = JournalHeader {
page_count: 0,
nonce: 0x1234_5678,
initial_db_size: 1,
sector_size: 4096,
page_size: PageSize::DEFAULT.get(),
}
.encode_padded();
mark_local_journal_header(&mut header);
journal.write(&cx, &header, 0).await.unwrap();
vfs.arm(JournalDurabilityFaultPlan {
ignore_zero_magic_write: true,
..JournalDurabilityFaultPlan::default()
});
let zero_error =
durable_invalidate_journal(&cx, &mut journal, JournalInvalidation::ZeroMagic)
.await
.unwrap_err();
assert!(
matches!(zero_error, FrankenError::DatabaseCorrupt { .. }),
"a successful zero-magic write must be rejected when readback remains hot"
);
let mut observed_magic = [0_u8; JOURNAL_MAGIC.len()];
journal.read(&cx, &mut observed_magic, 0).await.unwrap();
assert_eq!(observed_magic, JOURNAL_MAGIC);
vfs.arm(JournalDurabilityFaultPlan {
ignore_truncate: true,
..JournalDurabilityFaultPlan::default()
});
let truncate_error =
durable_invalidate_journal(&cx, &mut journal, JournalInvalidation::Truncate)
.await
.unwrap_err();
assert!(
matches!(truncate_error, FrankenError::DatabaseCorrupt { .. }),
"a successful truncate must be rejected when bytes remain"
);
assert_eq!(journal.file_size(&cx).unwrap(), header.len() as u64);
});
}
#[test]
fn rollback_commit_reports_indeterminate_when_all_marker_proofs_lie() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let directory = tempfile::tempdir().unwrap();
let path = directory.path().join("lying-journal-commit.db");
let journal_path = SimplePager::<JournalDurabilityFaultVfs>::journal_path(&path);
let vfs = JournalDurabilityFaultVfs::new(journal_path.clone());
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
vfs.enable_file_backed_protocol();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &vec![0x31; ps])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page_two
};
vfs.arm(JournalDurabilityFaultPlan {
ignore_zero_magic_write: true,
ignore_truncate: true,
corrupt_hot_header_restore: true,
..JournalDurabilityFaultPlan::default()
});
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x73; ps])
.await
.unwrap();
let commit_error = txn.commit(&cx).await.unwrap_err();
assert!(
commit_error
.to_string()
.contains("commit outcome is indeterminate"),
"all three failed readback proofs must surface an indeterminate outcome: {commit_error}"
);
let journal_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut retained_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
let mut observed_magic = [0_u8; JOURNAL_MAGIC.len()];
retained_journal
.read(&cx, &mut observed_magic, 0)
.await
.unwrap();
assert_eq!(
observed_magic, JOURNAL_MAGIC,
"failed marker proofs must leave a replay-selecting hot journal"
);
retained_journal.close(&cx).unwrap();
// Once the lying behavior is disabled, canonical rollback recovery
// must restore the pre-transaction image and durably clear the journal.
vfs.disarm();
txn.rollback(&cx).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
vec![0x31; ps]
);
});
}
#[test]
fn vacuum_activation_failure_retains_then_finishes_exact_owner() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let path = PathBuf::from("/vacuum-activation-owner.db");
let candidate_path = PathBuf::from("/vacuum-activation-owner-candidate.db");
let journal_path = SimplePager::<JournalDurabilityFaultVfs>::journal_path(&path);
let vfs = JournalDurabilityFaultVfs::new(journal_path.clone());
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
vfs.enable_file_backed_protocol();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &sample_page(0x35))
.await
.unwrap();
seed.commit(&cx).await.unwrap();
let source = pager.capture_vacuum_source_image(&cx).await.unwrap();
pager.copy_database_to(&cx, &candidate_path).await.unwrap();
let provisional = pager
.inspect_database_image(&cx, &candidate_path)
.await
.unwrap();
let next_change_counter = source.header.change_counter.wrapping_add(1).max(1);
let candidate = pager
.restore_vacuum_candidate_change_counter(
&cx,
&candidate_path,
&provisional,
next_change_counter,
)
.await
.unwrap();
vfs.arm(JournalDurabilityFaultPlan {
ignore_zero_magic_write: true,
ignore_truncate: true,
corrupt_hot_header_activation: true,
..JournalDurabilityFaultPlan::default()
});
let publication_error = pager
.publish_validated_database_image(&cx, &candidate_path, &source, &candidate)
.await
.expect_err("ambiguous activation must fail publication");
assert!(
publication_error
.to_string()
.contains("exact recovery finalization also failed"),
"failed eager recovery must remain visible: {publication_error}"
);
let recovery_owner = {
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::JournalActivationPending
));
let owner = inner
.rollback_journal_recovery_owner
.expect("ambiguous activation must retain its exact owner");
assert_eq!(
inner.maintenance_gate.rollback_recovery_owner(),
Some(owner)
);
owner
};
let sibling_error = match vfs.open_file_backed_pager(&path).await {
Ok(_) => panic!("sibling open bypassed ambiguous VACUUM recovery"),
Err(error) => error,
};
assert!(matches!(sibling_error, FrankenError::BusyRecovery));
assert_eq!(
pager.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner)
);
vfs.disarm();
pager.refresh_published_snapshot(&cx).await.unwrap();
let restored = pager.capture_vacuum_source_image(&cx).await.unwrap();
assert!(
restored == source,
"retrying ambiguous activation must restore the exact source receipt"
);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None);
assert_eq!(inner.maintenance_gate.rollback_recovery_owner(), None);
});
}
#[test]
fn vacuum_activation_failure_eagerly_finishes_exact_owner() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let path = PathBuf::from("/vacuum-activation-eager-finish.db");
let candidate_path = PathBuf::from("/vacuum-activation-eager-finish-candidate.db");
let journal_path = SimplePager::<JournalDurabilityFaultVfs>::journal_path(&path);
let vfs = JournalDurabilityFaultVfs::new(journal_path.clone());
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
vfs.enable_file_backed_protocol();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &sample_page(0x36))
.await
.unwrap();
seed.commit(&cx).await.unwrap();
let source = pager.capture_vacuum_source_image(&cx).await.unwrap();
pager.copy_database_to(&cx, &candidate_path).await.unwrap();
let provisional = pager
.inspect_database_image(&cx, &candidate_path)
.await
.unwrap();
let candidate = pager
.restore_vacuum_candidate_change_counter(
&cx,
&candidate_path,
&provisional,
source.header.change_counter.wrapping_add(1).max(1),
)
.await
.unwrap();
vfs.arm(JournalDurabilityFaultPlan {
ignore_zero_magic_write: true,
corrupt_hot_header_activation: true,
..JournalDurabilityFaultPlan::default()
});
let publication_error = pager
.publish_validated_database_image(&cx, &candidate_path, &source, &candidate)
.await
.expect_err("ambiguous activation must preserve its original publication error");
assert!(
!publication_error
.to_string()
.contains("exact recovery finalization also failed"),
"successful eager finalization must not replace the original error: {publication_error}"
);
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None);
assert!(inner.rollback_journal_recovery_namespace.is_none());
assert_eq!(inner.maintenance_gate.rollback_recovery_owner(), None);
}
vfs.disarm();
assert!(
pager.capture_vacuum_source_image(&cx).await.unwrap() == source,
"eager activation recovery must preserve the exact source receipt"
);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
let sibling = vfs.open_file_backed_pager(&path).await.unwrap();
let mut reader = sibling.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().as_ref(),
sample_page(0x36).as_slice()
);
reader.rollback(&cx).await.unwrap();
});
}
#[test]
fn construction_pending_recovery_rebuilds_metadata_and_reuses_abandoned_allocation() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/construction-pending-recovery.db");
let journal_path = SimplePager::<ObservedLockVfs>::journal_path(&path);
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let (original_db_size, original_next_page) = {
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
(inner.db_size, inner.next_page)
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let abandoned_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, abandoned_page, &sample_page(0x58))
.await
.unwrap();
let recovery_owner = {
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner
.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::JournalConstructionPending,
&journal_path,
)
.unwrap()
};
txn.owned_rollback_recovery = Some(recovery_owner);
let journal_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_JOURNAL;
let (mut construction_artifact, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
construction_artifact
.write(&cx, &[0_u8; JOURNAL_MAGIC.len()], 0)
.await
.unwrap();
construction_artifact.close(&cx).unwrap();
txn.rollback(&cx).await.unwrap();
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None);
assert_eq!(inner.maintenance_gate.rollback_recovery_owner(), None);
assert_eq!(inner.db_size, original_db_size);
assert_eq!(inner.next_page, original_next_page);
}
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
let mut retry = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused_page = retry.allocate_page(&cx).await.unwrap();
assert_eq!(
reused_page, abandoned_page,
"pre-main recovery must not leak the abandoned EOF allocation"
);
retry.rollback(&cx).await.unwrap();
});
}
#[test]
fn construction_pending_recovery_replays_replacement_hot_journal() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/construction-pending-replacement-hot.db");
let journal_path = SimplePager::<ObservedLockVfs>::journal_path(&path);
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let page_two = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &vec![0x11; page_size])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page_two
};
let _recovery_owner = {
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner
.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::JournalConstructionPending,
&journal_path,
)
.unwrap()
};
// Model another process replacing the original construction debris
// after the maintenance publisher released every native lock. Its
// valid hot journal protects page 2, whose main-file bytes reflect
// an uncommitted external write.
let header = JournalHeader {
page_count: 1,
nonce: 0x4142_4121,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let header_bytes = header.encode_padded();
let journal_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_JOURNAL;
let (mut foreign_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
foreign_journal.write(&cx, &header_bytes, 0).await.unwrap();
let record =
JournalPageRecord::new(page_two.get(), vec![0x11; page_size], header.nonce);
foreign_journal
.write(&cx, &record.encode(), header_bytes.len() as u64)
.await
.unwrap();
foreign_journal.sync(&cx, SyncFlags::NORMAL).unwrap();
foreign_journal.close(&cx).unwrap();
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut foreign_db, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
foreign_db
.write(
&cx,
&vec![0x99; page_size],
PageSize::DEFAULT.as_usize() as u64,
)
.await
.unwrap();
foreign_db.sync(&cx, SyncFlags::NORMAL).unwrap();
foreign_db.close(&cx).unwrap();
pager.refresh_published_snapshot(&cx).await.unwrap();
let (db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut restored = vec![0_u8; page_size];
let restored_len = db_file
.read(&cx, &mut restored, PageSize::DEFAULT.as_usize() as u64)
.await
.unwrap();
assert_eq!(restored_len, page_size);
assert_eq!(
restored,
vec![0x11; page_size],
"construction provenance must not authorize deleting a replacement hot journal"
);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None);
assert_eq!(inner.maintenance_gate.rollback_recovery_owner(), None);
});
}
#[test]
fn identityless_orphaned_recovery_cannot_be_adopted_by_a_new_open() {
let gate = Arc::new(PagerMaintenanceGate::default());
let owner = gate.claim_rollback_recovery_owner().unwrap();
gate.orphan_rollback_recovery_owner(
owner,
RollbackJournalRecoveryState::ReplayPending,
RollbackRecoveryNamespace {
db_path: PathBuf::from("/identityless-orphan.db"),
journal_path: PathBuf::from("/identityless-orphan.db-journal"),
db_identity: None,
journal_mode: JournalMode::Delete,
#[cfg(all(feature = "native", any(unix, windows)))]
namespace_binding: None,
},
)
.unwrap();
assert!(matches!(
gate.enter_readwrite_open_for_orphan_recovery(),
Err(FrankenError::BusyRecovery)
));
assert_eq!(gate.rollback_recovery_owner(), Some(owner));
let state = gate
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let retained = state
.orphaned_rollback_recovery
.as_ref()
.expect("refused identityless adoption must retain the exact receipt");
assert_eq!(retained.owner, owner);
assert!(matches!(
retained.recovery_state,
RollbackJournalRecoveryState::ReplayPending
));
}
#[test]
fn dropped_owner_pager_keeps_identity_recovery_armed_until_adoption() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/orphaned-recovery-owner.db");
let derived_journal_path = SimplePager::<ObservedLockVfs>::journal_path(&path);
// Model a receipt whose origin sidecar namespace differs from the
// survivor's derived name. Recovery must retain that exact journal
// path instead of silently switching to the survivor's sidecar.
let journal_path = PathBuf::from("/orphaned-recovery-origin.db-journal");
assert_ne!(journal_path, derived_journal_path);
let pager_a = vfs.open_file_backed_pager(&path).await.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let page_two = {
let mut seed = pager_a
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &vec![0x21; page_size])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page_two
};
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let delete_page_one = {
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut page_one = vec![0_u8; page_size];
let bytes_read = db_file.read(&cx, &mut page_one, 0).await.unwrap();
assert_eq!(bytes_read, page_size);
db_file.close(&cx).unwrap();
page_one
};
let readonly_pager = vfs
.open_file_backed_readonly_pager(&cx, &path)
.await
.unwrap();
let mut reader_b = readonly_pager
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
reader_b.get_page(&cx, page_two).await.unwrap().as_ref()[0],
0x21
);
// The dropped owner really observed WAL mode: both its cached
// metadata and the durable main header agree when it claims the
// receipt. A later external rollback-mode writer then replaces
// the sidecar and restores a Delete-mode page 1.
let mut wal_page_one = delete_page_one.clone();
wal_page_one[18] = 2;
wal_page_one[19] = 2;
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
db_file.write(&cx, &wal_page_one, 0).await.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
db_file.close(&cx).unwrap();
let recovery_owner = {
let mut inner = pager_a
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.journal_mode = JournalMode::Wal;
let owner = inner
.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::ReplayPending,
&journal_path,
)
.unwrap();
assert_eq!(
inner
.rollback_journal_recovery_namespace
.as_ref()
.expect("claimed recovery must retain its origin namespace")
.journal_mode,
JournalMode::Wal
);
owner
};
let header = JournalHeader {
page_count: 2,
nonce: 0x4F52_5048,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let header_bytes = header.encode_padded();
let journal_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_JOURNAL;
let (mut hot_journal, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
hot_journal.write(&cx, &header_bytes, 0).await.unwrap();
let page_one_record = JournalPageRecord::new(
PageNumber::ONE.get(),
delete_page_one.clone(),
header.nonce,
);
let encoded_page_one = page_one_record.encode();
hot_journal
.write(&cx, &encoded_page_one, header_bytes.len() as u64)
.await
.unwrap();
let page_two_record =
JournalPageRecord::new(page_two.get(), vec![0x21; page_size], header.nonce);
let page_two_offset = header_bytes.len() + encoded_page_one.len();
let encoded_page_two = page_two_record.encode();
hot_journal
.write(&cx, &encoded_page_two, page_two_offset as u64)
.await
.unwrap();
hot_journal.sync(&cx, SyncFlags::NORMAL).unwrap();
hot_journal.close(&cx).unwrap();
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
db_file.write(&cx, &delete_page_one, 0).await.unwrap();
db_file
.write(
&cx,
&vec![0x92; page_size],
PageSize::DEFAULT.as_usize() as u64,
)
.await
.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
db_file.close(&cx).unwrap();
drop(pager_a);
assert_eq!(
readonly_pager.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner),
"dropping one pager must not clear the identity-wide recovery barrier"
);
assert!(matches!(
reader_b.get_page(&cx, page_two).await,
Err(FrankenError::BusyRecovery)
));
assert!(matches!(
readonly_pager.begin(&cx, TransactionMode::ReadOnly).await,
Err(FrankenError::BusyRecovery)
));
reader_b.rollback(&cx).await.unwrap();
// With every pre-existing identity lease drained, this read-only
// pager is otherwise eligible to adopt. It must leave the orphan
// in the gate for a recovery-capable read-write sibling.
assert!(matches!(
readonly_pager.begin(&cx, TransactionMode::ReadOnly).await,
Err(FrankenError::BusyRecovery)
));
// No recovery-capable pager survived the owner. A newly opened
// read-write pager must claim and settle the orphan before it
// inspects the main-file header, while the read-only handle lives.
// First inject a failure while restoring the external maintenance
// lock after durable replay. The failed open must retain the exact
// owner in ExternalFinalizationPending rather than resurrecting a
// ReplayPending receipt whose journal is already gone.
vfs.external_restore_failures
.store(1, AtomicOrdering::Release);
match vfs.open_file_backed_pager(&path).await {
Ok(_) => panic!("injected external restoration failure was ignored"),
Err(error) => assert!(
error
.to_string()
.contains("injected external maintenance restoration failure")
),
}
assert_eq!(
readonly_pager.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner)
);
{
let state = readonly_pager
.maintenance_gate
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let orphaned = state
.orphaned_rollback_recovery
.as_ref()
.expect("failed open must restore its exact orphan receipt");
assert_eq!(orphaned.owner, recovery_owner);
assert!(matches!(
orphaned.recovery_state,
RollbackJournalRecoveryState::ExternalFinalizationPending
));
assert_eq!(orphaned.namespace.journal_mode, JournalMode::Delete);
}
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
let pager_c = vfs.open_file_backed_pager(&path).await.unwrap();
let mut recovered = pager_c.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
recovered.get_page(&cx, page_two).await.unwrap().into_vec(),
vec![0x21; page_size]
);
recovered.rollback(&cx).await.unwrap();
assert_eq!(pager_c.maintenance_gate.rollback_recovery_owner(), None);
let inner = pager_c
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None);
assert_eq!(inner.journal_mode, JournalMode::Delete);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
assert!(
!vfs.access(&cx, &derived_journal_path, AccessFlags::EXISTS)
.unwrap()
);
});
}
#[test]
fn orphaned_recovery_rejects_replaced_origin_namespace_before_replay() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/orphaned-origin-validation.db");
let replaced_origin_path = PathBuf::from("/replaced-origin-generation.db");
let journal_path = PathBuf::from("/replaced-origin-generation.db-journal");
let pager_a = vfs.open_file_backed_pager(&path).await.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let page_two = {
let mut seed = pager_a
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &vec![0x41; page_size])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page_two
};
let pager_b = vfs.open_file_backed_pager(&path).await.unwrap();
let (recovery_owner, expected_identity) = {
let mut inner = pager_a
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let owner = inner
.claim_rollback_journal_recovery(
RollbackJournalRecoveryState::ReplayPending,
&journal_path,
)
.unwrap();
let expected_identity = inner
.database_identity
.expect("file-protocol MemoryVfs exposes an exact main identity");
inner
.rollback_journal_recovery_namespace
.as_mut()
.expect("claimed recovery retains its origin namespace")
.db_path = replaced_origin_path.clone();
(owner, expected_identity)
};
let db_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_DB;
let (mut replacement_origin, _) = vfs
.open(&cx, Some(&replaced_origin_path), db_flags)
.unwrap();
assert_ne!(
replacement_origin.file_identity().unwrap(),
Some(expected_identity),
"keeper requires a replacement origin generation"
);
replacement_origin.close(&cx).unwrap();
let header = JournalHeader {
page_count: 1,
nonce: 0x5245_504C,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let header_bytes = header.encode_padded();
let journal_flags = VfsOpenFlags::CREATE
| VfsOpenFlags::EXCLUSIVE
| VfsOpenFlags::READWRITE
| VfsOpenFlags::MAIN_JOURNAL;
let (mut foreign_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
foreign_journal.write(&cx, &header_bytes, 0).await.unwrap();
let record =
JournalPageRecord::new(page_two.get(), vec![0x41; page_size], header.nonce);
foreign_journal
.write(&cx, &record.encode(), header_bytes.len() as u64)
.await
.unwrap();
foreign_journal.sync(&cx, SyncFlags::NORMAL).unwrap();
foreign_journal.close(&cx).unwrap();
let live_db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut live_db, _) = vfs.open(&cx, Some(&path), live_db_flags).unwrap();
live_db
.write(
&cx,
&vec![0x92; page_size],
PageSize::DEFAULT.as_usize() as u64,
)
.await
.unwrap();
live_db.sync(&cx, SyncFlags::NORMAL).unwrap();
live_db.close(&cx).unwrap();
drop(pager_a);
let recovery_error = match pager_b.begin(&cx, TransactionMode::ReadOnly).await {
Ok(_) => panic!("replacement origin identity was replayed into the live inode"),
Err(error) => error,
};
assert!(matches!(recovery_error, FrankenError::CannotOpen { .. }));
assert_eq!(
pager_b.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner)
);
assert!(vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
let (mut live_db, _) = vfs.open(&cx, Some(&path), live_db_flags).unwrap();
let mut still_unrecovered = vec![0_u8; page_size];
live_db
.read(
&cx,
&mut still_unrecovered,
PageSize::DEFAULT.as_usize() as u64,
)
.await
.unwrap();
assert_eq!(
still_unrecovered,
vec![0x92; page_size],
"a mismatched origin identity must fail before journal replay"
);
live_db.close(&cx).unwrap();
// Model restoration of the exact origin pathname and prove the
// retained owner can retry rather than becoming terminally stuck.
pager_b
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.rollback_journal_recovery_namespace
.as_mut()
.expect("failed validation retains the exact namespace")
.db_path = path.clone();
pager_b.refresh_published_snapshot(&cx).await.unwrap();
let mut reader = pager_b.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().as_ref(),
vec![0x41; page_size].as_slice()
);
reader.rollback(&cx).await.unwrap();
assert_eq!(pager_b.maintenance_gate.rollback_recovery_owner(), None);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
});
}
#[test]
fn test_commit_journal_short_preimage_read_errors() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/short_preimage.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
// Establish page 2 so the next commit must read a pre-image for it.
let page_two = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x11; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
p
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x22; ps])
.await
.unwrap();
// Simulate external truncation: pre-image read for page 2 becomes short.
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
db_file
.truncate(&cx, PageSize::DEFAULT.as_usize() as u64)
.unwrap();
let err = txn.commit(&cx).await.unwrap_err();
assert!(
matches!(err, FrankenError::DatabaseCorrupt { .. }),
"bead_id={BEAD_ID} case=short_preimage_read_is_corruption"
);
// Commit failure should keep the writer lock held on commit failure so no other writer
// can interleave while the caller decides to retry or roll back.
let Err(busy) = pager.begin(&cx, TransactionMode::Immediate).await else {
panic!("expected begin to fail while writer lock is still held");
};
assert!(
matches!(busy, FrankenError::Busy),
"bead_id={BEAD_ID} case=commit_error_keeps_writer_lock"
);
txn.rollback(&cx).await.unwrap();
let _next_writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
});
}
#[test]
fn test_partial_commit_failure_recovers_before_return_and_rollback_finalizes() {
asupersync::test_utils::run_test(|| async {
let path = PathBuf::from("/rollback_after_failed_commit.db");
let journal_path = SimplePager::<DbWriteFailOnceVfs>::journal_path(&path);
let vfs = DbWriteFailOnceVfs::new(path.clone());
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let original_two = vec![0x11; ps];
let original_three = vec![0x44; ps];
let (page_two, page_three) = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
let page_three = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &original_two).await.unwrap();
txn.write_page(&cx, page_three, &original_three)
.await
.unwrap();
txn.commit(&cx).await.unwrap();
(page_two, page_three)
};
let original_db_image = {
let flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let file_size = usize::try_from(db_file.file_size(&cx).unwrap()).unwrap();
let mut image = vec![0_u8; file_size];
assert_eq!(db_file.read(&cx, &mut image, 0).await.unwrap(), file_size);
db_file.close(&cx).unwrap();
image
};
// Page 1 and the first user page land before the second user-page
// write fails, proving recovery restores user data rather than
// merely the header change counter.
vfs.arm_after_db_writes(2);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x22; ps])
.await
.unwrap();
txn.write_page(&cx, page_three, &vec![0x55; ps])
.await
.unwrap();
let err = txn.commit(&cx).await.unwrap_err();
assert!(
matches!(&err, FrankenError::Io(_)),
"bead_id={BEAD_ID} case=partial_commit_surfaces_io_error"
);
assert!(
err.to_string().contains("simulated main-db write failure"),
"bead_id={BEAD_ID} case=partial_commit_preserves_original_injected_error error={err}"
);
let (successful_offsets, failed_offset) = vfs.db_write_fault_observation();
assert_eq!(
successful_offsets,
vec![0, u64::from(page_two.get() - 1) * ps as u64],
"bead_id={BEAD_ID} case=partial_commit_applies_header_and_first_user_page"
);
assert_eq!(
failed_offset,
Some(u64::from(page_three.get() - 1) * ps as u64),
"bead_id={BEAD_ID} case=partial_commit_fails_on_second_user_page"
);
// Capture at the exact database-write fault boundary: commit()
// synchronously consumes and deletes the live hot journal before
// returning the original I/O error.
let captured_journal = vfs
.captured_journal_bytes()
.expect("database-write fault must capture the complete hot journal");
assert!(
captured_journal.len() >= crate::journal::JOURNAL_HEADER_SIZE,
"bead_id={BEAD_ID} case=partial_commit_captures_complete_journal_header"
);
let hot_header =
JournalHeader::decode(&captured_journal[..crate::journal::JOURNAL_HEADER_SIZE])
.unwrap();
assert_eq!(hot_header.sector_size, 4096);
// The two user pages plus page 1's commit-counter update all require
// preimages; the header count must match the exact encoded records.
assert_eq!(hot_header.page_count, 3);
let record_size = 4 + ps + 4;
assert_eq!(captured_journal.len(), 4096 + 3 * record_size);
let mut preimages = HashMap::new();
for record_index in 0..3_usize {
let record_start = 4096 + record_index * record_size;
let record_end = record_start + record_size;
let record = JournalPageRecord::decode(
&captured_journal[record_start..record_end],
ps as u32,
)
.unwrap();
record.verify_checksum(hot_header.nonce).unwrap();
preimages.insert(record.page_number, record.content);
}
assert_eq!(
preimages
.get(&PageNumber::ONE.get())
.map(std::vec::Vec::as_slice),
Some(&original_db_image[..ps]),
"bead_id={BEAD_ID} case=partial_commit_journal_captures_page_one_preimage"
);
assert_eq!(preimages.get(&page_two.get()), Some(&original_two));
assert_eq!(preimages.get(&page_three.get()), Some(&original_three));
let recovered_db_image = {
let flags = VfsOpenFlags::READONLY | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let file_size = usize::try_from(db_file.file_size(&cx).unwrap()).unwrap();
let mut image = vec![0_u8; file_size];
assert_eq!(db_file.read(&cx, &mut image, 0).await.unwrap(), file_size);
db_file.close(&cx).unwrap();
image
};
assert_eq!(
recovered_db_image, original_db_image,
"bead_id={BEAD_ID} case=commit_error_returns_only_after_exact_image_recovery"
);
assert_eq!(
txn.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean,
"bead_id={BEAD_ID} case=commit_error_returns_after_metadata_recovery"
);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=commit_error_returns_after_journal_cleanup"
);
let Err(busy) = pager.begin(&cx, TransactionMode::Immediate).await else {
panic!("failed transaction must retain writer ownership until rollback");
};
assert!(
matches!(busy, FrankenError::Busy),
"bead_id={BEAD_ID} case=recovery_does_not_finalize_failed_transaction"
);
txn.rollback(&cx).await.unwrap();
let mut next_writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
next_writer.rollback(&cx).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
original_two
);
assert_eq!(
reader.get_page(&cx, page_three).await.unwrap().into_vec(),
original_three
);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=rollback_preserves_completed_journal_cleanup"
);
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
let reopened_reader = reopened
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
reopened_reader
.get_page(&cx, page_two)
.await
.unwrap()
.into_vec(),
original_two
);
assert_eq!(
reopened_reader
.get_page(&cx, page_three)
.await
.unwrap()
.into_vec(),
original_three
);
});
}
#[test]
fn test_begin_recovers_abandoned_failed_commit() {
asupersync::test_utils::run_test(|| async {
let path = PathBuf::from("/begin_recovers_failed_commit.db");
let journal_path = SimplePager::<DbWriteFailOnceVfs>::journal_path(&path);
let vfs = DbWriteFailOnceVfs::new(path.clone());
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let original_two = vec![0x61; ps];
let original_three = vec![0x73; ps];
let (page_two, page_three) = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
let page_three = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &original_two).await.unwrap();
txn.write_page(&cx, page_three, &original_three)
.await
.unwrap();
txn.commit(&cx).await.unwrap();
(page_two, page_three)
};
vfs.arm_after_db_writes(1);
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x62; ps])
.await
.unwrap();
txn.write_page(&cx, page_three, &vec![0x74; ps])
.await
.unwrap();
let err = txn.commit(&cx).await.unwrap_err();
assert!(
matches!(err, FrankenError::Io(_)),
"bead_id={BEAD_ID} case=abandoned_failed_commit_surfaces_io_error"
);
}
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
original_two
);
assert_eq!(
reader.get_page(&cx, page_three).await.unwrap().into_vec(),
original_three
);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=next_begin_cleans_abandoned_failed_commit_journal"
);
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
let reopened_reader = reopened
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
reopened_reader
.get_page(&cx, page_two)
.await
.unwrap()
.into_vec(),
vec![0x61; ps]
);
assert_eq!(
reopened_reader
.get_page(&cx, page_three)
.await
.unwrap()
.into_vec(),
vec![0x73; ps]
);
});
}
#[test]
fn test_commit_survives_journal_delete_failure() {
asupersync::test_utils::run_test(|| async {
let vfs = JournalDeleteFailVfs::new();
let path = PathBuf::from("/journal_delete_failure_commit.db");
let journal_path = SimplePager::<JournalDeleteFailVfs>::journal_path(&path);
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0xAB; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_two
};
drop(pager);
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
let reader = reopened
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
vec![0xAB; ps]
);
assert!(
vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=delete_failure_leaves_journal_inode"
);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (journal_file, _) = vfs.open(&cx, Some(&journal_path), flags).unwrap();
assert!(
journal_file.file_size(&cx).unwrap() >= JOURNAL_MAGIC.len() as u64,
"bead_id={BEAD_ID} case=delete_failure_retains_persist_journal_inode"
);
let mut commit_marker = [0xFF_u8; 1];
assert_eq!(
journal_file.read(&cx, &mut commit_marker, 0).await.unwrap(),
1
);
assert_eq!(
commit_marker[0], 0,
"bead_id={BEAD_ID} case=delete_failure_still_durably_invalidates_journal"
);
assert_eq!(
classify_rollback_journal_prefix(&cx, &journal_file)
.await
.unwrap()
.0,
RollbackJournalPrefixState::NonHot,
"bead_id={BEAD_ID} case=retained_persist_journal_is_proven_non_hot"
);
drop(journal_file);
drop(reader);
drop(reopened);
let fresh_reopen = vfs.open_file_backed_pager(&path).await.unwrap();
let fresh_reader = fresh_reopen
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
fresh_reader
.get_page(&cx, page_two)
.await
.unwrap()
.into_vec(),
vec![0xAB; ps],
"bead_id={BEAD_ID} case=repeated_reopen_accepts_only_revalidated_non_hot_leftover"
);
});
}
#[test]
fn test_hot_journal_recovery_survives_delete_failure() {
asupersync::test_utils::run_test(|| async {
let vfs = JournalDeleteFailVfs::new();
let path = PathBuf::from("/journal_delete_failure_recovery.db");
let journal_path = SimplePager::<JournalDeleteFailVfs>::journal_path(&path);
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
{
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let header = JournalHeader {
page_count: 1,
nonce: 0x4652_414E,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let hdr_bytes = header.encode_padded();
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut jrnl_file, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
jrnl_file.truncate(&cx, 0).unwrap();
jrnl_file.write(&cx, &hdr_bytes, 0).await.unwrap();
let record = JournalPageRecord::new(2, vec![0x11; ps], header.nonce);
jrnl_file
.write(&cx, &record.encode(), hdr_bytes.len() as u64)
.await
.unwrap();
jrnl_file.sync(&cx, SyncFlags::NORMAL).unwrap();
let page_offset = PageSize::DEFAULT.as_usize() as u64;
db_file
.write(&cx, &vec![0x22; ps], page_offset)
.await
.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
}
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
let reader = reopened
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
let page_two = PageNumber::new(2).unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
vec![0x11; ps]
);
assert!(
vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=recovery_delete_failure_leaves_journal_inode"
);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (journal_file, _) = vfs.open(&cx, Some(&journal_path), flags).unwrap();
assert_eq!(
journal_file.file_size(&cx).unwrap(),
0,
"bead_id={BEAD_ID} case=recovery_delete_failure_still_invalidates_journal"
);
drop(journal_file);
drop(reader);
drop(reopened);
let fresh_reopen = vfs.open_file_backed_pager(&path).await.unwrap();
let fresh_reader = fresh_reopen
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
fresh_reader
.get_page(&cx, page_two)
.await
.unwrap()
.into_vec(),
vec![0x11; ps],
"bead_id={BEAD_ID} case=fresh_reopen_observes_exact_recovered_preimage"
);
});
}
#[test]
fn test_commit_creates_and_deletes_journal() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/jrnl_test.db");
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
// Before commit, no journal.
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=no_journal_before_commit"
);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xAA; 4096]).await.unwrap();
txn.commit(&cx).await.unwrap();
// After commit, journal should be deleted.
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=journal_deleted_after_commit"
);
});
}
#[test]
fn test_private_memory_commit_skips_journal_creation() {
asupersync::test_utils::run_test(|| async {
let pager = private_memory_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let journal_path = SimplePager::<MemoryVfs>::journal_path(&pager.db_path);
assert!(
!pager
.vfs
.access(&cx, &journal_path, AccessFlags::EXISTS)
.unwrap(),
"bead_id={BEAD_ID} case=private_memory_commit_starts_without_journal"
);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0xA5; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
assert!(
!pager
.vfs
.access(&cx, &journal_path, AccessFlags::EXISTS)
.unwrap(),
"bead_id={BEAD_ID} case=private_memory_commit_avoids_journal_creation"
);
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_ref()[0],
0xA5,
"bead_id={BEAD_ID} case=private_memory_commit_persists_visible_page_image"
);
});
}
#[test]
fn test_hot_journal_recovery_restores_original_data() {
asupersync::test_utils::run_test(|| async {
// Simulate a crash: write data, manually create a journal with pre-images,
// then reopen. The journal should be replayed, restoring original data.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/crash_test.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Step 1: Create a database with known data via normal commit.
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
assert_eq!(p.get(), 2);
txn.write_page(&cx, p, &vec![0x11; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
}
// Step 2: Corrupt the database (simulate a partial write that crashed).
{
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let corrupt_data = vec![0x99; ps];
let offset = u64::from(2_u32 - 1) * ps as u64;
db_file.write(&cx, &corrupt_data, offset).await.unwrap();
}
// Step 3: Create a hot journal with the original pre-image.
{
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (jrnl, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
let nonce = 42;
let header = JournalHeader {
page_count: 1,
nonce,
initial_db_size: 2,
sector_size: 512,
page_size: 4096,
};
let hdr_bytes = header.encode_padded();
jrnl.write(&cx, &hdr_bytes, 0).await.unwrap();
let record = JournalPageRecord::new(2, vec![0x11; ps], nonce);
let rec_bytes = record.encode();
jrnl.write(&cx, &rec_bytes, hdr_bytes.len() as u64)
.await
.unwrap();
}
// Step 4: Reopen — should detect hot journal and replay.
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let page_no_2 = PageNumber::new(2).unwrap();
let data = txn.get_page(&cx, page_no_2).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=journal_recovery_restores"
);
assert_eq!(
data.as_ref()[ps - 1],
0x11,
"bead_id={BEAD_ID} case=journal_recovery_restores_last_byte"
);
}
// Step 5: Verify journal is deleted after recovery.
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=journal_deleted_after_recovery"
);
});
}
#[test]
fn test_long_lived_pager_recovers_external_hot_journal_on_begin() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/long_lived_hot_journal.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = PageNumber::new(2).unwrap();
// Keep one pager instance alive while another actor mutates the file.
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
{
let seed = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = seed.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
assert_eq!(page_two.get(), 2);
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let warm_reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
warm_reader
.get_page(&cx, page_two)
.await
.unwrap()
.into_vec(),
vec![0x11; ps],
"bead_id={BEAD_ID} case=existing_pager_populates_publication_before_hot_journal"
);
drop(warm_reader);
assert!(
pager.published_snapshot().page_set_size > 0,
"bead_id={BEAD_ID} case=existing_pager_has_published_pages_before_hot_journal"
);
{
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let header = JournalHeader {
page_count: 1,
nonce: 0x4652_414E,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let hdr_bytes = header.encode_padded();
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut jrnl_file, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
jrnl_file.write(&cx, &hdr_bytes, 0).await.unwrap();
let record = JournalPageRecord::new(2, vec![0x11; ps], header.nonce);
jrnl_file
.write(&cx, &record.encode(), hdr_bytes.len() as u64)
.await
.unwrap();
jrnl_file.sync(&cx, SyncFlags::NORMAL).unwrap();
let page_offset = PageSize::DEFAULT.as_usize() as u64;
db_file
.write(&cx, &vec![0x99; ps], page_offset)
.await
.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
}
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
pager.published_snapshot().page_set_size,
0,
"bead_id={BEAD_ID} case=existing_pager_clears_published_pages_after_hot_journal"
);
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
vec![0x11; ps],
"bead_id={BEAD_ID} case=existing_pager_recovers_hot_journal"
);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=existing_pager_removes_hot_journal"
);
});
}
#[test]
fn test_refresh_published_snapshot_recovers_external_hot_journal() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/published_refresh_hot_journal.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
assert_eq!(page_two.get(), 2);
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let page_two = PageNumber::new(2).unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().as_ref()[0],
0x11
);
drop(reader);
assert!(
pager.published_snapshot().page_set_size > 0,
"bead_id={BEAD_ID} case=publication_plane_populated_before_hot_journal_refresh"
);
{
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let header = JournalHeader {
page_count: 1,
nonce: 0x5245_4652,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let hdr_bytes = header.encode_padded();
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut jrnl_file, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
jrnl_file.write(&cx, &hdr_bytes, 0).await.unwrap();
let record = JournalPageRecord::new(2, vec![0x11; ps], header.nonce);
jrnl_file
.write(&cx, &record.encode(), hdr_bytes.len() as u64)
.await
.unwrap();
jrnl_file.sync(&cx, SyncFlags::NORMAL).unwrap();
let page_offset = PageSize::DEFAULT.as_usize() as u64;
db_file
.write(&cx, &vec![0x99; ps], page_offset)
.await
.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
}
let refreshed = pager.refresh_published_snapshot(&cx).await.unwrap();
assert_eq!(
refreshed.page_set_size, 0,
"bead_id={BEAD_ID} case=published_refresh_clears_published_pages_after_hot_journal"
);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let mut restored = vec![0u8; ps];
let bytes_read = db_file
.read(&cx, &mut restored, PageSize::DEFAULT.as_usize() as u64)
.await
.unwrap();
assert_eq!(bytes_read, ps);
assert_eq!(
restored,
vec![0x11; ps],
"bead_id={BEAD_ID} case=published_refresh_recovers_hot_journal_bytes"
);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=published_refresh_removes_hot_journal"
);
});
}
#[test]
fn test_hot_journal_truncated_record_stops_replay() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/trunc_jrnl.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Create DB with 2 pages.
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0xAB; ps]).await.unwrap();
txn.write_page(&cx, p2, &vec![0xBB; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
}
// Corrupt page 2.
{
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
db_file
.write(&cx, &vec![0xFF; ps], u64::from(2_u32 - 1) * ps as u64)
.await
.unwrap();
}
// Journal claims 2 records but second is truncated.
{
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (jrnl, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
let nonce = 7;
let header = JournalHeader {
page_count: 2,
nonce,
initial_db_size: 3,
sector_size: 512,
page_size: 4096,
};
let hdr_bytes = header.encode_padded();
jrnl.write(&cx, &hdr_bytes, 0).await.unwrap();
// First record: valid pre-image for page 3.
let rec1 = JournalPageRecord::new(3, vec![0xCC; ps], nonce);
let rec1_bytes = rec1.encode();
jrnl.write(&cx, &rec1_bytes, hdr_bytes.len() as u64)
.await
.unwrap();
// Second record: truncated.
let rec2 = JournalPageRecord::new(2, vec![0xBB; ps], nonce);
let rec2_bytes = rec2.encode();
let trunc_len = rec2_bytes.len() / 2;
let offset = hdr_bytes.len() as u64 + rec1_bytes.len() as u64;
jrnl.write(&cx, &rec2_bytes[..trunc_len], offset)
.await
.unwrap();
}
// Recovery validates the complete record surface before the first
// source write. A truncated hot journal therefore fails closed and
// does not partially replay its valid prefix.
assert!(
SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.is_err(),
"bead_id={BEAD_ID} case=truncated_journal_rejected_before_replay"
);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let mut page_two = vec![0_u8; ps];
db_file
.read(&cx, &mut page_two, u64::from(2_u32 - 1) * ps as u64)
.await
.unwrap();
assert_eq!(
page_two[0], 0xFF,
"bead_id={BEAD_ID} case=truncated_journal_does_not_partially_replay"
);
});
}
#[test]
fn test_hot_journal_checksum_mismatch_stops_replay() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/cksum_jrnl.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x55; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
}
// Corrupt page 2.
{
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
db_file
.write(&cx, &vec![0xEE; ps], u64::from(2_u32 - 1) * ps as u64)
.await
.unwrap();
}
// Journal with wrong nonce in record (checksum won't verify).
{
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (jrnl, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
let nonce = 99;
let header = JournalHeader {
page_count: 1,
nonce,
initial_db_size: 2,
sector_size: 512,
page_size: 4096,
};
let hdr_bytes = header.encode_padded();
jrnl.write(&cx, &hdr_bytes, 0).await.unwrap();
// Wrong nonce in record.
let record = JournalPageRecord::new(2, vec![0x55; ps], nonce + 1);
let rec_bytes = record.encode();
jrnl.write(&cx, &rec_bytes, hdr_bytes.len() as u64)
.await
.unwrap();
}
// A checksum mismatch is corruption, not an ignorable suffix. The
// opener must fail closed and leave the target image untouched.
assert!(
SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.is_err(),
"bead_id={BEAD_ID} case=bad_checksum_rejected"
);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let mut page_two = vec![0_u8; ps];
db_file
.read(&cx, &mut page_two, u64::from(2_u32 - 1) * ps as u64)
.await
.unwrap();
assert_eq!(
page_two[0], 0xEE,
"bead_id={BEAD_ID} case=bad_checksum_does_not_replay"
);
});
}
#[test]
fn test_truncated_zero_magic_is_non_hot_but_nonzero_prefix_fails_closed() {
asupersync::test_utils::run_test(|| async {
for journal_size in 1_usize..JOURNAL_MAGIC.len() {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from(format!("/short_magic_{journal_size}.db"));
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let original_size = db_file.file_size(&cx).unwrap();
let mut original_page_one = vec![0_u8; PageSize::DEFAULT.as_usize()];
db_file.read(&cx, &mut original_page_one, 0).await.unwrap();
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut zero_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
zero_journal.truncate(&cx, 0).unwrap();
zero_journal
.write(&cx, &vec![0_u8; journal_size], 0)
.await
.unwrap();
zero_journal.close(&cx).unwrap();
SimplePager::<MemoryVfs>::replay_journal(
&cx,
&vfs,
&mut db_file,
&journal_path,
PageSize::DEFAULT,
)
.await
.unwrap();
let readonly =
SimplePager::open_readonly_with_cx(&cx, vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
drop(readonly);
let (mut preserved_zero_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
assert_eq!(
preserved_zero_journal.file_size(&cx).unwrap(),
journal_size as u64
);
preserved_zero_journal.close(&cx).unwrap();
pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| {
Box::pin(async { Ok(()) })
})
.await
.unwrap();
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=maintenance_removes_zero_short_magic size={journal_size}"
);
let (mut nonzero_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
nonzero_journal.truncate(&cx, 0).unwrap();
let mut nonzero_prefix = vec![0_u8; journal_size];
nonzero_prefix[0] = 1;
nonzero_journal
.write(&cx, &nonzero_prefix, 0)
.await
.unwrap();
nonzero_journal.close(&cx).unwrap();
assert!(
SimplePager::<MemoryVfs>::replay_journal(
&cx,
&vfs,
&mut db_file,
&journal_path,
PageSize::DEFAULT,
)
.await
.is_err(),
"bead_id={BEAD_ID} case=nonzero_short_magic_rejected size={journal_size}"
);
let Err(readonly_error) =
SimplePager::open_readonly_with_cx(&cx, vfs.clone(), &path, PageSize::DEFAULT)
.await
else {
panic!(
"read-only open accepted nonzero short journal magic size={journal_size}"
);
};
assert!(matches!(
readonly_error,
FrankenError::DatabaseCorrupt { .. }
));
let nonzero_before = nonzero_prefix.clone();
assert!(
pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| {
Box::pin(async { Ok(()) })
})
.await
.is_err(),
"bead_id={BEAD_ID} case=maintenance_rejects_nonzero_short_magic size={journal_size}"
);
let (mut preserved_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
let mut nonzero_after = vec![0_u8; journal_size];
assert_eq!(
preserved_journal
.read(&cx, &mut nonzero_after, 0)
.await
.unwrap(),
journal_size
);
preserved_journal.close(&cx).unwrap();
assert_eq!(nonzero_after, nonzero_before);
let mut final_page_one = vec![0_u8; PageSize::DEFAULT.as_usize()];
db_file.read(&cx, &mut final_page_one, 0).await.unwrap();
assert_eq!(db_file.file_size(&cx).unwrap(), original_size);
assert_eq!(final_page_one, original_page_one);
}
});
}
#[test]
fn test_first_zero_magic_byte_marks_full_length_journal_non_hot() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/partial_zero_magic.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let mut partial_magic = JOURNAL_MAGIC;
partial_magic[0] = 0;
let flags = VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (journal, _) = vfs.open(&cx, Some(&journal_path), flags).unwrap();
journal.write(&cx, &partial_magic, 0).await.unwrap();
assert_eq!(
classify_rollback_journal_prefix(&cx, &journal)
.await
.unwrap(),
(
RollbackJournalPrefixState::NonHot,
JOURNAL_MAGIC.len() as u64
),
"a torn PERSIST marker with byte zero cleared must never be replayed"
);
});
}
#[test]
fn test_existing_open_recovers_torn_main_header_from_hot_journal_page_size() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/torn_main_header_hot_journal.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let page_size = PageSize::DEFAULT.as_usize();
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
assert_eq!(page_two.get(), 2);
txn.write_page(&cx, page_two, &vec![0x11; page_size])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut original_page_one = vec![0_u8; page_size];
let mut original_page_two = vec![0_u8; page_size];
assert_eq!(
db_file.read(&cx, &mut original_page_one, 0).await.unwrap(),
page_size
);
assert_eq!(
db_file
.read(&cx, &mut original_page_two, page_size as u64)
.await
.unwrap(),
page_size
);
let header = JournalHeader {
page_count: 2,
nonce: 0x1380_0001,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let header_bytes = header.encode_padded();
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal_file, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
journal_file.truncate(&cx, 0).unwrap();
journal_file.write(&cx, &header_bytes, 0).await.unwrap();
let page_one_record =
JournalPageRecord::new(1, original_page_one.clone(), header.nonce).encode();
let page_two_record =
JournalPageRecord::new(2, original_page_two.clone(), header.nonce).encode();
journal_file
.write(&cx, &page_one_record, header_bytes.len() as u64)
.await
.unwrap();
journal_file
.write(
&cx,
&page_two_record,
(header_bytes.len() + page_one_record.len()) as u64,
)
.await
.unwrap();
journal_file.sync(&cx, SyncFlags::NORMAL).unwrap();
journal_file.close(&cx).unwrap();
let mut torn_page_one = original_page_one.clone();
let original_header_bytes: &[u8; DATABASE_HEADER_SIZE] = original_page_one
[..DATABASE_HEADER_SIZE]
.try_into()
.unwrap();
let mut syntactically_valid_wrong_header =
DatabaseHeader::from_bytes(original_header_bytes).unwrap();
syntactically_valid_wrong_header.page_size = PageSize::new(8192).unwrap();
torn_page_one[..DATABASE_HEADER_SIZE]
.copy_from_slice(&syntactically_valid_wrong_header.to_bytes().unwrap());
let torn_header_bytes: &[u8; DATABASE_HEADER_SIZE] =
torn_page_one[..DATABASE_HEADER_SIZE].try_into().unwrap();
assert_eq!(
DatabaseHeader::from_bytes(torn_header_bytes)
.unwrap()
.page_size
.get(),
8192
);
db_file.write(&cx, &torn_page_one, 0).await.unwrap();
db_file
.write(&cx, &vec![0x99; page_size], page_size as u64)
.await
.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
db_file.close(&cx).unwrap();
let recovered = SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
vfs.clone(),
&path,
PageSize::DEFAULT,
None,
None,
)
.await
.expect("hot journal must repair a torn main header before final parsing");
drop(recovered);
let (mut restored_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut restored_page_one = vec![0_u8; page_size];
let mut restored_page_two = vec![0_u8; page_size];
restored_file
.read(&cx, &mut restored_page_one, 0)
.await
.unwrap();
restored_file
.read(&cx, &mut restored_page_two, page_size as u64)
.await
.unwrap();
restored_file.close(&cx).unwrap();
assert_eq!(restored_page_one, original_page_one);
assert_eq!(restored_page_two, original_page_two);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
});
}
#[test]
fn test_existing_open_rejects_malformed_hot_journal_without_mutating_torn_main() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/torn_main_malformed_hot_journal.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let page_size = PageSize::DEFAULT.as_usize();
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x33; page_size])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
db_file.write(&cx, &vec![0xA5; page_size], 0).await.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
let db_size = db_file.file_size(&cx).unwrap() as usize;
let mut main_before = vec![0_u8; db_size];
db_file.read(&cx, &mut main_before, 0).await.unwrap();
db_file.close(&cx).unwrap();
let malformed_header = JournalHeader {
page_count: 1,
nonce: 0x1380_0003,
initial_db_size: 2,
sector_size: 512,
page_size: 123,
}
.encode_padded();
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal_file, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
journal_file.truncate(&cx, 0).unwrap();
journal_file.write(&cx, &malformed_header, 0).await.unwrap();
journal_file.sync(&cx, SyncFlags::NORMAL).unwrap();
journal_file.close(&cx).unwrap();
let Err(error) = SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
vfs.clone(),
&path,
PageSize::DEFAULT,
None,
None,
)
.await
else {
panic!("malformed hot journal unexpectedly opened a torn main file");
};
assert!(matches!(error, FrankenError::DatabaseCorrupt { .. }));
let (mut main_after_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut main_after = vec![0_u8; db_size];
main_after_file.read(&cx, &mut main_after, 0).await.unwrap();
main_after_file.close(&cx).unwrap();
assert_eq!(main_after, main_before);
let (mut journal_after_file, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
let mut journal_after = vec![0_u8; malformed_header.len()];
journal_after_file
.read(&cx, &mut journal_after, 0)
.await
.unwrap();
journal_after_file.close(&cx).unwrap();
assert_eq!(journal_after, malformed_header);
});
}
#[test]
fn test_readonly_open_rejects_hot_journal_without_mutation_then_rw_recovers() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/readonly_hot_journal.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let page_size = PageSize::DEFAULT.as_usize();
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x22; page_size])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut original_page_two = vec![0_u8; page_size];
db_file
.read(&cx, &mut original_page_two, page_size as u64)
.await
.unwrap();
let header = JournalHeader {
page_count: 1,
nonce: 0x1380_0002,
initial_db_size: 2,
sector_size: 512,
page_size: PageSize::DEFAULT.get(),
};
let header_bytes = header.encode_padded();
let record =
JournalPageRecord::new(2, original_page_two.clone(), header.nonce).encode();
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal_file, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
journal_file.truncate(&cx, 0).unwrap();
journal_file.write(&cx, &header_bytes, 0).await.unwrap();
journal_file
.write(&cx, &record, header_bytes.len() as u64)
.await
.unwrap();
journal_file.sync(&cx, SyncFlags::NORMAL).unwrap();
let journal_size = journal_file.file_size(&cx).unwrap() as usize;
let mut journal_before = vec![0_u8; journal_size];
journal_file
.read(&cx, &mut journal_before, 0)
.await
.unwrap();
journal_file.close(&cx).unwrap();
db_file
.write(&cx, &vec![0x99; page_size], page_size as u64)
.await
.unwrap();
db_file.sync(&cx, SyncFlags::NORMAL).unwrap();
db_file.close(&cx).unwrap();
let Err(readonly_error) =
SimplePager::open_readonly_with_cx(&cx, vfs.clone(), &path, PageSize::DEFAULT)
.await
else {
panic!("read-only open must fail closed while a hot journal exists");
};
assert!(matches!(readonly_error, FrankenError::BusyRecovery));
let (mut unchanged_db, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut corrupted_page_two = vec![0_u8; page_size];
unchanged_db
.read(&cx, &mut corrupted_page_two, page_size as u64)
.await
.unwrap();
unchanged_db.close(&cx).unwrap();
assert_eq!(corrupted_page_two, vec![0x99; page_size]);
let (mut unchanged_journal, _) =
vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
let mut journal_after = vec![0_u8; journal_size];
unchanged_journal
.read(&cx, &mut journal_after, 0)
.await
.unwrap();
unchanged_journal.close(&cx).unwrap();
assert_eq!(journal_after, journal_before);
let recovered = SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
vfs.clone(),
&path,
PageSize::DEFAULT,
None,
None,
)
.await
.expect("read-write open must recover the retained hot journal");
drop(recovered);
let (mut restored_db, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut restored_page_two = vec![0_u8; page_size];
restored_db
.read(&cx, &mut restored_page_two, page_size as u64)
.await
.unwrap();
restored_db.close(&cx).unwrap();
assert_eq!(restored_page_two, original_page_two);
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
});
}
#[test]
fn test_hot_journal_invalid_page_number_errors() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/bad_pgno_jrnl.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
{
let _pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let jrnl_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (jrnl, _) = vfs.open(&cx, Some(&journal_path), jrnl_flags).unwrap();
let nonce = 321;
let header = JournalHeader {
page_count: 1,
nonce,
initial_db_size: 1,
sector_size: 512,
page_size: 4096,
};
let hdr_bytes = header.encode_padded();
jrnl.write(&cx, &hdr_bytes, 0).await.unwrap();
let record = JournalPageRecord::new(0, vec![0xAA; ps], nonce);
let rec_bytes = record.encode();
jrnl.write(&cx, &rec_bytes, hdr_bytes.len() as u64)
.await
.unwrap();
}
let Err(err) = SimplePager::open(vfs, &path, PageSize::DEFAULT).await else {
panic!("expected invalid journal page number error");
};
assert!(
matches!(err, FrankenError::DatabaseCorrupt { .. }),
"bead_id={BEAD_ID} case=invalid_journal_page_number_rejected"
);
});
}
#[test]
fn test_journal_not_created_for_readonly_commit() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let journal_path = SimplePager::<MemoryVfs>::journal_path(&pager.db_path);
let mut txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
txn.commit(&cx).await.unwrap();
assert!(
!pager
.vfs
.access(&cx, &journal_path, AccessFlags::EXISTS)
.unwrap(),
"bead_id={BEAD_ID} case=journal_deleted_for_readonly"
);
});
}
#[test]
fn physically_readonly_pager_rejects_every_writer_entry_path() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/physically_readonly_writer_rejection.db");
drop(
SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap(),
);
let pager = SimplePager::open_readonly_with_cx(&cx, vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
assert!(matches!(
pager.begin(&cx, TransactionMode::Immediate).await,
Err(FrankenError::ReadOnly)
));
assert!(matches!(
pager.begin(&cx, TransactionMode::Exclusive).await,
Err(FrankenError::ReadOnly)
));
let mut deferred = pager.begin(&cx, TransactionMode::Deferred).await.unwrap();
assert!(matches!(
deferred.allocate_page(&cx).await,
Err(FrankenError::ReadOnly)
));
deferred.rollback(&cx).await.unwrap();
let mut concurrent = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(matches!(
concurrent.allocate_page(&cx).await,
Err(FrankenError::ReadOnly)
));
concurrent.rollback(&cx).await.unwrap();
});
}
#[test]
fn malformed_hot_journal_is_fully_validated_before_first_database_write() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let malformed_cases = [
("duplicate", vec![1_u32, 1_u32], Vec::new()),
("beyond_initial", vec![2_u32], Vec::new()),
("trailing_bytes", vec![1_u32], vec![0xA5]),
];
for (name, record_pages, trailing) in malformed_cases {
let vfs = MemoryVfs::new();
let path = PathBuf::from(format!("/malformed_hot_journal_{name}.db"));
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
drop(
SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap(),
);
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut before_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let before_len = usize::try_from(before_file.file_size(&cx).unwrap()).unwrap();
let mut before = vec![0_u8; before_len];
before_file.read(&cx, &mut before, 0).await.unwrap();
before_file.close(&cx).unwrap();
let nonce = 0xA11C_E000_u32.wrapping_add(record_pages.len() as u32);
let header = JournalHeader {
page_count: i32::try_from(record_pages.len()).unwrap(),
nonce,
initial_db_size: 1,
sector_size: 4096,
page_size: PageSize::DEFAULT.get(),
};
let mut header_bytes = header.encode_padded();
if name == "trailing_bytes" {
mark_local_journal_header(&mut header_bytes);
}
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
journal.truncate(&cx, 0).unwrap();
journal.write(&cx, &header_bytes, 0).await.unwrap();
let mut offset = header_bytes.len() as u64;
for page_no in record_pages {
let record = JournalPageRecord::new(page_no, vec![0x5A; ps], nonce).encode();
journal.write(&cx, &record, offset).await.unwrap();
offset += record.len() as u64;
}
if !trailing.is_empty() {
journal.write(&cx, &trailing, offset).await.unwrap();
}
journal.close(&cx).unwrap();
let error = SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
vfs.clone(),
&path,
PageSize::DEFAULT,
None,
None,
)
.await
.err()
.unwrap_or_else(|| panic!("malformed case {name} unexpectedly opened"));
assert!(matches!(error, FrankenError::DatabaseCorrupt { .. }));
let (mut after_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut after = vec![0_u8; before_len];
after_file.read(&cx, &mut after, 0).await.unwrap();
after_file.close(&cx).unwrap();
assert_eq!(after, before, "case={name} changed the main file");
}
});
}
#[test]
fn external_journal_with_additional_structure_is_refused_without_replay() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/external_multisection_journal.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
drop(
SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap(),
);
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut main_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let main_len = usize::try_from(main_file.file_size(&cx).unwrap()).unwrap();
let mut before = vec![0_u8; main_len];
main_file.read(&cx, &mut before, 0).await.unwrap();
main_file.close(&cx).unwrap();
let header = JournalHeader {
page_count: 0,
nonce: 0xA11C_E222,
initial_db_size: 1,
sector_size: 4096,
page_size: PageSize::DEFAULT.get(),
};
let header_bytes = header.encode_padded();
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
journal.write(&cx, &header_bytes, 0).await.unwrap();
journal
.write(&cx, &vec![0_u8; 4096], header_bytes.len() as u64)
.await
.unwrap();
journal.close(&cx).unwrap();
let error = SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
vfs.clone(),
&path,
PageSize::DEFAULT,
None,
None,
)
.await
.err()
.expect("external multi-section journal must be refused");
assert!(matches!(error, FrankenError::Unsupported));
let (mut after_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
let mut after = vec![0_u8; main_len];
after_file.read(&cx, &mut after, 0).await.unwrap();
after_file.close(&cx).unwrap();
assert_eq!(after, before);
assert!(vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
});
}
#[test]
fn zero_record_hot_journal_rolls_back_pure_growth_by_truncation() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/zero_record_growth_rollback.db");
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let ps = PageSize::DEFAULT.as_usize();
drop(
SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap(),
);
let db_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
assert_eq!(db_file.file_size(&cx).unwrap(), ps as u64);
db_file
.write(&cx, &vec![0xCC; ps], ps as u64)
.await
.unwrap();
db_file.close(&cx).unwrap();
let header = JournalHeader {
page_count: 0,
nonce: 0xA11C_E111,
initial_db_size: 1,
sector_size: 4096,
page_size: PageSize::DEFAULT.get(),
};
let journal_flags =
VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_JOURNAL;
let (mut journal, _) = vfs.open(&cx, Some(&journal_path), journal_flags).unwrap();
journal
.write(&cx, &header.encode_padded(), 0)
.await
.unwrap();
journal.close(&cx).unwrap();
drop(
SimplePager::open_existing_with_cx_and_page_buffer_max(
&cx,
vfs.clone(),
&path,
PageSize::DEFAULT,
None,
None,
)
.await
.unwrap(),
);
let (mut restored, _) = vfs.open(&cx, Some(&path), db_flags).unwrap();
assert_eq!(restored.file_size(&cx).unwrap(), ps as u64);
restored.close(&cx).unwrap();
assert!(!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap());
});
}
#[test]
fn test_rollback_deletes_journal() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/rollback_jrnl.db");
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xDD; 4096]).await.unwrap();
txn.rollback(&cx).await.unwrap();
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=journal_deleted_on_rollback"
);
});
}
// ── Savepoint tests ────────────────────────────────────────────────
#[test]
fn test_savepoint_basic_rollback_to() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
// Create savepoint after first write.
txn.savepoint(&cx, "sp1").unwrap();
// Second write (after savepoint).
let p2 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p2, &vec![0x22; ps]).await.unwrap();
// Overwrite p1 after savepoint.
txn.write_page(&cx, p1, &vec![0x33; ps]).await.unwrap();
// Rollback to sp1 — should undo second write and p1 overwrite.
txn.rollback_to_savepoint(&cx, "sp1").unwrap();
// p1 should have the value from before the savepoint.
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=savepoint_rollback_restores_p1"
);
// p2 should no longer be in the write-set (reads zeros from disk).
let data2 = txn.get_page(&cx, p2).await.unwrap();
assert_eq!(
data2.as_ref()[0],
0x00,
"bead_id={BEAD_ID} case=savepoint_rollback_removes_p2"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_release_keeps_changes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0xAA; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
// Write after savepoint.
txn.write_page(&cx, p1, &vec![0xBB; ps]).await.unwrap();
// Release — changes after savepoint are kept.
txn.release_savepoint(&cx, "sp1").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0xBB,
"bead_id={BEAD_ID} case=release_keeps_changes"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_nested_rollback_to_inner() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "outer").unwrap();
txn.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
txn.savepoint(&cx, "inner").unwrap();
txn.write_page(&cx, p1, &vec![0x33; ps]).await.unwrap();
// Rollback to inner — should restore to 0x22 (state at "inner" creation).
txn.rollback_to_savepoint(&cx, "inner").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=nested_rollback_inner"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_nested_rollback_to_outer() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "outer").unwrap();
txn.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
txn.savepoint(&cx, "inner").unwrap();
txn.write_page(&cx, p1, &vec![0x33; ps]).await.unwrap();
// Rollback to outer — should restore to 0x11 and discard inner savepoint.
txn.rollback_to_savepoint(&cx, "outer").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=nested_rollback_outer"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_rollback_to_preserves_savepoint() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
// First modification + rollback.
txn.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
txn.rollback_to_savepoint(&cx, "sp1").unwrap();
// Should be back to 0x11.
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=rollback_to_preserves_savepoint"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_rollback_reclaims_allocated_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
// Initial state: 1 page (header)
let p1 = txn.allocate_page(&cx).await.unwrap(); // Page 2
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
// Allocate Page 3 inside savepoint
let p2 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p2, &vec![0x22; ps]).await.unwrap();
assert_eq!(p2.get(), p1.get() + 1, "Expected sequential allocation");
// Rollback to sp1. This should ideally "un-allocate" p2.
txn.rollback_to_savepoint(&cx, "sp1").unwrap();
// Allocate again. Should we get p2 again?
// If next_page wasn't reverted, we'll get p2 + 1 (Page 4), leaving Page 3 as a hole.
let p3 = txn.allocate_page(&cx).await.unwrap();
assert_eq!(
p3.get(),
p2.get(),
"bead_id={BEAD_ID} case=rollback_reclaims_allocation: expected page {} but got {}",
p2.get(),
p3.get()
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_rollback_to_preserves_savepoint_multi() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
// Modify again.
txn.write_page(&cx, p1, &vec![0x33; ps]).await.unwrap();
txn.rollback_to_savepoint(&cx, "sp1").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=rollback_to_preserves_savepoint_multi"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_freed_pages_restored_on_rollback() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0xAA; ps]).await.unwrap();
txn.write_page(&cx, p2, &vec![0xBB; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
// Free p2 after savepoint.
txn.free_page(&cx, p2).await.unwrap();
// Rollback — p2 should no longer be freed.
txn.rollback_to_savepoint(&cx, "sp1").unwrap();
// p2 should still be in the write-set (not freed).
let data = txn.get_page(&cx, p2).await.unwrap();
assert_eq!(
data.as_ref()[0],
0xBB,
"bead_id={BEAD_ID} case=freed_pages_restored"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_unknown_name_errors() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let result = txn.rollback_to_savepoint(&cx, "nonexistent");
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=rollback_to_unknown_savepoint_errors"
);
let result = txn.release_savepoint(&cx, "nonexistent");
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=release_unknown_savepoint_errors"
);
});
}
#[test]
fn test_savepoint_release_then_rollback_to_outer() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "outer").unwrap();
txn.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
txn.savepoint(&cx, "inner").unwrap();
txn.write_page(&cx, p1, &vec![0x33; ps]).await.unwrap();
// Release inner — changes kept, inner savepoint removed.
txn.release_savepoint(&cx, "inner").unwrap();
// Rollback to outer — should revert to 0x11.
txn.rollback_to_savepoint(&cx, "outer").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x11,
"bead_id={BEAD_ID} case=release_inner_then_rollback_outer"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_savepoint_commit_with_active_savepoints() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
txn.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
// Commit with active savepoint — all changes should be persisted.
txn.commit(&cx).await.unwrap();
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn2.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=commit_with_savepoints_persists_all"
);
});
}
#[test]
fn test_savepoint_full_rollback_clears_savepoints() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
txn.savepoint(&cx, "sp2").unwrap();
// Full rollback should clear all savepoints.
txn.rollback(&cx).await.unwrap();
// Trying to rollback to a savepoint after full rollback should error.
let result = txn.rollback_to_savepoint(&cx, "sp1");
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=full_rollback_clears_savepoints"
);
});
}
#[test]
fn test_savepoint_three_levels_deep() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
// Level 0: write 0x00
txn.write_page(&cx, p1, &vec![0x00; ps]).await.unwrap();
txn.savepoint(&cx, "L0").unwrap();
// Level 1: write 0x11
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "L1").unwrap();
// Level 2: write 0x22
txn.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
txn.savepoint(&cx, "L2").unwrap();
// Level 3: write 0x33
txn.write_page(&cx, p1, &vec![0x33; ps]).await.unwrap();
// Verify current state
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x33,
"bead_id={BEAD_ID} case=3level_current"
);
// Rollback to L2 → should see 0x22
txn.rollback_to_savepoint(&cx, "L2").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(data.as_ref()[0], 0x22, "bead_id={BEAD_ID} case=3level_L2");
// Rollback to L1 → should see 0x11
txn.rollback_to_savepoint(&cx, "L1").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(data.as_ref()[0], 0x11, "bead_id={BEAD_ID} case=3level_L1");
// Rollback to L0 → should see 0x00
txn.rollback_to_savepoint(&cx, "L0").unwrap();
let data = txn.get_page(&cx, p1).await.unwrap();
assert_eq!(data.as_ref()[0], 0x00, "bead_id={BEAD_ID} case=3level_L0");
txn.commit(&cx).await.unwrap();
// Verify committed value is 0x00 (state at L0).
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn2.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x00,
"bead_id={BEAD_ID} case=3level_committed"
);
});
}
// ── WAL mode integration tests ──────────────────────────────────────
use fsqlite_wal::checksum::{WAL_FRAME_HEADER_SIZE, WalChecksumTransform};
use fsqlite_wal::wal::WalAppendFrameRef;
use fsqlite_wal::{WalFile, WalSalts};
use serde_json::json;
use std::sync::{
Arc as StdArc, Condvar as StdCondvar, LazyLock as StdLazyLock, Mutex as StdMutex,
};
use std::time::Instant;
/// (page_number, page_data, db_size_if_commit)
type WalFrame = (u32, Vec<u8>, u32);
type SharedFrames = StdArc<StdMutex<Vec<WalFrame>>>;
type SharedCounter = StdArc<StdMutex<usize>>;
type SharedLockLevels = StdArc<StdMutex<Vec<LockLevel>>>;
type SharedGate = StdArc<(StdMutex<bool>, StdCondvar)>;
#[derive(Clone)]
struct MockPersistedParallelWalCommit {
certificate: ParallelWalCommitCertificate,
wal_frame_start: u64,
wal_frame_end: u64,
sync: bool,
}
type SharedPersistedParallelWalCommit =
StdArc<StdMutex<Option<MockPersistedParallelWalCommit>>>;
static PARALLEL_WAL_LANE_TEST_LOCK: StdLazyLock<StdMutex<()>> =
StdLazyLock::new(|| StdMutex::new(()));
fn signal_gate(gate: &SharedGate) {
let (lock, cvar) = &**gate;
*lock.lock().unwrap() = true;
cvar.notify_all();
}
fn wait_gate(gate: &SharedGate) {
let (lock, cvar) = &**gate;
let mut ready = lock.lock().unwrap();
while !*ready {
ready = cvar.wait(ready).unwrap();
}
}
fn wait_gate_timeout(gate: &SharedGate, timeout: Duration) -> bool {
let deadline = Instant::now() + timeout;
let (lock, cvar) = &**gate;
let mut ready = lock.lock().unwrap();
while !*ready {
let remaining = deadline.saturating_duration_since(Instant::now());
if remaining.is_zero() {
return false;
}
let (next_ready, wait_result) = cvar.wait_timeout(ready, remaining).unwrap();
ready = next_ready;
if wait_result.timed_out() && !*ready {
return false;
}
}
true
}
fn prepared_batch_from_frame_refs(
frames: &[crate::traits::WalFrameRef<'_>],
corrupt_first_payload: bool,
) -> crate::traits::PreparedWalFrameBatch {
let frame_size = WAL_FRAME_HEADER_SIZE + frames[0].page_data.len();
let mut frame_bytes = Vec::with_capacity(frame_size * frames.len());
let mut frame_metas = Vec::with_capacity(frames.len());
for frame in frames {
frame_metas.push(crate::traits::PreparedWalFrameMeta {
page_number: frame.page_number,
db_size_if_commit: frame.db_size_if_commit,
});
frame_bytes.extend_from_slice(&frame.page_number.to_be_bytes());
frame_bytes.extend_from_slice(&frame.db_size_if_commit.to_be_bytes());
frame_bytes.extend_from_slice(&[0_u8; WAL_FRAME_HEADER_SIZE - 8]);
frame_bytes.extend_from_slice(frame.page_data);
}
if corrupt_first_payload {
let payload_offset = WAL_FRAME_HEADER_SIZE;
frame_bytes[payload_offset] ^= 0xFF;
}
let mut prepared = crate::traits::PreparedWalFrameBatch {
frame_size,
page_data_offset: WAL_FRAME_HEADER_SIZE,
big_endian_checksum: false,
frame_metas,
checksum_transforms: Vec::new(),
frame_bytes,
last_commit_frame_offset: frames
.iter()
.enumerate()
.rev()
.find_map(|(offset, frame)| (frame.db_size_if_commit != 0).then_some(offset)),
finalized_for: None,
finalized_running_checksum: None,
};
prepared.recompute_checksum_transforms().unwrap();
prepared
}
fn lane_staged_batch_for_test(
batch_id: u64,
lane_id: u16,
frames: &[crate::traits::WalFrameRef<'_>],
) -> LaneStagedPreparedBatch {
LaneStagedPreparedBatch {
batch_id,
lane_id,
staged_frame_count: u32::try_from(frames.len()).unwrap_or(u32::MAX),
staging_elapsed_ns: u64::try_from(frames.len()).unwrap_or(u64::MAX) * 10,
shadow_verdict: ParallelWalShadowVerdict::NotRun,
payload: prepared_batch_from_frame_refs(frames, false),
}
}
const TRACK_C_BATCH_BENCH_BEAD_ID: &str = "bd-db300.3.1.4";
const TRACK_C_PUBLISH_WINDOW_INVENTORY_BEAD_ID: &str = "bd-db300.3.2.1";
const TRACK_C_PUBLISH_WINDOW_BENCH_BEAD_ID: &str = "bd-db300.3.2.3";
const TRACK_C_BATCH_BENCH_WARMUP_ITERS: usize = 5;
const TRACK_C_BATCH_BENCH_MEASURE_ITERS: usize = 25;
const TRACK_C_BATCH_BENCH_CASES: [(&str, usize); 3] = [
("page1_plus_1_new_page", 2),
("page1_plus_7_new_pages", 8),
("page1_plus_31_new_pages", 32),
];
const TRACK_C_PUBLISH_WINDOW_BENCH_WARMUP_ITERS: usize = 5;
const TRACK_C_PUBLISH_WINDOW_BENCH_MEASURE_ITERS: usize = 20;
const TRACK_C_PUBLISH_WINDOW_BENCH_CASES: [(&str, usize); 3] = [
("interior_only_1_dirty_page", 1),
("interior_only_7_dirty_pages", 7),
("interior_only_31_dirty_pages", 31),
];
const TRACK_C_METADATA_BENCH_BEAD_ID: &str = "bd-db300.3.3.3";
const TRACK_C_METADATA_BENCH_WARMUP_ITERS: usize = 5;
const TRACK_C_METADATA_BENCH_MEASURE_ITERS: usize = 25;
const TRACK_C_METADATA_BENCH_CASES: [(&str, usize); 3] = [
("interior_only_1_dirty_page", 1),
("interior_only_7_dirty_pages", 7),
("interior_only_31_dirty_pages", 31),
];
/// In-memory WAL backend for testing WAL-mode commit and page lookup.
struct MockWalBackend {
frames: SharedFrames,
begin_calls: SharedCounter,
batch_calls: SharedCounter,
sync_calls: SharedCounter,
read_page_calls: SharedCounter,
reconcile_calls: SharedCounter,
persisted_parallel_wal_commit: SharedPersistedParallelWalCommit,
fail_append_before_write: bool,
fail_sync_after_append: bool,
/// Read snapshot pinned at the most recent `begin_transaction`, used
/// by the WAL commit path's cross-connection conflict detection. Kept
/// per-backend (per pager handle) like the real adapter's pinned
/// snapshot.
pinned_snapshot: StdMutex<Option<traits::WalPublicationSnapshot>>,
publish_after_begin: Option<(Arc<PublishedPagerState>, PublishedPagerUpdate)>,
}
/// Derive a publication snapshot from the shared mock frame log: commit
/// frames are those carrying a nonzero `db_size_if_commit`, matching how
/// every mock-based test encodes commits.
fn mock_wal_publication_snapshot(
frames: &[(u32, Vec<u8>, u32)],
) -> traits::WalPublicationSnapshot {
let commit_count = frames.iter().filter(|frame| frame.2 > 0).count() as u64;
traits::WalPublicationSnapshot {
publication_seq: commit_count,
generation: fsqlite_wal::WalGenerationIdentity {
checkpoint_seq: 0,
salts: fsqlite_wal::checksum::WalSalts { salt1: 0, salt2: 0 },
},
last_commit_frame: frames.iter().rposition(|frame| frame.2 > 0),
commit_count,
latest_frame_entries: frames.len(),
index_is_partial: false,
}
}
impl MockWalBackend {
fn new() -> (Self, SharedFrames, SharedCounter, SharedCounter) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let begin_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let batch_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let sync_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let read_page_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let reconcile_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let persisted_parallel_wal_commit = StdArc::new(StdMutex::new(None));
(
Self {
frames: StdArc::clone(&frames),
begin_calls: StdArc::clone(&begin_calls),
batch_calls: StdArc::clone(&batch_calls),
sync_calls,
read_page_calls,
reconcile_calls,
persisted_parallel_wal_commit,
fail_append_before_write: false,
fail_sync_after_append: false,
pinned_snapshot: StdMutex::new(None),
publish_after_begin: None,
},
frames,
begin_calls,
batch_calls,
)
}
fn new_with_sync_tracking() -> (
Self,
SharedFrames,
SharedCounter,
SharedCounter,
SharedCounter,
) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let begin_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let batch_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let sync_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let read_page_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let reconcile_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let persisted_parallel_wal_commit = StdArc::new(StdMutex::new(None));
(
Self {
frames: StdArc::clone(&frames),
begin_calls: StdArc::clone(&begin_calls),
batch_calls: StdArc::clone(&batch_calls),
sync_calls: StdArc::clone(&sync_calls),
read_page_calls,
reconcile_calls,
persisted_parallel_wal_commit,
fail_append_before_write: false,
fail_sync_after_append: false,
pinned_snapshot: StdMutex::new(None),
publish_after_begin: None,
},
frames,
begin_calls,
batch_calls,
sync_calls,
)
}
fn with_shared_frames(frames: SharedFrames) -> (Self, SharedCounter, SharedCounter) {
let begin_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let batch_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let sync_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let read_page_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let reconcile_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let persisted_parallel_wal_commit = StdArc::new(StdMutex::new(None));
(
Self {
frames,
begin_calls: StdArc::clone(&begin_calls),
batch_calls: StdArc::clone(&batch_calls),
sync_calls,
read_page_calls,
reconcile_calls,
persisted_parallel_wal_commit,
fail_append_before_write: false,
fail_sync_after_append: false,
pinned_snapshot: StdMutex::new(None),
publish_after_begin: None,
},
begin_calls,
batch_calls,
)
}
fn new_with_read_tracking() -> (
Self,
SharedFrames,
SharedCounter,
SharedCounter,
SharedCounter,
) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let begin_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let batch_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let sync_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let read_page_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let reconcile_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let persisted_parallel_wal_commit = StdArc::new(StdMutex::new(None));
(
Self {
frames: StdArc::clone(&frames),
begin_calls: StdArc::clone(&begin_calls),
batch_calls: StdArc::clone(&batch_calls),
sync_calls,
read_page_calls: StdArc::clone(&read_page_calls),
reconcile_calls,
persisted_parallel_wal_commit,
fail_append_before_write: false,
fail_sync_after_append: false,
pinned_snapshot: StdMutex::new(None),
publish_after_begin: None,
},
frames,
begin_calls,
batch_calls,
read_page_calls,
)
}
fn new_with_failing_sync() -> (Self, SharedFrames, SharedCounter, SharedCounter) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let begin_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let batch_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let sync_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let read_page_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let reconcile_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let persisted_parallel_wal_commit = StdArc::new(StdMutex::new(None));
(
Self {
frames: StdArc::clone(&frames),
begin_calls,
batch_calls,
sync_calls: StdArc::clone(&sync_calls),
read_page_calls,
reconcile_calls: StdArc::clone(&reconcile_calls),
persisted_parallel_wal_commit,
fail_append_before_write: false,
fail_sync_after_append: true,
pinned_snapshot: StdMutex::new(None),
publish_after_begin: None,
},
frames,
sync_calls,
reconcile_calls,
)
}
fn new_with_failing_append_before_write()
-> (Self, SharedFrames, SharedCounter, SharedCounter) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let begin_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let batch_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let sync_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let read_page_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let reconcile_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let persisted_parallel_wal_commit = StdArc::new(StdMutex::new(None));
(
Self {
frames: StdArc::clone(&frames),
begin_calls,
batch_calls,
sync_calls: StdArc::clone(&sync_calls),
read_page_calls,
reconcile_calls: StdArc::clone(&reconcile_calls),
persisted_parallel_wal_commit,
fail_append_before_write: true,
fail_sync_after_append: false,
pinned_snapshot: StdMutex::new(None),
publish_after_begin: None,
},
frames,
sync_calls,
reconcile_calls,
)
}
fn with_publish_after_begin(
mut self,
published: Arc<PublishedPagerState>,
update: PublishedPagerUpdate,
) -> Self {
self.publish_after_begin = Some((published, update));
self
}
}
/// WAL backend whose frame source accepts and copies one tracked append,
/// then remains pending until the test-owned source completion is made
/// terminal. The returned future itself never reports success: dropping
/// it exercises the real group-commit ownership transfer rather than a
/// synthetic durability flag transition.
struct PendingAcceptedWalBackend {
inner: MockWalBackend,
append_entered: Arc<AtomicBool>,
source_completion: Arc<Mutex<Option<VfsWriteCompletion>>>,
}
impl PendingAcceptedWalBackend {
fn new() -> (
Self,
SharedFrames,
Arc<AtomicBool>,
Arc<Mutex<Option<VfsWriteCompletion>>>,
) {
let (inner, frames, _, _) = MockWalBackend::new();
let append_entered = Arc::new(AtomicBool::new(false));
let source_completion = Arc::new(Mutex::new(None));
(
Self {
inner,
append_entered: Arc::clone(&append_entered),
source_completion: Arc::clone(&source_completion),
},
frames,
append_entered,
source_completion,
)
}
}
struct FailingGroupCommitWalBackend {
append_frames_calls: SharedCounter,
}
impl FailingGroupCommitWalBackend {
fn new() -> (Self, SharedCounter) {
let append_frames_calls: SharedCounter = StdArc::new(StdMutex::new(0));
(
Self {
append_frames_calls: StdArc::clone(&append_frames_calls),
},
append_frames_calls,
)
}
}
struct FailingCheckpointWalBackend;
struct PreparedBatchObservedWalBackend {
frames: SharedFrames,
append_frames_calls: SharedCounter,
append_prepared_calls: SharedCounter,
prepare_lock_levels: SharedLockLevels,
append_lock_levels: SharedLockLevels,
observed_lock_level: Arc<Mutex<LockLevel>>,
}
impl PreparedBatchObservedWalBackend {
fn new(
observed_lock_level: Arc<Mutex<LockLevel>>,
) -> (
Self,
SharedFrames,
SharedCounter,
SharedCounter,
SharedLockLevels,
SharedLockLevels,
) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let append_frames_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let append_prepared_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let prepare_lock_levels: SharedLockLevels = StdArc::new(StdMutex::new(Vec::new()));
let append_lock_levels: SharedLockLevels = StdArc::new(StdMutex::new(Vec::new()));
(
Self {
frames: StdArc::clone(&frames),
append_frames_calls: StdArc::clone(&append_frames_calls),
append_prepared_calls: StdArc::clone(&append_prepared_calls),
prepare_lock_levels: StdArc::clone(&prepare_lock_levels),
append_lock_levels: StdArc::clone(&append_lock_levels),
observed_lock_level,
},
frames,
append_frames_calls,
append_prepared_calls,
prepare_lock_levels,
append_lock_levels,
)
}
}
struct ShadowCompareMismatchWalBackend {
frames: SharedFrames,
prepare_calls: SharedCounter,
append_frames_calls: SharedCounter,
append_prepared_calls: SharedCounter,
}
impl ShadowCompareMismatchWalBackend {
fn new() -> (
Self,
SharedFrames,
SharedCounter,
SharedCounter,
SharedCounter,
) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let prepare_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let append_frames_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let append_prepared_calls: SharedCounter = StdArc::new(StdMutex::new(0));
(
Self {
frames: StdArc::clone(&frames),
prepare_calls: StdArc::clone(&prepare_calls),
append_frames_calls: StdArc::clone(&append_frames_calls),
append_prepared_calls: StdArc::clone(&append_prepared_calls),
},
frames,
prepare_calls,
append_frames_calls,
append_prepared_calls,
)
}
}
struct BlockingFirstPrepareWalBackend {
frames: SharedFrames,
prepare_calls: SharedCounter,
first_prepare_entered: SharedGate,
release_first_prepare: SharedGate,
}
impl BlockingFirstPrepareWalBackend {
fn new() -> (Self, SharedFrames, SharedGate, SharedGate, SharedCounter) {
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let first_prepare_entered: SharedGate =
StdArc::new((StdMutex::new(false), StdCondvar::new()));
let release_first_prepare: SharedGate =
StdArc::new((StdMutex::new(false), StdCondvar::new()));
let prepare_calls: SharedCounter = StdArc::new(StdMutex::new(0));
(
Self {
frames: StdArc::clone(&frames),
prepare_calls: StdArc::clone(&prepare_calls),
first_prepare_entered: StdArc::clone(&first_prepare_entered),
release_first_prepare: StdArc::clone(&release_first_prepare),
},
frames,
first_prepare_entered,
release_first_prepare,
prepare_calls,
)
}
}
#[derive(Clone, Copy)]
enum TrackCBatchMode {
SingleFrame,
Batched,
}
impl TrackCBatchMode {
const fn as_str(self) -> &'static str {
match self {
Self::SingleFrame => "single_frame",
Self::Batched => "batch_append",
}
}
}
#[derive(Clone, Copy)]
enum TrackCPublishWindowMode {
InlinePrepareBaseline,
PreparedCandidate,
}
impl TrackCPublishWindowMode {
const fn as_str(self) -> &'static str {
match self {
Self::InlinePrepareBaseline => "inline_prepare_baseline",
Self::PreparedCandidate => "prepared_candidate",
}
}
}
#[derive(Clone, Copy)]
enum TrackCMetadataMode {
ForcedPageOneBaseline,
SemanticCleanupCandidate,
}
impl TrackCMetadataMode {
const fn as_str(self) -> &'static str {
match self {
Self::ForcedPageOneBaseline => "forced_page_one_baseline",
Self::SemanticCleanupCandidate => "semantic_cleanup_candidate",
}
}
}
struct TrackCBenchmarkWalBackend {
wal: WalFile<MemoryFile>,
mode: TrackCBatchMode,
}
struct TrackCPublishWindowBenchWalBackend {
wal: WalFile<BlockingObservedLockFile>,
mode: TrackCPublishWindowMode,
}
impl TrackCBenchmarkWalBackend {
async fn new(
vfs: &MemoryVfs,
cx: &Cx,
path: &std::path::Path,
mode: TrackCBatchMode,
) -> Self {
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::CREATE | VfsOpenFlags::WAL;
let (file, _) = vfs.open(cx, Some(path), flags).unwrap();
let wal = WalFile::create(
cx,
file,
PageSize::DEFAULT.get(),
0,
WalSalts {
salt1: 0xDB30_0314,
salt2: 0xC1C1_C1C1,
},
)
.await
.unwrap();
Self { wal, mode }
}
}
impl TrackCPublishWindowBenchWalBackend {
async fn new(
vfs: &BlockingObservedLockVfs,
cx: &Cx,
path: &std::path::Path,
mode: TrackCPublishWindowMode,
) -> Self {
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::CREATE | VfsOpenFlags::WAL;
let (file, _) = vfs.open(cx, Some(path), flags).unwrap();
let wal = WalFile::create(
cx,
file,
PageSize::DEFAULT.get(),
0,
WalSalts {
salt1: 0xDB30_0323,
salt2: 0xC2C3_C2C3,
},
)
.await
.unwrap();
Self { wal, mode }
}
}
impl crate::traits::WalBackend for TrackCBenchmarkWalBackend {
fn append_frame<'a>(
&'a mut self,
cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.wal
.append_frame(cx, page_number, page_data, db_size_if_commit)
.await
})
}
fn append_frames<'a>(
&'a mut self,
cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
match self.mode {
TrackCBatchMode::SingleFrame => {
for frame in frames {
self.wal
.append_frame(
cx,
frame.page_number,
frame.page_data,
frame.db_size_if_commit,
)
.await?;
}
Ok(())
}
TrackCBatchMode::Batched => {
let wal_frames: Vec<_> = frames
.iter()
.map(|frame| WalAppendFrameRef {
page_number: frame.page_number,
page_data: frame.page_data,
db_size_if_commit: frame.db_size_if_commit,
})
.collect();
self.wal.append_frames(cx, &wal_frames).await
}
}
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn committed_txn_count<'a>(&'a mut self, cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async move {
let Some(last_commit_frame) = self.wal.last_commit_frame(cx)? else {
return Ok(0);
};
let mut commit_count = 0_u64;
for frame_index in 0..=last_commit_frame {
if self
.wal
.read_frame_header(cx, frame_index)
.await?
.is_commit()
{
commit_count = commit_count.saturating_add(1);
}
}
Ok(commit_count)
})
}
fn sync(&mut self, cx: &Cx) -> fsqlite_error::Result<()> {
self.wal.sync(cx, SyncFlags::NORMAL)
}
fn frame_count(&self) -> usize {
self.wal.frame_count()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
let total_frames = u32::try_from(self.wal.frame_count()).unwrap_or(u32::MAX);
Ok(crate::traits::CheckpointResult {
total_frames,
frames_backfilled: 0,
completed: false,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
impl crate::traits::WalBackend for TrackCPublishWindowBenchWalBackend {
fn append_frame<'a>(
&'a mut self,
cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.wal.file_mut().lock(cx, LockLevel::Exclusive)?;
let append_result = self
.wal
.append_frame(cx, page_number, page_data, db_size_if_commit)
.await;
let unlock_result = self.wal.file_mut().unlock(cx, LockLevel::None);
match (append_result, unlock_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(append_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"publish-window WAL append failed and unlock failed: append={append_error}; unlock={unlock_error}"
))),
}
})
}
fn append_frames<'a>(
&'a mut self,
cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
let wal_frames: Vec<_> = frames
.iter()
.map(|frame| WalAppendFrameRef {
page_number: frame.page_number,
page_data: frame.page_data,
db_size_if_commit: frame.db_size_if_commit,
})
.collect();
self.wal.file_mut().lock(cx, LockLevel::Exclusive)?;
let append_result = self.wal.append_frames(cx, &wal_frames).await;
let unlock_result = self.wal.file_mut().unlock(cx, LockLevel::None);
match (append_result, unlock_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(append_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"publish-window WAL append failed and unlock failed: append={append_error}; unlock={unlock_error}"
))),
}
})
}
fn prepare_append_frames(
&self,
frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>> {
if matches!(self.mode, TrackCPublishWindowMode::InlinePrepareBaseline) {
return Ok(None);
}
if frames.is_empty() {
return Ok(None);
}
let wal_frames: Vec<_> = frames
.iter()
.map(|frame| WalAppendFrameRef {
page_number: frame.page_number,
page_data: frame.page_data,
db_size_if_commit: frame.db_size_if_commit,
})
.collect();
let frame_size = WAL_FRAME_HEADER_SIZE + wal_frames[0].page_data.len();
let frame_metas = wal_frames
.iter()
.map(|frame| crate::traits::PreparedWalFrameMeta {
page_number: frame.page_number,
db_size_if_commit: frame.db_size_if_commit,
})
.collect();
let frame_bytes = self.wal.prepare_frame_bytes(&wal_frames)?;
let checksum_transforms = wal_frames
.iter()
.enumerate()
.map(|(index, _)| {
let frame_start = index
.checked_mul(frame_size)
.expect("frame start fits usize");
let frame_end = frame_start
.checked_add(frame_size)
.expect("frame end fits usize");
let transform = WalChecksumTransform::for_wal_frame(
&frame_bytes[frame_start..frame_end],
self.wal.page_size(),
self.wal.big_endian_checksum(),
)?;
Ok(transform)
})
.collect::<Result<Vec<_>>>()?;
Ok(Some(crate::traits::PreparedWalFrameBatch {
frame_size,
page_data_offset: WAL_FRAME_HEADER_SIZE,
big_endian_checksum: self.wal.big_endian_checksum(),
frame_metas,
checksum_transforms,
frame_bytes,
last_commit_frame_offset: frames
.iter()
.enumerate()
.rev()
.find_map(|(offset, frame)| (frame.db_size_if_commit != 0).then_some(offset)),
finalized_for: None,
finalized_running_checksum: None,
}))
}
fn append_prepared_frames<'a>(
&'a mut self,
cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
let checksum_transforms: Vec<_> = prepared
.checksum_transforms
.iter()
.map(|transform| WalChecksumTransform {
a11: transform.a11,
a12: transform.a12,
a21: transform.a21,
a22: transform.a22,
c1: transform.c1,
c2: transform.c2,
})
.collect();
self.wal.file_mut().lock(cx, LockLevel::Exclusive)?;
let append_result = self
.wal
.append_prepared_frame_bytes(
cx,
&mut prepared.frame_bytes,
&checksum_transforms,
)
.await;
let unlock_result = self.wal.file_mut().unlock(cx, LockLevel::None);
match (append_result, unlock_result) {
(Ok(()), Ok(())) => Ok(()),
(Err(error), Ok(())) | (Ok(()), Err(error)) => Err(error),
(Err(append_error), Err(unlock_error)) => Err(FrankenError::internal(format!(
"publish-window prepared WAL append failed and unlock failed: append={append_error}; unlock={unlock_error}"
))),
}
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn committed_txn_count<'a>(&'a mut self, cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async move {
let Some(last_commit_frame) = self.wal.last_commit_frame(cx)? else {
return Ok(0);
};
let mut commit_count = 0_u64;
for frame_index in 0..=last_commit_frame {
if self
.wal
.read_frame_header(cx, frame_index)
.await?
.is_commit()
{
commit_count = commit_count.saturating_add(1);
}
}
Ok(commit_count)
})
}
fn sync(&mut self, cx: &Cx) -> fsqlite_error::Result<()> {
self.wal.sync(cx, SyncFlags::NORMAL)
}
fn frame_count(&self) -> usize {
self.wal.frame_count()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
let total_frames = u32::try_from(self.wal.frame_count()).unwrap_or(u32::MAX);
Ok(crate::traits::CheckpointResult {
total_frames,
frames_backfilled: 0,
completed: false,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
impl crate::traits::WalBackend for FailingCheckpointWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
// Report a non-empty WAL so `SimplePager::checkpoint` does
// not short-circuit via the zero-frame fast path: this
// fixture exists specifically to exercise how the pager
// propagates errors from the backend's `checkpoint()`
// implementation, and that code path is only reachable
// when there are frames to back-fill.
1
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async { Err(FrankenError::BusyRecovery) })
}
}
async fn track_c_prepared_commit(
mode: TrackCBatchMode,
dirty_pages: usize,
) -> (Cx, SimpleTransaction<MemoryVfs>) {
assert!(dirty_pages >= 2);
let cx = Cx::new();
let vfs = MemoryVfs::new();
let db_path = PathBuf::from(format!(
"/track_c_batch_commit_{}_{}.db",
mode.as_str(),
dirty_pages
));
let wal_path = PathBuf::from(format!(
"/track_c_batch_commit_{}_{}.db-wal",
mode.as_str(),
dirty_pages
));
let pager = SimplePager::open(vfs.clone(), &db_path, PageSize::DEFAULT)
.await
.unwrap();
let backend = TrackCBenchmarkWalBackend::new(&vfs, &cx, &wal_path, mode).await;
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_bytes = PageSize::DEFAULT.as_usize();
txn.write_page(&cx, PageNumber::ONE, &vec![0xA1; page_bytes])
.await
.unwrap();
for page_idx in 1..dirty_pages {
let page_no = txn.allocate_page(&cx).await.unwrap();
let fill = u8::try_from((page_idx % 251) + 1).unwrap();
txn.write_page(&cx, page_no, &vec![fill; page_bytes])
.await
.unwrap();
}
(cx, txn)
}
async fn track_c_measure_commit_ns(mode: TrackCBatchMode, dirty_pages: usize) -> Vec<u64> {
let total_iters = TRACK_C_BATCH_BENCH_WARMUP_ITERS + TRACK_C_BATCH_BENCH_MEASURE_ITERS;
let mut samples = Vec::with_capacity(TRACK_C_BATCH_BENCH_MEASURE_ITERS);
for iter_idx in 0..total_iters {
let (cx, mut txn) = track_c_prepared_commit(mode, dirty_pages).await;
let started = Instant::now();
txn.commit(&cx).await.unwrap();
if iter_idx >= TRACK_C_BATCH_BENCH_WARMUP_ITERS {
let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
samples.push(elapsed_ns);
}
}
samples
}
async fn track_c_metadata_seed_existing_pages(
pager: &SimplePager<MemoryVfs>,
cx: &Cx,
interior_dirty_pages: usize,
) {
assert!(interior_dirty_pages > 0);
let mut txn = pager.begin(cx, TransactionMode::Immediate).await.unwrap();
let page_bytes = PageSize::DEFAULT.as_usize();
for page_idx in 0..interior_dirty_pages {
let page_no = txn.allocate_page(cx).await.unwrap();
let fill = u8::try_from((page_idx % 251) + 1).unwrap();
txn.write_page(cx, page_no, &vec![fill; page_bytes])
.await
.unwrap();
}
txn.commit(cx).await.unwrap();
}
async fn track_c_metadata_apply_workload(
txn: &mut SimpleTransaction<MemoryVfs>,
cx: &Cx,
interior_dirty_pages: usize,
mode: TrackCMetadataMode,
) {
let page_bytes = PageSize::DEFAULT.as_usize();
for page_idx in 0..interior_dirty_pages {
let page_no = PageNumber::new(u32::try_from(page_idx + 2).unwrap()).unwrap();
let fill = u8::try_from(((page_idx + 17) % 251) + 1).unwrap();
txn.write_page(cx, page_no, &vec![fill; page_bytes])
.await
.unwrap();
}
if matches!(mode, TrackCMetadataMode::ForcedPageOneBaseline) {
let page_one = txn.get_page(cx, PageNumber::ONE).await.unwrap().into_vec();
txn.write_page(cx, PageNumber::ONE, &page_one)
.await
.unwrap();
}
}
async fn track_c_metadata_prepared_commit(
mode: TrackCMetadataMode,
interior_dirty_pages: usize,
) -> (Cx, SimpleTransaction<MemoryVfs>) {
assert!(interior_dirty_pages > 0);
let cx = Cx::new();
let vfs = MemoryVfs::new();
let db_path = PathBuf::from(format!(
"/track_c_metadata_cleanup_{}_{}.db",
mode.as_str(),
interior_dirty_pages
));
let wal_path = PathBuf::from(format!(
"/track_c_metadata_cleanup_{}_{}.db-wal",
mode.as_str(),
interior_dirty_pages
));
let pager = SimplePager::open(vfs.clone(), &db_path, PageSize::DEFAULT)
.await
.unwrap();
track_c_metadata_seed_existing_pages(&pager, &cx, interior_dirty_pages).await;
let backend =
TrackCBenchmarkWalBackend::new(&vfs, &cx, &wal_path, TrackCBatchMode::Batched).await;
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
track_c_metadata_apply_workload(&mut txn, &cx, interior_dirty_pages, mode).await;
(cx, txn)
}
async fn track_c_metadata_measure_commit_ns(
mode: TrackCMetadataMode,
interior_dirty_pages: usize,
) -> Vec<u64> {
let total_iters =
TRACK_C_METADATA_BENCH_WARMUP_ITERS + TRACK_C_METADATA_BENCH_MEASURE_ITERS;
let mut samples = Vec::with_capacity(TRACK_C_METADATA_BENCH_MEASURE_ITERS);
for iter_idx in 0..total_iters {
let (cx, mut txn) = track_c_metadata_prepared_commit(mode, interior_dirty_pages).await;
let started = Instant::now();
txn.commit(&cx).await.unwrap();
if iter_idx >= TRACK_C_METADATA_BENCH_WARMUP_ITERS {
let elapsed_ns = u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX);
samples.push(elapsed_ns);
}
}
samples
}
async fn track_c_metadata_capture_frame_pages(
mode: TrackCMetadataMode,
interior_dirty_pages: usize,
) -> Vec<u32> {
assert!(interior_dirty_pages > 0);
let cx = Cx::new();
let vfs = MemoryVfs::new();
let db_path = PathBuf::from(format!(
"/track_c_metadata_capture_{}_{}.db",
mode.as_str(),
interior_dirty_pages
));
let pager = SimplePager::open(vfs, &db_path, PageSize::DEFAULT)
.await
.unwrap();
track_c_metadata_seed_existing_pages(&pager, &cx, interior_dirty_pages).await;
let (backend, frames, _begin_calls, _batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
frames.lock().unwrap().clear();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
track_c_metadata_apply_workload(&mut txn, &cx, interior_dirty_pages, mode).await;
txn.commit(&cx).await.unwrap();
frames
.lock()
.unwrap()
.iter()
.map(|(page_number, _, _)| *page_number)
.collect()
}
async fn track_c_seed_existing_pages_blocking(
pager: &SimplePager<BlockingObservedLockVfs>,
cx: &Cx,
page_count: usize,
) {
assert!(page_count > 0);
let mut txn = pager.begin(cx, TransactionMode::Immediate).await.unwrap();
let page_bytes = PageSize::DEFAULT.as_usize();
for page_idx in 0..page_count {
let page_no = txn.allocate_page(cx).await.unwrap();
let fill = u8::try_from((page_idx % 251) + 1).unwrap();
txn.write_page(cx, page_no, &vec![fill; page_bytes])
.await
.unwrap();
}
txn.commit(cx).await.unwrap();
}
async fn track_c_write_existing_page_range(
txn: &mut SimpleTransaction<BlockingObservedLockVfs>,
cx: &Cx,
start_page_no: u32,
page_count: usize,
fill_offset: usize,
) {
let page_bytes = PageSize::DEFAULT.as_usize();
for page_idx in 0..page_count {
let page_no =
PageNumber::new(start_page_no + u32::try_from(page_idx).unwrap()).unwrap();
let fill = u8::try_from(((page_idx + fill_offset) % 251) + 1).unwrap();
txn.write_page(cx, page_no, &vec![fill; page_bytes])
.await
.unwrap();
}
}
async fn track_c_publish_window_prepared_commit(
mode: TrackCPublishWindowMode,
dirty_pages: usize,
) -> (
Cx,
SimpleTransaction<BlockingObservedLockVfs>,
BlockingObservedLockVfs,
) {
assert!(dirty_pages > 0);
let cx = Cx::new();
let vfs = BlockingObservedLockVfs::new();
let db_path = PathBuf::from(format!(
"/track_c_publish_window_hold_{}_{}.db",
mode.as_str(),
dirty_pages
));
let wal_path = PathBuf::from(format!(
"/track_c_publish_window_hold_{}_{}.db-wal",
mode.as_str(),
dirty_pages
));
let seed_pager = vfs.open_file_backed_pager(&db_path).await.unwrap();
track_c_seed_existing_pages_blocking(&seed_pager, &cx, dirty_pages).await;
drop(seed_pager);
let pager = vfs.open_file_backed_pager(&db_path).await.unwrap();
let backend = TrackCPublishWindowBenchWalBackend::new(&vfs, &cx, &wal_path, mode).await;
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
vfs.clear_exclusive_metrics();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
track_c_write_existing_page_range(&mut txn, &cx, 2, dirty_pages, 37).await;
(cx, txn, vfs)
}
async fn track_c_measure_publish_window_hold_ns(
mode: TrackCPublishWindowMode,
dirty_pages: usize,
) -> Vec<u64> {
let total_iters =
TRACK_C_PUBLISH_WINDOW_BENCH_WARMUP_ITERS + TRACK_C_PUBLISH_WINDOW_BENCH_MEASURE_ITERS;
let mut samples = Vec::with_capacity(TRACK_C_PUBLISH_WINDOW_BENCH_MEASURE_ITERS);
for iter_idx in 0..total_iters {
let (cx, mut txn, vfs) =
track_c_publish_window_prepared_commit(mode, dirty_pages).await;
txn.commit(&cx).await.unwrap();
let hold_ns = *vfs.exclusive_hold_samples_ns().last().unwrap_or(&0);
if iter_idx >= TRACK_C_PUBLISH_WINDOW_BENCH_WARMUP_ITERS {
samples.push(hold_ns);
}
}
samples
}
async fn track_c_open_contending_pagers(
mode: TrackCPublishWindowMode,
dirty_pages: usize,
) -> (
BlockingObservedLockVfs,
SimplePager<BlockingObservedLockVfs>,
SimplePager<BlockingObservedLockVfs>,
) {
assert!(dirty_pages > 0);
let vfs = BlockingObservedLockVfs::new();
let db_path = PathBuf::from(format!(
"/track_c_publish_window_contention_{}_{}.db",
mode.as_str(),
dirty_pages
));
let wal_path = PathBuf::from(format!(
"/track_c_publish_window_contention_{}_{}.db-wal",
mode.as_str(),
dirty_pages
));
let seed_cx = Cx::new();
let seed_pager = vfs.open_file_backed_pager(&db_path).await.unwrap();
track_c_seed_existing_pages_blocking(&seed_pager, &seed_cx, dirty_pages.saturating_mul(2))
.await;
drop(seed_pager);
let pager_a = vfs.open_file_backed_pager(&db_path).await.unwrap();
let pager_b = vfs.open_file_backed_pager(&db_path).await.unwrap();
let cx_a = Cx::new();
let cx_b = Cx::new();
pager_a
.set_wal_backend(Box::new(
TrackCPublishWindowBenchWalBackend::new(&vfs, &cx_a, &wal_path, mode).await,
))
.unwrap();
pager_b
.set_wal_backend(Box::new(
TrackCPublishWindowBenchWalBackend::new(&vfs, &cx_b, &wal_path, mode).await,
))
.unwrap();
pager_a
.set_journal_mode(&cx_a, JournalMode::Wal)
.await
.unwrap();
pager_b
.set_journal_mode(&cx_b, JournalMode::Wal)
.await
.unwrap();
vfs.clear_exclusive_metrics();
(vfs, pager_a, pager_b)
}
async fn track_c_measure_competing_writer_stall_ns(
mode: TrackCPublishWindowMode,
dirty_pages: usize,
) -> Vec<u64> {
let total_iters =
TRACK_C_PUBLISH_WINDOW_BENCH_WARMUP_ITERS + TRACK_C_PUBLISH_WINDOW_BENCH_MEASURE_ITERS;
let mut samples = Vec::with_capacity(TRACK_C_PUBLISH_WINDOW_BENCH_MEASURE_ITERS);
for iter_idx in 0..total_iters {
let (vfs, pager_a, pager_b) = track_c_open_contending_pagers(mode, dirty_pages).await;
let writer_a = std::thread::spawn(move || {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let mut txn = pager_a
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
track_c_write_existing_page_range(&mut txn, &cx, 2, dirty_pages, 53).await;
txn.commit(&cx).await.unwrap();
});
});
vfs.wait_for_exclusive_acquisitions(1);
let cx_b = Cx::new();
let mut txn_b = pager_b
.begin(&cx_b, TransactionMode::Immediate)
.await
.unwrap();
let contender_start_page = u32::try_from(dirty_pages).unwrap() + 2;
track_c_write_existing_page_range(
&mut txn_b,
&cx_b,
contender_start_page,
dirty_pages,
97,
)
.await;
txn_b.commit(&cx_b).await.unwrap();
writer_a.join().unwrap();
let stall_ns = vfs
.exclusive_wait_samples_ns()
.into_iter()
.max()
.unwrap_or(0);
if iter_idx >= TRACK_C_PUBLISH_WINDOW_BENCH_WARMUP_ITERS {
samples.push(stall_ns);
}
}
samples
}
fn track_c_percentile_ns(sorted_samples: &[u64], percentile: usize) -> u64 {
let last_idx = sorted_samples.len().saturating_sub(1);
let rank = last_idx.saturating_mul(percentile).div_ceil(100);
sorted_samples[rank]
}
fn track_c_sample_summary(samples: &[u64]) -> serde_json::Value {
let mut sorted = samples.to_vec();
sorted.sort_unstable();
let total_ns: u128 = sorted.iter().map(|value| u128::from(*value)).sum();
let mean_ns = (total_ns as f64) / (sorted.len() as f64);
json!({
"sample_count": sorted.len(),
"min_ns": sorted.first().copied().unwrap_or(0),
"median_ns": track_c_percentile_ns(&sorted, 50),
"p95_ns": track_c_percentile_ns(&sorted, 95),
"max_ns": sorted.last().copied().unwrap_or(0),
"mean_ns": mean_ns,
"raw_samples_ns": sorted,
})
}
impl crate::traits::WalBackend for MockWalBackend {
fn begin_transaction<'a>(&'a mut self, cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async move {
let mut begin_calls = self.begin_calls.lock().unwrap();
*begin_calls += 1;
drop(begin_calls);
// Pin the read snapshot for cross-connection conflict
// detection, mirroring the real adapter: the commit path
// compares each batch against the snapshot pinned when its
// transaction began.
let snapshot = mock_wal_publication_snapshot(&self.frames.lock().unwrap());
*self.pinned_snapshot.lock().unwrap() = Some(snapshot);
if let Some((published, update)) = &self.publish_after_begin {
published.publish_metadata_only(cx, *update);
}
Ok(())
})
}
fn pinned_read_snapshot(&self) -> Option<traits::WalPublicationSnapshot> {
*self.pinned_snapshot.lock().unwrap()
}
fn conflicting_pages_since_snapshot<'a>(
&'a mut self,
_cx: &'a Cx,
snapshot: TransactionConflictSnapshot,
page_numbers: &'a [u32],
_page_baselines: &'a [TransactionConflictPageBaseline],
) -> WalFuture<'a, Vec<u32>> {
Box::pin(async move {
let frames = self.frames.lock().unwrap();
let start = snapshot
.last_commit_frame
.map_or(0, |frame| frame.saturating_add(1));
if start >= frames.len() {
return Ok(Vec::new());
}
let mut conflicts = page_numbers
.iter()
.copied()
.filter(|page| frames[start..].iter().any(|frame| frame.0 == *page))
.collect::<Vec<_>>();
conflicts.sort_unstable();
conflicts.dedup();
Ok(conflicts)
})
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.frames.lock().unwrap().push((
page_number,
page_data.to_vec(),
db_size_if_commit,
));
Ok(())
})
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
let mut batch_calls = self.batch_calls.lock().unwrap();
*batch_calls += 1;
drop(batch_calls);
if self.fail_append_before_write {
self.fail_append_before_write = false;
return Err(FrankenError::internal(
"forced group-commit append failure before WAL write",
));
}
let mut written = self.frames.lock().unwrap();
for frame in frames {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
Ok(())
})
}
fn persist_parallel_wal_commit_certificate<'a>(
&'a mut self,
_cx: &'a Cx,
certificate: &'a ParallelWalCommitCertificate,
wal_frame_start: u64,
wal_frame_end: u64,
sync: bool,
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.persisted_parallel_wal_commit.lock().unwrap() =
Some(MockPersistedParallelWalCommit {
certificate: certificate.clone(),
wal_frame_start,
wal_frame_end,
sync,
});
Ok(())
})
}
fn reconcile_parallel_wal_commit<'a>(
&'a mut self,
cx: &'a Cx,
certificate: &'a ParallelWalCommitCertificate,
wal_frame_start: u64,
wal_frame_end: u64,
sync: bool,
) -> WalFuture<'a, crate::traits::ParallelWalCommitReconciliation> {
Box::pin(async move {
*self.reconcile_calls.lock().unwrap() += 1;
let persisted = self
.persisted_parallel_wal_commit
.lock()
.unwrap()
.clone()
.ok_or_else(|| {
FrankenError::internal(
"mock WAL recovery has no persisted commit certificate",
)
})?;
if &persisted.certificate != certificate
|| persisted.wal_frame_start != wal_frame_start
|| persisted.wal_frame_end != wal_frame_end
|| persisted.sync != sync
{
return Err(FrankenError::internal(
"mock WAL recovery certificate or interval mismatch",
));
}
let start_index =
usize::try_from(wal_frame_start.checked_sub(1).ok_or_else(|| {
FrankenError::internal("mock WAL recovery interval must be one-based")
})?)
.map_err(|_| FrankenError::internal("mock WAL frame start exceeds usize"))?;
let end_index = usize::try_from(wal_frame_end)
.map_err(|_| FrankenError::internal("mock WAL frame end exceeds usize"))?;
if end_index <= start_index {
return Err(FrankenError::internal(
"mock WAL recovery interval must be non-empty",
));
}
let mut frames = self.frames.lock().unwrap();
if frames.len() < start_index {
return Err(FrankenError::internal(
"mock WAL recovery committed prefix is missing",
));
}
if end_index > frames.len() {
frames.truncate(start_index);
return Ok(crate::traits::ParallelWalCommitReconciliation::NotCommitted);
}
if frames[start_index..end_index]
.last()
.is_none_or(|(_, _, db_size_if_commit)| *db_size_if_commit == 0)
{
frames.truncate(start_index);
return Ok(crate::traits::ParallelWalCommitReconciliation::NotCommitted);
}
let interval = &frames[start_index..end_index];
let mut digest = ParallelWalFramePayloadDigestBuilder::new();
for (page_number, page_data, db_size_if_commit) in interval {
let page_number = PageNumber::new(*page_number).ok_or_else(|| {
FrankenError::internal("mock WAL recovery interval contains page zero")
})?;
digest.update(page_number, *db_size_if_commit, page_data);
}
if digest.finalize() != certificate.wal_frame_payload_digest {
return Err(FrankenError::internal(
"mock WAL recovery frame payload digest mismatch",
));
}
drop(frames);
if sync {
self.sync(cx)?;
}
Ok(crate::traits::ParallelWalCommitReconciliation::Authorized)
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async move {
*self.read_page_calls.lock().unwrap() += 1;
let frames = self.frames.lock().unwrap();
// Scan backwards for the latest version of the page.
let result = frames
.iter()
.rev()
.find(|(pn, _, _)| *pn == page_number)
.map(|(_, data, _)| data.clone());
drop(frames);
Ok(result)
})
}
fn committed_txns_since_page<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, u64> {
Box::pin(async move {
let frames = self.frames.lock().unwrap();
let last_page_frame = frames.iter().rposition(|(pn, _, _)| *pn == page_number);
let Some(last_page_frame) = last_page_frame else {
return Ok(frames
.iter()
.filter(|(_, _, db_size_if_commit)| *db_size_if_commit > 0)
.count() as u64);
};
let mut page_commit_seen = false;
let mut committed_txns_after_page = 0_u64;
for (frame_index, (_, _, db_size_if_commit)) in frames.iter().enumerate() {
if *db_size_if_commit == 0 {
continue;
}
if !page_commit_seen && frame_index >= last_page_frame {
page_commit_seen = true;
continue;
}
if page_commit_seen {
committed_txns_after_page = committed_txns_after_page.saturating_add(1);
}
}
Ok(committed_txns_after_page)
})
}
fn committed_txn_count<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async move {
Ok(self
.frames
.lock()
.unwrap()
.iter()
.filter(|(_, _, db_size_if_commit)| *db_size_if_commit > 0)
.count() as u64)
})
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
*self.sync_calls.lock().unwrap() += 1;
if self.fail_sync_after_append {
self.fail_sync_after_append = false;
Err(FrankenError::internal(
"forced group-commit sync failure after full WAL append",
))
} else {
Ok(())
}
}
fn frame_count(&self) -> usize {
self.frames.lock().unwrap().len()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
let total_frames =
u32::try_from(self.frames.lock().unwrap().len()).map_err(|_| {
fsqlite_error::FrankenError::internal("mock wal frame count exceeds u32")
})?;
Ok(crate::traits::CheckpointResult {
total_frames,
frames_backfilled: total_frames,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
impl crate::traits::WalBackend for PendingAcceptedWalBackend {
fn begin_transaction<'a>(&'a mut self, cx: &'a Cx) -> WalFuture<'a, ()> {
self.inner.begin_transaction(cx)
}
fn append_frame<'a>(
&'a mut self,
cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
self.inner
.append_frame(cx, page_number, page_data, db_size_if_commit)
}
fn append_frames_tracked<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
completion: VfsWriteCompletion,
) -> WalFuture<'a, ()> {
let append_entered = Arc::clone(&self.append_entered);
let source_completion = Arc::clone(&self.source_completion);
let written_frames = Arc::clone(&self.inner.frames);
let batch_calls = Arc::clone(&self.inner.batch_calls);
let mut accepted = false;
Box::pin(std::future::poll_fn(move |_| {
if !accepted {
*batch_calls
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) += 1;
let mut written = written_frames
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for frame in frames {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
drop(written);
let replaced = source_completion
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.replace(completion.clone());
assert!(
replaced.is_none(),
"pending WAL fixture accepted more than one tracked append"
);
accepted = true;
append_entered.store(true, AtomicOrdering::Release);
}
std::task::Poll::Pending
}))
}
fn persist_parallel_wal_commit_certificate<'a>(
&'a mut self,
cx: &'a Cx,
certificate: &'a ParallelWalCommitCertificate,
wal_frame_start: u64,
wal_frame_end: u64,
sync: bool,
) -> WalFuture<'a, ()> {
self.inner.persist_parallel_wal_commit_certificate(
cx,
certificate,
wal_frame_start,
wal_frame_end,
sync,
)
}
fn reconcile_parallel_wal_commit<'a>(
&'a mut self,
cx: &'a Cx,
certificate: &'a ParallelWalCommitCertificate,
wal_frame_start: u64,
wal_frame_end: u64,
sync: bool,
) -> WalFuture<'a, crate::traits::ParallelWalCommitReconciliation> {
self.inner.reconcile_parallel_wal_commit(
cx,
certificate,
wal_frame_start,
wal_frame_end,
sync,
)
}
fn read_page<'a>(
&'a mut self,
cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
self.inner.read_page(cx, page_number)
}
fn committed_txns_since_page<'a>(
&'a mut self,
cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, u64> {
self.inner.committed_txns_since_page(cx, page_number)
}
fn committed_txn_count<'a>(&'a mut self, cx: &'a Cx) -> WalFuture<'a, u64> {
self.inner.committed_txn_count(cx)
}
fn sync(&mut self, cx: &Cx) -> Result<()> {
self.inner.sync(cx)
}
fn frame_count(&self) -> usize {
self.inner.frame_count()
}
fn checkpoint<'a>(
&'a mut self,
cx: &'a Cx,
mode: crate::traits::CheckpointMode,
writer: &'a mut dyn crate::traits::CheckpointPageWriter,
backfilled_frames: u32,
oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
self.inner
.checkpoint(cx, mode, writer, backfilled_frames, oldest_reader_frame)
}
}
impl crate::traits::WalBackend for FailingGroupCommitWalBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async {
Err(FrankenError::internal(
"forced single-frame group commit failure",
))
})
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
_frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
let mut append_frames_calls = self.append_frames_calls.lock().unwrap();
*append_frames_calls += 1;
drop(append_frames_calls);
std::thread::sleep(std::time::Duration::from_millis(1));
Err(FrankenError::internal(
"forced batched group commit append failure",
))
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn committed_txn_count<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async { Ok(0) })
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
0
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
impl crate::traits::WalBackend for PreparedBatchObservedWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.frames.lock().unwrap().push((
page_number,
page_data.to_vec(),
db_size_if_commit,
));
Ok(())
})
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.append_frames_calls.lock().unwrap() += 1;
let mut written = self.frames.lock().unwrap();
for frame in frames {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
Ok(())
})
}
fn prepare_append_frames(
&self,
frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>> {
self.prepare_lock_levels
.lock()
.unwrap()
.push(*self.observed_lock_level.lock().unwrap());
if frames.is_empty() {
return Ok(None);
}
let frame_size =
fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE + frames[0].page_data.len();
let mut frame_bytes = Vec::with_capacity(frame_size * frames.len());
let mut frame_metas = Vec::with_capacity(frames.len());
let mut checksum_transforms = Vec::with_capacity(frames.len());
for frame in frames {
frame_metas.push(crate::traits::PreparedWalFrameMeta {
page_number: frame.page_number,
db_size_if_commit: frame.db_size_if_commit,
});
checksum_transforms.push(crate::traits::PreparedWalChecksumTransform {
a11: 0,
a12: 0,
a21: 0,
a22: 0,
c1: 0,
c2: 0,
});
frame_bytes
.extend_from_slice(&[0_u8; fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE]);
frame_bytes.extend_from_slice(frame.page_data);
}
Ok(Some(crate::traits::PreparedWalFrameBatch {
frame_size,
page_data_offset: fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE,
big_endian_checksum: false,
frame_metas,
checksum_transforms,
frame_bytes,
last_commit_frame_offset: frames
.iter()
.enumerate()
.rev()
.find_map(|(offset, frame)| (frame.db_size_if_commit != 0).then_some(offset)),
finalized_for: None,
finalized_running_checksum: None,
}))
}
fn append_prepared_frames<'a>(
&'a mut self,
_cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.append_lock_levels
.lock()
.unwrap()
.push(*self.observed_lock_level.lock().unwrap());
*self.append_prepared_calls.lock().unwrap() += 1;
let mut written = self.frames.lock().unwrap();
for frame in prepared.frame_refs() {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
Ok(())
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async move {
let frames = self.frames.lock().unwrap();
Ok(frames
.iter()
.rev()
.find(|(pn, _, _)| *pn == page_number)
.map(|(_, data, _)| data.clone()))
})
}
fn committed_txn_count<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async move {
Ok(self
.frames
.lock()
.unwrap()
.iter()
.filter(|(_, _, db_size_if_commit)| *db_size_if_commit > 0)
.count() as u64)
})
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
self.frames.lock().unwrap().len()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
let total_frames =
u32::try_from(self.frames.lock().unwrap().len()).map_err(|_| {
fsqlite_error::FrankenError::internal(
"observed wal frame count exceeds u32",
)
})?;
Ok(crate::traits::CheckpointResult {
total_frames,
frames_backfilled: total_frames,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
impl crate::traits::WalBackend for ShadowCompareMismatchWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.frames.lock().unwrap().push((
page_number,
page_data.to_vec(),
db_size_if_commit,
));
Ok(())
})
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.append_frames_calls.lock().unwrap() += 1;
let mut written = self.frames.lock().unwrap();
for frame in frames {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
Ok(())
})
}
fn prepare_append_frames(
&self,
frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>> {
let mut prepare_calls = self.prepare_calls.lock().unwrap();
*prepare_calls += 1;
let call_index = *prepare_calls;
drop(prepare_calls);
if frames.is_empty() {
return Ok(None);
}
if call_index == 1 {
Ok(Some(prepared_batch_from_frame_refs(frames, true)))
} else {
Ok(None)
}
}
fn append_prepared_frames<'a>(
&'a mut self,
_cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.append_prepared_calls.lock().unwrap() += 1;
let mut written = self.frames.lock().unwrap();
for frame in prepared.frame_refs() {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
Ok(())
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async move {
let frames = self.frames.lock().unwrap();
Ok(frames
.iter()
.rev()
.find(|(pn, _, _)| *pn == page_number)
.map(|(_, data, _)| data.clone()))
})
}
fn committed_txn_count<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async move {
Ok(self
.frames
.lock()
.unwrap()
.iter()
.filter(|(_, _, db_size_if_commit)| *db_size_if_commit > 0)
.count() as u64)
})
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
self.frames.lock().unwrap().len()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
let total_frames =
u32::try_from(self.frames.lock().unwrap().len()).unwrap_or(u32::MAX);
Ok(crate::traits::CheckpointResult {
total_frames,
frames_backfilled: total_frames,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
impl crate::traits::WalBackend for BlockingFirstPrepareWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
page_data: &'a [u8],
db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.frames.lock().unwrap().push((
page_number,
page_data.to_vec(),
db_size_if_commit,
));
Ok(())
})
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
let mut written = self.frames.lock().unwrap();
for frame in frames {
written.push((
frame.page_number,
frame.page_data.to_vec(),
frame.db_size_if_commit,
));
}
Ok(())
})
}
fn prepare_append_frames(
&self,
_frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>> {
let call_index = {
let mut calls = self.prepare_calls.lock().unwrap();
*calls += 1;
*calls
};
if call_index == 1 {
signal_gate(&self.first_prepare_entered);
wait_gate(&self.release_first_prepare);
}
Ok(None)
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async move {
let frames = self.frames.lock().unwrap();
Ok(frames
.iter()
.rev()
.find(|(pn, _, _)| *pn == page_number)
.map(|(_, data, _)| data.clone()))
})
}
fn committed_txn_count<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, u64> {
Box::pin(async move {
Ok(self
.frames
.lock()
.unwrap()
.iter()
.filter(|(_, _, db_size_if_commit)| *db_size_if_commit > 0)
.count() as u64)
})
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
self.frames.lock().unwrap().len()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
let total_frames =
u32::try_from(self.frames.lock().unwrap().len()).unwrap_or(u32::MAX);
Ok(crate::traits::CheckpointResult {
total_frames,
frames_backfilled: total_frames,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
async fn wal_pager() -> (SimplePager<MemoryVfs>, SharedFrames) {
let (pager, frames, _, _) = wal_pager_with_tracking().await;
(pager, frames)
}
async fn wal_pager_with_tracking() -> (
SimplePager<MemoryVfs>,
SharedFrames,
SharedCounter,
SharedCounter,
) {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_test.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, begin_calls, batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
(pager, frames, begin_calls, batch_calls)
}
async fn wal_pager_with_read_tracking() -> (
SimplePager<MemoryVfs>,
SharedFrames,
SharedCounter,
SharedCounter,
SharedCounter,
) {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_read_tracking.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, begin_calls, batch_calls, read_page_calls) =
MockWalBackend::new_with_read_tracking();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
(pager, frames, begin_calls, batch_calls, read_page_calls)
}
async fn wal_pager_pair_with_shared_backend()
-> (SimplePager<MemoryVfs>, SimplePager<MemoryVfs>, SharedFrames) {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_shared_refresh.db");
let pager1 = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let pager2 = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let (backend1, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
let (backend2, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
pager1.set_wal_backend(Box::new(backend1)).unwrap();
pager2.set_wal_backend(Box::new(backend2)).unwrap();
pager1
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap();
pager2
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap();
(pager1, pager2, frames)
}
#[test]
fn test_journal_mode_default_is_delete() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
assert_eq!(
pager.journal_mode(),
JournalMode::Delete,
"bead_id={BEAD_ID} case=default_journal_mode"
);
});
}
#[test]
fn test_set_journal_mode_wal_requires_backend() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
// Without a WAL backend, switching to WAL should fail.
let result = pager.set_journal_mode(&cx, JournalMode::Wal).await;
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=wal_requires_backend"
);
});
}
#[test]
fn test_set_journal_mode_wal_with_backend() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let (backend, _frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
let mode = pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
assert_eq!(
mode,
JournalMode::Wal,
"bead_id={BEAD_ID} case=wal_mode_set"
);
assert_eq!(
pager.journal_mode(),
JournalMode::Wal,
"bead_id={BEAD_ID} case=wal_mode_persisted"
);
});
}
#[test]
fn test_wal_journal_mode_persists_across_reopen() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_journal_mode_reopen.db");
let cx = Cx::new();
{
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, _frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
}
let reopened = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
assert_eq!(
reopened.journal_mode(),
JournalMode::Wal,
"bead_id={BEAD_ID} case=wal_mode_reopen"
);
});
}
#[test]
fn test_set_journal_mode_updates_header_version_bytes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
// Verify default state: bytes 18-19 should be 1 (rollback journal).
{
let inner = pager.inner.lock().unwrap();
let mut page1 = vec![0u8; inner.page_size.as_usize()];
let db_file = shared_db_file_read(&inner.db_file, &cx).await.unwrap();
let n = db_file.read(&cx, &mut page1, 0).await.unwrap();
assert!(n >= DATABASE_HEADER_SIZE);
assert_eq!(page1[18], 1, "bead_id={BEAD_ID} case=default_write_version");
assert_eq!(page1[19], 1, "bead_id={BEAD_ID} case=default_read_version");
}
// Switch to WAL mode: bytes 18-19 should become 2.
let (backend, _frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
let mut page1 = vec![0u8; inner.page_size.as_usize()];
let db_file = shared_db_file_read(&inner.db_file, &cx).await.unwrap();
let n = db_file.read(&cx, &mut page1, 0).await.unwrap();
assert!(n >= DATABASE_HEADER_SIZE);
assert_eq!(page1[18], 2, "bead_id={BEAD_ID} case=wal_write_version");
assert_eq!(page1[19], 2, "bead_id={BEAD_ID} case=wal_read_version");
}
// Switch back to DELETE mode: bytes 18-19 should revert to 1.
pager
.set_journal_mode(&cx, JournalMode::Delete)
.await
.unwrap();
{
let inner = pager.inner.lock().unwrap();
let mut page1 = vec![0u8; inner.page_size.as_usize()];
let db_file = shared_db_file_read(&inner.db_file, &cx).await.unwrap();
let n = db_file.read(&cx, &mut page1, 0).await.unwrap();
assert!(n >= DATABASE_HEADER_SIZE);
assert_eq!(page1[18], 1, "bead_id={BEAD_ID} case=delete_write_version");
assert_eq!(page1[19], 1, "bead_id={BEAD_ID} case=delete_read_version");
}
});
}
#[test]
fn test_set_journal_mode_blocked_during_write() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let (backend, _frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
let result = pager.set_journal_mode(&cx, JournalMode::Wal).await;
assert!(
result.is_err(),
"bead_id={BEAD_ID} case=mode_switch_blocked_during_write"
);
});
}
#[test]
fn test_set_journal_mode_same_mode_succeeds_during_reader() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let _reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let mode = pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
assert_eq!(mode, JournalMode::Wal);
});
}
/// Fault VFS for checkpoint-writer failure-atomicity tests (GH #194/#195):
/// wraps `MemoryVfs` and fails the NEXT `write` or `sync` on the main DB
/// file when armed. Everything else delegates.
#[derive(Clone)]
struct CheckpointFaultVfs {
inner: MemoryVfs,
fail_next_write: Arc<AtomicBool>,
fail_next_sync: Arc<AtomicBool>,
/// Encoded durability request last observed on the main-DB file
/// (0 = none yet; see `encode_sync_request`). Used by the GH #198
/// regression to prove the checkpoint fence asks for `FullDurable`.
last_db_sync_request: Arc<std::sync::atomic::AtomicU8>,
}
/// Encoding for `last_db_sync_request`: plain `sync` calls record
/// `0x10 | flags`, `durable_sync` calls record `0x20 | kind`.
const SYNC_REQUEST_PLAIN: u8 = 0x10;
const SYNC_REQUEST_DURABLE: u8 = 0x20;
fn encode_durable_sync_kind(kind: SyncKind) -> u8 {
SYNC_REQUEST_DURABLE
| match kind {
SyncKind::DataOnly => 1,
SyncKind::DataAndMetadata => 2,
SyncKind::FullDurable => 3,
}
}
impl CheckpointFaultVfs {
fn new() -> Self {
Self {
inner: MemoryVfs::new(),
fail_next_write: Arc::new(AtomicBool::new(false)),
fail_next_sync: Arc::new(AtomicBool::new(false)),
last_db_sync_request: Arc::new(std::sync::atomic::AtomicU8::new(0)),
}
}
}
struct CheckpointFaultFile {
inner: MemoryFile,
is_main_db: bool,
fail_next_write: Arc<AtomicBool>,
fail_next_sync: Arc<AtomicBool>,
last_db_sync_request: Arc<std::sync::atomic::AtomicU8>,
}
impl Vfs for CheckpointFaultVfs {
type File = CheckpointFaultFile;
fn name(&self) -> &'static str {
"checkpoint-fault"
}
fn open(
&self,
cx: &Cx,
path: Option<&Path>,
flags: VfsOpenFlags,
) -> Result<(Self::File, VfsOpenFlags)> {
let (inner, actual_flags) = self.inner.open(cx, path, flags)?;
Ok((
CheckpointFaultFile {
inner,
is_main_db: flags.contains(VfsOpenFlags::MAIN_DB),
fail_next_write: Arc::clone(&self.fail_next_write),
fail_next_sync: Arc::clone(&self.fail_next_sync),
last_db_sync_request: Arc::clone(&self.last_db_sync_request),
},
actual_flags,
))
}
fn delete(&self, cx: &Cx, path: &Path, sync_dir: bool) -> Result<()> {
self.inner.delete(cx, path, sync_dir)
}
fn access(&self, cx: &Cx, path: &Path, flags: AccessFlags) -> Result<bool> {
self.inner.access(cx, path, flags)
}
fn full_pathname(&self, cx: &Cx, path: &Path) -> Result<PathBuf> {
self.inner.full_pathname(cx, path)
}
fn is_memory(&self) -> bool {
true
}
}
impl VfsFile for CheckpointFaultFile {
fn close(&mut self, cx: &Cx) -> Result<()> {
self.inner.close(cx)
}
fn read<'a>(
&'a self,
cx: &'a Cx,
buf: &'a mut [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<usize>> + Send + 'a {
self.inner.read(cx, buf, offset)
}
fn write<'a>(
&'a self,
cx: &'a Cx,
buf: &'a [u8],
offset: u64,
) -> impl std::future::Future<Output = Result<()>> + Send + 'a {
async move {
if self.is_main_db && self.fail_next_write.swap(false, AtomicOrdering::AcqRel) {
return Err(FrankenError::internal(
"injected checkpoint db write failure",
));
}
self.inner.write(cx, buf, offset).await
}
}
fn truncate(&mut self, cx: &Cx, size: u64) -> Result<()> {
self.inner.truncate(cx, size)
}
fn sync(&mut self, cx: &Cx, flags: SyncFlags) -> Result<()> {
if self.is_main_db {
self.last_db_sync_request
.store(SYNC_REQUEST_PLAIN | flags.bits(), AtomicOrdering::Release);
if self.fail_next_sync.swap(false, AtomicOrdering::AcqRel) {
return Err(FrankenError::internal(
"injected checkpoint db sync failure",
));
}
}
self.inner.sync(cx, flags)
}
fn durable_sync(&mut self, cx: &Cx, kind: SyncKind) -> Result<()> {
if self.is_main_db {
self.last_db_sync_request
.store(encode_durable_sync_kind(kind), AtomicOrdering::Release);
if self.fail_next_sync.swap(false, AtomicOrdering::AcqRel) {
return Err(FrankenError::internal(
"injected checkpoint db sync failure",
));
}
}
self.inner.durable_sync(cx, kind)
}
fn file_size(&self, cx: &Cx) -> Result<u64> {
self.inner.file_size(cx)
}
fn lock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
self.inner.lock(cx, level)
}
fn unlock(&mut self, cx: &Cx, level: LockLevel) -> Result<()> {
self.inner.unlock(cx, level)
}
fn lock_external_shared_snapshot(&mut self, cx: &Cx) -> Result<()> {
self.inner.lock_external_shared_snapshot(cx)
}
fn restore_external_shared_snapshot_attempt(&mut self, cx: &Cx) -> Result<()> {
self.inner.restore_external_shared_snapshot_attempt(cx)
}
fn lock_external_maintenance(&mut self, cx: &Cx, wal_mode: bool) -> Result<()> {
self.inner.lock_external_maintenance(cx, wal_mode)
}
fn restore_external_maintenance_attempt(&mut self, cx: &Cx) -> Result<()> {
self.inner.restore_external_maintenance_attempt(cx)
}
fn check_reserved_lock(&self, cx: &Cx) -> Result<bool> {
self.inner.check_reserved_lock(cx)
}
fn shm_map(
&mut self,
cx: &Cx,
region: u32,
size: u32,
extend: bool,
) -> Result<fsqlite_vfs::ShmRegion> {
self.inner.shm_map(cx, region, size, extend)
}
fn shm_lock(&mut self, cx: &Cx, offset: u32, n: u32, flags: u32) -> Result<()> {
self.inner.shm_lock(cx, offset, n, flags)
}
fn shm_barrier(&self) {
self.inner.shm_barrier();
}
fn shm_unmap(&mut self, cx: &Cx, delete: bool) -> Result<()> {
self.inner.shm_unmap(cx, delete)
}
}
/// GH #194: a failed checkpoint page write must not leave `db_size`
/// advanced. Pre-fix, `write_page` bumped `inner.db_size` BEFORE the
/// fallible file write, so an I/O error left an inflated size behind for
/// the checkpoint error path to publish and for a later `sync()` to
/// stamp into the page-1 header as a page count past the true file end.
#[test]
fn test_checkpoint_write_failure_does_not_advance_db_size() {
asupersync::test_utils::run_test(|| async {
use crate::traits::CheckpointPageWriter as _;
let cx = Cx::new();
let vfs = CheckpointFaultVfs::new();
let fail_write = Arc::clone(&vfs.fail_next_write);
let pager = SimplePager::open(vfs, Path::new("/ckpt-fault-194.db"), PageSize::DEFAULT)
.await
.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let mut writer = pager.checkpoint_writer();
// Baseline: successful checkpoint writes advance db_size normally.
let data = vec![0u8; page_size];
writer
.write_page(&cx, PageNumber::new(2).unwrap(), &data)
.await
.unwrap();
let baseline = pager.inner.lock().unwrap().db_size;
assert!(baseline >= 2, "baseline write must advance db_size");
// Injected write failure for a page far past the current end must
// leave db_size exactly where it was.
fail_write.store(true, AtomicOrdering::Release);
let err = writer
.write_page(&cx, PageNumber::new(50).unwrap(), &data)
.await;
assert!(err.is_err(), "injected write failure must surface");
assert_eq!(
pager.inner.lock().unwrap().db_size,
baseline,
"failed checkpoint write must not advance db_size (GH #194)"
);
// After the fault clears, the same write succeeds and db_size moves.
writer
.write_page(&cx, PageNumber::new(50).unwrap(), &data)
.await
.unwrap();
assert_eq!(pager.inner.lock().unwrap().db_size, 50);
});
}
/// GH #195: `sync()` must not publish checkpoint state to the shared
/// pager plane before the durability barrier succeeds. Pre-fix, the
/// publish preceded `db_file.sync`, so a sync failure left published
/// metadata advertising checkpoint state whose writes had no barrier.
#[test]
fn test_checkpoint_sync_failure_publishes_nothing() {
asupersync::test_utils::run_test(|| async {
use crate::traits::CheckpointPageWriter as _;
let cx = Cx::new();
let vfs = CheckpointFaultVfs::new();
let fail_sync = Arc::clone(&vfs.fail_next_sync);
let pager = SimplePager::open(vfs, Path::new("/ckpt-fault-195.db"), PageSize::DEFAULT)
.await
.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let mut writer = pager.checkpoint_writer();
// Complete one full checkpoint write+sync so page 1 exists on disk
// and the published plane holds a coherent baseline.
let mut page1 = vec![0u8; page_size];
page1[..16].copy_from_slice(b"SQLite format 3\0");
writer
.write_page(&cx, PageNumber::ONE, &page1)
.await
.unwrap();
writer.sync(&cx).await.unwrap();
let baseline = pager.published_snapshot();
// Grow the database, then fail the durability barrier: the published
// plane must still show the baseline db_size, not the new one.
let data = vec![0u8; page_size];
writer
.write_page(&cx, PageNumber::new(30).unwrap(), &data)
.await
.unwrap();
fail_sync.store(true, AtomicOrdering::Release);
assert!(
writer.sync(&cx).await.is_err(),
"injected sync failure must surface"
);
let after_failure = pager.published_snapshot();
assert_eq!(
after_failure.db_size, baseline.db_size,
"publication must not precede the sync barrier (GH #195)"
);
// Once the barrier succeeds, the new state publishes.
writer.sync(&cx).await.unwrap();
assert_eq!(pager.published_snapshot().db_size, 30);
});
}
/// GH #198: the checkpoint writer's durability barrier is the recovery
/// fence executed before WAL invalidation, so the request that crosses
/// the VFS boundary must be `durable_sync(FullDurable)` — the only entry
/// point that reaches the strongest platform barrier (F_FULLFSYNC on
/// macOS) — not the relaxed `sync(NORMAL)` mode and not plain
/// `sync(FULL)`, which stops at fsync.
#[test]
fn test_checkpoint_writer_sync_uses_full_durability_barrier() {
asupersync::test_utils::run_test(|| async {
use crate::traits::CheckpointPageWriter as _;
let cx = Cx::new();
let vfs = CheckpointFaultVfs::new();
let observed_request = Arc::clone(&vfs.last_db_sync_request);
let pager =
SimplePager::open(vfs, Path::new("/ckpt-sync-flag-198.db"), PageSize::DEFAULT)
.await
.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let mut writer = pager.checkpoint_writer();
let mut page1 = vec![0u8; page_size];
page1[..16].copy_from_slice(b"SQLite format 3\0");
writer
.write_page(&cx, PageNumber::ONE, &page1)
.await
.unwrap();
writer.sync(&cx).await.unwrap();
let request = observed_request.load(AtomicOrdering::Acquire);
assert_eq!(
request,
encode_durable_sync_kind(SyncKind::FullDurable),
"checkpoint durability fence must cross the VFS as \
durable_sync(FullDurable) (GH #198), got encoded request {request:#04x}"
);
});
}
#[test]
fn test_checkpoint_busy_with_active_reader() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let err = pager
.checkpoint(&cx, crate::traits::CheckpointMode::Passive)
.await
.expect_err("checkpoint should be blocked by active reader");
assert!(matches!(err, FrankenError::Busy));
drop(reader);
// After reader ends, checkpoint should proceed.
let result = pager
.checkpoint(&cx, crate::traits::CheckpointMode::Passive)
.await
.expect("checkpoint should succeed after reader closes");
assert_eq!(result.total_frames, 0);
});
}
#[test]
fn test_checkpoint_busy_with_active_writer() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let _writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let err = pager
.checkpoint(&cx, crate::traits::CheckpointMode::Passive)
.await
.expect_err("checkpoint should be blocked by active writer");
assert!(matches!(err, FrankenError::Busy));
});
}
#[test]
fn test_checkpoint_coordination_blocks_new_writer_when_checkpoint_active() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
{
let mut inner = pager
.inner
.lock()
.expect("checkpoint coordination test lock poisoned");
inner.checkpoint_active = true;
}
let err = match pager.begin(&cx, TransactionMode::Concurrent).await {
Ok(_) => panic!("checkpoint-active pager should reject foreground writer begin"),
Err(err) => err,
};
assert!(
matches!(err, FrankenError::Busy),
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_active_blocks_writer_begin error={err}"
);
});
}
#[test]
fn test_checkpoint_coordination_reports_idle_queue_on_success() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let result = pager
.checkpoint(&cx, crate::traits::CheckpointMode::Passive)
.await
.expect("idle checkpoint should succeed");
let queue_snapshot =
checkpoint_coordination_queue_snapshot(&pager_group_commit_queue(&pager));
assert_eq!(
queue_snapshot.queue_phase, "filling",
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_success_queue_phase"
);
assert_eq!(
queue_snapshot.pending_batch_count, 0,
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_success_queue_pending_count"
);
assert_eq!(
result.total_frames, 0,
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_success_total_frames"
);
});
}
#[test]
fn test_checkpoint_empty_wal_fast_path_does_not_extract_backend() {
// The zero-frames checkpoint fast path must:
// 1. Return a zero-work `CheckpointResult` that preserves the
// caller-requested mode (so PRAGMA wal_checkpoint(TRUNCATE)
// does not surface as Passive in a no-op case).
// 2. NOT take the WAL backend out of the SharedWalBackend
// RwLock — otherwise peer readers / commits serialize
// behind a gratuitous `write()` lock for no real work.
//
// This test verifies both: we install a sentinel backend,
// invoke `pager.checkpoint` on a freshly-opened (empty) WAL,
// and assert that the sentinel is still reachable via the
// read-only helper immediately after. A concurrent
// `with_wal_backend_read` probe racing against the checkpoint
// would block if the fast path still extracted the backend.
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _frames) = wal_pager().await;
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(1)));
let cx = Cx::new();
for mode in [
crate::traits::CheckpointMode::Passive,
crate::traits::CheckpointMode::Full,
crate::traits::CheckpointMode::Restart,
crate::traits::CheckpointMode::Truncate,
] {
let result = pager
.checkpoint(&cx, mode)
.await
.expect("empty-WAL checkpoint should succeed as a no-op");
assert_eq!(result.total_frames, 0);
assert_eq!(result.frames_backfilled, 0);
assert!(result.completed);
assert!(!result.wal_was_reset);
assert_eq!(result.requested_mode, mode);
assert_eq!(result.effective_mode, mode);
}
// The fast path must leave the SharedWalBackend occupied — a
// `with_wal_backend_read` probe immediately after must still
// see the backend mounted.
let frame_count_after = with_wal_backend_read(&pager.wal_backend, &cx, |wal, _| {
Box::pin(async move { Ok(wal.frame_count()) })
})
.await
.expect("SharedWalBackend must still hold the installed backend after fast-path");
assert_eq!(
frame_count_after, 0,
"fast path must not disturb the installed WAL backend state"
);
});
}
#[test]
fn test_checkpoint_coordination_preserves_foreground_isolation_on_checkpoint_error() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
pager
.set_wal_backend(Box::new(FailingCheckpointWalBackend))
.expect("install failing checkpoint wal backend");
let err = pager
.checkpoint(&cx, crate::traits::CheckpointMode::Passive)
.await
.expect_err("failing checkpoint backend should surface error");
assert!(
matches!(err, FrankenError::BusyRecovery),
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_error_surfaces_backend_failure error={err}"
);
let queue_snapshot =
checkpoint_coordination_queue_snapshot(&pager_group_commit_queue(&pager));
assert_eq!(
queue_snapshot.pending_batch_count, 0,
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_error_keeps_foreground_queue_idle"
);
let metadata = pager.published_snapshot();
assert!(
!metadata.checkpoint_active,
"bead_id={CHECKPOINT_DECOUPLING_BEAD_ID} case=checkpoint_error_releases_checkpoint_flag"
);
});
}
#[test]
fn test_checkpoint_writer_sync_repairs_page1_header_after_late_growth() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0xCD; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
// Snapshot current page 1 as a realistic checkpoint payload.
let mut page1_data = {
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec()
};
let header: [u8; DATABASE_HEADER_SIZE] = page1_data[..DATABASE_HEADER_SIZE]
.try_into()
.expect("page 1 header must be present");
let expected_change_counter = DatabaseHeader::from_bytes(&header)
.expect("header must parse")
.change_counter;
page1_data[24..28].copy_from_slice(&0_u32.to_be_bytes());
page1_data[92..96].copy_from_slice(&0_u32.to_be_bytes());
let mut writer = pager.checkpoint_writer();
// Simulate checkpoint replay order where page 1 arrives first, then a
// higher page extends the DB. Without final header repair this can
// leave header page_count stale.
crate::traits::CheckpointPageWriter::write_page(
&mut writer,
&cx,
PageNumber::ONE,
&page1_data,
)
.await
.unwrap();
let page_three = PageNumber::new(3).unwrap();
crate::traits::CheckpointPageWriter::write_page(
&mut writer,
&cx,
page_three,
&vec![0xAB; ps],
)
.await
.unwrap();
crate::traits::CheckpointPageWriter::sync(&mut writer, &cx)
.await
.unwrap();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw_page1 = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
let header: [u8; DATABASE_HEADER_SIZE] = raw_page1[..DATABASE_HEADER_SIZE]
.try_into()
.expect("page 1 header must be present");
let parsed = DatabaseHeader::from_bytes(&header).expect("header must parse");
assert_eq!(
parsed.page_count,
page_three.get(),
"bead_id={BEAD_ID} case=checkpoint_sync_repairs_page_count"
);
assert_eq!(
parsed.change_counter, expected_change_counter,
"bead_id={BEAD_ID} case=checkpoint_sync_repairs_change_counter"
);
assert_eq!(
parsed.version_valid_for, parsed.change_counter,
"bead_id={BEAD_ID} case=checkpoint_sync_repairs_version_valid_for"
);
});
}
#[test]
fn test_wal_commit_appends_frames() {
asupersync::test_utils::run_test(|| async {
let (pager, frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
let data = vec![0xAA_u8; ps];
txn.write_page(&cx, p1, &data).await.unwrap();
txn.commit(&cx).await.unwrap();
let locked_frames = frames.lock().unwrap();
assert_eq!(
locked_frames.len(),
2,
"bead_id={BEAD_ID} case=wal_two_frames_appended_including_header"
);
let p1_frame = locked_frames.iter().find(|f| f.0 == p1.get()).unwrap();
assert_eq!(p1_frame.1[0], 0xAA, "bead_id={BEAD_ID} case=wal_frame_data");
// Commit frame should have db_size > 0.
let commit_count = locked_frames.iter().filter(|f| f.2 > 0).count();
assert_eq!(commit_count, 1, "bead_id={BEAD_ID} case=wal_commit_marker");
drop(locked_frames);
});
}
#[test]
fn test_wal_commit_multi_page() {
asupersync::test_utils::run_test(|| async {
let (pager, frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.write_page(&cx, p2, &vec![0x22; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let locked_frames = frames.lock().unwrap();
assert_eq!(
locked_frames.len(),
3,
"bead_id={BEAD_ID} case=wal_multi_page_count"
);
// Exactly one frame should be the commit frame (db_size > 0).
let commit_count = locked_frames.iter().filter(|f| f.2 > 0).count();
drop(locked_frames);
assert_eq!(
commit_count, 1,
"bead_id={BEAD_ID} case=wal_exactly_one_commit_marker"
);
});
}
#[test]
fn test_wal_commit_uses_single_batch_append() {
asupersync::test_utils::run_test(|| async {
let (pager, frames, _begin_calls, batch_calls) = wal_pager_with_tracking().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let new_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, PageNumber::ONE, &vec![0x11; ps])
.await
.unwrap();
txn.write_page(&cx, new_page, &vec![0x22; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
assert_eq!(
*batch_calls.lock().unwrap(),
1,
"bead_id={BEAD_ID} case=wal_batch_append_single_call"
);
let frames = frames.lock().unwrap();
assert_eq!(
frames.len(),
2,
"bead_id={BEAD_ID} case=wal_batch_append_frame_count"
);
assert_eq!(
frames[0].0,
PageNumber::ONE.get(),
"bead_id={BEAD_ID} case=wal_batch_append_sorted_page1"
);
assert_eq!(
frames[1].0,
new_page.get(),
"bead_id={BEAD_ID} case=wal_batch_append_sorted_new_page"
);
assert_eq!(
frames[0].2, 0,
"bead_id={BEAD_ID} case=wal_batch_append_non_commit_first"
);
assert_eq!(
frames[1].2,
new_page.get(),
"bead_id={BEAD_ID} case=wal_batch_append_commit_marker_last"
);
});
}
#[test]
fn test_wal_preparation_happens_before_reserved_publish_lock() {
asupersync::test_utils::run_test(|| async {
let (pager, observed_lock_level) = observed_lock_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let (
backend,
frames,
append_frames_calls,
append_prepared_calls,
prepare_lock_levels,
append_lock_levels,
) = PreparedBatchObservedWalBackend::new(Arc::clone(&observed_lock_level));
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let new_page = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
PageNumber::ONE,
StagedPage::from_bytes(&pager.pool, &vec![0x11; ps]).unwrap(),
);
write_set.insert(
new_page,
StagedPage::from_bytes(&pager.pool, &vec![0x22; ps]).unwrap(),
);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(1),
..ParallelWalControlSurface::default()
},
));
SimpleTransaction::<ObservedLockVfs>::commit_wal_group_commit(
&cx,
&pager.wal_backend,
&pager.inner,
&write_set,
&[PageNumber::ONE, new_page],
&[],
&queue,
)
.await
.unwrap();
assert_eq!(
*append_frames_calls.lock().unwrap(),
0,
"bead_id=bd-db300.3.2 case=prepared_path_skips_fallback_append"
);
assert_eq!(
*append_prepared_calls.lock().unwrap(),
1,
"bead_id=bd-db300.3.2 case=prepared_path_uses_prepared_append"
);
assert_eq!(
prepare_lock_levels.lock().unwrap().as_slice(),
&[LockLevel::None],
"bead_id=bd-db300.3.2 case=prepare_runs_before_reserved_publish"
);
assert_eq!(
append_lock_levels.lock().unwrap().as_slice(),
&[LockLevel::Reserved],
"bead_id=bd-db300.3.2 case=prepared_append_runs_inside_reserved_publish"
);
let frames = frames.lock().unwrap();
assert_eq!(
frames.len(),
2,
"bead_id=bd-db300.3.2 case=prepared_path_preserves_frame_count"
);
assert_eq!(
frames[0].0,
PageNumber::ONE.get(),
"bead_id=bd-db300.3.2 case=prepared_path_preserves_sorted_page_one"
);
assert_eq!(
frames[1].0,
new_page.get(),
"bead_id=bd-db300.3.2 case=prepared_path_preserves_sorted_commit_frame"
);
});
}
#[test]
fn test_wal_commit_sync_policy_deferred_skips_sync() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_deferred_sync_policy.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, _begin_calls, _batch_calls, sync_calls) =
MockWalBackend::new_with_sync_tracking();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::Deferred)
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x55; PageSize::DEFAULT.as_usize()])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
assert_eq!(
*sync_calls.lock().unwrap(),
0,
"bead_id={BEAD_ID} case=wal_sync_policy_deferred_skips_commit_sync"
);
assert_eq!(
frames.lock().unwrap().len(),
2,
"bead_id={BEAD_ID} case=wal_sync_policy_deferred_still_appends_wal_frames"
);
});
}
#[test]
fn test_wal_commit_sync_policy_per_commit_syncs() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_per_commit_sync_policy.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, _begin_calls, _batch_calls, sync_calls) =
MockWalBackend::new_with_sync_tracking();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::PerCommit)
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x66; PageSize::DEFAULT.as_usize()])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
assert_eq!(
*sync_calls.lock().unwrap(),
1,
"bead_id={BEAD_ID} case=wal_sync_policy_per_commit_runs_commit_sync"
);
assert_eq!(
frames.lock().unwrap().len(),
2,
"bead_id={BEAD_ID} case=wal_sync_policy_per_commit_appends_wal_frames"
);
});
}
#[test]
fn test_group_commit_in_doubt_failure_fails_waiter_closed() {
asupersync::test_utils::run_test(|| async {
for attempt in 0..32 {
let vfs = MemoryVfs::new();
let path = PathBuf::from(format!("/wal_group_commit_failure_waiter_{attempt}.db"));
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, append_frames_calls) = FailingGroupCommitWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::Deferred)
.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig {
max_group_delay: Duration::from_millis(10),
max_group_delay_ceiling: Duration::from_millis(10),
..GroupCommitConfig::default()
}));
let pool = pager.pool.clone();
let start = StdArc::new(std::sync::Barrier::new(3));
let spawn_commit = |page_number: u32, fill: u8| {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
let start = StdArc::clone(&start);
std::thread::spawn(move || {
let mut outcome = None;
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let page_no = PageNumber::new(page_number).unwrap();
let page = StagedPage::from_bytes(&pool, &vec![fill; pool.page_size()])
.unwrap();
let mut write_set = HashMap::new();
write_set.insert(page_no, page);
let write_pages_sorted = vec![page_no];
start.wait();
outcome = Some(
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&write_pages_sorted,
&[],
&queue,
)
.await,
);
});
outcome.expect("group-commit thread must record an outcome")
})
};
let writer_a = spawn_commit(2, 0x11);
let writer_b = spawn_commit(3, 0x22);
start.wait();
let result_a = writer_a.join().unwrap();
let result_b = writer_b.join().unwrap();
let append_call_count = *append_frames_calls.lock().unwrap();
if append_call_count != 1 {
continue;
}
let error_a = result_a.expect_err("flusher should observe append failure");
let error_b = result_b.expect_err("waiter should fail closed");
let saw_fail_closed_recovery = matches!(
&error_a,
FrankenError::BusyRecovery | FrankenError::Unsupported
) || matches!(
&error_b,
FrankenError::BusyRecovery | FrankenError::Unsupported
);
let error_a = error_a.to_string();
let error_b = error_b.to_string();
assert!(
error_a.contains("forced batched group commit append failure")
|| error_b.contains("forced batched group commit append failure"),
"bead_id={BEAD_ID} case=group_commit_flusher_reports_backend_failure error_a={error_a} error_b={error_b}"
);
assert!(
saw_fail_closed_recovery,
"bead_id={BEAD_ID} case=group_commit_waiter_fails_closed error_a={error_a} error_b={error_b}"
);
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let flush_epoch = consolidator.epoch();
assert_eq!(consolidator.phase(), ConsolidationPhase::Flushing);
drop(consolidator);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"an awaited in-doubt failure must not publish Abort"
);
return;
}
panic!(
"bead_id={BEAD_ID} case=group_commit_in_doubt_failure_fails_waiter_closed could not coalesce flusher+waiter in allotted attempts"
);
});
}
#[test]
fn test_group_commit_awaited_error_separates_physical_and_logical_unlocks() {
asupersync::test_utils::run_test(|| async {
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let observed_unlock_trace_ids = vfs.observed_unlock_trace_ids();
let path = PathBuf::from("/wal_group_commit_awaited_error_separate_unlocks.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let (backend, frames, sync_calls, reconcile_calls) =
MockWalBackend::new_with_failing_sync();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::PerCommit)
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
let committed_page = vec![0x77; PageSize::DEFAULT.as_usize()];
txn.write_page(&cx, page, &committed_page).await.unwrap();
observed_unlock_trace_ids.lock().unwrap().clear();
let error = txn
.commit(&cx)
.await
.expect_err("post-append sync failure must surface its original error");
assert!(
error
.to_string()
.contains("forced group-commit sync failure after full WAL append"),
"awaited-error path must preserve the original callback error: {error}"
);
assert_eq!(
*sync_calls.lock().unwrap(),
1,
"the injected awaited error must occur at the first sync"
);
let frames = frames.lock().unwrap();
assert!(
frames.last().is_some_and(|frame| frame.2 != 0),
"the failure fixture must contain a full WAL commit marker"
);
drop(frames);
let flush_epoch = {
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
consolidator.phase(),
ConsolidationPhase::Flushing,
"awaited in-doubt error must leave the epoch FLUSHING"
);
consolidator.epoch()
};
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"awaited in-doubt error must not publish Abort"
);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"database-lock obligation must transfer ownership to the queue"
);
assert!(
queue.has_unresolved_in_doubt_epoch(),
"queued awaited error must enter BusyRecovery"
);
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&flush_epoch),
Some(&2),
"physical recovery and logical Phase C must each retain an epoch consumer after caller error"
);
assert!(
observed_unlock_trace_ids.lock().unwrap().is_empty(),
"normal error handling may not call unlock before reconciliation"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"awaited in-doubt error must retain RESERVED"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"the stranded physical restoration must own one process root while the transaction remains attached"
);
drop(txn);
assert!(
observed_unlock_trace_ids.lock().unwrap().is_empty(),
"transaction Drop must enqueue its rooted logical exit while physical recovery owns RESERVED"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"queued obligation must still own RESERVED after caller Drop"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"physical recovery and detached logical exit must own independent roots"
);
assert!(
queue.resolve_one_pending_external_unlock().await.unwrap(),
"durable reconciliation must claim the queued external lock"
);
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&flush_epoch),
Some(&1),
"physical reconciliation must leave the detached logical owner intact"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"physical recovery must restore only the flusher handle's captured RESERVED baseline"
);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
1,
"physical recovery must perform its own baseline restoration"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"terminal physical restoration must leave only the logical root"
);
settle_pending_group_commit_finalization(&queue)
.await
.expect("detached logical Phase C must settle its own transaction exit");
assert_eq!(
*reconcile_calls.lock().unwrap(),
1,
"the retained recovery object must reconcile the exact certificate and WAL interval"
);
assert_eq!(
*sync_calls.lock().unwrap(),
2,
"authorized synchronous recovery must re-establish the WAL durability fence"
);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
2,
"physical baseline restoration and logical transaction exit are distinct unlock transitions"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"recorded final target must release the last snapshot fence"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"terminal logical exit must release the final process root"
);
assert!(queue.is_epoch_complete(flush_epoch));
assert!(
queue.persisted_epoch_for(flush_epoch).is_none(),
"authorized recovery must reclaim the receipt after its final owner consumes it"
);
assert!(
!queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"authorized recovery must release its final epoch consumer"
);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"durable reconciliation must complete without publishing Abort"
);
assert!(!queue.has_unresolved_in_doubt_epoch());
let reader = pager
.begin(&cx, TransactionMode::ReadOnly)
.await
.expect("post-reconciliation begin must not require a BusyRecovery retry");
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_bytes(),
committed_page.as_slice(),
"authorized recovery must publish the retained page batch"
);
drop(reader);
let queue_weak = Arc::downgrade(&queue);
drop(pager);
drop(queue);
assert!(
queue_weak.upgrade().is_none(),
"terminal recovery must not retain the group-commit queue"
);
});
}
#[test]
fn test_group_commit_logical_owner_outlives_physical_recovery_and_128_epochs() {
asupersync::test_utils::run_test(|| async {
const BEAD: &str = "bd-vn2ea";
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/wal_group_commit_logical_owner_evidence.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let (backend, _frames, _sync_calls, _reconcile_calls) =
MockWalBackend::new_with_failing_sync();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::PerCommit)
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x78; PageSize::DEFAULT.as_usize()])
.await
.unwrap();
txn.commit(&cx)
.await
.expect_err("sync failure must transfer evidence ownership");
let flush_epoch = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.epoch();
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&flush_epoch),
Some(&2),
"bead_id={BEAD} case=physical_and_logical_owners_own_ambiguous_epoch"
);
assert!(
queue.resolve_one_pending_external_unlock().await.unwrap(),
"physical recovery must reach an authorized terminal verdict"
);
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&flush_epoch),
Some(&1),
"bead_id={BEAD} case=logical_owner_outlives_physical_recovery"
);
assert!(
queue.persisted_epoch_for(flush_epoch).is_some(),
"bead_id={BEAD} case=logical_owner_retains_recovered_certificate"
);
for epoch in (flush_epoch + 1)..=(flush_epoch + 129) {
queue.publish_completed_epoch(epoch, false);
}
assert!(
queue.persisted_epoch_for(flush_epoch).is_some(),
"bead_id={BEAD} case=logical_owner_evidence_outlives_128_later_epochs"
);
drop(txn);
assert!(
queue.persisted_epoch_for(flush_epoch).is_some(),
"bead_id={BEAD} case=drop_keeps_evidence_until_rooted_logical_cleanup"
);
assert_eq!(
queue.pending_logical_cleanup_count(),
1,
"bead_id={BEAD} case=drop_enqueues_exact_logical_cleanup"
);
settle_pending_group_commit_finalization(&queue)
.await
.expect("bead_id={BEAD} case=rooted_logical_cleanup_reaches_terminal");
assert!(
queue.persisted_epoch_for(flush_epoch).is_none(),
"bead_id={BEAD} case=logical_finalization_reclaims_recovered_certificate"
);
assert!(
!queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"bead_id={BEAD} case=logical_final_owner_releases_epoch_count"
);
});
}
#[test]
fn test_group_commit_nonterminal_recovery_is_rooted_until_terminal() {
asupersync::test_utils::run_test(|| async {
const BEAD: &str = "bd-6xjma";
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/wal_group_commit_nonterminal_recovery_rooted_queue.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let (backend, _frames, sync_calls, reconcile_calls) =
MockWalBackend::new_with_failing_sync();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::PerCommit)
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
queue.bind_finalization_path(&path);
let queue_weak = Arc::downgrade(&queue);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x7A; PageSize::DEFAULT.as_usize()])
.await
.unwrap();
let error = txn
.commit(&cx)
.await
.expect_err("sync failure must leave a nonterminal recovery owner");
assert!(
error
.to_string()
.contains("forced group-commit sync failure after full WAL append"),
"bead_id={BEAD} case=nonterminal_recovery_fixture error={error}"
);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"bead_id={BEAD} case=nonterminal_recovery_is_queued"
);
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"bead_id={BEAD} case=nonterminal_recovery_retains_epoch_consumer"
);
// Emulate the narrow TOCTOU in which a replacement backend is
// installed after the old backend accepted the certified interval
// but before process-root recovery reconciles it. Recovery must
// retain and address the exact accepting backend, never reacquire
// the pager's replaceable outer slot.
let (replacement, _, _, _) = MockWalBackend::new();
let replacement_reconcile_calls = Arc::clone(&replacement.reconcile_calls);
*pager
.wal_backend
.write()
.unwrap_or_else(std::sync::PoisonError::into_inner) = Some(Arc::new(
AsyncRwLock::with_name("replacement_wal_backend", Box::new(replacement)),
));
drop(txn);
drop(pager);
drop(queue);
drop(
queue_weak.upgrade().expect(
"bead_id={BEAD} case=nonterminal_recovery_keeps_process_root_ownership",
),
);
settle_process_root_finalizations_for_path(&path)
.await
.expect("bead_id={BEAD} case=same_path_gate_reaches_terminal_resolution");
assert_eq!(
*reconcile_calls.lock().unwrap(),
1,
"bead_id={BEAD} case=rooted_recovery_reconciles_exact_wal_interval"
);
assert_eq!(
*replacement_reconcile_calls.lock().unwrap(),
0,
"bead_id={BEAD} case=recovery_never_reconciles_a_replacement_backend"
);
assert_eq!(
*sync_calls.lock().unwrap(),
2,
"bead_id={BEAD} case=rooted_recovery_reestablishes_durability_fence"
);
assert!(
queue_weak.upgrade().is_none(),
"bead_id={BEAD} case=terminal_recovery_releases_process_root_without_cycle"
);
});
}
#[test]
fn test_dropped_pending_accepted_wal_commit_keeps_source_and_process_ownership() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("pending accepted WAL test runtime should build");
runtime.block_on(async {
const BEAD: &str = "bd-6xjma";
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let path = PathBuf::from("/wal-pending-accepted-drop-ownership.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let (backend, frames, append_entered, source_completion) =
PendingAcceptedWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::Deferred)
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
let committed_page = vec![0x7B; PageSize::DEFAULT.as_usize()];
txn.write_page(&cx, page, &committed_page).await.unwrap();
let mut commit = Box::pin(txn.commit(&cx));
std::future::poll_fn(|poll_cx| match commit.as_mut().poll(poll_cx) {
std::task::Poll::Pending if append_entered.load(AtomicOrdering::Acquire) => {
std::task::Poll::Ready(())
}
std::task::Poll::Pending => {
poll_cx.waker().wake_by_ref();
std::task::Poll::Pending
}
std::task::Poll::Ready(result) => {
panic!("pending WAL commit unexpectedly completed: {result:?}")
}
})
.await;
let accepted_completion = source_completion
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.as_ref()
.cloned()
.expect("accepted WAL append must publish its source completion token");
assert_eq!(
accepted_completion.state(),
VfsWriteCompletionState::Pending,
"bead_id={BEAD} case=accepted_write_is_genuinely_pending"
);
assert!(
frames
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.last()
.is_some_and(|frame| frame.2 != 0),
"bead_id={BEAD} case=accepted_source_copied_complete_wal_interval"
);
drop(commit);
assert_eq!(
accepted_completion.state(),
VfsWriteCompletionState::Pending,
"bead_id={BEAD} case=future_drop_does_not_forge_source_terminal_state"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"bead_id={BEAD} case=pending_source_retains_reserved_fence"
);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"bead_id={BEAD} case=dropped_commit_transfers_physical_owner"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"bead_id={BEAD} case=pending_physical_owner_has_process_root"
);
drop(txn);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"bead_id={BEAD} case=dropped_transaction_adds_independent_logical_root"
);
assert!(matches!(
settle_pending_group_commit_finalization(&queue).await,
Err(FrankenError::BusyRecovery)
));
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"bead_id={BEAD} case=pending_source_fails_closed_during_retry"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"bead_id={BEAD} case=nonterminal_retry_preserves_both_roots"
);
assert!(
accepted_completion.complete_success(),
"bead_id={BEAD} case=source_reports_its_own_terminal_success"
);
settle_pending_group_commit_finalization(&queue)
.await
.expect("terminal source must permit exact recovery and logical cleanup");
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"bead_id={BEAD} case=terminal_recovery_restores_final_lock_baseline"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"bead_id={BEAD} case=terminal_recovery_releases_all_process_roots"
);
assert!(!queue.has_unresolved_in_doubt_epoch());
let reader = pager
.begin(&cx, TransactionMode::ReadOnly)
.await
.expect("terminal recovery must reopen admission");
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_bytes(),
committed_page.as_slice(),
"bead_id={BEAD} case=terminal_recovery_publishes_accepted_wal_batch"
);
});
}
#[test]
fn test_group_commit_prewrite_error_separates_physical_and_logical_unlocks() {
asupersync::test_utils::run_test(|| async {
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let observed_unlock_trace_ids = vfs.observed_unlock_trace_ids();
let path = PathBuf::from("/wal_group_commit_prewrite_error_reconciliation.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let (backend, frames, sync_calls, reconcile_calls) =
MockWalBackend::new_with_failing_append_before_write();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::PerCommit)
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x55; PageSize::DEFAULT.as_usize()])
.await
.unwrap();
observed_unlock_trace_ids.lock().unwrap().clear();
let error = txn
.commit(&cx)
.await
.expect_err("pre-write WAL append failure must surface");
assert!(
error
.to_string()
.contains("forced group-commit append failure before WAL write"),
"pre-write reconciliation fixture must preserve its source error: {error}"
);
assert!(
frames.lock().unwrap().is_empty(),
"pre-write failure must leave the certified WAL interval absent"
);
assert_eq!(
*sync_calls.lock().unwrap(),
0,
"an absent WAL interval must never reach the commit sync"
);
let flush_epoch = {
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(consolidator.phase(), ConsolidationPhase::Flushing);
consolidator.epoch()
};
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"pre-write callback error must retain RESERVED until exact reconciliation"
);
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&flush_epoch),
Some(&2),
"physical recovery and logical Phase C must each own the admitted epoch after the caller returns"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"NotCommitted reconciliation must remain process-rooted until its terminal verdict"
);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::Reserved);
assert!(observed_unlock_trace_ids.lock().unwrap().is_empty());
drop(txn);
assert!(
observed_unlock_trace_ids.lock().unwrap().is_empty(),
"transaction Drop must queue its exact logical exit behind physical recovery"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"NotCommitted physical recovery and detached logical exit must own independent roots"
);
assert!(
queue.resolve_one_pending_external_unlock().await.unwrap(),
"exact reconciliation must classify and release the absent interval"
);
assert_eq!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.get(&flush_epoch),
Some(&1),
"physical NotCommitted reconciliation must leave logical cleanup ownership intact"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"physical NotCommitted recovery must restore only the captured RESERVED baseline"
);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
1,
"physical NotCommitted recovery must perform its own baseline restoration"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"terminal physical NotCommitted recovery must leave only the logical root"
);
settle_pending_group_commit_finalization(&queue)
.await
.expect("detached NotCommitted logical cleanup must settle its own exit");
assert_eq!(
*reconcile_calls.lock().unwrap(),
1,
"the retained recovery object must reconcile the exact absent interval"
);
assert_eq!(
*sync_calls.lock().unwrap(),
0,
"NotCommitted recovery must not manufacture a durability fence"
);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
2,
"NotCommitted physical restoration and logical exit are distinct unlock transitions"
);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"terminal NotCommitted logical exit must release the final process root"
);
assert!(
queue.persisted_epoch_for(flush_epoch).is_none(),
"NotCommitted recovery must not publish a durability receipt"
);
assert!(!queue.is_epoch_complete(flush_epoch));
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"NotCommitted recovery must reclaim Abort after its final owner consumes it"
);
assert!(
!queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"NotCommitted recovery must release its final epoch consumer"
);
assert!(!queue.has_unresolved_in_doubt_epoch());
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"NotCommitted reconciliation must release its exact root only after unlock and epoch abort"
);
});
}
#[test]
fn test_group_commit_fault_hook_after_durability_completes_without_abort_and_records_context() {
asupersync::test_utils::run_test(|| async {
let _guard = FAULT_HOOK_TEST_GUARD
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for attempt in 0..32 {
crate::fault_hooks::clear();
let vfs = MemoryVfs::new();
let path = PathBuf::from(format!("/wal_group_commit_publish_hook_{attempt}.db"));
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, _frames, _begin_calls, batch_calls, sync_calls) =
MockWalBackend::new_with_sync_tracking();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
pager
.set_wal_commit_sync_policy(WalCommitSyncPolicy::PerCommit)
.unwrap();
crate::fault_hooks::arm_after_flush_before_publish(
crate::fault_hooks::FaultHookArm::new(
"bd-db300.7.2.2-after-flush-before-publish",
"GROUP-COMMIT-PUBLISH",
"group_commit_publish_recovery",
),
);
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let pool = pager.pool.clone();
let start = StdArc::new(std::sync::Barrier::new(3));
let fault_participant = _guard.participant();
let spawn_commit = |page_number: u32, fill: u8| {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
let start = StdArc::clone(&start);
std::thread::spawn(move || {
let _fault_participation = fault_participant.enter();
let mut outcome = None;
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let page_no = PageNumber::new(page_number).unwrap();
let page = StagedPage::from_bytes(&pool, &vec![fill; pool.page_size()])
.unwrap();
let mut write_set = HashMap::new();
write_set.insert(page_no, page);
let write_pages_sorted = vec![page_no];
start.wait();
outcome = Some(
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&write_pages_sorted,
&[],
&queue,
)
.await,
);
});
outcome.expect("group-commit thread must record an outcome")
})
};
let writer_a = spawn_commit(2, 0x11);
let writer_b = spawn_commit(3, 0x22);
start.wait();
let result_a = writer_a.join().unwrap();
let result_b = writer_b.join().unwrap();
if *batch_calls.lock().unwrap() != 1 {
continue;
}
assert_eq!(
*sync_calls.lock().unwrap(),
1,
"bead_id={BEAD_ID} case=group_commit_publish_hook_runs_after_real_sync"
);
let error = match (result_a, result_b) {
(Err(error), Ok(())) | (Ok(()), Err(error)) => error.to_string(),
(Err(error_a), Err(error_b)) => panic!(
"exactly the flusher should surface the post-durable local fault: \
error_a={error_a} error_b={error_b}"
),
(Ok(()), Ok(())) => {
panic!("the post-durable local fault must surface to its flusher")
}
};
assert!(
error.contains("fault_inject:after_flush_before_publish"),
"bead_id={BEAD_ID} case=group_commit_publish_hook_reports_primary_failure error={error}"
);
let completed_epoch = queue.completed_epoch.load(AtomicOrdering::Acquire);
assert!(
completed_epoch >= 1,
"bead_id={BEAD_ID} case=group_commit_post_durable_fault_completes_epoch completed_epoch={completed_epoch}"
);
assert!(
queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"bead_id={BEAD_ID} case=group_commit_post_durable_fault_must_not_publish_abort"
);
assert!(
queue
.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"bead_id={BEAD_ID} case=group_commit_post_durable_final_owner_reclaims_receipt"
);
assert_eq!(
queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.phase(),
ConsolidationPhase::Complete,
"post-durable publication fault must not strand the epoch in Flushing"
);
let records = crate::fault_hooks::take_records();
assert_eq!(records.len(), 1, "publish hook should record exactly once");
assert_eq!(records[0].point, "after_flush_before_publish");
assert_eq!(
records[0].run_id,
"bd-db300.7.2.2-after-flush-before-publish"
);
assert_eq!(records[0].scenario_id, "GROUP-COMMIT-PUBLISH");
assert_eq!(records[0].invariant_family, "group_commit_publish_recovery");
assert!(
records[0].detail.contains("batch_count=2"),
"record should capture coalesced batch context: {}",
records[0].detail
);
assert!(
records[0].detail.contains("frame_count=2"),
"record should capture coalesced frame context: {}",
records[0].detail
);
crate::fault_hooks::clear();
return;
}
panic!(
"bead_id={BEAD_ID} case=group_commit_publish_hook_wakes_waiters could not coalesce flusher+waiter in allotted attempts"
);
});
}
/// H11 / F11: Suppressed waiter notification — the flusher still publishes
/// the completed epoch and advances the keyed generation, but intentionally
/// omits direct delivery. The active async waiter must recover after its
/// bounded timeout by rechecking the eventcount and completed epoch.
///
/// Proof obligation: once the publisher reaches the completed-epoch seam,
/// the waiter recovers through its bounded timeout. The publisher is joined
/// before post-scenario assertions so no worker is detached. Arbitrary
/// synchronous publisher deadlocks require a separate subprocess keeper.
///
/// Replay: `cargo test -p fsqlite-pager --lib -- test_fault_drop_waiter_notify --nocapture`
#[test]
fn test_fault_drop_waiter_notify_recovers_via_timeout() {
asupersync::test_utils::run_test(|| async {
let _guard = FAULT_HOOK_TEST_GUARD
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
crate::fault_hooks::clear();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let target_epoch = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let first = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: vec![0x33; PageSize::DEFAULT.as_usize()],
db_size_if_commit: 2,
}]))
.unwrap();
let second = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 3,
page_data: vec![0x44; PageSize::DEFAULT.as_usize()],
db_size_if_commit: 3,
}]))
.unwrap();
assert_eq!(first.outcome, SubmitOutcome::Flusher);
assert_eq!(second.outcome, SubmitOutcome::Waiter);
assert_eq!(first.target_epoch, second.target_epoch);
let batches = consolidator.begin_flush().unwrap();
assert_eq!(
batches.len(),
2,
"bead_id=bd-db300.7.2.2 case=drop_notify_exact_epoch_membership"
);
assert_eq!(consolidator.phase(), ConsolidationPhase::Flushing);
first.target_epoch
};
let slot = queue.epoch_waiters.slot(target_epoch);
let observed_generation = slot.generation();
let rendezvous = Arc::new(KeyedWaitTestRendezvous::default());
slot.arm_async_wait_rendezvous(Arc::clone(&rendezvous));
let publisher_queue = Arc::clone(&queue);
let publisher_slot = Arc::clone(&slot);
let publisher_rendezvous = Arc::clone(&rendezvous);
let fault_participant = _guard.participant();
let publisher = std::thread::spawn(move || {
let publish_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
let _fault_participation = fault_participant.enter();
if !publisher_rendezvous.wait_until_entered(Duration::from_secs(2)) {
return Err(
"async waiter did not enter the keyed wait boundary before publication"
.to_owned(),
);
}
if publisher_slot.active_async_waiter_count() != 1 {
return Err(format!(
"keyed wait boundary had {} active waiters instead of exactly one",
publisher_slot.active_async_waiter_count()
));
}
let completion_error = {
let mut consolidator = publisher_queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
match consolidator.complete_flush() {
Ok(has_promoted)
if !has_promoted
&& consolidator.epoch() == target_epoch
&& consolidator.phase() == ConsolidationPhase::Complete =>
{
None
}
Ok(has_promoted) => Some(format!(
"unexpected completion state: promoted={has_promoted} epoch={} phase={:?}",
consolidator.epoch(),
consolidator.phase()
)),
Err(error) => Some(format!("complete_flush failed: {error}")),
}
};
if let Some(error) = completion_error {
return Err(error);
}
crate::fault_hooks::arm_drop_waiter_notify(
crate::fault_hooks::FaultHookArm::new(
"bd-db300.7.2.2-drop-waiter-notify",
"DROP-WAITER-NOTIFY",
"liveness_under_fault",
),
);
publisher_queue.publish_completed_epoch(target_epoch, false);
Ok(())
}));
// Release the test-only boundary on every normal, error, and
// caught-panic path. If publication failed, also publish a
// synthetic terminal state solely to let the scoped waiter
// return before the test reports the publisher failure.
let publish_result = match publish_result {
Ok(result) => result,
Err(_) => Err("publisher panicked while injecting dropped delivery".to_owned()),
};
if publish_result.is_err() {
publisher_queue
.completed_epoch
.store(target_epoch, AtomicOrdering::Release);
publisher_slot.signal();
}
publisher_rendezvous.release();
publish_result
});
let waiter_cx = Cx::new();
let waiter_result = asupersync::time::timeout(
asupersync::time::wall_now(),
Duration::from_secs(4),
queue.wait_for_epoch_outcome_async(&waiter_cx, target_epoch),
)
.await;
let publisher_result = publisher
.join()
.expect("dropped-delivery publisher thread must not panic outside its guard");
publisher_result
.expect("dropped-delivery publisher must complete the exact target epoch");
let outcome = waiter_result
.expect("async epoch waiter must finish within the outer liveness bound")
.expect("async epoch waiter must observe a terminal result");
assert!(
matches!(outcome, WaitForEpochOutcome::Completed),
"bead_id=bd-db300.7.2.2 case=drop_notify_waiter_observes_completion outcome={outcome:?}"
);
assert_eq!(
queue.completed_epoch.load(AtomicOrdering::Acquire),
target_epoch,
"completed epoch must remain published while delivery is suppressed"
);
assert_eq!(
slot.generation(),
observed_generation.wrapping_add(1),
"suppressed delivery must still advance the keyed eventcount"
);
assert_eq!(
slot.timeout_recovery_count(),
1,
"active keyed waiter must recover exactly once after timeout"
);
assert_eq!(
slot.active_async_waiter_count(),
0,
"completed keyed waiter must leave the async wait boundary"
);
let records = crate::fault_hooks::take_records();
assert_eq!(records.len(), 1, "drop-notify hook should fire once");
assert_eq!(records[0].point, "drop_waiter_notify");
assert_eq!(records[0].run_id, "bd-db300.7.2.2-drop-waiter-notify");
assert_eq!(records[0].scenario_id, "DROP-WAITER-NOTIFY");
assert_eq!(records[0].invariant_family, "liveness_under_fault");
assert_eq!(
records[0].detail,
format!(
"completed_epoch={target_epoch} wait_strategy=keyed_eventcount \
suppressed_delivery=keyed_notify"
),
"record must capture the exact suppressed publication"
);
crate::fault_hooks::clear();
});
}
#[test]
fn test_epoch_waiter_rechecks_identity_wide_root_after_wake() {
asupersync::test_utils::run_test(|| async {
struct SuspendedIdentityWideRestore {
entered: Arc<AtomicBool>,
released: Arc<AtomicBool>,
}
impl PendingExternalUnlockOperation for SuspendedIdentityWideRestore {
fn restore(&mut self) -> LocalPagerFuture<'_, ()> {
let entered = Arc::clone(&self.entered);
let released = Arc::clone(&self.released);
Box::pin(std::future::poll_fn(move |_| {
entered.store(true, AtomicOrdering::Release);
if released.load(AtomicOrdering::Acquire) {
std::task::Poll::Ready(Ok(()))
} else {
std::task::Poll::Pending
}
}))
}
fn try_restore(&mut self) -> Result<bool> {
Ok(false)
}
}
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let target_epoch = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let submission = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: vec![0x5A; PageSize::DEFAULT.as_usize()],
db_size_if_commit: 2,
}]))
.expect("keeper batch must enter the exact target epoch");
assert_eq!(submission.outcome, SubmitOutcome::Flusher);
assert_eq!(consolidator.begin_flush().unwrap().len(), 1);
submission.target_epoch
};
let slot = queue.epoch_waiters.slot(target_epoch);
let waiter_cx = Cx::new();
let mut waiter = Box::pin(queue.wait_for_epoch_outcome_async(&waiter_cx, target_epoch));
let initial_poll = asupersync::time::timeout(
asupersync::time::wall_now(),
Duration::from_secs(2),
std::future::poll_fn(|poll_cx| match waiter.as_mut().poll(poll_cx) {
std::task::Poll::Pending if slot.active_async_waiter_count() == 1 => {
std::task::Poll::Ready(None)
}
std::task::Poll::Pending => {
poll_cx.waker().wake_by_ref();
std::task::Poll::Pending
}
std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)),
}),
)
.await
.expect("epoch waiter must enter its keyed slot within the keeper bound");
assert!(
initial_poll.is_none(),
"epoch waiter completed before entering its keyed slot: {initial_poll:?}"
);
let restore_entered = Arc::new(AtomicBool::new(false));
let restore_released = Arc::new(AtomicBool::new(false));
queue.enqueue_pending_external_unlock(PendingExternalUnlock {
sequence: None,
scope: ProcessRootFinalizationScope::IdentityWide,
epoch: None,
durability_started: Arc::new(AtomicBool::new(false)),
durable_io_completed: Arc::new(AtomicBool::new(false)),
coordination_owner: None,
recovery: None,
root_attempt: None,
operation: Box::new(SuspendedIdentityWideRestore {
entered: Arc::clone(&restore_entered),
released: Arc::clone(&restore_released),
}),
});
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(!consolidator.complete_flush().unwrap());
assert_eq!(consolidator.phase(), ConsolidationPhase::Complete);
}
queue.publish_completed_epoch(target_epoch, false);
let post_wake_poll = asupersync::time::timeout(
asupersync::time::wall_now(),
Duration::from_secs(2),
std::future::poll_fn(|poll_cx| match waiter.as_mut().poll(poll_cx) {
std::task::Poll::Pending if restore_entered.load(AtomicOrdering::Acquire) => {
std::task::Poll::Ready(None)
}
std::task::Poll::Pending => {
poll_cx.waker().wake_by_ref();
std::task::Poll::Pending
}
std::task::Poll::Ready(result) => std::task::Poll::Ready(Some(result)),
}),
)
.await;
match post_wake_poll {
Ok(None) => {}
Ok(Some(result)) => {
drop(waiter);
restore_released.store(true, AtomicOrdering::Release);
settle_identity_wide_group_commit_finalization(&queue)
.await
.expect("unexpected completion path must still release its test root");
panic!(
"identity-wide restoration must settle before completion observation: {result:?}"
);
}
Err(error) => {
drop(waiter);
restore_released.store(true, AtomicOrdering::Release);
settle_identity_wide_group_commit_finalization(&queue)
.await
.expect("timed-out keeper path must still release its test root");
panic!("post-wake settlement did not enter within the keeper bound: {error}");
}
}
let pending_was_empty_while_claimed = queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty();
let claims_while_suspended = queue
.identity_wide_external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire);
drop(waiter);
let wake_drop_count = queue
.unaccounted_epoch_wake_drops
.load(AtomicOrdering::Acquire);
let active_waiters_after_drop = slot.active_async_waiter_count();
let claims_after_drop = queue
.identity_wide_external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire);
let pending_after_drop = queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len();
let root_present_after_drop = queue.has_identity_wide_process_root();
restore_released.store(true, AtomicOrdering::Release);
let retry_cx = Cx::new();
let retry_result = asupersync::time::timeout(
asupersync::time::wall_now(),
Duration::from_secs(2),
queue.wait_for_epoch_outcome_async(&retry_cx, target_epoch),
)
.await;
let cleanup_result = settle_identity_wide_group_commit_finalization(&queue).await;
let final_quiescent = queue.identity_wide_finalization_is_quiescent();
let final_pending_empty = queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty();
assert!(
pending_was_empty_while_claimed,
"suspended post-wake settlement must own its queue record"
);
assert_eq!(
claims_while_suspended, 1,
"suspended post-wake settlement must publish its identity-wide claim"
);
assert_eq!(
wake_drop_count, 1,
"dropping suspended settlement must classify its resolved wake exactly once"
);
assert_eq!(active_waiters_after_drop, 0);
assert_eq!(
claims_after_drop, 0,
"dropping settlement must release its in-flight claim publication"
);
assert_eq!(
pending_after_drop, 1,
"dropping settlement must requeue the exact restoration"
);
assert!(root_present_after_drop);
assert!(
matches!(&retry_result, Ok(Ok(WaitForEpochOutcome::Completed))),
"retry must settle restoration before completion: {retry_result:?}"
);
assert!(
cleanup_result.is_ok(),
"keeper cleanup must be terminal: {cleanup_result:?}"
);
assert!(final_quiescent);
assert!(
final_pending_empty,
"terminal retry must consume the requeued restoration"
);
});
}
#[test]
fn test_epoch_waiter_retained_failure_precedes_unrelated_identity_root() {
asupersync::test_utils::run_test(|| async {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let target_epoch = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let submission = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: vec![0x6B; PageSize::DEFAULT.as_usize()],
db_size_if_commit: 2,
}]))
.expect("failure keeper batch must enter its target epoch");
assert_eq!(submission.outcome, SubmitOutcome::Flusher);
let _ = consolidator.begin_flush().unwrap();
submission.target_epoch
};
let consumer = queue.register_epoch_consumer(target_epoch);
queue
.abort_flushing_epoch_as_failed(target_epoch, &FrankenError::Abort)
.expect("keeper must retain an exact failed-epoch verdict");
assert!(
queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&target_epoch),
"consumer lease must retain the failed-epoch evidence"
);
let unrelated_root = ProcessRootFinalizationAttempt::register(&queue);
let waiter_cx = Cx::new();
let waiter_result = asupersync::time::timeout(
asupersync::time::wall_now(),
Duration::from_secs(2),
queue.wait_for_epoch_outcome_async(&waiter_cx, target_epoch),
)
.await;
unrelated_root.release_after_terminal();
assert!(
matches!(&waiter_result, Ok(Err(FrankenError::Abort))),
"exact failed epoch must take precedence over unrelated BusyRecovery: {waiter_result:?}"
);
drop(consumer);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&target_epoch),
"final consumer release must reclaim retained failure evidence"
);
});
}
/// H4 / F4: Crash during Phase C — after WAL frames are durable and
/// commit_seq is updated, but before snapshot publish completes.
///
/// Proof obligation: the first commit returns `Err`, but the pager reports
/// that durability is already authorized and retains the exact publication
/// obligation. Retrying the same handle must finish publication exactly
/// once without losing the durable WAL frames.
///
/// Replay: `cargo test -p fsqlite-pager --lib -- test_fault_during_phase_c --nocapture`
#[test]
fn test_fault_during_phase_c_returns_error_and_wal_frames_survive() {
asupersync::test_utils::run_test(|| async {
let _guard = FAULT_HOOK_TEST_GUARD
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
crate::fault_hooks::clear();
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/fault_phase_c_test.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, frames, _begin_calls, _batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
// Arm the Phase C hook.
crate::fault_hooks::arm_during_phase_c(crate::fault_hooks::FaultHookArm::new(
"bd-db300.7.2.2-phase-c",
"PHASE-C-CRASH",
"commit_publish_recovery",
));
// Begin a writer transaction and dirty a page.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_bytes = PageSize::DEFAULT.as_usize();
txn.write_page(&cx, PageNumber::ONE, &vec![0xCC; page_bytes])
.await
.unwrap();
// Commit should fail — the hook fires after WAL I/O but before publish.
let err = txn
.commit(&cx)
.await
.expect_err("Phase C fault hook should cause commit to fail");
assert!(
err.to_string().contains("fault_inject:during_phase_c"),
"error should identify the Phase C hook: {err}"
);
assert_eq!(
txn.pager_commit_state(),
PagerCommitState::DurableNeedsPublication,
"a post-WAL Phase C error must retain its publication obligation"
);
txn.commit(&cx)
.await
.expect("retrying the same authorized attempt must finish publication");
assert_eq!(
txn.pager_commit_state(),
PagerCommitState::Committed,
"the retained publication obligation must settle as committed"
);
// WAL frames should have been written (the error is AFTER WAL I/O).
let written_frames = frames.lock().unwrap();
assert!(
!written_frames.is_empty(),
"WAL frames should survive — the fault fires after WAL I/O completes"
);
// Verify injection record.
let records = crate::fault_hooks::take_records();
assert_eq!(records.len(), 1, "exactly one Phase C fault should fire");
assert_eq!(records[0].point, "during_phase_c");
assert_eq!(records[0].run_id, "bd-db300.7.2.2-phase-c");
assert_eq!(records[0].scenario_id, "PHASE-C-CRASH");
assert!(
records[0].detail.contains("commit_seq="),
"record should capture commit_seq: {}",
records[0].detail
);
crate::fault_hooks::clear();
});
}
#[test]
fn test_durable_rollback_journal_phase_c_receipt_finishes_exactly_once() {
asupersync::test_utils::run_test(|| async {
#[derive(Clone, Copy)]
enum Completion {
RetryCommit,
RejectRollback,
Drop,
}
let _guard = FAULT_HOOK_TEST_GUARD
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
for (case, completion, fill) in [
("retry_commit", Completion::RetryCommit, 0x61),
("reject_rollback", Completion::RejectRollback, 0x62),
("drop", Completion::Drop, 0x63),
] {
crate::fault_hooks::clear();
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let path = PathBuf::from(format!("/rollback-phase-c-{case}.db"));
let journal_path = SimplePager::<ObservedLockVfs>::journal_path(&path);
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let baseline_commit_seq = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.commit_seq;
crate::fault_hooks::arm_during_phase_c(crate::fault_hooks::FaultHookArm::new(
format!("rollback-phase-c-{case}"),
case,
"durable_rollback_commit_finalization",
));
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
let expected = sample_page(fill);
txn.write_page(&cx, page, &expected).await.unwrap();
let commit_error = txn
.commit(&cx)
.await
.expect_err("Phase C hook must interrupt publication after durable commit");
assert!(
commit_error
.to_string()
.contains("fault_inject:during_phase_c"),
"case={case} expected Phase C injection, got {commit_error}"
);
assert!(txn.rollback_commit_finalization_pending, "case={case}");
assert!(txn.committed, "case={case}");
assert!(txn.maintenance_lease.is_none(), "case={case}");
let recovery_owner = txn
.owned_rollback_recovery
.expect("durable rollback commit must retain its exact owner");
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
));
assert_eq!(
inner.rollback_journal_recovery_owner,
Some(recovery_owner),
"case={case}"
);
assert_eq!(
inner.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner),
"case={case}"
);
assert_eq!(
inner
.rollback_recovery_pending
.load(AtomicOrdering::Acquire),
recovery_owner.get(),
"case={case}"
);
assert_eq!(inner.active_transactions, 0, "case={case}");
assert!(!inner.writer_active, "case={case}");
assert_eq!(
inner.commit_seq.get(),
baseline_commit_seq.get().saturating_add(1),
"case={case}"
);
}
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
let begin_error = match pager.begin(&cx, TransactionMode::ReadOnly).await {
Ok(_) => panic!("case={case} sibling begin stole a durable recovery owner"),
Err(error) => error,
};
assert!(
matches!(begin_error, FrankenError::BusyRecovery),
"case={case} unexpected sibling begin error: {begin_error}"
);
let refresh_error = pager
.refresh_published_snapshot(&cx)
.await
.expect_err("sibling refresh must not take over a durable recovery owner");
assert!(
matches!(refresh_error, FrankenError::BusyRecovery),
"case={case} unexpected sibling refresh error: {refresh_error}"
);
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
inner.rollback_journal_recovery_owner,
Some(recovery_owner),
"case={case}"
);
assert_eq!(
inner.commit_seq.get(),
baseline_commit_seq.get().saturating_add(1),
"case={case}"
);
}
crate::fault_hooks::clear();
match completion {
Completion::RetryCommit => {
txn.commit(&cx).await.unwrap();
drop(txn);
}
Completion::RejectRollback => {
let rollback_error = txn
.rollback(&cx)
.await
.expect_err("rollback must not undo an already-durable commit");
assert!(
rollback_error.to_string().contains("already durable"),
"case={case} unexpected rollback error: {rollback_error}"
);
drop(txn);
}
Completion::Drop => drop(txn),
}
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None, "case={case}");
assert_eq!(
inner.maintenance_gate.rollback_recovery_owner(),
None,
"case={case}"
);
assert_eq!(
inner
.rollback_recovery_pending
.load(AtomicOrdering::Acquire),
0,
"case={case}"
);
assert_eq!(inner.active_transactions, 0, "case={case}");
assert!(!inner.writer_active, "case={case}");
assert_eq!(
inner.commit_seq.get(),
baseline_commit_seq.get().saturating_add(1),
"case={case}"
);
}
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"case={case} durable commit must not leave a hot journal"
);
assert_eq!(
pager.published_snapshot().visible_commit_seq.get(),
baseline_commit_seq.get().saturating_add(1),
"case={case}"
);
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_ref(),
expected.as_slice(),
"case={case}"
);
reader.rollback(&cx).await.unwrap();
}
crate::fault_hooks::clear();
});
}
#[test]
fn test_durable_rollback_owner_keeps_identity_gate_alive_after_pager_drop() {
asupersync::test_utils::run_test(|| async {
let _guard = FAULT_HOOK_TEST_GUARD
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
crate::fault_hooks::clear();
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let path = PathBuf::from("/rollback-owner-gate-lifetime.db");
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let weak_gate = Arc::downgrade(&pager.maintenance_gate);
crate::fault_hooks::arm_during_phase_c(crate::fault_hooks::FaultHookArm::new(
"rollback-owner-gate-lifetime",
"drop-pager-before-owner-finalization",
"durable_rollback_commit_finalization",
));
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x73)).await.unwrap();
txn.commit(&cx)
.await
.expect_err("Phase C hook must retain a durable exact owner");
let recovery_owner = txn
.owned_rollback_recovery
.expect("durable transaction must retain its exact owner");
drop(pager);
let retained_gate = weak_gate
.upgrade()
.expect("live transaction must strongly retain the identity gate");
assert_eq!(
retained_gate.rollback_recovery_owner(),
Some(recovery_owner)
);
let reopen_error = match vfs.open_file_backed_pager(&path).await {
Ok(_) => panic!("reopen bypassed a live durable recovery owner"),
Err(error) => error,
};
assert!(
matches!(reopen_error, FrankenError::BusyRecovery),
"unexpected reopen error: {reopen_error}"
);
crate::fault_hooks::clear();
txn.commit(&cx).await.unwrap();
drop(txn);
drop(retained_gate);
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
assert_eq!(reopened.maintenance_gate.rollback_recovery_owner(), None);
crate::fault_hooks::clear();
});
}
#[test]
fn test_dropped_durable_rollback_commit_defers_and_finishes_exact_exit() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
// The commit consumes the first restoration failure. Drop consumes
// the second, forcing exact-handle cleanup into the rooted queue.
// The detached finalizer consumes the third, is requeued with the
// same exact owner, and then succeeds on its next retry.
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let path = PathBuf::from("/dropped-durable-rollback-commit-exit.db");
let journal_path = SimplePager::<ObservedLockVfs>::journal_path(&path);
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let baseline_commit_seq = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.commit_seq;
let handle_key = {
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
shared_db_file_key(&inner.db_file)
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
let expected = sample_page(0x74);
txn.write_page(&cx, page, &expected).await.unwrap();
vfs.external_restore_failures
.store(3, AtomicOrdering::Release);
let commit_error = txn
.commit(&cx)
.await
.expect_err("injected snapshot restoration failure must interrupt Phase C exit");
assert!(
commit_error
.to_string()
.contains("external snapshot restoration failure"),
"unexpected commit error: {commit_error}"
);
assert!(txn.rollback_commit_finalization_pending);
assert!(txn.committed);
assert!(txn.maintenance_lease.is_some());
let recovery_owner = txn
.owned_rollback_recovery
.expect("durable rollback commit must retain its exact owner");
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::DurableCommitFinalizationPending
));
assert_eq!(inner.rollback_journal_recovery_owner, Some(recovery_owner));
assert_eq!(
inner.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner)
);
assert_eq!(inner.active_transactions, 1);
assert!(inner.writer_active);
assert_eq!(
inner.commit_seq.get(),
baseline_commit_seq.get().saturating_add(1)
);
}
drop(txn);
assert_eq!(pager.group_commit_queue.pending_logical_cleanup_count(), 1);
assert!(
pager
.group_commit_queue
.has_process_root_finalization_attempt()
);
assert_eq!(
pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.rollback_recovery_pending
.load(AtomicOrdering::Acquire),
recovery_owner.get()
);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::Exclusive);
let retry_error = settle_pending_group_commit_finalization_for_handle(
&pager.group_commit_queue,
handle_key,
)
.await
.expect_err("first detached durable finalization must be requeued");
assert!(
retry_error
.to_string()
.contains("external snapshot restoration failure"),
"unexpected detached retry error: {retry_error}"
);
assert_eq!(pager.group_commit_queue.pending_logical_cleanup_count(), 1);
assert!(
pager
.group_commit_queue
.has_process_root_finalization_attempt()
);
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(inner.rollback_journal_recovery_owner, Some(recovery_owner));
assert_eq!(
inner.maintenance_gate.rollback_recovery_owner(),
Some(recovery_owner)
);
assert_eq!(inner.active_transactions, 1);
assert!(inner.writer_active);
}
settle_pending_group_commit_finalization_for_handle(
&pager.group_commit_queue,
handle_key,
)
.await
.unwrap();
assert_eq!(pager.group_commit_queue.pending_logical_cleanup_count(), 0);
assert!(
!pager
.group_commit_queue
.has_process_root_finalization_attempt()
);
{
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
inner.rollback_journal_recovery_state,
RollbackJournalRecoveryState::Clean
));
assert_eq!(inner.rollback_journal_recovery_owner, None);
assert_eq!(inner.maintenance_gate.rollback_recovery_owner(), None);
assert_eq!(
inner
.rollback_recovery_pending
.load(AtomicOrdering::Acquire),
0
);
assert_eq!(inner.active_transactions, 0);
assert!(!inner.writer_active);
assert_eq!(
inner.commit_seq.get(),
baseline_commit_seq.get().saturating_add(1)
);
}
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"durable deferred cleanup must not leave a hot journal"
);
assert_eq!(
pager.published_snapshot().visible_commit_seq.get(),
baseline_commit_seq.get().saturating_add(1)
);
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_ref(),
expected.as_slice()
);
reader.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_publish_window_measurement_captures_exclusive_hold_sample() {
asupersync::test_utils::run_test(|| async {
let (cx, mut txn, vfs) = track_c_publish_window_prepared_commit(
TrackCPublishWindowMode::PreparedCandidate,
7,
)
.await;
txn.commit(&cx).await.unwrap();
let hold_samples = vfs.exclusive_hold_samples_ns();
assert_eq!(
hold_samples.len(),
1,
"bead_id={TRACK_C_PUBLISH_WINDOW_BENCH_BEAD_ID} case=hold_sample_count"
);
assert!(
hold_samples[0] > 0,
"bead_id={TRACK_C_PUBLISH_WINDOW_BENCH_BEAD_ID} case=hold_sample_positive"
);
});
}
#[test]
fn test_publish_window_contention_measurement_captures_competing_writer_wait() {
asupersync::test_utils::run_test(|| async {
let (vfs, pager_a, pager_b) =
track_c_open_contending_pagers(TrackCPublishWindowMode::PreparedCandidate, 7).await;
let writer_a = std::thread::spawn(move || {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let mut txn = pager_a
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
track_c_write_existing_page_range(&mut txn, &cx, 2, 7, 53).await;
txn.commit(&cx).await.unwrap();
});
});
vfs.wait_for_exclusive_acquisitions(1);
let cx_b = Cx::new();
let mut txn_b = pager_b
.begin(&cx_b, TransactionMode::Immediate)
.await
.unwrap();
track_c_write_existing_page_range(&mut txn_b, &cx_b, 9, 7, 97).await;
txn_b.commit(&cx_b).await.unwrap();
writer_a.join().unwrap();
let wait_samples = vfs.exclusive_wait_samples_ns();
assert_eq!(
wait_samples.len(),
2,
"bead_id={TRACK_C_PUBLISH_WINDOW_BENCH_BEAD_ID} case=wait_sample_count"
);
assert!(
wait_samples.iter().copied().max().unwrap_or(0) > 0,
"bead_id={TRACK_C_PUBLISH_WINDOW_BENCH_BEAD_ID} case=contending_writer_wait_positive"
);
});
}
#[test]
fn test_collect_wal_commit_batch_keeps_sorted_order_and_single_commit_boundary() {
let page_three = PageNumber::new(3).unwrap();
let mut write_set = HashMap::new();
let mut page1 = PageBuf::new(PageSize::DEFAULT);
page1.fill(0x11);
write_set.insert(PageNumber::ONE, StagedPage::from_buf(page1));
let mut page3 = PageBuf::new(PageSize::DEFAULT);
page3.fill(0x33);
write_set.insert(page_three, StagedPage::from_buf(page3));
let write_pages_sorted = vec![PageNumber::ONE, page_three];
let batch = collect_wal_commit_batch(2, &write_set, &write_pages_sorted)
.unwrap()
.expect("non-empty write set should yield a WAL batch");
assert_eq!(
batch.new_db_size,
page_three.get(),
"bead_id={BEAD_ID} case=wal_batch_helper_new_db_size"
);
assert_eq!(
batch.frames.len(),
2,
"bead_id={BEAD_ID} case=wal_batch_helper_frame_count"
);
assert_eq!(
batch.frames[0].page_number,
PageNumber::ONE.get(),
"bead_id={BEAD_ID} case=wal_batch_helper_sorted_first"
);
assert_eq!(
batch.frames[0].db_size_if_commit, 0,
"bead_id={BEAD_ID} case=wal_batch_helper_non_commit_prefix"
);
assert_eq!(
batch.frames[1].page_number,
page_three.get(),
"bead_id={BEAD_ID} case=wal_batch_helper_sorted_last"
);
assert_eq!(
batch.frames[1].db_size_if_commit,
page_three.get(),
"bead_id={BEAD_ID} case=wal_batch_helper_commit_boundary_last"
);
}
#[test]
fn test_collect_wal_commit_batch_preserves_existing_db_size_for_interior_updates() {
let page_two = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
let mut page2 = PageBuf::new(PageSize::DEFAULT);
page2.fill(0x22);
write_set.insert(page_two, StagedPage::from_buf(page2));
let batch = collect_wal_commit_batch(9, &write_set, &[page_two])
.unwrap()
.expect("single dirty page should yield a WAL batch");
assert_eq!(
batch.new_db_size, 9,
"bead_id={BEAD_ID} case=wal_batch_helper_preserve_db_size"
);
assert_eq!(
batch.frames[0].db_size_if_commit, 9,
"bead_id={BEAD_ID} case=wal_batch_helper_commit_marker_uses_existing_db_size"
);
}
#[test]
fn test_collect_wal_commit_batch_returns_none_for_empty_write_set() {
let write_set = HashMap::new();
let batch = collect_wal_commit_batch(4, &write_set, &[]).unwrap();
assert!(
batch.is_none(),
"bead_id={BEAD_ID} case=wal_batch_helper_empty_batch"
);
}
#[test]
fn test_collect_wal_commit_batch_errors_when_sorted_page_missing_from_write_set() {
let missing_page = PageNumber::new(4).unwrap();
let write_set = HashMap::new();
let err = match collect_wal_commit_batch(1, &write_set, &[missing_page]) {
Ok(_) => panic!(
"bead_id={BEAD_ID} case=wal_batch_helper_missing_page expected helper to fail"
),
Err(err) => err,
};
let message = err.to_string();
assert!(
message.contains("missing page 4"),
"bead_id={BEAD_ID} case=wal_batch_helper_missing_page error={message}"
);
}
#[test]
fn test_build_group_commit_batch_clones_owned_frames_and_commit_boundary() {
let page_two = PageNumber::new(2).unwrap();
let page_three = PageNumber::new(3).unwrap();
let mut write_set = HashMap::new();
let mut page2 = PageBuf::new(PageSize::DEFAULT);
page2.fill(0x22);
write_set.insert(page_two, StagedPage::from_buf(page2));
let mut page3 = PageBuf::new(PageSize::DEFAULT);
page3.fill(0x33);
write_set.insert(page_three, StagedPage::from_buf(page3));
let (batch, new_db_size) = build_group_commit_batch(2, &write_set, &[page_two, page_three])
.unwrap()
.expect("sorted write set should yield a group commit batch");
drop(write_set);
assert_eq!(
new_db_size, 3,
"bead_id={BEAD_ID} case=group_commit_batch_helper_new_db_size"
);
assert_eq!(
batch.frames.len(),
2,
"bead_id={BEAD_ID} case=group_commit_batch_helper_frame_count"
);
assert_eq!(
batch.frames[0].page_number,
page_two.get(),
"bead_id={BEAD_ID} case=group_commit_batch_helper_sorted_first"
);
assert_eq!(
batch.frames[0].db_size_if_commit, 0,
"bead_id={BEAD_ID} case=group_commit_batch_helper_non_commit_first"
);
assert_eq!(
batch.frames[1].page_number,
page_three.get(),
"bead_id={BEAD_ID} case=group_commit_batch_helper_sorted_last"
);
assert_eq!(
batch.frames[1].db_size_if_commit,
page_three.get(),
"bead_id={BEAD_ID} case=group_commit_batch_helper_commit_marker_last"
);
assert_eq!(
batch.frames[0].page_data[0], 0x22,
"bead_id={BEAD_ID} case=group_commit_batch_helper_preserves_first_payload_after_source_drop"
);
assert_eq!(
batch.frames[1].page_data[0], 0x33,
"bead_id={BEAD_ID} case=group_commit_batch_helper_preserves_second_payload_after_source_drop"
);
}
#[test]
fn test_build_group_commit_batch_returns_none_for_empty_write_set() {
let write_set = HashMap::new();
let batch = build_group_commit_batch(4, &write_set, &[]).unwrap();
assert!(
batch.is_none(),
"bead_id={BEAD_ID} case=group_commit_batch_helper_empty_batch"
);
}
#[test]
fn test_build_group_commit_batch_errors_when_sorted_page_missing_from_write_set() {
let missing_page = PageNumber::new(4).unwrap();
let write_set = HashMap::new();
let err = match build_group_commit_batch(1, &write_set, &[missing_page]) {
Ok(_) => panic!(
"bead_id={BEAD_ID} case=group_commit_batch_helper_missing_page expected helper to fail"
),
Err(err) => err,
};
let message = err.to_string();
assert!(
message.contains("missing page 4"),
"bead_id={BEAD_ID} case=group_commit_batch_helper_missing_page error={message}"
);
}
#[test]
fn test_parallel_wal_control_override_applies_to_new_queue() {
let control = ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Conservative,
lane_count_override: Some(3),
max_parallel_commit_bytes: Some(4096),
..ParallelWalControlSurface::default()
};
set_parallel_wal_control_override(Some(control.clone()));
let queue = GroupCommitQueue::new(GroupCommitConfig::default());
assert_eq!(
queue.parallel_wal_control(),
&control,
"bead_id=bd-3wop3.1.2 case=parallel_wal_control_override_applies_to_new_queue"
);
set_parallel_wal_control_override(None);
}
#[test]
fn test_parallel_wal_lane_identity_is_stable_within_thread() {
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let queue = GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(4),
..ParallelWalControlSurface::default()
},
);
let first = queue.current_parallel_wal_lane_id();
let second = queue.current_parallel_wal_lane_id();
assert_eq!(
first, second,
"bead_id=bd-3wop3.1.2 case=lane_identity_stable_within_thread"
);
assert!(
usize::from(first) < 4,
"bead_id=bd-3wop3.1.2 case=lane_identity_respects_override lane_id={first}"
);
}
#[test]
fn test_parallel_wal_lane_reuse_after_worker_churn() {
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
const LANE_COUNT: usize = 2;
let queue = StdArc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(LANE_COUNT),
..ParallelWalControlSurface::default()
},
));
let spawn_wave = || {
let mut lanes = Vec::new();
for _ in 0..2 {
let queue = StdArc::clone(&queue);
lanes.push(std::thread::spawn(move || {
queue.current_parallel_wal_lane_id()
}));
}
let mut observed = lanes
.into_iter()
.map(|handle| handle.join().unwrap())
.collect::<Vec<_>>();
observed.sort_unstable();
observed
};
let first_wave = spawn_wave();
let second_wave = spawn_wave();
for (wave_name, wave) in [("first", &first_wave), ("second", &second_wave)] {
assert_eq!(
wave.len(),
LANE_COUNT,
"bead_id=bd-3wop3.1.2 case=lane_reuse_{wave_name}_wave_size"
);
assert!(
wave.iter().all(|lane| usize::from(*lane) < LANE_COUNT),
"bead_id=bd-3wop3.1.2 case=lane_reuse_{wave_name}_wave_bounds lanes={wave:?}"
);
}
}
#[test]
fn test_parallel_wal_same_lane_order_mismatch_returns_none_without_drain() {
let queue = GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
);
let page_a = sample_page(0xA1);
let page_b = sample_page(0xB2);
let frame_refs_a = [crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_a,
db_size_if_commit: 2,
}];
let frame_refs_b = [crate::traits::WalFrameRef {
page_number: 3,
page_data: &page_b,
db_size_if_commit: 3,
}];
assert_eq!(
queue.record_prepared_batch(lane_staged_batch_for_test(10, 0, &frame_refs_a)),
1
);
assert_eq!(
queue.record_prepared_batch(lane_staged_batch_for_test(11, 0, &frame_refs_b)),
2
);
let out_of_order = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 3,
page_data: page_b.clone(),
db_size_if_commit: 3,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 11,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: page_a.clone(),
db_size_if_commit: 2,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 10,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
];
assert!(
queue
.take_prepared_batches_for_flush(&out_of_order)
.is_none(),
"bead_id=bd-3wop3.1.2 case=same_lane_order_mismatch_forces_fallback"
);
assert_eq!(
queue.current_lane_backlog(0),
2,
"bead_id=bd-3wop3.1.2 case=same_lane_order_mismatch_does_not_drain"
);
let in_order = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: page_a,
db_size_if_commit: 2,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 10,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 3,
page_data: page_b,
db_size_if_commit: 3,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 11,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
];
let drained = queue
.take_prepared_batches_for_flush(&in_order)
.expect("verified in-order same-lane batches should drain");
assert_eq!(
drained.len(),
2,
"bead_id=bd-3wop3.1.2 case=same_lane_order_in_order_drains"
);
assert_eq!(
queue.current_lane_backlog(0),
0,
"bead_id=bd-3wop3.1.2 case=same_lane_order_drain_clears_backlog"
);
}
#[test]
fn test_parallel_wal_raw_fallback_discard_removes_stale_prepared_batches() {
let queue = GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
);
let page_a = sample_page(0xC1);
let page_b = sample_page(0xD2);
let frame_refs_a = [crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_a,
db_size_if_commit: 2,
}];
let frame_refs_b = [crate::traits::WalFrameRef {
page_number: 3,
page_data: &page_b,
db_size_if_commit: 3,
}];
assert_eq!(
queue.record_prepared_batch(lane_staged_batch_for_test(10, 0, &frame_refs_a)),
1
);
assert_eq!(
queue.record_prepared_batch(lane_staged_batch_for_test(11, 0, &frame_refs_b)),
2
);
let raw_fallback_batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 3,
page_data: page_b,
db_size_if_commit: 3,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 11,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: page_a,
db_size_if_commit: 2,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 10,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
];
assert!(
queue
.take_prepared_batches_for_flush(&raw_fallback_batches)
.is_none(),
"bead_id=bd-3wop3.1.2 case=raw_fallback_order_mismatch_forces_frame_ref_path"
);
assert_eq!(
queue.current_lane_backlog(0),
2,
"bead_id=bd-3wop3.1.2 case=raw_fallback_mismatch_leaves_prepared_batches_until_explicit_cleanup"
);
assert_eq!(
queue.discard_prepared_batches_for_flush(&raw_fallback_batches),
2,
"bead_id=bd-3wop3.1.2 case=raw_fallback_discards_exact_flushed_prepared_batches"
);
assert_eq!(
queue.current_lane_backlog(0),
0,
"bead_id=bd-3wop3.1.2 case=raw_fallback_discard_clears_stale_lane_backlog"
);
}
#[test]
fn test_parallel_wal_prepared_merge_clears_intermediate_commit_headers() {
let page_a = sample_page(0xC3);
let page_b = sample_page(0xD4);
let frame_refs_a = [crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_a,
db_size_if_commit: 2,
}];
let frame_refs_b = [crate::traits::WalFrameRef {
page_number: 3,
page_data: &page_b,
db_size_if_commit: 3,
}];
let staged = vec![
lane_staged_batch_for_test(10, 0, &frame_refs_a),
lane_staged_batch_for_test(11, 1, &frame_refs_b),
];
let merged = merge_prepared_group_commit_batches(staged, 3).unwrap();
let db_size_headers = (0..merged.frame_count())
.map(|index| {
let frame = merged.frame_slice(index);
u32::from_be_bytes([frame[4], frame[5], frame[6], frame[7]])
})
.collect::<Vec<_>>();
assert_eq!(
merged
.frame_metas
.iter()
.map(|meta| meta.db_size_if_commit)
.collect::<Vec<_>>(),
vec![0, 3],
"bead_id=bd-3wop3.8 case=prepared_merge_metadata_has_single_group_commit_marker"
);
assert_eq!(
db_size_headers,
vec![0, 3],
"bead_id=bd-3wop3.8 case=prepared_merge_bytes_clear_hidden_intermediate_commit_marker"
);
assert_eq!(
merged.last_commit_frame_offset,
Some(1),
"bead_id=bd-3wop3.8 case=prepared_merge_last_commit_points_to_group_tail"
);
}
#[test]
fn test_parallel_wal_prepared_merge_reuses_canonical_single_batch() {
let page_a = sample_page(0xA7);
let page_b = sample_page(0xB8);
let frame_refs = [
crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_a,
db_size_if_commit: 0,
},
crate::traits::WalFrameRef {
page_number: 3,
page_data: &page_b,
db_size_if_commit: 3,
},
];
let mut staged_batch = lane_staged_batch_for_test(12, 0, &frame_refs);
staged_batch.payload.finalized_for = Some(crate::traits::PreparedWalFinalizationState {
salt1: 11,
salt2: 22,
checkpoint_seq: 0,
start_frame_index: 0,
seed: crate::traits::PreparedWalChecksumSeed { s1: 33, s2: 44 },
});
staged_batch.payload.finalized_running_checksum =
Some(crate::traits::PreparedWalChecksumSeed { s1: 55, s2: 66 });
let staged = vec![staged_batch.clone()];
let merged = merge_prepared_group_commit_batches(staged, 3).unwrap();
assert_eq!(
merged, staged_batch.payload,
"case=prepared_merge_single_batch_reuses_payload_without_checksum_recompute"
);
assert_eq!(
merged.finalized_for, staged_batch.payload.finalized_for,
"case=prepared_merge_single_batch_preserves_prelock_finalization"
);
}
#[test]
fn test_parallel_wal_prepared_merge_rejects_hidden_single_batch_commit_header() {
let page_a = sample_page(0xA9);
let page_b = sample_page(0xBA);
let frame_refs = [
crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_a,
db_size_if_commit: 0,
},
crate::traits::WalFrameRef {
page_number: 3,
page_data: &page_b,
db_size_if_commit: 3,
},
];
let mut staged_batch = lane_staged_batch_for_test(13, 0, &frame_refs);
staged_batch.payload.frame_bytes[4..8].copy_from_slice(&99_u32.to_be_bytes());
staged_batch.payload.finalized_for = Some(crate::traits::PreparedWalFinalizationState {
salt1: 11,
salt2: 22,
checkpoint_seq: 0,
start_frame_index: 0,
seed: crate::traits::PreparedWalChecksumSeed { s1: 33, s2: 44 },
});
let staged = vec![staged_batch];
let merged = merge_prepared_group_commit_batches(staged, 3).unwrap();
let first_frame = merged.frame_slice(0);
assert_eq!(
u32::from_be_bytes([
first_frame[4],
first_frame[5],
first_frame[6],
first_frame[7],
]),
0,
"case=prepared_merge_single_batch_fallback_clears_hidden_commit_header"
);
assert_eq!(
merged.finalized_for, None,
"case=prepared_merge_single_batch_fallback_invalidates_stale_finalization"
);
}
/// Stress test for the in-process Waiter path of group commit.
///
/// The cross-process `swarm_multiprocess` binary mostly exercises the
/// Flusher path because each process has its own `GroupCommitQueue`
/// (the consolidator is process-local). This test drives many threads
/// through a single shared `GroupCommitQueue` under contention so the
/// merge/flush path is hit with batches of varying sizes and concurrent
/// Waiter wakes. It would have failed prior to `04812db3` because a
/// merged batch with N input commits would leave N byte-level commit
/// markers — the invariant below is "at most one commit marker per
/// flushed prepared batch" at both meta and byte levels.
#[test]
fn test_concurrent_commits_in_process_waiter_path_no_ghost_commits_stress() {
asupersync::test_utils::run_test(|| async {
type BatchMarkerCounts = StdArc<StdMutex<Vec<(usize, usize, usize)>>>;
struct WaiterStressBackend {
// Per-call tuples: (meta_commits, byte_commits, frame_count).
per_batch: BatchMarkerCounts,
total_frames: SharedCounter,
append_prepared_calls: SharedCounter,
}
impl crate::traits::WalBackend for WaiterStressBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
// Non-prepared fallback path: count meta-only (no frame bytes to inspect).
let meta_commits =
frames.iter().filter(|f| f.db_size_if_commit != 0).count();
self.per_batch.lock().unwrap().push((
meta_commits,
meta_commits,
frames.len(),
));
*self.total_frames.lock().unwrap() += frames.len();
Ok(())
})
}
fn prepare_append_frames(
&self,
frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>>
{
if frames.is_empty() {
return Ok(None);
}
let frame_size =
fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE + frames[0].page_data.len();
let mut frame_bytes = Vec::with_capacity(frame_size * frames.len());
let mut frame_metas = Vec::with_capacity(frames.len());
let mut checksum_transforms = Vec::with_capacity(frames.len());
let mut last_commit: Option<usize> = None;
for (i, frame) in frames.iter().enumerate() {
frame_metas.push(crate::traits::PreparedWalFrameMeta {
page_number: frame.page_number,
db_size_if_commit: frame.db_size_if_commit,
});
checksum_transforms.push(crate::traits::PreparedWalChecksumTransform {
a11: 0,
a12: 0,
a21: 0,
a22: 0,
c1: 0,
c2: 0,
});
let mut header = [0_u8; fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE];
header[0..4].copy_from_slice(&frame.page_number.to_be_bytes());
header[4..8].copy_from_slice(&frame.db_size_if_commit.to_be_bytes());
frame_bytes.extend_from_slice(&header);
frame_bytes.extend_from_slice(frame.page_data);
if frame.db_size_if_commit != 0 {
last_commit = Some(i);
}
}
Ok(Some(crate::traits::PreparedWalFrameBatch {
frame_size,
page_data_offset: fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE,
big_endian_checksum: false,
frame_metas,
checksum_transforms,
frame_bytes,
last_commit_frame_offset: last_commit,
finalized_for: None,
finalized_running_checksum: None,
}))
}
fn append_prepared_frames<'a>(
&'a mut self,
_cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.append_prepared_calls.lock().unwrap() += 1;
let meta_commits = prepared
.frame_metas
.iter()
.filter(|meta| meta.db_size_if_commit != 0)
.count();
let mut byte_commits = 0_usize;
for i in 0..prepared.frame_count() {
let slice = prepared.frame_slice(i);
let byte_db_size =
u32::from_be_bytes([slice[4], slice[5], slice[6], slice[7]]);
if byte_db_size != 0 {
byte_commits += 1;
}
}
self.per_batch.lock().unwrap().push((
meta_commits,
byte_commits,
prepared.frame_count(),
));
*self.total_frames.lock().unwrap() += prepared.frame_count();
Ok(())
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
*self.total_frames.lock().unwrap()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let vfs = MemoryVfs::new();
let path = PathBuf::from("/waiter_path_stress.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let per_batch: BatchMarkerCounts = StdArc::new(StdMutex::new(Vec::new()));
let total_frames: SharedCounter = StdArc::new(StdMutex::new(0));
let append_prepared_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let backend = WaiterStressBackend {
per_batch: StdArc::clone(&per_batch),
total_frames: StdArc::clone(&total_frames),
append_prepared_calls: StdArc::clone(&append_prepared_calls),
};
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface::default(),
));
let pool = pager.pool.clone();
const WORKERS: u32 = 8;
const ITERS: u32 = 50;
let start = StdArc::new(std::sync::Barrier::new(WORKERS as usize));
let mut handles = Vec::with_capacity(WORKERS as usize);
for worker_id in 0..WORKERS {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
let start = StdArc::clone(&start);
let handle = std::thread::spawn(move || {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
start.wait();
for iter in 0..ITERS {
// Each (worker, iter) owns two disjoint page numbers so
// concurrent batches never conflict on pages.
let page_base = 2 + worker_id * ITERS * 2 + iter * 2;
let page_a = PageNumber::new(page_base).unwrap();
let page_b = PageNumber::new(page_base + 1).unwrap();
let fill_a = u8::try_from((worker_id * 31 + iter) & 0xff).unwrap();
let fill_b = u8::try_from((worker_id * 47 + iter) & 0xff).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_a,
StagedPage::from_bytes(&pool, &sample_page(fill_a)).unwrap(),
);
write_set.insert(
page_b,
StagedPage::from_bytes(&pool, &sample_page(fill_b)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_a, page_b],
&[],
&queue,
)
.await
.expect("commit_wal_group_commit succeeded under contention");
}
});
});
handles.push(handle);
}
for handle in handles {
handle.join().expect("worker thread joined cleanly");
}
let per_batch = per_batch.lock().unwrap().clone();
// Per-batch invariants: at most one commit marker in meta AND bytes,
// and meta/bytes must agree. A regression that re-introduces the
// pre-04812db3 ghost-commit behavior would leave intermediate
// commit markers in `frame_bytes` and fail the byte-level check.
//
// Total marker count may be *less* than submitted txns when batches
// merge (which is precisely the Waiter-path coverage this test adds);
// we assert `merged_batches >= 1` so the merge path actually fires.
let mut merged_batches = 0_usize;
for (i, (meta_commits, byte_commits, n)) in per_batch.iter().enumerate() {
assert!(
*meta_commits <= 1,
"bead_id=bd-3wop3.8 case=waiter_stress_batch_{i}_meta_commits_exceed_one meta={meta_commits} frames={n}"
);
assert!(
*byte_commits <= 1,
"bead_id=bd-3wop3.8 case=waiter_stress_batch_{i}_byte_commits_exceed_one byte={byte_commits} frames={n}"
);
assert_eq!(
meta_commits, byte_commits,
"bead_id=bd-3wop3.8 case=waiter_stress_batch_{i}_meta_byte_disagree meta={meta_commits} byte={byte_commits} frames={n}"
);
// A batch that contains >2 frames (more than one txn's contribution
// of 2 pages) is evidence that merge happened on this call.
if *n > 2 {
merged_batches += 1;
}
}
assert!(
merged_batches >= 1,
"bead_id=bd-3wop3.8 case=waiter_stress_no_merge_observed_test_did_not_exercise_waiter_path per_batch_len={} batches={:?}",
per_batch.len(),
per_batch
);
let expected_frames = (WORKERS * ITERS * 2) as usize;
assert_eq!(
*total_frames.lock().unwrap(),
expected_frames,
"bead_id=bd-3wop3.8 case=waiter_stress_total_frames_equals_submitted"
);
});
}
#[test]
fn test_group_commit_overlap_abort_discards_stale_prepared_batches() {
let queue = GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
);
let page_a = sample_page(0xE1);
let page_b = sample_page(0xE2);
let frame_refs_a = [crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_a,
db_size_if_commit: 2,
}];
let frame_refs_b = [crate::traits::WalFrameRef {
page_number: 2,
page_data: &page_b,
db_size_if_commit: 2,
}];
assert_eq!(
queue.record_prepared_batch(lane_staged_batch_for_test(20, 0, &frame_refs_a)),
1
);
assert_eq!(
queue.record_prepared_batch(lane_staged_batch_for_test(21, 0, &frame_refs_b)),
2
);
let overlapping_batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: page_a,
db_size_if_commit: 2,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 20,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: page_b,
db_size_if_commit: 2,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 21,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 10,
}),
];
assert_eq!(
conflicting_pages_across_group_commit_batches(&overlapping_batches),
vec![2],
"bead_id=bd-3wop3.1.2 case=overlap_abort_detects_same_page_batches"
);
assert_eq!(
queue.discard_prepared_batches_for_flush(&overlapping_batches),
2,
"bead_id=bd-3wop3.1.2 case=overlap_abort_discards_failed_prepared_batches"
);
assert_eq!(
queue.current_lane_backlog(0),
0,
"bead_id=bd-3wop3.1.2 case=overlap_abort_clears_stale_lane_backlog"
);
}
#[test]
fn test_parallel_wal_lane_staging_does_not_need_consolidator_lock() {
asupersync::test_utils::run_test(|| async {
let queue = GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
);
let wal_backend = new_shared_wal_backend();
let observed_lock_level = Arc::new(Mutex::new(LockLevel::Reserved));
let (
backend,
_frames,
_append_frames_calls,
_append_prepared_calls,
_prepare_lock_levels,
_append_lock_levels,
) = PreparedBatchObservedWalBackend::new(observed_lock_level);
*wal_backend.write().unwrap() = Some(Arc::new(AsyncRwLock::with_name(
"wal_backend",
Box::new(backend) as Box<dyn crate::traits::WalBackend>,
)));
let _guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batch = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x42),
db_size_if_commit: 2,
}]);
let prepared = prepare_group_commit_batch_for_lane(
&Cx::new(),
&wal_backend,
&batch,
1,
0,
queue.parallel_wal_control(),
)
.await
.unwrap();
assert!(
prepared.is_some(),
"bead_id=bd-3wop3.1.2 case=ordinary_lane_staging_avoids_consolidator_gate"
);
});
}
#[test]
fn test_parallel_wal_control_modes_preserve_commit_equivalence() {
asupersync::test_utils::run_test(|| async {
async fn run_commit(mode: ParallelWalOperatingMode) -> SharedFrames {
let vfs = MemoryVfs::new();
let path = PathBuf::from(format!("/parallel_wal_control_mode_{:?}.db", mode));
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let observed_lock_level = Arc::new(Mutex::new(LockLevel::Reserved));
let (
backend,
frames,
_append_frames_calls,
_append_prepared_calls,
_prepare_lock_levels,
_append_lock_levels,
) = PreparedBatchObservedWalBackend::new(observed_lock_level);
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
));
let page_two = PageNumber::new(2).unwrap();
let page_three = PageNumber::new(3).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_two,
StagedPage::from_bytes(&pager.pool, &sample_page(0x11)).unwrap(),
);
write_set.insert(
page_three,
StagedPage::from_bytes(&pager.pool, &sample_page(0x22)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_two, page_three],
&[],
&queue,
)
.await
.unwrap();
frames
}
let auto_frames = run_commit(ParallelWalOperatingMode::Auto).await;
let conservative_frames = run_commit(ParallelWalOperatingMode::Conservative).await;
assert_eq!(
*auto_frames.lock().unwrap(),
*conservative_frames.lock().unwrap(),
"bead_id=bd-3wop3.1.2 case=control_mode_equivalence"
);
});
}
#[test]
fn test_process_root_phase_c_wal_submission_ignores_unrelated_exact_handle_root() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let pager = SimplePager::open(
MemoryVfs::new(),
Path::new("/wal-unrelated-exact-root.db"),
PageSize::DEFAULT,
)
.await
.unwrap();
let observed_lock_level = Arc::new(Mutex::new(LockLevel::Reserved));
let (
backend,
frames,
_append_frames_calls,
_append_prepared_calls,
_prepare_lock_levels,
_append_lock_levels,
) = PreparedBatchObservedWalBackend::new(observed_lock_level);
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let (unrelated_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/wal-unrelated-exact-root-other.db"));
let unrelated_key = shared_db_file_key(&unrelated_file);
let current_key = {
let inner = pager.inner.lock().unwrap();
shared_db_file_key(&inner.db_file)
};
assert_ne!(unrelated_key, current_key);
let unrelated_root =
ProcessRootFinalizationAttempt::register_exact_handle(&queue, unrelated_key);
let page_two = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_two,
StagedPage::from_bytes(&pager.pool, &sample_page(0x73)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&pager.wal_backend,
&pager.inner,
&write_set,
&[page_two],
&[],
&queue,
)
.await
.expect("an unrelated exact-handle root must not convoy WAL submission");
assert_eq!(frames.lock().unwrap().len(), 1);
assert!(queue.has_relevant_process_root(unrelated_key));
assert!(!queue.has_relevant_process_root(current_key));
unrelated_root.release_after_terminal();
});
}
#[test]
fn test_parallel_wal_shadow_compare_divergence_falls_back_to_append_frames() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/parallel_wal_shadow_compare_fallback.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, prepare_calls, append_frames_calls, append_prepared_calls) =
ShadowCompareMismatchWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::ShadowCompare,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
));
let page_two = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_two,
StagedPage::from_bytes(&pager.pool, &sample_page(0x5A)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_two],
&[],
&queue,
)
.await
.unwrap();
assert_eq!(
*prepare_calls.lock().unwrap(),
2,
"bead_id=bd-3wop3.1.2 case=shadow_compare_divergence_rebuilds_conservative_candidate"
);
assert_eq!(
*append_frames_calls.lock().unwrap(),
1,
"bead_id=bd-3wop3.1.2 case=shadow_compare_divergence_falls_back_to_append_frames"
);
assert_eq!(
*append_prepared_calls.lock().unwrap(),
0,
"bead_id=bd-3wop3.1.2 case=shadow_compare_divergence_skips_bad_prepared_append"
);
let written = frames.lock().unwrap();
assert_eq!(written.len(), 1);
assert_eq!(written[0].0, 2);
assert_eq!(
written[0].1,
sample_page(0x5A),
"bead_id=bd-3wop3.1.2 case=shadow_compare_fallback_preserves_committed_payload"
);
});
}
#[test]
fn test_parallel_wal_auto_shadow_sampling_can_force_compare_fallback() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/parallel_wal_auto_shadow_sampled_fallback.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, prepare_calls, append_frames_calls, append_prepared_calls) =
ShadowCompareMismatchWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
shadow_compare_sampling_per_mille: Some(1_000),
..ParallelWalControlSurface::default()
},
));
let page_two = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_two,
StagedPage::from_bytes(&pager.pool, &sample_page(0x6C)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_two],
&[],
&queue,
)
.await
.unwrap();
assert_eq!(
*prepare_calls.lock().unwrap(),
2,
"bead_id=bd-3wop3.1.2 case=auto_shadow_sampling_rebuilds_conservative_candidate"
);
assert_eq!(
*append_frames_calls.lock().unwrap(),
1,
"bead_id=bd-3wop3.1.2 case=auto_shadow_sampling_falls_back_to_append_frames"
);
assert_eq!(
*append_prepared_calls.lock().unwrap(),
0,
"bead_id=bd-3wop3.1.2 case=auto_shadow_sampling_skips_bad_prepared_append"
);
let written = frames.lock().unwrap();
assert_eq!(written.len(), 1);
assert_eq!(written[0].0, 2);
assert_eq!(
written[0].1,
sample_page(0x6C),
"bead_id=bd-3wop3.1.2 case=auto_shadow_sampling_preserves_committed_payload"
);
});
}
#[test]
fn test_parallel_wal_concurrent_writers_on_disjoint_lanes_commit_successfully() {
asupersync::test_utils::run_test(|| async {
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let vfs = MemoryVfs::new();
let path = PathBuf::from("/parallel_wal_disjoint_lane_commit.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let observed_lock_level = Arc::new(Mutex::new(LockLevel::Reserved));
let (
backend,
frames,
_append_frames_calls,
_append_prepared_calls,
_prepare_lock_levels,
_append_lock_levels,
) = PreparedBatchObservedWalBackend::new(observed_lock_level);
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
));
let pool = pager.pool.clone();
let start = StdArc::new(std::sync::Barrier::new(3));
let spawn_commit = |page_number: u32, fill: u8| {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
let start = StdArc::clone(&start);
std::thread::spawn(move || {
let lane_id = queue.current_parallel_wal_lane_id();
let mut outcome = None;
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let page_no = PageNumber::new(page_number).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_no,
StagedPage::from_bytes(&pool, &sample_page(fill)).unwrap(),
);
start.wait();
outcome = Some(
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_no],
&[],
&queue,
)
.await,
);
});
(
lane_id,
outcome.expect("group-commit thread must record an outcome"),
)
})
};
let writer_a = spawn_commit(2, 0x31);
let writer_b = spawn_commit(3, 0x47);
start.wait();
let (lane_a, result_a) = writer_a.join().unwrap();
let (lane_b, result_b) = writer_b.join().unwrap();
result_a.unwrap();
result_b.unwrap();
assert_ne!(
lane_a, lane_b,
"bead_id=bd-3wop3.1.2 case=disjoint_writers_receive_distinct_lanes"
);
let written = frames.lock().unwrap();
assert_eq!(
written.len(),
2,
"bead_id=bd-3wop3.1.2 case=disjoint_writers_commit_all_frames"
);
});
}
#[test]
fn test_group_commit_promoted_epoch_uses_live_db_size_floor() {
asupersync::test_utils::run_test(|| async {
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let vfs = MemoryVfs::new();
let path = PathBuf::from("/group_commit_promoted_epoch_live_db_size_floor.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, frames, first_prepare_entered, release_first_prepare, _prepare_calls) =
BlockingFirstPrepareWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
// Force the lane-prepared path to opt out before submit, so
// the flusher's raw fallback prepare hook becomes the
// deterministic "phase is FLUSHING" synchronization point.
max_parallel_commit_bytes: Some(0),
..ParallelWalControlSurface::default()
},
));
let pool = pager.pool.clone();
let spawn_commit = |page_number: u32, fill: u8| {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
std::thread::spawn(move || {
let mut outcome = None;
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let page_no = PageNumber::new(page_number).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_no,
StagedPage::from_bytes(&pool, &sample_page(fill)).unwrap(),
);
outcome = Some(
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_no],
&[],
&queue,
)
.await,
);
});
outcome.expect("group-commit thread must record an outcome")
})
};
let writer_a = spawn_commit(5, 0x51);
if !wait_gate_timeout(&first_prepare_entered, Duration::from_secs(1)) {
signal_gate(&release_first_prepare);
let _ = writer_a.join();
panic!("bead_id={BEAD_ID} case=promoted_epoch_first_flush_reached_prepare");
}
let writer_b = spawn_commit(2, 0x22);
let deadline = Instant::now() + Duration::from_secs(1);
let mut pipelined = false;
while Instant::now() < deadline {
let observed_pipelined = {
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator.has_pipelined_batches()
};
if observed_pipelined {
pipelined = true;
break;
}
std::thread::yield_now();
}
if !pipelined {
signal_gate(&release_first_prepare);
let _ = writer_a.join();
let _ = writer_b.join();
panic!("bead_id={BEAD_ID} case=promoted_epoch_second_writer_pipelined");
}
signal_gate(&release_first_prepare);
writer_a
.join()
.expect("first writer thread should not panic")
.unwrap();
writer_b
.join()
.expect("second writer thread should not panic")
.unwrap();
let written = frames.lock().unwrap();
let commit_sizes = written
.iter()
.map(|(_, _, db_size_if_commit)| *db_size_if_commit)
.collect::<Vec<_>>();
assert_eq!(
commit_sizes,
vec![5, 5],
"bead_id={BEAD_ID} case=promoted_epoch_commit_marker_must_not_shrink_db_size"
);
assert_eq!(
inner.lock().unwrap().db_size,
5,
"bead_id={BEAD_ID} case=promoted_epoch_inner_db_size_must_not_shrink"
);
});
}
#[test]
fn test_group_commit_queue_retains_failed_epoch_for_late_waiter() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let consumer = queue.register_epoch_consumer(1);
queue.publish_failed_epoch(
1,
&FrankenError::internal("forced group commit flush failure"),
false,
);
queue.publish_completed_epoch(2, false);
let guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let err = queue.wait_for_epoch_outcome(guard, 1).unwrap_err();
let message = err.to_string();
assert!(
message.contains("epoch 1"),
"bead_id={BEAD_ID} case=group_commit_failed_epoch_late_waiter_mentions_epoch message={message}"
);
assert!(
message.contains("forced group commit flush failure"),
"bead_id={BEAD_ID} case=group_commit_failed_epoch_late_waiter_preserves_detail message={message}"
);
drop(consumer);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&1),
"bead_id={BEAD_ID} case=group_commit_failed_epoch_final_owner_reclaims"
);
}
#[test]
fn test_group_commit_queue_success_not_poisoned_by_other_failed_epoch() {
let queue = GroupCommitQueue::new(GroupCommitConfig::default());
queue.publish_failed_epoch(
1,
&FrankenError::internal("forced group commit flush failure"),
false,
);
queue.publish_completed_epoch(2, false);
let guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
queue.wait_for_epoch_outcome(guard, 2).unwrap(),
WaitForEpochOutcome::Completed
));
}
#[test]
fn test_group_commit_filling_obligation_drop_aborts_exact_target_epoch() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let (receipt, consumer) = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let receipt = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x61),
db_size_if_commit: 1,
}]))
.unwrap();
let consumer = queue.register_epoch_consumer(receipt.target_epoch);
(receipt, consumer)
};
assert_eq!(receipt.outcome, SubmitOutcome::Flusher);
drop(GroupCommitFillingObligation::new(
&queue,
receipt.target_epoch,
));
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(consolidator.phase(), ConsolidationPhase::Complete);
assert_eq!(consolidator.epoch(), receipt.target_epoch);
assert_eq!(consolidator.pending_batch_count(), 0);
drop(consolidator);
assert!(
queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&receipt.target_epoch),
"cancelled filling epoch must publish one atomic Abort outcome"
);
drop(consumer);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&receipt.target_epoch),
"cancelled filling epoch must reclaim Abort after its final owner releases"
);
}
#[test]
fn test_group_commit_durable_signal_survives_future_drop_before_await_returns() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x62),
db_size_if_commit: 1,
}]))
.unwrap();
let _ = consolidator.begin_flush().unwrap();
consolidator.epoch()
};
let obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
obligation
.durable_io_signal()
.store(true, AtomicOrdering::Release);
drop(obligation);
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(consolidator.phase(), ConsolidationPhase::Complete);
drop(consolidator);
assert!(queue.is_epoch_complete(flush_epoch));
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"durable cancellation must never be published as Abort"
);
}
#[test]
fn test_group_commit_duplicate_in_doubt_evidence_requires_every_completion() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x64),
db_size_if_commit: 1,
}]))
.unwrap();
let _ = consolidator.begin_flush().unwrap();
consolidator.epoch()
};
let completed_first = Arc::new(AtomicBool::new(true));
let completed_second = Arc::new(AtomicBool::new(false));
queue.defer_pending_epoch_resolution(
flush_epoch,
GroupCommitFlushDurability::InDoubt,
Arc::clone(&completed_first),
None,
);
queue.defer_pending_epoch_resolution(
flush_epoch,
GroupCommitFlushDurability::InDoubt,
Arc::clone(&completed_second),
None,
);
queue.resolve_pending_epoch_resolutions().unwrap();
assert_eq!(
queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.phase(),
ConsolidationPhase::Flushing,
"one true completion must not authorize an epoch with a second unresolved write"
);
assert!(queue.has_process_root_finalization_attempt());
completed_second.store(true, AtomicOrdering::Release);
queue.resolve_pending_epoch_resolutions().unwrap();
assert!(queue.is_epoch_complete(flush_epoch));
assert!(
!queue.has_process_root_finalization_attempt(),
"terminal resolution must release the single surviving process-root owner"
);
}
#[test]
fn test_group_commit_durable_obligation_does_not_complete_before_external_unlock() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x63),
db_size_if_commit: 1,
}]))
.unwrap();
let _ = consolidator.begin_flush().unwrap();
consolidator.epoch()
};
let mut obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
obligation
.external_lock_restored
.store(false, AtomicOrdering::Release);
obligation.mark_durable();
drop(obligation);
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(consolidator.phase(), ConsolidationPhase::Flushing);
drop(consolidator);
assert!(!queue.is_epoch_complete(flush_epoch));
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"an unresolved durable cleanup is neither complete nor Abort"
);
}
fn begin_pending_unlock_test_epoch(queue: &GroupCommitQueueRef) -> u64 {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x64),
db_size_if_commit: 1,
}]))
.unwrap();
let _ = consolidator.begin_flush().unwrap();
consolidator.epoch()
}
fn pending_unlock_test_db_file(
cx: &Cx,
path: &Path,
) -> (
SharedDbFile<ObservedLockFile>,
ObservedLockLevel,
ObservedUnlockTraceIds,
) {
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let observed_unlock_trace_ids = vfs.observed_unlock_trace_ids();
let flags = VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut file, _) = vfs.open(cx, Some(path), flags).unwrap();
file.lock(cx, LockLevel::Reserved).unwrap();
assert_eq!(
*observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
LockLevel::Reserved,
"test fixture must begin with an external RESERVED lock"
);
(
Arc::new(AsyncRwLock::new(file)),
observed_lock_level,
observed_unlock_trace_ids,
)
}
#[test]
fn test_group_commit_external_lock_coordination_is_exact_handle_scoped() {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let (first_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/exact-handle-first.db"));
let (second_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/exact-handle-second.db"));
let first_key = shared_db_file_key(&first_file);
let second_key = shared_db_file_key(&second_file);
assert_ne!(
first_key, second_key,
"distinct shared file handles require distinct coordination keys"
);
let first_physical = GroupCommitPhysicalLockWindow::register(&queue, first_key).unwrap();
assert!(
GroupCommitPhysicalLockWindow::try_register(&queue, first_key).is_none(),
"a second physical owner on the same handle must be rejected"
);
assert!(
GroupCommitLogicalExitClaim::try_register(&queue, first_key).is_none(),
"a logical transition on the same handle must wait for physical restoration"
);
let second_logical = GroupCommitLogicalExitClaim::try_register(&queue, second_key)
.expect("a distinct handle must remain logically admissible");
assert!(
GroupCommitPhysicalLockWindow::try_register(&queue, second_key).is_none(),
"the distinct handle's own logical owner must still exclude its physical owner"
);
drop(second_logical);
let second_physical = GroupCommitPhysicalLockWindow::register(&queue, second_key).unwrap();
drop(second_physical);
drop(first_physical);
let first_logical = GroupCommitLogicalExitClaim::try_register(&queue, first_key)
.expect("terminal physical restoration must release the exact handle");
drop(first_logical);
let coordination = queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(coordination.physical_lock_windows.is_empty());
assert!(coordination.logical_exit_in_flight.is_empty());
}
#[test]
fn test_queued_physical_restoration_pins_exact_file_handle_until_terminal() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("queued physical lifetime test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
let (db_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/queued-handle-lifetime.db"));
let weak_file = Arc::downgrade(&db_file);
let held_file = db_file
.try_write()
.expect("test must contend the exact shared file handle");
let flush_obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
let db_lock_obligation = GroupCommitDbLockObligation::new(
&queue,
flush_epoch,
&db_file,
&cx,
LockLevel::Shared,
flush_obligation.durability_started_signal(),
flush_obligation.durable_io_signal(),
flush_obligation.external_lock_state(),
GroupCommitPhysicalLockWindow::register(&queue, shared_db_file_key(&db_file))
.unwrap(),
);
drop(db_lock_obligation);
drop(flush_obligation);
assert!(
weak_file.upgrade().is_some(),
"queued restoration must retain the exact file handle"
);
drop(held_file);
drop(db_file);
assert!(
weak_file.upgrade().is_some(),
"the queue record must be the final strong owner before restoration"
);
assert!(queue.resolve_one_pending_external_unlock().await.unwrap());
assert!(
weak_file.upgrade().is_none(),
"terminal restoration must release the queue's exact file-handle pin"
);
});
}
#[test]
fn test_identity_wide_external_claim_fences_every_exact_handle_lane() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
let (global_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/global-restore.db"));
let (exact_file, exact_lock_level, _) =
pending_unlock_test_db_file(&cx, Path::new("/exact-restore.db"));
let held_global_file = global_file
.try_write()
.expect("test must initially contend the identity-wide restoration");
let flush_obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
let global_obligation = GroupCommitDbLockObligation::new(
&queue,
flush_epoch,
&global_file,
&cx,
LockLevel::Shared,
flush_obligation.durability_started_signal(),
flush_obligation.durable_io_signal(),
flush_obligation.external_lock_state(),
GroupCommitPhysicalLockWindow::register(&queue, shared_db_file_key(&global_file))
.unwrap(),
);
drop(global_obligation);
drop(flush_obligation);
let exact_key = shared_db_file_key(&exact_file);
queue.enqueue_pending_external_unlock(PendingExternalUnlock {
sequence: None,
scope: ProcessRootFinalizationScope::ExactHandle(exact_key),
epoch: None,
durability_started: Arc::new(AtomicBool::new(false)),
durable_io_completed: Arc::new(AtomicBool::new(false)),
coordination_owner: Some(GroupCommitExternalLockOwner::Physical(
GroupCommitPhysicalLockWindow::register(&queue, exact_key).unwrap(),
)),
recovery: None,
root_attempt: None,
operation: Box::new(SharedDbPendingExternalUnlock {
db_file: Arc::clone(&exact_file),
cleanup_cx: cleanup_child_cx(&cx),
restore_target: PendingExternalUnlockTarget::LockLevel(LockLevel::Shared),
restored: Arc::new(AtomicBool::new(false)),
}),
});
let global_claim = queue
.claim_pending_external_unlock_for(ProcessRootFinalizationSelector::IdentityWide)
.expect("identity-wide restoration must be claimable");
assert_eq!(
queue
.identity_wide_external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire),
1
);
assert!(
queue
.claim_pending_external_unlock_for(
ProcessRootFinalizationSelector::ExactHandle(exact_key),
)
.is_none(),
"an in-flight identity-wide restoration must fence every exact handle"
);
drop(global_claim);
assert!(
queue
.claim_pending_external_unlock_for(
ProcessRootFinalizationSelector::ExactHandle(exact_key),
)
.is_none(),
"a queued identity-wide restoration must fence exact handles before claim"
);
drop(held_global_file);
assert!(
queue
.resolve_one_pending_external_unlock_for(
ProcessRootFinalizationSelector::IdentityWide,
)
.await
.unwrap()
);
assert!(
queue
.resolve_one_pending_external_unlock_for_handle(exact_key)
.await
.unwrap()
);
assert_eq!(*exact_lock_level.lock().unwrap(), LockLevel::Shared);
assert!(!queue.has_process_root_finalization_attempt());
});
}
#[test]
fn test_process_root_phase_c_any_claim_prioritizes_identity_scope_and_preserves_exact_fifo() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let (exact_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/any-priority-exact.db"));
let (global_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/any-priority-global.db"));
let (independent_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/any-priority-independent.db"));
let exact_key = shared_db_file_key(&exact_file);
let independent_key = shared_db_file_key(&independent_file);
let enqueue = |db_file: &SharedDbFile<ObservedLockFile>,
scope: ProcessRootFinalizationScope| {
queue.enqueue_pending_external_unlock(PendingExternalUnlock {
sequence: None,
scope,
epoch: None,
durability_started: Arc::new(AtomicBool::new(false)),
durable_io_completed: Arc::new(AtomicBool::new(false)),
coordination_owner: None,
recovery: None,
root_attempt: None,
operation: Box::new(SharedDbPendingExternalUnlock {
db_file: Arc::clone(db_file),
cleanup_cx: cleanup_child_cx(&cx),
restore_target: PendingExternalUnlockTarget::LockLevel(LockLevel::Shared),
restored: Arc::new(AtomicBool::new(false)),
}),
});
};
enqueue(
&exact_file,
ProcessRootFinalizationScope::ExactHandle(exact_key),
);
enqueue(
&independent_file,
ProcessRootFinalizationScope::ExactHandle(independent_key),
);
enqueue(&global_file, ProcessRootFinalizationScope::IdentityWide);
enqueue(
&exact_file,
ProcessRootFinalizationScope::ExactHandle(exact_key),
);
let global_sequence = {
let global_claim = queue
.claim_pending_external_unlock()
.expect("Any must claim the admitted identity-wide restoration first");
assert_eq!(
global_claim.scope,
ProcessRootFinalizationScope::IdentityWide
);
assert!(
queue
.claim_pending_external_unlock_for(
ProcessRootFinalizationSelector::ExactHandle(exact_key),
)
.is_none(),
"an in-flight identity-wide claim must fence every exact lane"
);
global_claim
.pending
.as_ref()
.and_then(|pending| pending.sequence)
.expect("queued restoration must retain its sequence")
};
let global_claim = queue
.claim_pending_external_unlock()
.expect("cancelled Any claim must requeue the identity-wide restoration");
assert_eq!(
global_claim.scope,
ProcessRootFinalizationScope::IdentityWide
);
assert_eq!(
global_claim
.pending
.as_ref()
.and_then(|pending| pending.sequence),
Some(global_sequence),
"claim cancellation must preserve the identity-wide record's sequence"
);
drop(global_claim);
assert!(
queue
.resolve_one_pending_external_unlock_for(
ProcessRootFinalizationSelector::IdentityWide,
)
.await
.unwrap(),
"the identity-wide restoration must become terminal before exact work proceeds"
);
let first_exact_sequence = {
let first_exact = queue
.claim_pending_external_unlock_for(
ProcessRootFinalizationSelector::ExactHandle(exact_key),
)
.expect("the first exact restoration must be claimable");
let sequence = first_exact
.pending
.as_ref()
.and_then(|pending| pending.sequence)
.expect("the first exact restoration must retain its sequence");
assert!(
queue
.claim_pending_external_unlock_for(
ProcessRootFinalizationSelector::ExactHandle(exact_key),
)
.is_none(),
"a live exact-handle claim must fence the younger record for that handle"
);
let independent = queue
.claim_pending_external_unlock()
.expect("Any must skip a leased handle and claim independent exact work");
assert_eq!(
independent.scope,
ProcessRootFinalizationScope::ExactHandle(independent_key)
);
drop(independent);
drop(first_exact);
sequence
};
let first_exact = queue
.claim_pending_external_unlock_for(ProcessRootFinalizationSelector::ExactHandle(
exact_key,
))
.expect("cancelled exact claim must return to the front of its lane");
assert_eq!(
first_exact
.pending
.as_ref()
.and_then(|pending| pending.sequence),
Some(first_exact_sequence),
"exact-handle cancellation must preserve same-lane FIFO"
);
drop(first_exact);
assert!(
queue
.resolve_one_pending_external_unlock_for_handle(exact_key)
.await
.unwrap()
);
assert!(
queue
.resolve_one_pending_external_unlock_for_handle(exact_key)
.await
.unwrap()
);
assert!(
queue
.resolve_one_pending_external_unlock_for_handle(independent_key)
.await
.unwrap()
);
assert!(!queue.has_process_root_finalization_attempt());
});
}
#[test]
fn test_process_root_phase_c_dropped_maintenance_clears_activity_before_reentry() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("dropped maintenance test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let pager = vfs
.open_file_backed_pager(Path::new("/dropped-maintenance-activity.db"))
.await
.unwrap();
let entered = Arc::new(AtomicBool::new(false));
let mut state = ();
let mut maintenance =
Box::pin(
pager.with_exclusive_maintenance(&cx, &mut state, |_, _, _, ()| {
let entered = Arc::clone(&entered);
Box::pin(std::future::poll_fn(move |_| {
entered.store(true, AtomicOrdering::Release);
std::task::Poll::<Result<()>>::Pending
}))
}),
);
std::future::poll_fn(|poll_cx| match maintenance.as_mut().poll(poll_cx) {
std::task::Poll::Pending => std::task::Poll::Ready(()),
std::task::Poll::Ready(result) => {
panic!("maintenance unexpectedly completed: {result:?}")
}
})
.await;
assert!(entered.load(AtomicOrdering::Acquire));
assert!(
pager.published_snapshot().checkpoint_active,
"a suspended whole-image operation must publish maintenance as active"
);
drop(maintenance);
assert!(
!pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.checkpoint_active,
"dropping the admitted future must clear the internal maintenance gate"
);
assert!(
!pager.published_snapshot().checkpoint_active,
"dropping the admitted future must clear the published maintenance gate"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"drop cleanup must restore the exact external lock baseline"
);
assert!(
!pager
.group_commit_queue
.has_process_root_finalization_attempt(),
"uncontended drop cleanup must not leave a process root"
);
pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| Box::pin(async { Ok(()) }))
.await
.expect("a later maintenance entrant must not remain fenced");
});
}
#[test]
fn test_dropped_pending_recovery_upgrade_retains_transaction_lease_receipt() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("pending recovery-upgrade test runtime should build");
runtime.block_on(async {
const BEAD: &str = "bd-6xjma";
let gate = Arc::new(PagerMaintenanceGate::default());
let mut lease = gate
.enter_transaction()
.expect("fixture must begin with one transaction lease");
let entered = Arc::new(AtomicBool::new(false));
let pending_entered = Arc::clone(&entered);
let mut recovery = Box::pin(async {
let prior = lease.upgrade_to_exclusive(None)?;
std::future::poll_fn(move |_| {
pending_entered.store(true, AtomicOrdering::Release);
std::task::Poll::<()>::Pending
})
.await;
lease.downgrade_from_exclusive(prior)?;
Result::<()>::Ok(())
});
std::future::poll_fn(|poll_cx| match recovery.as_mut().poll(poll_cx) {
std::task::Poll::Pending if entered.load(AtomicOrdering::Acquire) => {
std::task::Poll::Ready(())
}
std::task::Poll::Pending => {
poll_cx.waker().wake_by_ref();
std::task::Poll::Pending
}
std::task::Poll::Ready(result) => {
panic!("pending recovery upgrade unexpectedly completed: {result:?}")
}
})
.await;
drop(recovery);
assert!(matches!(lease.kind, PagerMaintenanceLeaseKind::Exclusive));
assert!(matches!(
lease.exclusive_upgrade_prior,
Some(PagerMaintenanceLeaseKind::Transaction)
));
{
let state = gate
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
state.maintenance_active,
"bead_id={BEAD} case=dropped_recovery_fails_closed"
);
assert_eq!(
state.active_openers, 0,
"bead_id={BEAD} case=upgrade_removes_open_admission"
);
assert_eq!(
state.active_transactions, 0,
"bead_id={BEAD} case=upgrade_removes_transaction_admission"
);
}
assert!(matches!(gate.enter_transaction(), Err(FrankenError::Busy)));
let prior = lease
.upgrade_to_exclusive(None)
.expect("same persistent lease must be able to retry recovery");
assert!(matches!(prior, PagerMaintenanceLeaseKind::Transaction));
lease
.downgrade_from_exclusive(prior)
.expect("terminal retry must restore the recorded transaction lease");
assert!(matches!(lease.kind, PagerMaintenanceLeaseKind::Transaction));
assert!(lease.exclusive_upgrade_prior.is_none());
{
let state = gate
.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
!state.maintenance_active,
"bead_id={BEAD} case=terminal_retry_reopens_gate"
);
assert_eq!(
state.active_transactions, 1,
"bead_id={BEAD} case=terminal_retry_restores_exact_prior_count"
);
}
drop(lease);
assert_eq!(
gate.state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.active_transactions,
0,
"bead_id={BEAD} case=restored_transaction_lease_drops_exactly_once"
);
});
}
#[test]
fn test_duplicate_maintenance_rejection_preserves_pending_owner_until_drop() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("duplicate pending maintenance test runtime should build");
runtime.block_on(async {
const BEAD: &str = "bd-6xjma";
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let pager = vfs
.open_file_backed_pager(Path::new("/duplicate-pending-maintenance.db"))
.await
.unwrap();
let entered = Arc::new(AtomicBool::new(false));
let mut state = ();
let mut first =
Box::pin(
pager.with_exclusive_maintenance(&cx, &mut state, |_, _, _, ()| {
let entered = Arc::clone(&entered);
Box::pin(std::future::poll_fn(move |_| {
entered.store(true, AtomicOrdering::Release);
std::task::Poll::<Result<()>>::Pending
}))
}),
);
std::future::poll_fn(|poll_cx| match first.as_mut().poll(poll_cx) {
std::task::Poll::Pending if entered.load(AtomicOrdering::Acquire) => {
std::task::Poll::Ready(())
}
std::task::Poll::Pending => {
poll_cx.waker().wake_by_ref();
std::task::Poll::Pending
}
std::task::Poll::Ready(result) => {
panic!("first maintenance unexpectedly completed: {result:?}")
}
})
.await;
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Exclusive,
"bead_id={BEAD} case=pending_owner_holds_external_exclusive"
);
assert!(pager.published_snapshot().checkpoint_active);
let duplicate = pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| Box::pin(async { Ok(()) }))
.await;
assert!(matches!(duplicate, Err(FrankenError::Busy)));
assert!(
pager.published_snapshot().checkpoint_active,
"bead_id={BEAD} case=duplicate_rejection_preserves_owner_publication"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Exclusive,
"bead_id={BEAD} case=duplicate_rejection_does_not_unlock_owner"
);
drop(first);
assert!(
!pager.published_snapshot().checkpoint_active,
"bead_id={BEAD} case=owner_drop_clears_publication"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"bead_id={BEAD} case=owner_drop_restores_external_baseline"
);
assert!(
!pager
.group_commit_queue
.has_process_root_finalization_attempt(),
"bead_id={BEAD} case=uncontended_owner_drop_needs_no_root"
);
pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| Box::pin(async { Ok(()) }))
.await
.expect("a terminal owner drop must admit later maintenance");
});
}
#[test]
fn test_exclusive_maintenance_stays_active_until_failed_restore_is_rooted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let pager = vfs
.open_file_backed_pager(Path::new("/maintenance-root-before-inactive.db"))
.await
.unwrap();
*vfs.external_restore_publication_probe
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(Arc::downgrade(&pager.published));
vfs.external_restore_failures
.store(2, AtomicOrdering::Release);
let error = pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| Box::pin(async { Ok(()) }))
.await
.expect_err("the injected external restoration must fail the operation");
assert!(
error
.to_string()
.contains("external maintenance restoration failure")
);
assert_eq!(
*vfs.external_restore_checkpoint_observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![true, true],
"both the explicit restore and Drop retry must run before inactivity is published"
);
assert!(!pager.published_snapshot().checkpoint_active);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Exclusive,
"failed restoration must retain the physical maintenance fence"
);
assert_eq!(
pager
.group_commit_queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"inactivity may be published only after the retry has a process root"
);
assert_eq!(
pager
.group_commit_queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1
);
assert!(
pager
.group_commit_queue
.resolve_one_pending_external_unlock()
.await
.unwrap()
);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
assert!(
!pager
.group_commit_queue
.has_process_root_finalization_attempt()
);
});
}
#[test]
fn test_exclusive_maintenance_acquire_failure_roots_before_publishing_inactive() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let pager = vfs
.open_file_backed_pager(Path::new("/maintenance-acquire-root-before-inactive.db"))
.await
.unwrap();
*vfs.external_restore_publication_probe
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(Arc::downgrade(&pager.published));
vfs.external_maintenance_acquire_failures
.store(1, AtomicOrdering::Release);
vfs.external_restore_failures
.store(1, AtomicOrdering::Release);
let error = pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| Box::pin(async { Ok(()) }))
.await
.expect_err("the injected partial maintenance acquisition must fail");
assert!(
error
.to_string()
.contains("external maintenance acquisition failure")
);
assert_eq!(
*vfs.external_restore_checkpoint_observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![true],
"the failed-acquisition Drop retry must run before inactivity is published"
);
assert!(!pager.published_snapshot().checkpoint_active);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::Exclusive);
assert_eq!(
pager
.group_commit_queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1
);
assert!(
pager
.group_commit_queue
.resolve_one_pending_external_unlock()
.await
.unwrap()
);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
assert!(
!pager
.group_commit_queue
.has_process_root_finalization_attempt()
);
});
}
#[test]
fn test_checkpoint_stays_active_until_failed_restore_is_rooted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let pager = vfs
.open_file_backed_pager(Path::new("/checkpoint-root-before-inactive.db"))
.await
.unwrap();
let (backend, _, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
*vfs.external_restore_publication_probe
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner) =
Some(Arc::downgrade(&pager.published));
let db_file = Arc::clone(
&pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.db_file,
);
let baseline_file_owners = Arc::strong_count(&db_file);
vfs.external_restore_failures
.store(2, AtomicOrdering::Release);
let error = pager
.checkpoint(&cx, traits::CheckpointMode::Passive)
.await
.expect_err("the injected external restoration must fail the checkpoint");
assert!(
error
.to_string()
.contains("external maintenance restoration failure")
);
assert_eq!(
*vfs.external_restore_checkpoint_observations
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![true, true],
"explicit restoration and the guard's Drop retry must precede inactivity"
);
assert!(!pager.published_snapshot().checkpoint_active);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::Exclusive);
assert_eq!(
pager
.group_commit_queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1
);
assert_eq!(
pager
.group_commit_queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1
);
assert_eq!(
Arc::strong_count(&db_file),
baseline_file_owners + 1,
"the rooted restoration must retain the exact database file"
);
assert!(
pager
.group_commit_queue
.resolve_one_pending_external_unlock()
.await
.unwrap()
);
assert_eq!(*observed_lock_level.lock().unwrap(), LockLevel::None);
assert!(
!pager
.group_commit_queue
.has_process_root_finalization_attempt()
);
assert!(
pager
.group_commit_queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty()
);
assert_eq!(Arc::strong_count(&db_file), baseline_file_owners);
});
}
async fn assert_observed_external_attempt_retry_is_rooted(
queue: &Arc<GroupCommitQueue>,
db_file: SharedDbFile<ObservedLockFile>,
expected_scope: ProcessRootFinalizationScope,
) {
let weak_file = Arc::downgrade(&db_file);
drop(db_file);
{
let pending = queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(pending.len(), 1);
assert_eq!(
pending.front().map(|record| record.scope),
Some(expected_scope)
);
}
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"failed Drop restoration must publish exactly one process root"
);
assert!(
weak_file.upgrade().is_some(),
"the rooted retry must retain the exact opened file"
);
assert!(
queue.resolve_one_pending_external_unlock().await.is_err(),
"the first structured retry must surface the second injected restoration failure"
);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"a failed structured retry must requeue the same obligation"
);
assert!(queue.has_process_root_finalization_attempt());
assert!(weak_file.upgrade().is_some());
assert!(
queue.resolve_one_pending_external_unlock().await.unwrap(),
"the retained attempt must become terminal after faults are exhausted"
);
assert!(!queue.has_process_root_finalization_attempt());
assert!(
weak_file.upgrade().is_none(),
"terminal cleanup must release the retained file owner"
);
}
fn observed_external_attempt_test_file(
cx: &Cx,
vfs: &ObservedLockVfs,
path: &Path,
) -> SharedDbFile<ObservedLockFile> {
let flags = VfsOpenFlags::CREATE | VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (file, _) = vfs.open(cx, Some(path), flags).unwrap();
Arc::new(AsyncRwLock::with_name(
"observed_external_attempt_test_file",
file,
))
}
#[test]
fn test_open_snapshot_partial_acquire_cleanup_is_rooted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::with_external_attempt_failures(1, 0, 2);
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let db_file = observed_external_attempt_test_file(
&cx,
&vfs,
Path::new("/snapshot-partial-acquire-root.db"),
);
let mut attempt = BeginExternalLockState::new(&queue, Arc::clone(&db_file), &cx);
assert!(attempt.acquire_snapshot(&cx).await.is_err());
drop(attempt);
assert_observed_external_attempt_retry_is_rooted(
&queue,
db_file,
ProcessRootFinalizationScope::IdentityWide,
)
.await;
});
}
#[test]
fn test_open_snapshot_drop_restore_failure_is_rooted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::with_external_attempt_failures(0, 0, 2);
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let db_file = observed_external_attempt_test_file(
&cx,
&vfs,
Path::new("/snapshot-drop-restore-root.db"),
);
let handle_key = shared_db_file_key(&db_file);
let mut attempt = BeginExternalLockState::new(&queue, Arc::clone(&db_file), &cx);
attempt.acquire_snapshot(&cx).await.unwrap();
drop(attempt);
assert_observed_external_attempt_retry_is_rooted(
&queue,
db_file,
ProcessRootFinalizationScope::ExactHandle(handle_key),
)
.await;
});
}
#[test]
fn test_open_recovery_maintenance_partial_acquire_cleanup_is_rooted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::with_external_attempt_failures(0, 1, 2);
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let db_file = observed_external_attempt_test_file(
&cx,
&vfs,
Path::new("/maintenance-partial-acquire-root.db"),
);
let mut attempt = BeginExternalLockState::new(&queue, Arc::clone(&db_file), &cx);
assert!(attempt.acquire_maintenance(&cx, true).await.is_err());
drop(attempt);
assert_observed_external_attempt_retry_is_rooted(
&queue,
db_file,
ProcessRootFinalizationScope::IdentityWide,
)
.await;
});
}
#[test]
fn test_open_recovery_maintenance_drop_restore_failure_is_rooted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = ObservedLockVfs::with_external_attempt_failures(0, 0, 2);
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let db_file = observed_external_attempt_test_file(
&cx,
&vfs,
Path::new("/maintenance-drop-restore-root.db"),
);
let mut attempt = BeginExternalLockState::new(&queue, Arc::clone(&db_file), &cx);
attempt.acquire_maintenance(&cx, true).await.unwrap();
drop(attempt);
assert_observed_external_attempt_retry_is_rooted(
&queue,
db_file,
ProcessRootFinalizationScope::IdentityWide,
)
.await;
});
}
struct CountingLogicalCleanup {
resolutions: Arc<AtomicUsize>,
db_file: SharedDbFile<ObservedLockFile>,
}
impl PendingGroupCommitLogicalCleanupOperation for CountingLogicalCleanup {
fn handle_key(&self) -> SharedDbFileKey {
shared_db_file_key(&self.db_file)
}
fn resolve<'a>(
&'a mut self,
_logical_exit_claim: &'a GroupCommitLogicalExitClaim,
) -> LocalPagerFuture<'a, bool> {
Box::pin(async move {
self.resolutions.fetch_add(1, AtomicOrdering::AcqRel);
Ok(true)
})
}
}
struct OrderedLogicalCleanup {
id: u8,
order: Arc<Mutex<Vec<u8>>>,
db_file: SharedDbFile<ObservedLockFile>,
}
impl PendingGroupCommitLogicalCleanupOperation for OrderedLogicalCleanup {
fn handle_key(&self) -> SharedDbFileKey {
shared_db_file_key(&self.db_file)
}
fn resolve<'a>(
&'a mut self,
_logical_exit_claim: &'a GroupCommitLogicalExitClaim,
) -> LocalPagerFuture<'a, bool> {
Box::pin(async move {
self.order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.push(self.id);
Ok(true)
})
}
}
#[test]
fn test_logical_cleanup_claim_is_single_flight_and_cancellation_preserves_fifo() {
asupersync::test_utils::run_test(|| async {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let order = Arc::new(Mutex::new(Vec::new()));
let cx = Cx::new();
let (db_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/logical-cleanup-fifo.db"));
for id in [1, 2] {
queue.enqueue_pending_logical_cleanup(PendingGroupCommitLogicalCleanup::new(
None,
Box::new(OrderedLogicalCleanup {
id,
order: Arc::clone(&order),
db_file: Arc::clone(&db_file),
}),
));
}
let first_claim = queue
.claim_pending_logical_cleanup()
.expect("front logical cleanup must be claimable");
assert!(
queue.claim_pending_logical_cleanup().is_none(),
"exactly one logical exit may be in flight"
);
drop(first_claim);
assert_eq!(
queue.pending_logical_cleanup_count(),
2,
"claim cancellation must requeue the front operation"
);
assert!(queue.resolve_one_pending_logical_cleanup().await.unwrap());
assert!(queue.resolve_one_pending_logical_cleanup().await.unwrap());
assert_eq!(
*order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![1, 2],
"cancelled front cleanup must not move behind a later exit"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"both logical process roots must release after FIFO completion"
);
});
}
#[test]
fn test_blocked_logical_cleanup_does_not_convoy_a_distinct_handle_lane() {
asupersync::test_utils::run_test(|| async {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let order = Arc::new(Mutex::new(Vec::new()));
let cx = Cx::new();
let (blocked_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/logical-lane-blocked.db"));
let (ready_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/logical-lane-ready.db"));
let blocked_key = shared_db_file_key(&blocked_file);
let blocker = GroupCommitPhysicalLockWindow::register(&queue, blocked_key)
.expect("test must own the blocked handle's physical window");
for (id, db_file) in [(1, blocked_file), (2, ready_file)] {
queue.enqueue_pending_logical_cleanup(PendingGroupCommitLogicalCleanup::new(
None,
Box::new(OrderedLogicalCleanup {
id,
order: Arc::clone(&order),
db_file,
}),
));
}
assert!(
queue.resolve_one_pending_logical_cleanup().await.unwrap(),
"a ready handle must not wait behind another handle's physical window"
);
assert_eq!(
*order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![2],
"only the unrelated ready lane may run while the first handle is blocked"
);
assert_eq!(queue.pending_logical_cleanup_count(), 1);
drop(blocker);
assert!(queue.resolve_one_pending_logical_cleanup().await.unwrap());
assert_eq!(
*order
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
vec![2, 1]
);
assert!(!queue.has_process_root_finalization_attempt());
});
}
#[test]
fn test_in_flight_epoch_resolution_claim_fences_logical_cleanup() {
asupersync::test_utils::run_test(|| async {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let cx = Cx::new();
let (db_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/epoch-logical-cleanup-fence.db"));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
queue.defer_pending_epoch_resolution(
flush_epoch,
GroupCommitFlushDurability::PreDurable,
Arc::new(AtomicBool::new(false)),
None,
);
let epoch_claim = queue
.claim_pending_epoch_resolution(flush_epoch)
.expect("deferred epoch resolution must be claimable");
assert!(
queue
.in_doubt_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"the claimant must temporarily own the only epoch record"
);
assert_eq!(
queue
.epoch_resolution_claims_in_flight
.load(AtomicOrdering::Acquire),
1,
"claim publication must remain visible while the map is empty"
);
let logical_resolutions = Arc::new(AtomicUsize::new(0));
queue.enqueue_pending_logical_cleanup(PendingGroupCommitLogicalCleanup::new(
None,
Box::new(CountingLogicalCleanup {
resolutions: Arc::clone(&logical_resolutions),
db_file: Arc::clone(&db_file),
}),
));
assert!(
!queue.resolve_one_pending_logical_cleanup().await.unwrap(),
"logical cleanup must not overtake an in-flight epoch transition"
);
assert_eq!(
logical_resolutions.load(AtomicOrdering::Acquire),
0,
"the fenced logical cleanup must remain unpolled"
);
drop(epoch_claim);
assert!(
queue
.in_doubt_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"claim cancellation must requeue the exact epoch record"
);
assert_eq!(
queue
.epoch_resolution_claims_in_flight
.load(AtomicOrdering::Acquire),
0,
"requeue must precede clearing the claim publication"
);
queue.resolve_pending_epoch_resolutions().unwrap();
assert!(
queue.resolve_one_pending_logical_cleanup().await.unwrap(),
"logical cleanup may run after the epoch transition is terminal"
);
assert_eq!(
logical_resolutions.load(AtomicOrdering::Acquire),
1,
"logical cleanup must run exactly once"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"epoch and logical roots must both release after terminal work"
);
});
}
#[test]
fn test_in_flight_external_unlock_claim_fences_logical_cleanup() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("in-flight external unlock test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
let (db_file, observed_lock_level, _) =
pending_unlock_test_db_file(&cx, Path::new("/pending-unlock-claim-fence.db"));
let held_file = db_file
.try_write()
.expect("test should hold the shared database-file handle");
let flush_obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
let db_lock_obligation = GroupCommitDbLockObligation::new(
&queue,
flush_epoch,
&db_file,
&cx,
LockLevel::Shared,
flush_obligation.durability_started_signal(),
flush_obligation.durable_io_signal(),
flush_obligation.external_lock_state(),
GroupCommitPhysicalLockWindow::register(&queue, shared_db_file_key(&db_file))
.unwrap(),
);
drop(db_lock_obligation);
drop(flush_obligation);
let physical_claim = queue
.claim_pending_external_unlock()
.expect("physical restoration must be claimable");
assert_eq!(
queue
.external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire),
1,
"claim publication must precede removal from the visible queue"
);
assert!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"the physical claimant must hold the only external-unlock record"
);
let logical_resolutions = Arc::new(AtomicUsize::new(0));
queue.enqueue_pending_logical_cleanup(PendingGroupCommitLogicalCleanup::new(
None,
Box::new(CountingLogicalCleanup {
resolutions: Arc::clone(&logical_resolutions),
db_file: Arc::clone(&db_file),
}),
));
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"physical and logical cleanup must own independent process roots"
);
assert!(
!queue.resolve_one_pending_logical_cleanup().await.unwrap(),
"logical cleanup must not overtake an in-flight physical claimant"
);
assert_eq!(
logical_resolutions.load(AtomicOrdering::Acquire),
0,
"the fenced logical operation must remain unpolled"
);
drop(physical_claim);
assert_eq!(
queue
.external_unlock_claims_in_flight
.load(AtomicOrdering::Acquire),
0,
"claim cancellation must clear the in-flight publication"
);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"claim cancellation must requeue the exact physical obligation"
);
drop(held_file);
assert!(
queue.resolve_one_pending_external_unlock().await.unwrap(),
"requeued physical restoration must reach a terminal result"
);
assert_eq!(
*observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
LockLevel::Shared,
"physical restoration must restore its captured baseline"
);
assert!(
queue.resolve_one_pending_logical_cleanup().await.unwrap(),
"logical cleanup may run after physical restoration is terminal"
);
assert_eq!(
logical_resolutions.load(AtomicOrdering::Acquire),
1,
"logical cleanup must run exactly once"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"both process-root obligations must release only after terminal cleanup"
);
});
}
#[test]
fn test_contended_writer_drop_retains_state_until_exact_handle_exit() {
asupersync::test_utils::run_test(|| async {
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let observed_unlock_trace_ids = vfs.observed_unlock_trace_ids();
let pager = vfs
.open_file_backed_pager(Path::new("/contended-transaction-drop.db"))
.await
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
let cx = Cx::new();
let txn = pager
.begin(&cx, TransactionMode::Immediate)
.await
.expect("immediate writer transaction must begin");
let inner_state = Arc::clone(&txn.inner);
let db_file = {
let inner = txn
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Arc::clone(&inner.db_file)
};
observed_unlock_trace_ids.lock().unwrap().clear();
let held_file = shared_db_file_write(&db_file, &cx)
.await
.expect("test must contend the exact database-file handle");
drop(txn);
assert_eq!(
queue.pending_logical_cleanup_count(),
1,
"contended Drop must preserve its exact-handle unlock as queued work"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"the deferred transaction exit must own a process root"
);
assert!(
observed_unlock_trace_ids.lock().unwrap().is_empty(),
"Drop may not claim an unlock completed while the exact handle is contended"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::Reserved,
"the writer lock must remain held until rooted cleanup runs"
);
{
let inner = inner_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
inner.active_transactions, 1,
"contended Drop must retain its active-transaction slot"
);
assert!(
inner.writer_active,
"contended Drop must retain the writer baton until exact unlock succeeds"
);
}
drop(held_file);
assert!(
queue.resolve_one_pending_logical_cleanup().await.unwrap(),
"rooted exact-handle cleanup must finish after contention clears"
);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
1,
"the deferred transaction exit must perform exactly one unlock"
);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"the last transaction exit must release the snapshot fence"
);
{
let inner = inner_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
inner.active_transactions, 0,
"terminal detached exit must release the active-transaction slot"
);
assert!(
!inner.writer_active,
"terminal detached exit must release the writer baton"
);
}
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"terminal exact-handle cleanup must release its process root"
);
});
}
#[test]
fn test_process_root_phase_c_transaction_drop_recovers_poison_and_releases_root_terminally() {
asupersync::test_utils::run_test(|| async {
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let pager = vfs
.open_file_backed_pager(Path::new("/poisoned-transaction-drop.db"))
.await
.unwrap();
let queue = Arc::clone(&pager.group_commit_queue);
let cx = Cx::new();
let txn = pager
.begin(&cx, TransactionMode::Immediate)
.await
.expect("writer transaction must begin before poisoning its state mutex");
let inner_state = Arc::clone(&txn.inner);
let poison_result = std::panic::catch_unwind(std::panic::AssertUnwindSafe({
let inner_state = Arc::clone(&inner_state);
move || {
let _guard = inner_state.lock().unwrap();
panic!("intentional PagerInner poison for Drop recovery coverage");
}
}));
assert!(poison_result.is_err());
drop(txn);
let inner = inner_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
inner.active_transactions, 0,
"poison recovery must still release the active-transaction slot"
);
assert!(
!inner.writer_active,
"poison recovery must still release the writer baton"
);
drop(inner);
assert_eq!(
*observed_lock_level.lock().unwrap(),
LockLevel::None,
"poison recovery must restore the exact external snapshot baseline"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"the Drop root may release only after all exit state is terminal"
);
assert!(!queue.has_process_root_finalization_attempt());
});
}
#[test]
fn test_dropped_pending_unlock_claim_requeues_until_waiter_restores_lock() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("pending external unlock test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
let epoch_consumer = queue.register_epoch_consumer(flush_epoch);
let (db_file, observed_lock_level, _) =
pending_unlock_test_db_file(&cx, Path::new("/pending-unlock-requeue.db"));
let held_file = db_file
.try_write()
.expect("test should hold the shared database-file handle");
let flush_obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
let db_lock_obligation = GroupCommitDbLockObligation::new(
&queue,
flush_epoch,
&db_file,
&cx,
LockLevel::Shared,
flush_obligation.durability_started_signal(),
flush_obligation.durable_io_signal(),
flush_obligation.external_lock_state(),
GroupCommitPhysicalLockWindow::register(&queue, shared_db_file_key(&db_file))
.unwrap(),
);
drop(db_lock_obligation);
drop(flush_obligation);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"contended Drop must transfer its external unlock into the queue"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"queued cleanup must install exactly one process-root attempt"
);
let mut first_claim = Box::pin(queue.resolve_one_pending_external_unlock());
std::future::poll_fn(|poll_cx| match first_claim.as_mut().poll(poll_cx) {
std::task::Poll::Pending => std::task::Poll::Ready(()),
std::task::Poll::Ready(result) => {
panic!("contended pending unlock unexpectedly completed: {result:?}")
}
})
.await;
assert!(
queue.claim_pending_external_unlock().is_none(),
"exactly one cleanup path may own the pending unlock"
);
drop(first_claim);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"dropping the cleanup future must requeue its leased obligation"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"claim cancellation must retain the original root rather than duplicate it"
);
drop(held_file);
assert!(queue.resolve_one_pending_external_unlock().await.unwrap());
assert_eq!(
*observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
LockLevel::Shared,
"structured cleanup must downgrade the external RESERVED lock"
);
assert!(
queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"a pre-side-effect dropped flusher resolves as Abort after unlock"
);
drop(epoch_consumer);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"the final epoch consumer must reclaim the retained Abort"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"terminal restoration must release the exact root attempt"
);
});
}
#[test]
fn test_process_root_attempts_release_individually_and_unknown_release_fails_closed() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let path = PathBuf::from("/process-root-attempt-accounting.db");
queue.bind_finalization_path(&path);
let queue_weak = Arc::downgrade(&queue);
let queue_id = queue.queue_id;
let first = ProcessRootFinalizationAttempt::register(&queue);
let second = ProcessRootFinalizationAttempt::register(&queue);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"two admitted attempts require two independently owned root tokens"
);
assert!(
!release_process_root_finalization_attempt(queue_id, 0),
"an unknown token must never release a process-root queue"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
2,
"unknown release must preserve the fail-closed root count"
);
first.release_after_terminal();
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"one terminal attempt must not unpin its sibling"
);
drop(queue);
assert!(
queue_weak.upgrade().is_some(),
"the remaining attempt must retain the queue at process root"
);
second.release_after_terminal();
assert!(
queue_weak.upgrade().is_none(),
"the final terminal attempt must remove the root without an Arc cycle"
);
}
#[test]
fn test_process_root_scope_check_revalidates_after_terminal_release() {
fn exercise(scope: ProcessRootFinalizationScope, queried_handle: Option<SharedDbFileKey>) {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let attempt = ProcessRootFinalizationAttempt::register_scope(&queue, scope);
let queue_id = queue.queue_id;
let checker_queue = Arc::clone(&queue);
let (observed_tx, observed_rx) = std::sync::mpsc::sync_channel(0);
let (resume_tx, resume_rx) = std::sync::mpsc::sync_channel(0);
let checker = std::thread::spawn(move || {
checker_queue.has_process_root_scope_with_hook(queried_handle, || {
observed_tx
.send(())
.expect("scope checker reports its optimistic observation");
resume_rx
.recv()
.expect("scope checker receives permission to revalidate");
})
});
observed_rx
.recv_timeout(Duration::from_secs(5))
.expect("scope checker observes the published root");
let (released_tx, released_rx) = std::sync::mpsc::sync_channel(1);
let releaser = std::thread::spawn(move || {
attempt.release_after_terminal();
released_tx
.send(())
.expect("terminal releaser reports completion");
});
if let Err(error) = released_rx.recv_timeout(Duration::from_secs(5)) {
let _ = resume_tx.send(());
let _ = checker.join();
let _ = releaser.join();
panic!(
"terminal release blocked behind a scope checker before its registry revalidation: {error}"
);
}
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0,
"terminal release must clear the optimistic queue-local root"
);
assert!(
!process_root_finalization_registry()
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.by_queue
.contains_key(&queue_id),
"terminal release must remove the authoritative registry entry"
);
resume_tx
.send(())
.expect("resume scope checker after terminal release");
assert!(
!checker.join().expect("join scope checker"),
"a stale optimistic observation must revalidate to no live root"
);
releaser.join().expect("join terminal releaser");
}
exercise(ProcessRootFinalizationScope::IdentityWide, None);
let handle_key = SharedDbFileKey(17);
exercise(
ProcessRootFinalizationScope::ExactHandle(handle_key),
Some(handle_key),
);
}
#[test]
fn test_exact_handle_process_root_does_not_convoy_unrelated_handle_settlement() {
asupersync::test_utils::run_test(|| async {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let cx = Cx::new();
let (first_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/root-scope-first.db"));
let (second_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/root-scope-second.db"));
let first_key = shared_db_file_key(&first_file);
let second_key = shared_db_file_key(&second_file);
let root = ProcessRootFinalizationAttempt::register_exact_handle(&queue, first_key);
assert!(queue.has_relevant_process_root(first_key));
assert!(!queue.has_identity_wide_process_root());
assert!(
!queue.has_relevant_process_root(second_key),
"an exact-handle root must not become an identity-wide admission fence"
);
settle_pending_group_commit_finalization_for_handle(&queue, second_key)
.await
.expect("an unrelated handle must not wait behind exact-handle cleanup");
assert!(
matches!(
settle_pending_group_commit_finalization_for_handle(&queue, first_key).await,
Err(FrankenError::BusyRecovery)
),
"the rooted handle itself must remain fail closed"
);
root.release_after_terminal();
assert!(!queue.has_process_root_finalization_attempt());
});
}
#[test]
fn test_process_root_queue_fence_and_path_registry_publish_atomically() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let path = PathBuf::from("/process-root-publication-linearization.db");
queue.bind_finalization_path(&path);
let fence_published = Arc::new(std::sync::Barrier::new(2));
let allow_registry_publish = Arc::new(std::sync::Barrier::new(2));
let (attempt_tx, attempt_rx) = std::sync::mpsc::sync_channel(1);
let register_queue = Arc::clone(&queue);
let register_fence = Arc::clone(&fence_published);
let register_release = Arc::clone(&allow_registry_publish);
let register_thread = std::thread::spawn(move || {
let attempt = ProcessRootFinalizationAttempt::register_with_publication_hook(
®ister_queue,
|| {
register_fence.wait();
register_release.wait();
},
);
attempt_tx.send(attempt).unwrap();
});
fence_published.wait();
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1,
"the queue-local admission fence must publish before the global indexes"
);
let lookup_started = Arc::new(std::sync::Barrier::new(2));
let (lookup_tx, lookup_rx) = std::sync::mpsc::sync_channel(1);
let lookup_path = path.clone();
let lookup_barrier = Arc::clone(&lookup_started);
let lookup_thread = std::thread::spawn(move || {
lookup_barrier.wait();
let queues = process_root_finalization_queues_for_path(&lookup_path);
lookup_tx.send(queues).unwrap();
});
lookup_started.wait();
assert!(
matches!(
lookup_rx.try_recv(),
Err(std::sync::mpsc::TryRecvError::Empty)
),
"a same-path replacement lookup must block during atomic-to-registry publication"
);
allow_registry_publish.wait();
let rooted_queues = lookup_rx.recv().unwrap();
assert_eq!(rooted_queues.len(), 1);
assert!(
Arc::ptr_eq(&rooted_queues[0], &queue),
"the unblocked path gate must observe the exact rooted queue"
);
let attempt = attempt_rx.recv().unwrap();
register_thread.join().unwrap();
lookup_thread.join().unwrap();
attempt.release_after_terminal();
}
#[test]
fn test_private_memory_same_synthetic_path_does_not_share_process_root_gate() {
asupersync::test_utils::run_test(|| async {
let path = Path::new("/private-memory-process-root-isolation.db");
let first = SimplePager::open(MemoryVfs::new(), path, PageSize::DEFAULT)
.await
.unwrap();
let root = ProcessRootFinalizationAttempt::register(&first.group_commit_queue);
let second = SimplePager::open(MemoryVfs::new(), path, PageSize::DEFAULT)
.await
.expect("an unrelated private-memory identity must not inherit a path root");
assert!(
!Arc::ptr_eq(&first.group_commit_queue, &second.group_commit_queue),
"private memory databases with the same synthetic path require distinct queues"
);
root.release_after_terminal();
});
}
#[test]
fn test_same_database_entry_points_fail_closed_while_root_attempt_is_admitted() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let pager = SimplePager::open(
MemoryVfs::new(),
Path::new("/process-root-entry-gates.db"),
PageSize::DEFAULT,
)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Deferred).await.unwrap();
let rollback_root = ProcessRootFinalizationAttempt::register(&pager.group_commit_queue);
assert!(
matches!(txn.rollback(&cx).await, Err(FrankenError::BusyRecovery)),
"rollback must not pass an admitted physical finalization"
);
rollback_root.release_after_terminal();
txn.rollback(&cx).await.unwrap();
let root = ProcessRootFinalizationAttempt::register(&pager.group_commit_queue);
assert!(
matches!(
pager.begin(&cx, TransactionMode::Deferred).await,
Err(FrankenError::BusyRecovery)
),
"begin must fail closed while an admitted root has no terminal receipt"
);
assert!(
matches!(
pager.set_journal_mode(&cx, JournalMode::Wal).await,
Err(FrankenError::BusyRecovery)
),
"journal-mode transitions must settle rooted work first"
);
assert!(
matches!(
pager.checkpoint(&cx, traits::CheckpointMode::Passive).await,
Err(FrankenError::BusyRecovery)
),
"checkpoint must settle rooted work before inspecting WAL state"
);
assert!(
matches!(
pager.export_database_bytes(&cx).await,
Err(FrankenError::BusyRecovery)
),
"export must settle rooted work before reading the database image"
);
assert!(
matches!(
pager
.with_exclusive_maintenance(&cx, &mut (), |_, _, _, ()| {
Box::pin(async { Ok(()) })
})
.await,
Err(FrankenError::BusyRecovery)
),
"whole-database maintenance must settle rooted work before entering"
);
let copy_target = Path::new("/process-root-entry-gates-copy.db");
assert!(
matches!(
pager.copy_database_to(&cx, copy_target).await,
Err(FrankenError::BusyRecovery)
),
"database copy must settle rooted work before creating its target"
);
assert!(
!pager
.vfs
.access(&cx, copy_target, AccessFlags::EXISTS)
.unwrap(),
"a rejected copy must not create its target"
);
assert!(
matches!(
pager.refresh_published_snapshot(&cx).await,
Err(FrankenError::BusyRecovery)
),
"snapshot refresh must settle rooted work before inspecting durable state"
);
let (backend, _, _, _) = MockWalBackend::new();
assert!(
matches!(
pager.set_wal_backend(Box::new(backend)),
Err(FrankenError::BusyRecovery)
),
"synchronous WAL replacement must reject unresolved rooted work"
);
let (owned_backend, _, _, _) = MockWalBackend::new();
let owned_result = pager.set_wal_backend_owned(owned_backend);
assert!(
matches!(owned_result, Err((FrankenError::BusyRecovery, _))),
"ownership-preserving WAL replacement must return the backend on BusyRecovery"
);
root.release_after_terminal();
});
}
#[test]
fn test_pending_unlock_stays_queued_without_unlock_while_in_doubt() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("in-doubt pending unlock test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
let (db_file, observed_lock_level, observed_unlock_trace_ids) =
pending_unlock_test_db_file(&cx, Path::new("/pending-unlock-in-doubt.db"));
let flush_obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
let durability_started = flush_obligation.durability_started_signal();
let durable_io_completed = flush_obligation.durable_io_signal();
durability_started.store(true, AtomicOrdering::Release);
let db_lock_obligation = GroupCommitDbLockObligation::new(
&queue,
flush_epoch,
&db_file,
&cx,
LockLevel::Shared,
Arc::clone(&durability_started),
Arc::clone(&durable_io_completed),
flush_obligation.external_lock_state(),
GroupCommitPhysicalLockWindow::register(&queue, shared_db_file_key(&db_file))
.unwrap(),
);
drop(db_lock_obligation);
drop(flush_obligation);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"an in-doubt Drop must queue even when the shared handle is uncontended"
);
assert!(
!queue.try_resolve_one_pending_external_unlock().unwrap(),
"nonblocking cleanup must leave an in-doubt unlock queued"
);
assert!(
!queue.resolve_one_pending_external_unlock().await.unwrap(),
"async cleanup must leave an in-doubt unlock queued"
);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"both cleanup claim types must requeue the same unlock obligation"
);
assert!(
observed_unlock_trace_ids.lock().unwrap().is_empty(),
"no external unlock operation may run while durability is in doubt"
);
assert_eq!(
*observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
LockLevel::Reserved,
"RESERVED must remain held while the lower WAL write may still run"
);
assert!(
queue.has_unresolved_in_doubt_epoch(),
"a queued in-doubt unlock must block new pager work"
);
let consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(consolidator.phase(), ConsolidationPhase::Flushing);
drop(consolidator);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&flush_epoch),
"an in-doubt side effect must never be reinterpreted as Abort"
);
durable_io_completed.store(true, AtomicOrdering::Release);
assert!(queue.resolve_one_pending_external_unlock().await.unwrap());
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
1,
"durable completion permits exactly one external unlock operation"
);
assert_eq!(
*observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
LockLevel::Shared,
"durable cleanup may downgrade RESERVED after the lower write is terminal"
);
assert!(queue.is_epoch_complete(flush_epoch));
assert!(!queue.has_unresolved_in_doubt_epoch());
});
}
#[test]
fn test_pending_unlock_completes_durable_epoch_only_after_lock_restoration() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("durable pending unlock test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let flush_epoch = begin_pending_unlock_test_epoch(&queue);
let (db_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/pending-unlock-durable.db"));
let held_file = db_file
.try_write()
.expect("test should hold the shared database-file handle");
let flush_obligation = GroupCommitFlushObligation::new(&queue, flush_epoch);
let durability_started = flush_obligation.durability_started_signal();
let durable_io_completed = flush_obligation.durable_io_signal();
durability_started.store(true, AtomicOrdering::Release);
durable_io_completed.store(true, AtomicOrdering::Release);
let db_lock_obligation = GroupCommitDbLockObligation::new(
&queue,
flush_epoch,
&db_file,
&cx,
LockLevel::Shared,
durability_started,
durable_io_completed,
flush_obligation.external_lock_state(),
GroupCommitPhysicalLockWindow::register(&queue, shared_db_file_key(&db_file))
.unwrap(),
);
drop(db_lock_obligation);
drop(flush_obligation);
assert!(
!queue.is_epoch_complete(flush_epoch),
"durability alone must not publish before external unlock"
);
drop(held_file);
assert!(queue.resolve_one_pending_external_unlock().await.unwrap());
assert!(
queue.is_epoch_complete(flush_epoch),
"the cleanup claimant publishes a durable epoch after restoring the lock"
);
});
}
#[test]
fn test_group_commit_queue_publish_synchronizes_with_waiter_mutex() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let (started_tx, started_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let publish_queue = Arc::clone(&queue);
let handle = std::thread::spawn(move || {
started_tx
.send(())
.expect("publisher thread should signal start");
publish_queue.publish_completed_epoch(1, false);
done_tx
.send(())
.expect("publisher thread should signal completion");
});
started_rx
.recv_timeout(std::time::Duration::from_secs(1))
.expect("publisher thread should start while waiter holds the mutex");
assert!(
done_rx
.recv_timeout(std::time::Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=group_commit_publish_must_block_behind_waiter_mutex"
);
drop(guard);
done_rx
.recv_timeout(std::time::Duration::from_secs(1))
.expect("publisher thread should complete once the waiter mutex is released");
handle.join().unwrap();
let guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(matches!(
queue.wait_for_epoch_outcome(guard, 1).unwrap(),
WaitForEpochOutcome::Completed
));
}
#[test]
fn test_group_commit_admission_registers_consumer_before_terminal_publication() {
const BEAD: &str = "bd-vn2ea";
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let mut guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let receipt = guard
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0xD1),
db_size_if_commit: 1,
}]))
.expect("test batch should be admitted");
let consumer = queue.register_epoch_consumer(receipt.target_epoch);
let target_epoch = receipt.target_epoch;
let (started_tx, started_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let publish_queue = Arc::clone(&queue);
let handle = std::thread::spawn(move || {
started_tx
.send(())
.expect("publisher thread should signal start");
publish_queue.publish_failed_epoch(
target_epoch,
&FrankenError::internal("admission ordering failure"),
false,
);
done_tx
.send(())
.expect("publisher thread should signal completion");
});
started_rx
.recv_timeout(std::time::Duration::from_secs(1))
.expect("publisher should start while admission owns the consolidator");
assert!(
done_rx
.recv_timeout(std::time::Duration::from_millis(20))
.is_err(),
"bead_id={BEAD} case=terminal_publish_cannot_overtake_consumer_registration"
);
drop(guard);
done_rx
.recv_timeout(std::time::Duration::from_secs(1))
.expect("publisher should complete after admission releases the consolidator");
handle.join().expect("publisher thread should not panic");
assert!(
queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&target_epoch),
"bead_id={BEAD} case=terminal_failure_retained_for_admitted_consumer"
);
drop(consumer);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&target_epoch),
"bead_id={BEAD} case=admitted_consumer_final_drop_reclaims_failure"
);
}
#[test]
fn test_group_commit_queue_waiter_takes_over_promoted_epoch_vacancy() {
let queue = GroupCommitQueue::new(GroupCommitConfig::default());
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batch1 = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x01),
db_size_if_commit: 1,
}]);
let receipt = consolidator.submit_batch(batch1).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Flusher);
assert_eq!(receipt.target_epoch, 1);
let _ = consolidator.begin_flush().unwrap();
let pipelined_batch = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x02),
db_size_if_commit: 2,
}]);
let receipt = consolidator.submit_batch(pipelined_batch).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Waiter);
assert_eq!(receipt.target_epoch, 2);
}
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(consolidator.complete_flush().unwrap());
}
queue.publish_completed_epoch(1, true);
let guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let outcome = queue.wait_for_epoch_outcome(guard, 2).unwrap();
let WaitForEpochOutcome::TakeOverFlusher {
flush_epoch,
batches,
} = outcome
else {
panic!("promoted epoch waiter should take over the flusher vacancy");
};
assert_eq!(flush_epoch, 2);
assert_eq!(batches.len(), 1);
}
#[test]
fn test_process_root_phase_c_async_epoch_takeover_precedes_exact_logical_cleanup() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let (db_file, _, _) =
pending_unlock_test_db_file(&cx, Path::new("/async-epoch-takeover.db"));
let handle_key = shared_db_file_key(&db_file);
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let first = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x71),
db_size_if_commit: 1,
}]))
.unwrap();
assert_eq!(first.outcome, SubmitOutcome::Flusher);
let _ = consolidator.begin_flush().unwrap();
let promoted = consolidator
.submit_batch(TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x72),
db_size_if_commit: 2,
}]))
.unwrap();
assert_eq!(promoted.outcome, SubmitOutcome::Waiter);
assert_eq!(promoted.target_epoch, 2);
assert!(consolidator.complete_flush().unwrap());
}
queue.publish_completed_epoch(1, true);
let physical_owner = GroupCommitPhysicalLockWindow::register(&queue, handle_key)
.expect("test must hold the exact handle's physical lane");
let resolutions = Arc::new(AtomicUsize::new(0));
queue.enqueue_pending_logical_cleanup(PendingGroupCommitLogicalCleanup::new(
None,
Box::new(CountingLogicalCleanup {
resolutions: Arc::clone(&resolutions),
db_file,
}),
));
let outcome = queue
.wait_for_epoch_outcome_async(&cx, 2)
.await
.expect("a promoted waiter must not depend on its exact logical cleanup");
let WaitForEpochOutcome::TakeOverFlusher {
flush_epoch,
batches,
} = outcome
else {
panic!("the production async waiter must take over the promoted epoch");
};
assert_eq!(flush_epoch, 2);
assert_eq!(batches.len(), 1);
assert_eq!(
resolutions.load(AtomicOrdering::Acquire),
0,
"epoch observation must not poll exact logical cleanup first"
);
drop(physical_owner);
assert!(queue.resolve_one_pending_logical_cleanup().await.unwrap());
assert_eq!(resolutions.load(AtomicOrdering::Acquire), 1);
});
}
#[test]
fn test_keyed_wait_registry_signals_only_target_key() {
let registry = Arc::new(KeyedWaitRegistry::new());
let slot_a = registry.slot(11);
let slot_b = registry.slot(17);
let generation_a = slot_a.generation();
let generation_b = slot_b.generation();
let (done_a_tx, done_a_rx) = std::sync::mpsc::channel();
let (done_b_tx, done_b_rx) = std::sync::mpsc::channel();
let waiter_a = std::thread::spawn(move || {
done_a_tx
.send(slot_a.wait_for_change(generation_a, Duration::from_secs(1)))
.expect("waiter A should report");
});
let waiter_b = std::thread::spawn(move || {
done_b_tx
.send(slot_b.wait_for_change(generation_b, Duration::from_secs(1)))
.expect("waiter B should report");
});
assert!(registry.signal(11));
assert_eq!(
done_a_rx
.recv_timeout(Duration::from_millis(100))
.expect("targeted waiter should wake"),
KeyedWaitResult::Signaled,
"bead_id={BEAD_ID} case=keyed_wait_registry_wakes_target_key"
);
assert!(
done_b_rx.recv_timeout(Duration::from_millis(20)).is_err(),
"bead_id={BEAD_ID} case=keyed_wait_registry_does_not_herd_wake_other_keys"
);
assert!(registry.signal(17));
assert_eq!(
done_b_rx
.recv_timeout(Duration::from_millis(100))
.expect("second waiter should wake once signaled"),
KeyedWaitResult::Signaled,
"bead_id={BEAD_ID} case=keyed_wait_registry_wakes_second_key"
);
waiter_a.join().unwrap();
waiter_b.join().unwrap();
}
#[test]
fn test_keyed_wait_slot_returns_signaled_after_generation_advance() {
let slot = KeyedWaitSlot::default();
let observed_generation = slot.generation();
slot.signal();
assert_eq!(
slot.wait_for_change(observed_generation, Duration::from_millis(1)),
KeyedWaitResult::Signaled,
"bead_id={BEAD_ID} case=keyed_wait_slot_pre_signaled_generation_must_not_timeout"
);
}
#[test]
fn test_epoch_wake_reason_classification_is_mutually_exclusive() {
assert_eq!(
completed_epoch_wake_reason(None),
EpochWakeReason::Notify,
"a completion visible before parking retains the normal completion bucket"
);
assert_eq!(
completed_epoch_wake_reason(Some(KeyedWaitResult::Signaled)),
EpochWakeReason::Notify,
"a directly delivered completion records exactly one notify wake"
);
assert_eq!(
completed_epoch_wake_reason(Some(KeyedWaitResult::RecoveredAfterTimeout)),
EpochWakeReason::Timeout,
"a dropped delivery recovered by generation recheck records timeout, not notify"
);
assert_eq!(
completed_epoch_wake_reason(Some(KeyedWaitResult::TimedOut)),
EpochWakeReason::Timeout,
"a completion discovered after an ordinary timeout records exactly one timeout wake"
);
assert_eq!(
nonterminal_epoch_wake_reason(KeyedWaitResult::Signaled),
EpochWakeReason::BusyRetry,
"a direct wake without a terminal outcome is a busy retry"
);
assert_eq!(
nonterminal_epoch_wake_reason(KeyedWaitResult::RecoveredAfterTimeout),
EpochWakeReason::Timeout,
"a nonterminal generation recovery remains a timeout wake"
);
}
#[test]
fn test_keyed_wait_slot_async_rechecks_generation_after_dropped_notify_race() {
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("async keyed-wait test runtime should build");
runtime.block_on(async {
let slot = KeyedWaitSlot::default();
let observed_generation = slot.generation();
// Inject the publication after the async waiter's fast-path
// generation check, but deliberately omit Notify. The bounded
// fallback must recheck the eventcount and classify this as a
// signal rather than lose the wake forever.
slot.arm_drop_next_async_notify();
assert_eq!(
slot.wait_for_change_async(observed_generation).await,
KeyedWaitResult::RecoveredAfterTimeout,
"bead_id={BEAD_ID} case=async_keyed_waiter_recovers_dropped_notify_race"
);
assert_ne!(slot.generation(), observed_generation);
assert_eq!(
slot.timeout_recovery_count(),
1,
"dropped async delivery must be classified as one timeout recovery"
);
assert_eq!(slot.active_async_waiter_count(), 0);
});
}
#[test]
fn test_keyed_wait_registry_prunes_stale_slots_after_last_waiter_drops() {
let registry = KeyedWaitRegistry::new();
{
let _slot = registry.slot(23);
assert!(
registry.has_slot(23),
"bead_id={BEAD_ID} case=keyed_wait_registry_live_slot_visible_while_held"
);
}
assert!(
!registry.signal(23),
"bead_id={BEAD_ID} case=keyed_wait_registry_stale_slot_must_not_report_signal"
);
assert!(
!registry.has_slot(23),
"bead_id={BEAD_ID} case=keyed_wait_registry_prunes_stale_slot_after_signal_attempt"
);
}
#[test]
fn test_group_commit_queue_promoted_waiter_wakes_on_targeted_publish() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batch1 = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x10),
db_size_if_commit: 1,
}]);
let receipt = consolidator.submit_batch(batch1).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Flusher);
assert_eq!(receipt.target_epoch, 1);
let _ = consolidator.begin_flush().unwrap();
let pipelined_batch = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x20),
db_size_if_commit: 2,
}]);
let receipt = consolidator.submit_batch(pipelined_batch).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Waiter);
assert_eq!(receipt.target_epoch, 2);
}
let waiter_queue = Arc::clone(&queue);
let (ready_tx, ready_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let waiter = std::thread::spawn(move || {
let guard = waiter_queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
// Pre-register the targeted slot before publishing readiness so
// this test isolates the targeted wake behavior instead of racing
// the scheduler on when the waiter first touches the registry.
let _registered_slot = waiter_queue.epoch_waiters.slot(2);
ready_tx.send(()).expect("waiter should signal readiness");
let outcome = waiter_queue.wait_for_epoch_outcome(guard, 2);
done_tx.send(outcome).expect("waiter should report outcome");
});
ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("waiter should start");
assert!(
done_rx.recv_timeout(Duration::from_millis(20)).is_err(),
"bead_id={BEAD_ID} case=group_commit_promoted_waiter_stays_parked_until_targeted_publish"
);
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(consolidator.complete_flush().unwrap());
}
queue.publish_completed_epoch(1, true);
let outcome = done_rx
.recv_timeout(Duration::from_millis(100))
.expect("targeted publish should wake the promoted waiter")
.expect("waiter should succeed");
let WaitForEpochOutcome::TakeOverFlusher {
flush_epoch,
batches,
} = outcome
else {
panic!("promoted epoch waiter should take over after targeted wake");
};
assert_eq!(flush_epoch, 2);
assert_eq!(batches.len(), 1);
waiter.join().unwrap();
}
#[test]
fn test_group_commit_queue_completed_publish_wakes_target_and_next_epoch_waiters() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batch1 = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x31),
db_size_if_commit: 1,
}]);
let receipt = consolidator.submit_batch(batch1).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Flusher);
assert_eq!(receipt.target_epoch, 1);
let _ = consolidator.begin_flush().unwrap();
let pipelined_batch = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x32),
db_size_if_commit: 2,
}]);
let receipt = consolidator.submit_batch(pipelined_batch).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Waiter);
assert_eq!(receipt.target_epoch, 2);
}
let _target_slot = queue.epoch_waiters.slot(1);
let _next_slot = queue.epoch_waiters.slot(2);
let spawn_waiter = |target_epoch: u64| {
let waiter_queue = Arc::clone(&queue);
let (ready_tx, ready_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
let guard = waiter_queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
ready_tx.send(()).expect("waiter should signal readiness");
let outcome = waiter_queue.wait_for_epoch_outcome(guard, target_epoch);
done_tx.send(outcome).expect("waiter should report outcome");
});
(handle, ready_rx, done_rx)
};
let (target_handle, target_ready_rx, target_done_rx) = spawn_waiter(1);
let (next_handle, next_ready_rx, next_done_rx) = spawn_waiter(2);
target_ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("target waiter should start");
next_ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("next-epoch waiter should start");
assert!(
target_done_rx
.recv_timeout(Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=group_commit_completed_publish_target_waiter_stays_parked_until_publish"
);
assert!(
next_done_rx
.recv_timeout(Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=group_commit_completed_publish_next_waiter_stays_parked_until_publish"
);
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
consolidator.complete_flush().unwrap(),
"bead_id={BEAD_ID} case=group_commit_completed_publish_must_promote_next_epoch"
);
}
queue.publish_completed_epoch(1, true);
let target_outcome = target_done_rx
.recv_timeout(Duration::from_millis(100))
.expect("target waiter should wake")
.expect("target waiter should succeed");
assert!(
matches!(target_outcome, WaitForEpochOutcome::Completed),
"bead_id={BEAD_ID} case=group_commit_completed_publish_target_waiter_observes_completed_epoch outcome={target_outcome:?}"
);
let next_outcome = next_done_rx
.recv_timeout(Duration::from_millis(100))
.expect("next-epoch waiter should wake")
.expect("next-epoch waiter should succeed");
let WaitForEpochOutcome::TakeOverFlusher {
flush_epoch,
batches,
} = next_outcome
else {
panic!(
"bead_id={BEAD_ID} case=group_commit_completed_publish_next_waiter_must_take_over_flusher outcome={next_outcome:?}"
);
};
assert_eq!(
flush_epoch, 2,
"bead_id={BEAD_ID} case=group_commit_completed_publish_next_waiter_flush_epoch"
);
assert_eq!(
batches.len(),
1,
"bead_id={BEAD_ID} case=group_commit_completed_publish_next_waiter_batch_count"
);
target_handle.join().unwrap();
next_handle.join().unwrap();
}
#[test]
fn test_group_commit_queue_claimed_promoted_epoch_blocks_parallel_takeover() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batch1 = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x51),
db_size_if_commit: 1,
}]);
let receipt = consolidator.submit_batch(batch1).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Flusher);
assert_eq!(receipt.target_epoch, 1);
let _ = consolidator.begin_flush().unwrap();
let pipelined_batch = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x52),
db_size_if_commit: 2,
}]);
let receipt = consolidator.submit_batch(pipelined_batch).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Waiter);
assert_eq!(receipt.target_epoch, 2);
}
let waiter_queue = Arc::clone(&queue);
let (ready_tx, ready_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let waiter = std::thread::spawn(move || {
let guard = waiter_queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let _registered_slot = waiter_queue.epoch_waiters.slot(2);
ready_tx.send(()).expect("waiter should signal readiness");
let outcome = waiter_queue.wait_for_epoch_outcome(guard, 2);
done_tx.send(outcome).expect("waiter should report outcome");
});
ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("next-epoch waiter should start");
assert!(
done_rx.recv_timeout(Duration::from_millis(20)).is_err(),
"bead_id={BEAD_ID} case=group_commit_claimed_promoted_epoch_waiter_stays_parked_before_publish"
);
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(consolidator.complete_flush().unwrap());
assert!(
consolidator.claim_flusher_vacancy(),
"bead_id={BEAD_ID} case=group_commit_claimed_promoted_epoch_original_flusher_claims"
);
}
queue.publish_completed_epoch(1, true);
assert!(
done_rx.recv_timeout(Duration::from_millis(20)).is_err(),
"bead_id={BEAD_ID} case=group_commit_claimed_promoted_epoch_waiter_must_not_take_over"
);
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batches = consolidator.begin_flush().unwrap();
assert_eq!(batches.len(), 1);
assert_eq!(consolidator.epoch(), 2);
assert!(!consolidator.complete_flush().unwrap());
}
queue.publish_completed_epoch(2, false);
let outcome = done_rx
.recv_timeout(Duration::from_millis(100))
.expect("next-epoch waiter should wake after its epoch completes")
.expect("next-epoch waiter should succeed");
assert!(
matches!(outcome, WaitForEpochOutcome::Completed),
"bead_id={BEAD_ID} case=group_commit_claimed_promoted_epoch_waiter_observes_completion outcome={outcome:?}"
);
waiter.join().unwrap();
}
#[test]
fn test_group_commit_queue_failed_publish_wakes_failed_and_next_epoch_waiters() {
let queue = Arc::new(GroupCommitQueue::new(GroupCommitConfig::default()));
let (failed_consumer, next_consumer) = {
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let batch1 = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: sample_page(0x41),
db_size_if_commit: 1,
}]);
let receipt = consolidator.submit_batch(batch1).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Flusher);
assert_eq!(receipt.target_epoch, 1);
let failed_consumer = queue.register_epoch_consumer(receipt.target_epoch);
let _ = consolidator.begin_flush().unwrap();
let pipelined_batch = TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x42),
db_size_if_commit: 2,
}]);
let receipt = consolidator.submit_batch(pipelined_batch).unwrap();
assert_eq!(receipt.outcome, SubmitOutcome::Waiter);
assert_eq!(receipt.target_epoch, 2);
let next_consumer = queue.register_epoch_consumer(receipt.target_epoch);
(failed_consumer, next_consumer)
};
let _failed_slot = queue.epoch_waiters.slot(1);
let _next_slot = queue.epoch_waiters.slot(2);
let spawn_waiter = |target_epoch: u64| {
let waiter_queue = Arc::clone(&queue);
let (ready_tx, ready_rx) = std::sync::mpsc::channel();
let (done_tx, done_rx) = std::sync::mpsc::channel();
let handle = std::thread::spawn(move || {
let guard = waiter_queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
ready_tx.send(()).expect("waiter should signal readiness");
let outcome = waiter_queue.wait_for_epoch_outcome(guard, target_epoch);
done_tx.send(outcome).expect("waiter should report outcome");
});
(handle, ready_rx, done_rx)
};
let (failed_handle, failed_ready_rx, failed_done_rx) = spawn_waiter(1);
let (next_handle, next_ready_rx, next_done_rx) = spawn_waiter(2);
failed_ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("failed-epoch waiter should start");
next_ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("next-epoch waiter should start");
assert!(
failed_done_rx
.recv_timeout(Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=group_commit_failed_publish_failed_waiter_stays_parked_until_publish"
);
assert!(
next_done_rx
.recv_timeout(Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=group_commit_failed_publish_next_waiter_stays_parked_until_publish"
);
{
let mut consolidator = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
consolidator.abort_flush().unwrap();
assert!(
consolidator.has_flusher_vacancy(),
"bead_id={BEAD_ID} case=group_commit_failed_publish_abort_must_expose_flusher_vacancy"
);
}
let failure = FrankenError::internal("forced keyed failed epoch for proof coverage");
queue.publish_failed_epoch(1, &failure, true);
let failed_error = failed_done_rx
.recv_timeout(Duration::from_millis(100))
.expect("failed-epoch waiter should wake")
.expect_err("failed-epoch waiter should surface the flush failure")
.to_string();
assert!(
failed_error.contains("epoch 1"),
"bead_id={BEAD_ID} case=group_commit_failed_publish_failed_waiter_mentions_epoch error={failed_error}"
);
assert!(
failed_error.contains("forced keyed failed epoch for proof coverage"),
"bead_id={BEAD_ID} case=group_commit_failed_publish_failed_waiter_preserves_detail error={failed_error}"
);
let next_outcome = next_done_rx
.recv_timeout(Duration::from_millis(100))
.expect("next-epoch waiter should wake")
.expect("next-epoch waiter should succeed");
let WaitForEpochOutcome::TakeOverFlusher {
flush_epoch,
batches,
} = next_outcome
else {
panic!(
"bead_id={BEAD_ID} case=group_commit_failed_publish_next_waiter_must_take_over_flusher outcome={next_outcome:?}"
);
};
assert_eq!(
flush_epoch, 2,
"bead_id={BEAD_ID} case=group_commit_failed_publish_next_waiter_flush_epoch"
);
assert_eq!(
batches.len(),
1,
"bead_id={BEAD_ID} case=group_commit_failed_publish_next_waiter_batch_count"
);
failed_handle.join().unwrap();
next_handle.join().unwrap();
drop(failed_consumer);
drop(next_consumer);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&1),
"failed epoch must be reclaimed after its final admitted owner releases"
);
}
#[test]
fn test_commit_service_policy_sparse_queue_prefers_low_latency_mode() {
let max_wait = Duration::from_micros(250);
let fill_age = Duration::from_micros(3);
let fairness_budget = default_commit_service_fairness_budget(max_wait);
let decision = decide_group_commit_arrival_wait(
Some(ArrivalWaitObservation {
pending_batch_count: 1,
should_flush_now: false,
fill_age,
}),
max_wait,
fairness_budget,
fill_age,
CommitServiceMode::Balanced,
1,
);
assert_eq!(
decision.mode,
CommitServiceMode::LowLatency,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=sparse_queue_prefers_low_latency_mode"
);
assert_eq!(
decision.wait_budget, GROUP_COMMIT_SPARSE_ARRIVAL_WAIT,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=sparse_queue_uses_low_latency_budget"
);
assert_eq!(
decision.reason, "sparse_queue",
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=sparse_queue_reason"
);
assert!(
decision.used_legacy_fallback,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=sparse_queue_marks_sparse_fallback"
);
}
#[test]
fn test_commit_service_policy_throughput_hysteresis_stays_enabled_for_bursty_backlog() {
let max_wait = Duration::from_micros(250);
let fill_age = Duration::from_micros(7);
let fairness_budget = default_commit_service_fairness_budget(max_wait);
let queue_age_p95 = Duration::from_micros(30);
let decision = decide_group_commit_arrival_wait(
Some(ArrivalWaitObservation {
pending_batch_count: 3,
should_flush_now: false,
fill_age,
}),
max_wait,
fairness_budget,
queue_age_p95,
CommitServiceMode::Throughput,
2,
);
assert_eq!(
decision.mode,
CommitServiceMode::Throughput,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=bursty_backlog_stays_in_throughput_mode"
);
assert_eq!(
decision.reason, "throughput_hysteresis",
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=bursty_backlog_hysteresis_reason"
);
assert!(
decision.wait_budget <= GROUP_COMMIT_BURST_ARRIVAL_WAIT,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=bursty_backlog_wait_budget_is_bounded"
);
assert!(
!decision.starvation_prevented,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=bursty_backlog_does_not_trip_starvation_guard"
);
}
#[test]
fn test_commit_service_policy_starvation_guard_skips_wait_under_tail_pressure() {
let max_wait = Duration::from_micros(120);
let fill_age = Duration::from_micros(40);
let fairness_budget = default_commit_service_fairness_budget(max_wait);
let decision = decide_group_commit_arrival_wait(
Some(ArrivalWaitObservation {
pending_batch_count: 2,
should_flush_now: false,
fill_age,
}),
max_wait,
fairness_budget,
Duration::from_micros(180),
CommitServiceMode::Throughput,
3,
);
assert_eq!(
decision.wait_budget,
Duration::ZERO,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=tail_pressure_zeroes_wait_budget"
);
assert_eq!(
decision.mode,
CommitServiceMode::LowLatency,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=tail_pressure_falls_back_to_low_latency"
);
assert_eq!(
decision.reason, "tail_latency_pressure",
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=tail_pressure_reason"
);
assert!(
decision.starvation_prevented,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=tail_pressure_sets_starvation_guard"
);
}
#[test]
fn test_commit_service_policy_skips_promoted_follow_on_flushes() {
let max_wait = Duration::from_micros(250);
let fairness_budget = default_commit_service_fairness_budget(max_wait);
let decision = decide_group_commit_arrival_wait(
None,
max_wait,
fairness_budget,
Duration::ZERO,
CommitServiceMode::Balanced,
4,
);
assert_eq!(
decision.wait_budget,
Duration::ZERO,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=promoted_follow_on_budget"
);
assert_eq!(
decision.reason, "promoted_follow_on",
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=promoted_follow_on_reason"
);
assert_eq!(
decision.max_wait, max_wait,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=promoted_follow_on_preserves_max_wait"
);
}
#[test]
fn test_commit_service_policy_records_wait_contract_metadata() {
let max_wait = Duration::from_micros(250);
let fill_age = Duration::from_micros(3);
let fairness_budget = default_commit_service_fairness_budget(max_wait);
let queue_age_p95 = Duration::from_micros(11);
let decision = decide_group_commit_arrival_wait(
Some(ArrivalWaitObservation {
pending_batch_count: 1,
should_flush_now: false,
fill_age,
}),
max_wait,
fairness_budget,
queue_age_p95,
CommitServiceMode::Balanced,
9,
);
assert_eq!(
decision.control_epoch, 9,
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=metadata_control_epoch"
);
assert_eq!(
decision.target_wait_ns(),
u64::try_from(GROUP_COMMIT_SPARSE_ARRIVAL_WAIT.as_nanos()).unwrap_or(u64::MAX),
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=metadata_target_wait_ns"
);
assert_eq!(
decision.max_wait_ns(),
u64::try_from(max_wait.as_nanos()).unwrap_or(u64::MAX),
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=metadata_max_wait_ns"
);
assert_eq!(
decision.fairness_budget_ns(),
u64::try_from(fairness_budget.as_nanos()).unwrap_or(u64::MAX),
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=metadata_fairness_budget_ns"
);
assert_eq!(
decision.queue_age_p95_ns(),
u64::try_from(queue_age_p95.as_nanos()).unwrap_or(u64::MAX),
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=metadata_queue_age_p95_ns"
);
assert_eq!(
decision.queue_delay_ns(),
u64::try_from(fill_age.as_nanos()).unwrap_or(u64::MAX),
"bead_id={COMMIT_SERVICE_POLICY_BEAD_ID} case=metadata_queue_delay_ns"
);
}
#[test]
fn test_commit_service_policy_logs_sparse_queue_metadata() {
asupersync::test_utils::run_test(|| async {
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
GLOBAL_CONSOLIDATION_METRICS.reset();
init_publication_test_tracing();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/commit_service_policy_sparse_queue.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let observed_lock_level = Arc::new(Mutex::new(LockLevel::Reserved));
let (
backend,
_frames,
_append_frames_calls,
_append_prepared_calls,
_prepare_lock_levels,
_append_lock_levels,
) = PreparedBatchObservedWalBackend::new(observed_lock_level);
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
..ParallelWalControlSurface::default()
},
));
let page_two = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_two,
StagedPage::from_bytes(&pager.pool, &sample_page(0x71)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_two],
&[],
&queue,
)
.await
.unwrap();
GLOBAL_CONSOLIDATION_METRICS.reset();
});
}
#[test]
fn test_commit_service_policy_logs_tail_guard_metadata() {
asupersync::test_utils::run_test(|| async {
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
GLOBAL_CONSOLIDATION_METRICS.reset();
for _ in 0..8 {
GLOBAL_CONSOLIDATION_METRICS
.record_phase_timing(0, 0, 0, true, 2_000, 0, 0, 0, 0, 0);
}
init_publication_test_tracing();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/commit_service_policy_tail_guard.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let observed_lock_level = Arc::new(Mutex::new(LockLevel::Reserved));
let (
backend,
_frames,
_append_frames_calls,
_append_prepared_calls,
_prepare_lock_levels,
_append_lock_levels,
) = PreparedBatchObservedWalBackend::new(observed_lock_level);
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface {
mode: ParallelWalOperatingMode::Auto,
lane_count_override: Some(2),
max_flush_delay_ms: Some(1),
..ParallelWalControlSurface::default()
},
));
let page_two = PageNumber::new(2).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_two,
StagedPage::from_bytes(&pager.pool, &sample_page(0x83)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_two],
&[],
&queue,
)
.await
.unwrap();
GLOBAL_CONSOLIDATION_METRICS.reset();
});
}
#[test]
fn test_physical_writer_batch_membership_preserves_submission_order() {
let batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: sample_page(0x55),
db_size_if_commit: 2,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 41,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 17,
}),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 3,
page_data: sample_page(0x66),
db_size_if_commit: 3,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 42,
lane_id: 1,
staged_frame_count: 1,
staging_elapsed_ns: 19,
}),
];
assert_eq!(
physical_writer_primary_batch_id(&batches),
41,
"bead_id=bd-1dp9.6.7.9.1 case=physical_writer_primary_batch_id_tracks_head"
);
assert_eq!(
physical_writer_batch_membership(&batches),
"41,42",
"bead_id=bd-1dp9.6.7.9.1 case=physical_writer_batch_membership_tracks_order"
);
}
#[test]
fn test_physical_writer_rollback_mode_active_tracks_disable_paths() {
assert!(
physical_writer_rollback_mode_active(ParallelWalOperatingMode::Conservative, None),
"bead_id=bd-1dp9.6.7.9.1 case=physical_writer_rollback_mode_active_operator_forced"
);
assert!(
physical_writer_rollback_mode_active(
ParallelWalOperatingMode::Auto,
Some(ParallelWalFallbackReason::LaneOverflow),
),
"bead_id=bd-1dp9.6.7.9.1 case=physical_writer_rollback_mode_active_runtime_fallback"
);
assert!(
!physical_writer_rollback_mode_active(ParallelWalOperatingMode::Auto, None),
"bead_id=bd-1dp9.6.7.9.1 case=physical_writer_rollback_mode_active_clean_auto_mode"
);
}
#[test]
#[ignore = "benchmark evidence only"]
fn wal_publish_window_shrink_benchmark_report() {
asupersync::test_utils::run_test(|| async {
let mut cases: Vec<serde_json::Value> =
Vec::with_capacity(TRACK_C_PUBLISH_WINDOW_BENCH_CASES.len());
for (scenario_id, dirty_pages) in TRACK_C_PUBLISH_WINDOW_BENCH_CASES.iter() {
let baseline_hold_samples = track_c_measure_publish_window_hold_ns(
TrackCPublishWindowMode::InlinePrepareBaseline,
*dirty_pages,
)
.await;
let candidate_hold_samples = track_c_measure_publish_window_hold_ns(
TrackCPublishWindowMode::PreparedCandidate,
*dirty_pages,
)
.await;
let baseline_stall_samples = track_c_measure_competing_writer_stall_ns(
TrackCPublishWindowMode::InlinePrepareBaseline,
*dirty_pages,
)
.await;
let candidate_stall_samples = track_c_measure_competing_writer_stall_ns(
TrackCPublishWindowMode::PreparedCandidate,
*dirty_pages,
)
.await;
let baseline_hold_summary = track_c_sample_summary(&baseline_hold_samples);
let candidate_hold_summary = track_c_sample_summary(&candidate_hold_samples);
let baseline_stall_summary = track_c_sample_summary(&baseline_stall_samples);
let candidate_stall_summary = track_c_sample_summary(&candidate_stall_samples);
let baseline_hold_median = baseline_hold_summary["median_ns"].as_u64().unwrap_or(0);
let candidate_hold_median =
candidate_hold_summary["median_ns"].as_u64().unwrap_or(0);
let baseline_stall_median =
baseline_stall_summary["median_ns"].as_u64().unwrap_or(0);
let candidate_stall_median =
candidate_stall_summary["median_ns"].as_u64().unwrap_or(0);
cases.push(json!({
"scenario_id": scenario_id,
"dirty_pages": dirty_pages,
"exclusive_window_hold_baseline": baseline_hold_summary,
"exclusive_window_hold_candidate": candidate_hold_summary,
"contending_writer_stall_baseline": baseline_stall_summary,
"contending_writer_stall_candidate": candidate_stall_summary,
"hold_reduction_ratio_median": if baseline_hold_median == 0 {
0.0
} else {
1.0 - (candidate_hold_median as f64 / baseline_hold_median as f64)
},
"stall_reduction_ratio_median": if baseline_stall_median == 0 {
0.0
} else {
1.0 - (candidate_stall_median as f64 / baseline_stall_median as f64)
},
"faster_variant_by_hold_median": if candidate_hold_median <= baseline_hold_median {
"prepared_candidate"
} else {
"inline_prepare_baseline"
},
"faster_variant_by_stall_median": if candidate_stall_median <= baseline_stall_median {
"prepared_candidate"
} else {
"inline_prepare_baseline"
},
}));
}
let report = json!({
"schema_version": "fsqlite.track_c.publish_window_benchmark.v1",
"bead_id": TRACK_C_PUBLISH_WINDOW_BENCH_BEAD_ID,
"parent_bead_id": "bd-db300.3.2",
"measured_operation": "pager_commit_wal_publish_window",
"warmup_iterations": TRACK_C_PUBLISH_WINDOW_BENCH_WARMUP_ITERS,
"measurement_iterations": TRACK_C_PUBLISH_WINDOW_BENCH_MEASURE_ITERS,
"vfs": "blocking_memory_vfs",
"baseline_variant": "inline_prepare_under_exclusive_lock",
"candidate_variant": "prepared_batch_before_exclusive_lock",
"cases": cases,
});
println!("BEGIN_BD_DB300_3_2_3_REPORT");
println!("{}", serde_json::to_string_pretty(&report).unwrap());
println!("END_BD_DB300_3_2_3_REPORT");
});
}
#[test]
#[ignore = "benchmark evidence only"]
fn wal_commit_batch_benchmark_report() {
asupersync::test_utils::run_test(|| async {
let mut cases: Vec<serde_json::Value> =
Vec::with_capacity(TRACK_C_BATCH_BENCH_CASES.len());
for (scenario_id, dirty_pages) in TRACK_C_BATCH_BENCH_CASES.iter() {
let single_samples =
track_c_measure_commit_ns(TrackCBatchMode::SingleFrame, *dirty_pages).await;
let batch_samples =
track_c_measure_commit_ns(TrackCBatchMode::Batched, *dirty_pages).await;
let single_summary = track_c_sample_summary(&single_samples);
let batch_summary = track_c_sample_summary(&batch_samples);
let single_median = single_summary["median_ns"].as_u64().unwrap_or(0);
let batch_median = batch_summary["median_ns"].as_u64().unwrap_or(0);
let single_mean = single_summary["mean_ns"].as_f64().unwrap_or(0.0);
let batch_mean = batch_summary["mean_ns"].as_f64().unwrap_or(0.0);
cases.push(json!({
"scenario_id": scenario_id,
"dirty_pages": dirty_pages,
"single_frame": single_summary,
"batch_append": batch_summary,
"speedup_vs_single_median": if batch_median == 0 {
0.0
} else {
(single_median as f64) / (batch_median as f64)
},
"speedup_vs_single_mean": if batch_mean == 0.0 {
0.0
} else {
single_mean / batch_mean
},
"faster_variant_by_median": if batch_median <= single_median {
"batch_append"
} else {
"single_frame"
},
}));
}
let report = json!({
"schema_version": "fsqlite.track_c.batch_commit_benchmark.v1",
"bead_id": TRACK_C_BATCH_BENCH_BEAD_ID,
"parent_bead_id": "bd-db300.3.1",
"measured_operation": "pager_commit_wal_path",
"warmup_iterations": TRACK_C_BATCH_BENCH_WARMUP_ITERS,
"measurement_iterations": TRACK_C_BATCH_BENCH_MEASURE_ITERS,
"vfs": "memory",
"sync_mode": "normal_noop_memory_vfs",
"baseline_variant": "single_frame_append_loop",
"candidate_variant": "transaction_wide_batch_append",
"cases": cases,
});
println!("BEGIN_BD_DB300_3_1_4_REPORT");
println!("{}", serde_json::to_string_pretty(&report).unwrap());
println!("END_BD_DB300_3_1_4_REPORT");
});
}
#[test]
#[ignore = "inventory evidence only"]
fn wal_publish_window_inventory_report() {
let outside_window = vec![
json!({
"component": "collect_wal_commit_batch",
"location": "crates/fsqlite-pager/src/pager.rs::commit_wal",
"classification": "already_outside_publish_window",
"move_candidate": "not_applicable",
"rationale": "frame ordering, commit-marker boundary, and new_db_size derivation happen before EXCLUSIVE lock acquisition",
}),
json!({
"component": "wal_adapter_frame_ref_copy",
"location": "crates/fsqlite-core/src/wal_adapter.rs::prepare_append_frames",
"classification": "pure_copy",
"move_candidate": "completed",
"rationale": "borrowed WalFrameRef values are copied into owned batch metadata before the exclusive publish window starts",
}),
json!({
"component": "wal_batch_buffer_allocation",
"location": "crates/fsqlite-wal/src/wal.rs::prepare_frame_bytes",
"classification": "allocation",
"move_candidate": "completed",
"rationale": "contiguous frame buffer allocation now happens in the prepare phase before EXCLUSIVE lock acquisition",
}),
json!({
"component": "wal_header_and_payload_copy",
"location": "crates/fsqlite-wal/src/wal.rs::prepare_frame_bytes",
"classification": "pure_copy",
"move_candidate": "completed",
"rationale": "page-number/db-size stamping, salt staging, and payload copies are fully serialized before publish time",
}),
json!({
"component": "wal_checksum_transform_precompute",
"location": "crates/fsqlite-core/src/wal_adapter.rs::prepare_append_frames",
"classification": "pure_compute",
"move_candidate": "completed",
"rationale": "per-frame checksum transforms are now derived outside the lock so publish only rebinds the live seed",
}),
json!({
"component": "wal_prelock_checksum_finalize",
"location": "crates/fsqlite-core/src/wal_adapter.rs::finalize_prepared_frames",
"classification": "pure_compute",
"move_candidate": "completed",
"rationale": "prepared batches now refresh/publish the base snapshot and stamp checksum fields before EXCLUSIVE when the live append window is still open to optimistic reuse",
}),
];
let inside_window = vec![
json!({
"component": "lock_exclusive",
"location": "crates/fsqlite-pager/src/pager.rs::commit_wal",
"classification": "serialized_boundary",
"move_candidate": "no",
"rationale": "cross-process WAL append exclusion is the start of the current publish window",
}),
json!({
"component": "wal_refresh_before_append",
"location": "crates/fsqlite-core/src/wal_adapter.rs::append_prepared_frames",
"classification": "state_refresh_read_only",
"move_candidate": "conditional",
"rationale": "prepared batches can skip the lock-held refresh when the pre-lock append-window token still matches on disk, but stale windows still require a refresh before durable append",
}),
json!({
"component": "wal_append_window_validation",
"location": "crates/fsqlite-core/src/wal_adapter.rs::append_prepared_frames",
"classification": "state_validation",
"move_candidate": "conditional",
"rationale": "the lock-held path now performs a cheap file-size/header check to decide whether the pre-lock finalized batch can be reused or whether it must fall back to refresh/re-finalize",
}),
json!({
"component": "wal_checksum_seed_rebind_fallback",
"location": "crates/fsqlite-core/src/wal_adapter.rs::append_prepared_frames",
"classification": "publish_seed_binding",
"move_candidate": "conditional",
"rationale": "checksum rebinding remains inside the publish window only for stale-window fallback cases where another writer changed the live append seed before EXCLUSIVE was acquired",
}),
json!({
"component": "wal_file_write",
"location": "crates/fsqlite-wal/src/wal.rs::append_finalized_prepared_frame_bytes",
"classification": "durable_state_transition",
"move_candidate": "no",
"rationale": "single contiguous file write is the core serialized append that must observe the authoritative WAL end",
}),
json!({
"component": "wal_state_advance_after_write",
"location": "crates/fsqlite-wal/src/wal.rs::append_finalized_prepared_frame_bytes",
"classification": "durable_state_transition",
"move_candidate": "no",
"rationale": "frame_count/running_checksum advancement must match the durable append that just occurred",
}),
json!({
"component": "fec_hook_on_frame",
"location": "crates/fsqlite-core/src/wal_adapter.rs::append_prepared_frames",
"classification": "post_append_compute",
"move_candidate": "yes",
"rationale": "FEC hook work remains explicitly non-fatal and still runs after append while the publish window is open",
}),
json!({
"component": "wal_sync",
"location": "crates/fsqlite-pager/src/pager.rs::commit_wal",
"classification": "durability_barrier",
"move_candidate": "no",
"rationale": "sync is the durability barrier for the commit and therefore part of the required serialized state transition",
}),
json!({
"component": "inner_db_size_update",
"location": "crates/fsqlite-pager/src/pager.rs::commit_wal",
"classification": "pager_state_publish",
"move_candidate": "no",
"rationale": "pager-visible db_size must only advance after the WAL append and sync succeed",
}),
];
let definitely_movable = inside_window
.iter()
.filter(|entry| entry["move_candidate"] == "yes")
.count();
let conditionally_movable = inside_window
.iter()
.filter(|entry| entry["move_candidate"] == "conditional")
.count();
let required_serialized = inside_window
.iter()
.filter(|entry| entry["move_candidate"] == "no")
.count();
let report = json!({
"schema_version": "fsqlite.track_c.publish_window_inventory.v1",
"bead_id": TRACK_C_PUBLISH_WINDOW_INVENTORY_BEAD_ID,
"parent_bead_id": "bd-db300.3.2",
"measured_operation": "pager_commit_wal_path",
"measurement_anchor_bead_id": TRACK_C_BATCH_BENCH_BEAD_ID,
"measurement_anchor_report": "fsqlite.track_c.batch_commit_benchmark.v1",
"current_window": {
"entry_function": "crates/fsqlite-pager/src/pager.rs::commit_wal",
"window_begins_at": "inner.db_file.lock(cx, LockLevel::Exclusive)?",
"window_ends_after": "inner.db_size = batch.new_db_size",
},
"outside_window": outside_window,
"inside_window": inside_window,
"summary": {
"outside_window_steps": outside_window.len(),
"inside_window_steps": inside_window.len(),
"definitely_movable_inside_window_steps": definitely_movable,
"conditionally_movable_inside_window_steps": conditionally_movable,
"required_serialized_inside_window_steps": required_serialized,
},
});
println!("BEGIN_BD_DB300_3_2_1_REPORT");
println!("{}", serde_json::to_string_pretty(&report).unwrap());
println!("END_BD_DB300_3_2_1_REPORT");
assert_eq!(
report["measured_operation"], "pager_commit_wal_path",
"bead_id={TRACK_C_PUBLISH_WINDOW_INVENTORY_BEAD_ID} case=measured_operation_anchor"
);
assert_eq!(
report["summary"]["definitely_movable_inside_window_steps"], 5,
"bead_id={TRACK_C_PUBLISH_WINDOW_INVENTORY_BEAD_ID} case=movable_step_count"
);
assert_eq!(
report["summary"]["conditionally_movable_inside_window_steps"], 1,
"bead_id={TRACK_C_PUBLISH_WINDOW_INVENTORY_BEAD_ID} case=conditional_step_count"
);
assert_eq!(
report["summary"]["required_serialized_inside_window_steps"], 5,
"bead_id={TRACK_C_PUBLISH_WINDOW_INVENTORY_BEAD_ID} case=required_step_count"
);
}
#[test]
#[ignore = "benchmark evidence only"]
fn wal_metadata_cleanup_benchmark_report() {
asupersync::test_utils::run_test(|| async {
let mut cases: Vec<serde_json::Value> =
Vec::with_capacity(TRACK_C_METADATA_BENCH_CASES.len());
for (scenario_id, interior_dirty_pages) in TRACK_C_METADATA_BENCH_CASES.iter() {
let baseline_samples = track_c_metadata_measure_commit_ns(
TrackCMetadataMode::ForcedPageOneBaseline,
*interior_dirty_pages,
)
.await;
let candidate_samples = track_c_metadata_measure_commit_ns(
TrackCMetadataMode::SemanticCleanupCandidate,
*interior_dirty_pages,
)
.await;
let baseline_summary = track_c_sample_summary(&baseline_samples);
let candidate_summary = track_c_sample_summary(&candidate_samples);
let baseline_median = baseline_summary["median_ns"].as_u64().unwrap_or(0);
let candidate_median = candidate_summary["median_ns"].as_u64().unwrap_or(0);
let baseline_mean = baseline_summary["mean_ns"].as_f64().unwrap_or(0.0);
let candidate_mean = candidate_summary["mean_ns"].as_f64().unwrap_or(0.0);
let baseline_frame_pages = track_c_metadata_capture_frame_pages(
TrackCMetadataMode::ForcedPageOneBaseline,
*interior_dirty_pages,
)
.await;
let candidate_frame_pages = track_c_metadata_capture_frame_pages(
TrackCMetadataMode::SemanticCleanupCandidate,
*interior_dirty_pages,
)
.await;
let baseline_page_one_frames = baseline_frame_pages
.iter()
.filter(|page_number| **page_number == PageNumber::ONE.get())
.count();
let candidate_page_one_frames = candidate_frame_pages
.iter()
.filter(|page_number| **page_number == PageNumber::ONE.get())
.count();
cases.push(json!({
"scenario_id": scenario_id,
"interior_dirty_pages": interior_dirty_pages,
"forced_page_one_baseline": baseline_summary,
"semantic_cleanup_candidate": candidate_summary,
"baseline_frame_pages": baseline_frame_pages,
"candidate_frame_pages": candidate_frame_pages,
"baseline_total_frames_per_commit": baseline_frame_pages.len(),
"candidate_total_frames_per_commit": candidate_frame_pages.len(),
"baseline_page_one_frames_per_commit": baseline_page_one_frames,
"candidate_page_one_frames_per_commit": candidate_page_one_frames,
"frame_count_reduction_per_commit": baseline_frame_pages.len().saturating_sub(candidate_frame_pages.len()),
"page_one_exposure_reduction_per_commit": baseline_page_one_frames.saturating_sub(candidate_page_one_frames),
"speedup_vs_baseline_median": if candidate_median == 0 {
0.0
} else {
(baseline_median as f64) / (candidate_median as f64)
},
"speedup_vs_baseline_mean": if candidate_mean == 0.0 {
0.0
} else {
baseline_mean / candidate_mean
},
"faster_variant_by_median": if candidate_median <= baseline_median {
"semantic_cleanup_candidate"
} else {
"forced_page_one_baseline"
},
}));
}
let report = json!({
"schema_version": "fsqlite.track_c.metadata_cleanup_benchmark.v1",
"bead_id": TRACK_C_METADATA_BENCH_BEAD_ID,
"parent_bead_id": "bd-db300.3.3",
"measured_operation": "wal_commit_interior_only_workload",
"warmup_iterations": TRACK_C_METADATA_BENCH_WARMUP_ITERS,
"measurement_iterations": TRACK_C_METADATA_BENCH_MEASURE_ITERS,
"vfs": "memory",
"sync_mode": "normal_noop_memory_vfs",
"baseline_variant": "forced_page_one_rewrite_every_commit",
"candidate_variant": "semantic_trigger_page_one_cleanup",
"cases": cases,
});
println!("BEGIN_BD_DB300_3_3_3_REPORT");
println!("{}", serde_json::to_string_pretty(&report).unwrap());
println!("END_BD_DB300_3_3_3_REPORT");
});
}
#[test]
fn test_wal_commit_preserves_sorted_unique_frame_order() {
asupersync::test_utils::run_test(|| async {
let (pager, frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
let p3 = txn.allocate_page(&cx).await.unwrap();
let p4 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p4, &vec![0x44; ps]).await.unwrap();
txn.write_page(&cx, p2, &vec![0x22; ps]).await.unwrap();
txn.write_page(&cx, p3, &vec![0x33; ps]).await.unwrap();
txn.write_page(&cx, p4, &vec![0x55; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let frame_pages: Vec<u32> =
frames.lock().unwrap().iter().map(|frame| frame.0).collect();
assert_eq!(
frame_pages,
vec![PageNumber::ONE.get(), p2.get(), p3.get(), p4.get()],
"bead_id={BEAD_ID} case=wal_frames_sorted_and_unique"
);
});
}
#[test]
fn test_wal_read_page_from_wal() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Write and commit via WAL.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
let data = vec![0xBB_u8; ps];
txn.write_page(&cx, p1, &data).await.unwrap();
txn.commit(&cx).await.unwrap();
// Read back in a new transaction — should find the page in WAL.
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let read_back = txn2.get_page(&cx, p1).await.unwrap();
assert_eq!(
read_back.as_ref()[0],
0xBB,
"bead_id={BEAD_ID} case=wal_read_back_from_wal"
);
});
}
#[test]
fn test_wal_no_journal_file_created() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_no_jrnl.db");
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let (backend, _frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0xFF; 4096]).await.unwrap();
txn.commit(&cx).await.unwrap();
// In WAL mode, no journal file should be created.
let journal_path = SimplePager::<MemoryVfs>::journal_path(&path);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={BEAD_ID} case=wal_no_journal_created"
);
});
}
#[test]
fn test_wal_begin_not_called_for_rejected_eager_writer() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let (backend, _frames, begin_calls, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let _writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
assert_eq!(
*begin_calls.lock().unwrap(),
1,
"bead_id={BEAD_ID} case=first_writer_initializes_wal_snapshot"
);
let err = pager
.begin(&cx, TransactionMode::Immediate)
.await
.err()
.expect("second eager writer should be rejected");
assert!(matches!(err, FrankenError::Busy));
assert_eq!(
*begin_calls.lock().unwrap(),
1,
"bead_id={BEAD_ID} case=rejected_writer_must_not_mutate_wal_state"
);
});
}
#[test]
fn test_cancelled_begin_after_reserved_retains_ownership_until_exact_restore() {
use crate::traits::{CheckpointMode, CheckpointPageWriter, CheckpointResult, WalBackend};
struct PendingBeginWalBackend {
begin_calls: Arc<AtomicUsize>,
pending_entered: Arc<AtomicBool>,
}
impl WalBackend for PendingBeginWalBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
let call_index = self.begin_calls.fetch_add(1, AtomicOrdering::AcqRel);
if call_index == 0 {
Box::pin(async { Ok(()) })
} else {
Box::pin(std::future::poll_fn(move |_| {
self.pending_entered.store(true, AtomicOrdering::Release);
std::task::Poll::Pending
}))
}
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
0
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: CheckpointMode,
_writer: &'a mut dyn CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, CheckpointResult> {
Box::pin(async move {
Ok(CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let runtime = asupersync::runtime::RuntimeBuilder::current_thread()
.blocking_threads(1, 1)
.build()
.expect("cancelled begin test runtime should build");
runtime.block_on(async {
let cx = Cx::new();
let vfs = ObservedLockVfs::new();
let observed_lock_level = vfs.observed_lock_level();
let observed_unlock_trace_ids = vfs.observed_unlock_trace_ids();
let pager = vfs
.open_file_backed_pager(Path::new("/cancelled-wal-begin.db"))
.await
.unwrap();
let begin_calls = Arc::new(AtomicUsize::new(0));
let pending_entered = Arc::new(AtomicBool::new(false));
pager
.set_wal_backend(Box::new(PendingBeginWalBackend {
begin_calls: Arc::clone(&begin_calls),
pending_entered: Arc::clone(&pending_entered),
}))
.unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(begin_calls.load(AtomicOrdering::Acquire), 1);
let reader_lock_level = *observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(reader_lock_level, LockLevel::Shared);
let queue = Arc::clone(&pager.group_commit_queue);
let inner_state = Arc::clone(&pager.inner);
let db_file = {
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
Arc::clone(&inner.db_file)
};
let handle_key = shared_db_file_key(&db_file);
observed_unlock_trace_ids.lock().unwrap().clear();
let mut begin = Box::pin(pager.begin(&cx, TransactionMode::Immediate));
std::future::poll_fn(|poll_cx| match begin.as_mut().poll(poll_cx) {
std::task::Poll::Pending if pending_entered.load(AtomicOrdering::Acquire) => {
std::task::Poll::Ready(())
}
std::task::Poll::Pending => {
poll_cx.waker().wake_by_ref();
std::task::Poll::Pending
}
std::task::Poll::Ready(Ok(_)) => panic!("WAL begin unexpectedly succeeded"),
std::task::Poll::Ready(Err(error)) => {
panic!("WAL begin unexpectedly failed: {error}")
}
})
.await;
assert!(pending_entered.load(AtomicOrdering::Acquire));
assert_eq!(begin_calls.load(AtomicOrdering::Acquire), 2);
let suspended_lock_level = *observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(
suspended_lock_level,
LockLevel::Reserved,
"the suspended eager begin must own RESERVED"
);
let held_file = db_file
.try_write()
.expect("test must contend the exact file during cancellation");
drop(begin);
assert_eq!(
queue
.pending_external_unlocks
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len(),
1,
"cancellation must root one exact-handle restoration before PagerInner is visible"
);
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
1
);
{
let coordination = queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(
coordination.logical_exit_in_flight.contains(&handle_key),
"same-handle admission must remain fenced until restoration is terminal"
);
}
{
let inner = inner_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(inner.active_transactions, 1);
assert!(
inner.writer_active,
"the writer baton must remain owned while RESERVED is stranded"
);
}
drop(held_file);
assert!(queue.resolve_one_pending_external_unlock().await.unwrap());
let restored_lock_level = *observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(restored_lock_level, LockLevel::Shared);
assert_eq!(
observed_unlock_trace_ids.lock().unwrap().len(),
1,
"the cancelled begin must perform exactly one terminal RESERVED release"
);
{
let inner = inner_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(inner.active_transactions, 1);
assert!(!inner.writer_active);
}
assert_eq!(
queue
.rooted_finalization_attempts
.load(AtomicOrdering::Acquire),
0
);
{
let coordination = queue
.external_lock_coordination
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert!(!coordination.logical_exit_in_flight.contains(&handle_key));
}
reader.rollback(&cx).await.unwrap();
let final_lock_level = *observed_lock_level
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(final_lock_level, LockLevel::None);
assert_eq!(observed_unlock_trace_ids.lock().unwrap().len(), 2);
let inner = inner_state
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
assert_eq!(inner.active_transactions, 0);
assert!(!inner.writer_active);
});
}
#[test]
fn test_wal_mode_switch_back_to_delete() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
assert_eq!(pager.journal_mode(), JournalMode::Wal);
let mode = pager
.set_journal_mode(&cx, JournalMode::Delete)
.await
.unwrap();
assert_eq!(
mode,
JournalMode::Delete,
"bead_id={BEAD_ID} case=switch_back_to_delete"
);
});
}
#[test]
fn test_wal_overwrite_page_reads_latest() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// First commit: write 0x11.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0x11; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
// Second commit: overwrite with 0x22.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.write_page(&cx, p1, &vec![0x22; ps]).await.unwrap();
txn2.commit(&cx).await.unwrap();
// Read should see 0x22 (latest WAL entry).
let txn3 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn3.get_page(&cx, p1).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=wal_latest_version"
);
});
}
#[test]
fn test_wal_interior_update_skips_page1_when_metadata_unchanged() {
asupersync::test_utils::run_test(|| async {
let (pager, frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_two
};
let frames_before = frames.lock().unwrap().len();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x22; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
let frames = frames.lock().unwrap();
let appended = &frames[frames_before..];
assert_eq!(
appended.len(),
1,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_frame_count"
);
assert_eq!(
appended[0].0,
page_two.get(),
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_page"
);
assert_eq!(
appended[0].2,
page_two.get(),
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_commit_marker_uses_existing_db_size"
);
});
}
#[test]
fn test_wal_page_one_write_plan_for_interior_update_has_no_trigger() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_two
};
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x22; ps])
.await
.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
assert_eq!(
plan,
WalPageOneWritePlan {
max_written: page_two.get(),
page_one_dirty: false,
freelist_metadata_dirty: false,
db_growth: false,
},
"bead_id={BEAD_ID} case=wal_page1_plan_interior_update"
);
assert!(
!plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=wal_page1_plan_interior_update_no_rewrite"
);
assert!(
!plan.requires_page_count_advance(),
"bead_id={BEAD_ID} case=wal_page1_plan_interior_update_no_growth"
);
});
}
#[test]
fn test_wal_page_one_write_plan_marks_page_one_dirty_trigger() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, PageNumber::ONE, &vec![0x77; ps])
.await
.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
assert_eq!(
plan,
WalPageOneWritePlan {
max_written: PageNumber::ONE.get(),
page_one_dirty: true,
freelist_metadata_dirty: false,
db_growth: false,
},
"bead_id={BEAD_ID} case=wal_page1_plan_page1_dirty"
);
assert!(
plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=wal_page1_plan_page1_dirty_requires_rewrite"
);
assert!(
!plan.requires_page_count_advance(),
"bead_id={BEAD_ID} case=wal_page1_plan_page1_dirty_no_growth"
);
});
}
#[test]
fn test_wal_page_one_write_plan_freelist_trigger_defers_page_one() {
asupersync::test_utils::run_test(|| async {
// bd-3wop3.8 (D1-CRITICAL): Verify that freelist changes do NOT trigger
// Page 1 rewrite in WAL mode. The WAL frames implicitly capture freelist
// state through the allocated/freed pages. Page 1 is reconstructed at
// checkpoint time. This eliminates MVCC conflicts from concurrent
// freelist operations (page_lease batch allocations).
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_two
};
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.free_page(&cx, page_two).await.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
// Note: freelist_metadata_dirty may be true or false depending on internal
// pager state, but the key assertion is that requires_page_one_rewrite()
// returns false for pure freelist changes (page_one_dirty is false).
assert!(
!plan.page_one_dirty,
"bead_id={BEAD_ID} case=wal_page1_plan_freelist_page1_not_dirty"
);
// D1-CRITICAL: Pure freelist changes do NOT require Page 1 rewrite
assert!(
!plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=wal_page1_plan_freelist_defers_page_one"
);
assert!(
!plan.requires_page_count_advance(),
"bead_id={BEAD_ID} case=wal_page1_plan_freelist_no_growth"
);
});
}
#[test]
fn test_wal_page_one_write_plan_marks_database_growth_trigger() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
assert_eq!(
plan,
WalPageOneWritePlan {
max_written: page_two.get(),
page_one_dirty: false,
freelist_metadata_dirty: false,
db_growth: true,
},
"bead_id={BEAD_ID} case=wal_page1_plan_db_growth"
);
// D1-CRITICAL: Pure db_growth does NOT require Page 1 rewrite in WAL mode.
// The WAL frame's db_size_if_commit captures the database size, so Page 1
// update can be deferred to checkpoint. This eliminates MVCC conflicts.
assert!(
!plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=wal_page1_plan_db_growth_skips_page_one_rewrite"
);
assert!(
plan.requires_page_count_advance(),
"bead_id={BEAD_ID} case=wal_page1_plan_db_growth_advances_count"
);
});
}
#[test]
fn test_wal_page_one_write_plan_clears_net_zero_freelist_reuse() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_two
};
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.free_page(&cx, page_two).await.unwrap();
txn.commit(&cx).await.unwrap();
}
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused = txn.allocate_page(&cx).await.unwrap();
assert_eq!(
reused, page_two,
"bead_id={BEAD_ID} case=wal_page1_plan_net_zero_reuse_reclaims_committed_freelist_page"
);
txn.free_page(&cx, reused).await.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
assert_eq!(
plan,
WalPageOneWritePlan {
max_written: 0,
page_one_dirty: false,
freelist_metadata_dirty: false,
db_growth: false,
},
"bead_id={BEAD_ID} case=wal_page1_plan_net_zero_reuse_has_no_trigger"
);
assert!(
!plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=wal_page1_plan_net_zero_reuse_skips_page_one"
);
assert!(
!txn.has_pending_writes(),
"bead_id={BEAD_ID} case=wal_page1_plan_net_zero_reuse_has_no_pending_writes"
);
assert!(
txn.pending_commit_pages().unwrap().is_empty(),
"bead_id={BEAD_ID} case=wal_page1_plan_net_zero_reuse_has_no_commit_pages"
);
});
}
#[test]
fn test_wal_net_zero_eof_allocate_free_does_not_append_frames_or_advance_seq() {
asupersync::test_utils::run_test(|| async {
let (pager, frames) = wal_pager().await;
let cx = Cx::new();
let seq_before = pager.published_snapshot().visible_commit_seq;
let frames_before = frames.lock().unwrap().len();
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.free_page(&cx, page_two).await.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
assert_eq!(
plan,
WalPageOneWritePlan {
max_written: 0,
page_one_dirty: false,
freelist_metadata_dirty: false,
db_growth: false,
},
"bead_id={BEAD_ID} case=wal_page1_plan_eof_allocate_then_free_has_no_trigger"
);
assert!(
!txn.has_pending_writes(),
"bead_id={BEAD_ID} case=wal_net_zero_eof_allocate_free_has_no_pending_writes"
);
assert!(
txn.pending_commit_pages().unwrap().is_empty(),
"bead_id={BEAD_ID} case=wal_net_zero_eof_allocate_free_has_no_commit_pages"
);
txn.commit(&cx).await.unwrap();
assert_eq!(
pager.published_snapshot().visible_commit_seq,
seq_before,
"bead_id={BEAD_ID} case=wal_net_zero_eof_allocate_free_keeps_visible_commit_seq"
);
assert_eq!(
frames.lock().unwrap().len(),
frames_before,
"bead_id={BEAD_ID} case=wal_net_zero_eof_allocate_free_appends_no_frames"
);
});
}
#[test]
fn shared_memory_pagers_commit_disjoint_concurrent_writers_without_file_locking() {
asupersync::test_utils::run_test(|| async {
let (pager1, pager2, frames) = wal_pager_pair_with_shared_backend().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let (page_two, page_three) = {
let mut seed = pager1
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
let page_three = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
seed.write_page(&cx, page_three, &vec![0x22; ps])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
(page_two, page_three)
};
let mut first = pager1
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let mut second = pager2
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
first
.write_page(&cx, page_two, &vec![0x31; ps])
.await
.unwrap();
second
.write_page(&cx, page_three, &vec![0x42; ps])
.await
.unwrap();
first.commit(&cx).await.unwrap();
second.commit(&cx).await.unwrap();
let committed = frames.lock().unwrap();
assert!(committed.iter().any(|(page, bytes, _)| {
*page == page_two.get() && bytes.first() == Some(&0x31)
}));
assert!(committed.iter().any(|(page, bytes, _)| {
*page == page_three.get() && bytes.first() == Some(&0x42)
}));
assert!(Arc::ptr_eq(
&pager1.group_commit_queue,
&pager2.group_commit_queue
));
});
}
#[test]
fn test_wal_external_refresh_tracks_headerless_interior_commit() {
asupersync::test_utils::run_test(|| async {
let (pager1, pager2, frames) = wal_pager_pair_with_shared_backend().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page_two = {
let mut txn = pager1.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_two
};
let seq_before = pager1.published_snapshot().visible_commit_seq;
let mut txn = pager1.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x22; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
let latest_seq = pager1.published_snapshot().visible_commit_seq;
assert!(
latest_seq > seq_before,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_advances_local_seq"
);
assert_eq!(
frames
.lock()
.unwrap()
.iter()
.filter(|(_, _, db_size_if_commit)| *db_size_if_commit > 0)
.count(),
2,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_emits_commit_markers"
);
{
let inner = pager2.inner.lock().unwrap();
assert_eq!(
inner.journal_mode,
JournalMode::Wal,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_follower_in_wal_mode"
);
drop(inner);
let committed_txn_count = with_wal_backend(&pager2.wal_backend, &cx, |wal, cx| {
wal.committed_txn_count(cx)
})
.await
.expect("WAL backend should stay installed");
assert_eq!(
committed_txn_count, 2,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_backend_reports_commit_count"
);
}
let reader = pager2.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
{
let inner = pager2.inner.lock().unwrap();
assert_eq!(
inner.commit_seq, latest_seq,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_refreshes_inner_commit_seq"
);
}
let refreshed = pager2.published_snapshot();
assert_eq!(
refreshed.visible_commit_seq, latest_seq,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_refreshes_visible_seq"
);
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=wal_headerless_interior_commit_refreshes_latest_page"
);
});
}
#[test]
fn test_wal_rollback_does_not_append_frames() {
asupersync::test_utils::run_test(|| async {
let (pager, frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p1 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p1, &vec![0xDD; ps]).await.unwrap();
txn.rollback(&cx).await.unwrap();
assert_eq!(
frames.lock().unwrap().len(),
0,
"bead_id={BEAD_ID} case=wal_rollback_no_frames"
);
});
}
#[test]
fn test_wal_begin_skips_committed_page1_reload_when_seq_and_file_size_hold() {
asupersync::test_utils::run_test(|| async {
let (pager, frames, _, _, read_page_calls) = wal_pager_with_read_tracking().await;
let cx = Cx::new();
let page1 = {
let inner = pager.inner.lock().unwrap();
let mut page = vec![0_u8; inner.page_size.as_usize()];
let db_file = shared_db_file_read(&inner.db_file, &cx).await.unwrap();
let bytes_read = db_file.read(&cx, &mut page, 0).await.unwrap();
assert_eq!(
bytes_read,
inner.page_size.as_usize(),
"bead_id={BEAD_ID} case=wal_begin_read_tracking_reads_page1"
);
page
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, PageNumber::ONE, &page1).await.unwrap();
txn.commit(&cx).await.unwrap();
assert!(
frames
.lock()
.unwrap()
.iter()
.any(|(page_number, _, _)| *page_number == PageNumber::ONE.get()),
"bead_id={BEAD_ID} case=wal_begin_read_tracking_commits_page1_into_wal"
);
let read_page_calls_before_begin = *read_page_calls.lock().unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let read_page_calls_after_begin = *read_page_calls.lock().unwrap();
drop(reader);
assert_eq!(
read_page_calls_after_begin, read_page_calls_before_begin,
"bead_id={BEAD_ID} case=wal_begin_skips_page1_reload_when_commit_seq_and_file_size_match"
);
});
}
#[test]
fn test_local_commit_synchronizes_durable_identity_without_self_invalidation() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = writer.allocate_page(&cx).await.unwrap();
writer.write_page(&cx, page, &vec![0xAB; ps]).await.unwrap();
writer.commit(&cx).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.committed_db_change_counter,
inner.commit_seq.get() & u64::from(u32::MAX),
"a rollback-journal commit must update the cached durable identity"
);
}
let cache_before_begin = pager.cache_metrics_snapshot().unwrap();
let published_before_begin = pager.published_snapshot();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let cache_after_begin = pager.cache_metrics_snapshot().unwrap();
let published_after_begin = pager.published_snapshot();
assert_eq!(
cache_after_begin, cache_before_begin,
"the next begin must not invalidate cache state for this pager's own commit"
);
assert_eq!(
published_after_begin.visible_commit_seq,
published_before_begin.visible_commit_seq
);
assert_eq!(
published_after_begin.db_size,
published_before_begin.db_size
);
assert_eq!(
published_after_begin.page_set_size, published_before_begin.page_set_size,
"the next begin must not clear published pages for this pager's own commit"
);
assert!(
pager.published.try_get_page(page).is_some(),
"the next begin must retain the locally committed published image"
);
assert_eq!(reader.get_page(&cx, page).await.unwrap().as_ref()[0], 0xAB);
reader.commit(&cx).await.unwrap();
});
}
// ── 5A.1: Page 1 initialization tests (bd-2yy6) ───────────────────
const BEAD_5A1: &str = "bd-2yy6";
#[test]
fn test_page1_database_header_all_fields() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
eprintln!(
"[5A1][test=page1_database_header_all_fields][step=parse] page_len={}",
raw.len()
);
let hdr_bytes: [u8; DATABASE_HEADER_SIZE] = raw[..DATABASE_HEADER_SIZE]
.try_into()
.expect("page 1 must have 100-byte header");
let hdr = DatabaseHeader::from_bytes(&hdr_bytes).expect("header must parse");
// Verify each field matches the expected new-database defaults.
assert_eq!(
hdr.page_size,
PageSize::DEFAULT,
"bead_id={BEAD_5A1} case=page_size"
);
assert_eq!(hdr.page_count, 1, "bead_id={BEAD_5A1} case=page_count");
assert_eq!(
hdr.sqlite_version, FRANKENSQLITE_SQLITE_VERSION_NUMBER,
"bead_id={BEAD_5A1} case=sqlite_version"
);
assert_eq!(
hdr.schema_format, 4,
"bead_id={BEAD_5A1} case=schema_format"
);
assert_eq!(
hdr.freelist_trunk, 0,
"bead_id={BEAD_5A1} case=freelist_trunk"
);
assert_eq!(
hdr.freelist_count, 0,
"bead_id={BEAD_5A1} case=freelist_count"
);
assert_eq!(
hdr.schema_cookie, 0,
"bead_id={BEAD_5A1} case=schema_cookie"
);
assert_eq!(
hdr.text_encoding,
fsqlite_types::TextEncoding::Utf8,
"bead_id={BEAD_5A1} case=text_encoding"
);
assert_eq!(hdr.user_version, 0, "bead_id={BEAD_5A1} case=user_version");
assert_eq!(
hdr.application_id, 0,
"bead_id={BEAD_5A1} case=application_id"
);
assert_eq!(
hdr.change_counter, 0,
"bead_id={BEAD_5A1} case=change_counter"
);
// Magic string bytes 0..16.
assert_eq!(
&raw[..16],
b"SQLite format 3\0",
"bead_id={BEAD_5A1} case=magic_string"
);
// Payload fractions at bytes 21/22/23.
assert_eq!(raw[21], 64, "bead_id={BEAD_5A1} case=max_payload_fraction");
assert_eq!(raw[22], 32, "bead_id={BEAD_5A1} case=min_payload_fraction");
assert_eq!(raw[23], 32, "bead_id={BEAD_5A1} case=leaf_payload_fraction");
eprintln!(
"[5A1][test=page1_database_header_all_fields][step=verify] \
page_size={} page_count={} schema_format={} encoding=UTF8 \u{2713}",
hdr.page_size.get(),
hdr.page_count,
hdr.schema_format
);
});
}
#[test]
fn test_page1_btree_header_is_valid_empty_leaf_table() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
let btree_hdr =
BTreePageHeader::parse(&raw, PageSize::DEFAULT, 0, true).expect("btree header");
assert_eq!(
btree_hdr.page_type,
fsqlite_types::BTreePageType::LeafTable,
"bead_id={BEAD_5A1} case=btree_page_type"
);
assert_eq!(
btree_hdr.cell_count, 0,
"bead_id={BEAD_5A1} case=btree_cell_count"
);
assert_eq!(
btree_hdr.cell_content_start,
PageSize::DEFAULT.get(),
"bead_id={BEAD_5A1} case=btree_content_start"
);
assert_eq!(
btree_hdr.first_freeblock, 0,
"bead_id={BEAD_5A1} case=btree_first_freeblock"
);
assert_eq!(
btree_hdr.fragmented_free_bytes, 0,
"bead_id={BEAD_5A1} case=btree_fragmented_free"
);
assert_eq!(
btree_hdr.header_offset, DATABASE_HEADER_SIZE,
"bead_id={BEAD_5A1} case=btree_header_offset"
);
assert!(
btree_hdr.right_most_child.is_none(),
"bead_id={BEAD_5A1} case=leaf_no_child"
);
eprintln!(
"[5A1][test=page1_btree_header][step=verify] empty_leaf_table valid \u{2713}"
);
});
}
#[test]
fn test_page1_rest_is_zeroed() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
// After the B-tree header (8 bytes starting at offset 100), the rest of
// the page should be all zeros (no cells, no cell pointers, no data).
let btree_header_end = DATABASE_HEADER_SIZE + 8;
let trailing = &raw[btree_header_end..];
let non_zero_count = trailing.iter().filter(|&&b| b != 0).count();
assert_eq!(
non_zero_count, 0,
"bead_id={BEAD_5A1} case=trailing_bytes_zeroed non_zero_count={non_zero_count}"
);
eprintln!(
"[5A1][test=page1_rest_is_zeroed][step=verify] \
trailing_bytes={} all_zero=true \u{2713}",
trailing.len()
);
});
}
#[test]
fn test_page1_various_page_sizes() {
asupersync::test_utils::run_test(|| async {
for &ps_val in &[512u32, 1024, 2048, 4096, 8192, 16384, 32768, 65536] {
let page_size = PageSize::new(ps_val).unwrap();
let vfs = MemoryVfs::new();
let path = PathBuf::from(format!("/test_{ps_val}.db"));
let pager = SimplePager::open(vfs, &path, page_size).await.unwrap();
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
eprintln!(
"[5A1][test=page1_various_page_sizes][step=open] page_size={ps_val} page_len={}",
raw.len()
);
assert_eq!(
raw.len(),
ps_val as usize,
"bead_id={BEAD_5A1} case=page_len ps={ps_val}"
);
// Verify database header parses.
let hdr_bytes: [u8; DATABASE_HEADER_SIZE] =
raw[..DATABASE_HEADER_SIZE].try_into().unwrap();
let hdr = DatabaseHeader::from_bytes(&hdr_bytes).unwrap_or_else(|e| {
panic!("bead_id={BEAD_5A1} case=hdr_parse ps={ps_val} err={e}")
});
assert_eq!(
hdr.page_size, page_size,
"bead_id={BEAD_5A1} case=hdr_page_size ps={ps_val}"
);
// Verify B-tree header parses.
let btree = BTreePageHeader::parse(&raw, page_size, 0, true).unwrap_or_else(|e| {
panic!("bead_id={BEAD_5A1} case=btree_parse ps={ps_val} err={e}")
});
assert_eq!(
btree.cell_count, 0,
"bead_id={BEAD_5A1} case=empty_cells ps={ps_val}"
);
// Content offset should be usable_size (= page_size when reserved=0).
let expected_content = ps_val;
assert_eq!(
btree.cell_content_start, expected_content,
"bead_id={BEAD_5A1} case=content_start ps={ps_val}"
);
eprintln!(
"[5A1][test=page1_various_page_sizes][step=verify] \
page_size={ps_val} content_start={} \u{2713}",
btree.cell_content_start
);
}
});
}
#[test]
fn test_write_empty_leaf_table_roundtrip() {
// Verify that write_empty_leaf_table produces bytes that parse back
// correctly via BTreePageHeader::parse().
let page_size = PageSize::DEFAULT;
let mut page = vec![0u8; page_size.as_usize()];
// Write at offset 0 (non-page-1 case).
BTreePageHeader::write_empty_leaf_table(&mut page, 0, page_size.get());
let parsed = BTreePageHeader::parse(&page, page_size, 0, false)
.expect("bead_id=bd-2yy6 written page must parse");
assert_eq!(parsed.page_type, fsqlite_types::BTreePageType::LeafTable);
assert_eq!(parsed.cell_count, 0);
assert_eq!(parsed.first_freeblock, 0);
assert_eq!(parsed.fragmented_free_bytes, 0);
assert_eq!(parsed.cell_content_start, page_size.get());
assert_eq!(parsed.header_offset, 0);
eprintln!(
"[5A1][test=write_empty_leaf_roundtrip][step=verify] \
non_page1 roundtrip \u{2713}"
);
// Write at offset 100 (page-1 case).
let mut page1 = vec![0u8; page_size.as_usize()];
BTreePageHeader::write_empty_leaf_table(&mut page1, DATABASE_HEADER_SIZE, page_size.get());
// Need to also write a valid database header for parse to succeed.
let hdr = DatabaseHeader {
page_size,
page_count: 1,
sqlite_version: FRANKENSQLITE_SQLITE_VERSION_NUMBER,
..DatabaseHeader::default()
};
let hdr_bytes = hdr.to_bytes().unwrap();
page1[..DATABASE_HEADER_SIZE].copy_from_slice(&hdr_bytes);
let parsed1 = BTreePageHeader::parse(&page1, page_size, 0, true)
.expect("bead_id=bd-2yy6 page1 written page must parse");
assert_eq!(parsed1.page_type, fsqlite_types::BTreePageType::LeafTable);
assert_eq!(parsed1.cell_count, 0);
assert_eq!(parsed1.header_offset, DATABASE_HEADER_SIZE);
eprintln!(
"[5A1][test=write_empty_leaf_roundtrip][step=verify] \
page1 roundtrip \u{2713}"
);
}
#[test]
fn test_write_empty_leaf_table_65536_page_size() {
let page_size = PageSize::new(65536).unwrap();
let mut page = vec![0u8; page_size.as_usize()];
BTreePageHeader::write_empty_leaf_table(&mut page, 0, page_size.get());
// The raw content offset bytes should be 0x00 0x00 (0 encodes 65536).
assert_eq!(page[5], 0x00);
assert_eq!(page[6], 0x00);
let parsed =
BTreePageHeader::parse(&page, page_size, 0, false).expect("65536 page must parse");
assert_eq!(parsed.cell_content_start, 65536);
eprintln!(
"[5A1][test=write_empty_leaf_65536][step=verify] \
content_start=65536 encoding=0x0000 \u{2713}"
);
}
#[test]
fn test_freelist_leak_on_rollback() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// 1. Allocate a page and commit.
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn1.allocate_page(&cx).await.unwrap();
txn1.write_page(&cx, p, &vec![0xAA; ps]).await.unwrap();
txn1.commit(&cx).await.unwrap();
// 2. Free the page and commit -> moves to freelist.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p).await.unwrap();
txn2.commit(&cx).await.unwrap();
// Verify freelist has the page.
{
let inner = pager.inner.lock().unwrap();
assert_eq!(inner.freelist.len(), 1);
assert_eq!(inner.freelist[0], p);
drop(inner);
}
// 3. Allocate the page again (pops from freelist).
let mut txn3 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = txn3.allocate_page(&cx).await.unwrap();
assert_eq!(
p2,
p,
"bead_id={BEAD_ID} case=freelist_reuse p3={} p1={}",
p2.get(),
p.get()
);
txn3.write_page(&cx, p2, &vec![0xBB; ps]).await.unwrap();
// Verify freelist is empty (in-flight).
{
let inner = pager.inner.lock().unwrap();
assert!(inner.freelist.is_empty());
drop(inner);
}
// 4. Rollback.
txn3.rollback(&cx).await.unwrap();
// 5. Verify freelist has the page again (no leak).
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.freelist.len(),
1,
"bead_id={BEAD_ID} case=freelist_leak_on_rollback"
);
assert_eq!(inner.freelist[0], p);
drop(inner);
}
});
}
#[test]
fn test_concurrent_rollback_reclaims_eof_allocations_without_holes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut concurrent = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let p2 = concurrent.allocate_page(&cx).await.unwrap();
let p3 = concurrent.allocate_page(&cx).await.unwrap();
concurrent
.write_page(&cx, p2, &vec![0xAA; ps])
.await
.unwrap();
concurrent
.write_page(&cx, p3, &vec![0xBB; ps])
.await
.unwrap();
concurrent.rollback(&cx).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&p2) && inner.freelist.contains(&p3),
"bead_id={BEAD_ID} case=concurrent_rollback_restores_eof_pages freelist={:?}",
inner.freelist
);
}
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused1 = txn.allocate_page(&cx).await.unwrap();
let reused2 = txn.allocate_page(&cx).await.unwrap();
assert!(
reused1 != reused2 && [p2, p3].contains(&reused1) && [p2, p3].contains(&reused2),
"bead_id={BEAD_ID} case=concurrent_rollback_reuses_eof_pages reused=({}, {}) expected=({}, {})",
reused1.get(),
reused2.get(),
p2.get(),
p3.get()
);
txn.write_page(&cx, reused1, &vec![0xCC; ps]).await.unwrap();
txn.write_page(&cx, reused2, &vec![0xDD; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.db_size, 3,
"bead_id={BEAD_ID} case=concurrent_rollback_does_not_skip_page_numbers"
);
});
}
#[test]
fn test_concurrent_allocate_ignores_global_freelist_pages() {
asupersync::test_utils::run_test(|| async {
// GH#302 changed the concurrent allocator: committed freelist
// pages ARE reused when this transaction is the pager's only
// live transaction AND its snapshot is current (see
// test_concurrent_allocate_reuses_committed_freelist_when_sole_
// current_snapshot). This keeper now pins the stale-snapshot
// half of that gate: a page freed AFTER this transaction
// captured its snapshot is still live tree content in this
// transaction's own view and must NOT be reused, even though no
// other transaction is active by allocation time.
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x11; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x22; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
// The concurrent transaction captures its snapshot BEFORE the
// free commits, so p2 is live content in its snapshot.
let mut concurrent = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let mut free_txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
free_txn.free_page(&cx, p2).await.unwrap();
free_txn.commit(&cx).await.unwrap();
let allocated = concurrent.allocate_page(&cx).await.unwrap();
assert_eq!(
allocated.get(),
p3.get() + 1,
"bead_id={BEAD_ID} case=concurrent_allocate_must_not_reuse_global_freelist_pages"
);
concurrent.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_immediate_allocate_ignores_global_freelist_pages_while_reader_active() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x11; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x22; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
let mut free_txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
free_txn.free_page(&cx, p2).await.unwrap();
free_txn.commit(&cx).await.unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let snapshot_page = reader.get_page(&cx, p3).await.unwrap();
assert_eq!(
snapshot_page.as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=reader_snapshot_established_before_writer_allocate"
);
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let allocated = writer.allocate_page(&cx).await.unwrap();
assert_eq!(
allocated.get(),
p3.get() + 1,
"bead_id={BEAD_ID} case=immediate_allocate_must_not_reuse_snapshot_pinned_global_freelist_pages"
);
writer.rollback(&cx).await.unwrap();
reader.commit(&cx).await.unwrap();
});
}
#[test]
fn test_reader_snapshot_must_not_refresh_on_new_page_access() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let baseline_page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, baseline_page, &vec![0x11; ps])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let captured_db_size = reader.snapshot_db_size();
let captured_commit_seq = reader.published_visible_commit_seq.get();
assert_eq!(captured_db_size, baseline_page.get());
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let later_page = writer.allocate_page(&cx).await.unwrap();
assert_eq!(writer.visible_db_size_bound(), later_page.get());
writer
.write_page(&cx, later_page, &vec![0xAA; ps])
.await
.unwrap();
writer.commit(&cx).await.unwrap();
assert!(later_page.get() > captured_db_size);
assert!(
reader.live_db_size() > reader.snapshot_db_size(),
"the mutable live extent must not be mistaken for the reader's fixed snapshot bound"
);
assert_eq!(
reader.visible_db_size_bound(),
reader.snapshot_db_size(),
"an unrelated commit must not widen the reader-visible bound"
);
let error = reader
.get_page(&cx, later_page)
.await
.expect_err("a reader must not observe a page committed after its snapshot");
assert!(
matches!(error, FrankenError::BusySnapshot { .. }),
"expected a fixed-snapshot refusal, got {error}"
);
assert_eq!(
reader.snapshot_db_size(),
captured_db_size,
"get_page must not expand the reader's captured db_size"
);
assert_eq!(
reader.published_visible_commit_seq.get(),
captured_commit_seq,
"get_page must not advance the reader's captured commit sequence"
);
reader.commit(&cx).await.unwrap();
});
}
#[test]
fn transaction_visible_bound_excludes_an_allocation_freed_before_reload() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let snapshot_db_size = txn.snapshot_db_size();
let allocated = txn.allocate_page(&cx).await.unwrap();
assert!(allocated.get() > snapshot_db_size);
assert_eq!(txn.visible_db_size_bound(), allocated.get());
txn.free_page(&cx, allocated).await.unwrap();
assert_eq!(
txn.visible_db_size_bound(),
snapshot_db_size,
"a freed transaction-owned page must not widen the catalog root bound"
);
assert!(
!txn.live_freelist_pages().contains(&allocated),
"a returned EOF allocation above the committed extent is fenced by the visible \
bound, not exposed as a durable freelist page"
);
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_reader_must_not_observe_mixed_snapshot_after_concurrent_commit() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let baseline_page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, baseline_page, &vec![0x11; ps])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, baseline_page).await.unwrap().as_ref()[0],
0x11,
"reader must establish the baseline image before the concurrent commit"
);
let captured_db_size = reader.published_db_size.get();
let captured_commit_seq = reader.published_visible_commit_seq.get();
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
writer
.write_page(&cx, baseline_page, &vec![0xEE; ps])
.await
.unwrap();
let later_page = writer.allocate_page(&cx).await.unwrap();
writer
.write_page(&cx, later_page, &vec![0xAA; ps])
.await
.unwrap();
writer.commit(&cx).await.unwrap();
let error = reader
.get_page(&cx, later_page)
.await
.expect_err("a post-snapshot page must not refresh the reader");
assert!(
matches!(error, FrankenError::BusySnapshot { .. }),
"expected a fixed-snapshot refusal, got {error}"
);
assert_eq!(
reader.get_page(&cx, baseline_page).await.unwrap().as_ref()[0],
0x11,
"reader must keep the already-observed baseline image after rejecting a post-snapshot page"
);
assert_eq!(reader.published_db_size.get(), captured_db_size);
assert_eq!(
reader.published_visible_commit_seq.get(),
captured_commit_seq
);
reader.commit(&cx).await.unwrap();
});
}
#[test]
fn test_commit_keeps_beyond_db_size_freelist_entries_volatile_only() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x11; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x22; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let p4 = abandoned.allocate_page(&cx).await.unwrap();
abandoned
.write_page(&cx, p4, &vec![0x33; ps])
.await
.unwrap();
abandoned.rollback(&cx).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&p4),
"bead_id={BEAD_ID} case=aborted_eof_page_retained_in_memory"
);
}
let mut free_txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
free_txn.free_page(&cx, p2).await.unwrap();
free_txn.commit(&cx).await.unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let committed_page = reader.get_page(&cx, p3).await.unwrap();
assert_eq!(
committed_page.as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=durable_refresh_survives_filtered_beyond_db_size_freelist"
);
let page_one = reader
.get_page(&cx, PageNumber::ONE)
.await
.unwrap()
.into_vec();
let hdr_bytes: [u8; DATABASE_HEADER_SIZE] =
page_one[..DATABASE_HEADER_SIZE].try_into().unwrap();
let hdr = DatabaseHeader::from_bytes(&hdr_bytes).unwrap();
assert_eq!(
hdr.freelist_count, 1,
"bead_id={BEAD_ID} case=durable_freelist_header_excludes_volatile_eof_page"
);
reader.commit(&cx).await.unwrap();
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&p2),
"bead_id={BEAD_ID} case=durable_freelist_keeps_committed_free_page"
);
assert!(
inner.freelist.contains(&p4),
"bead_id={BEAD_ID} case=volatile_freelist_keeps_beyond_db_size_page_for_reuse"
);
});
}
#[test]
fn test_committed_concurrent_page_lease_pages_are_reused_before_eof_growth() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let first = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, first, &vec![0x11; ps]).await.unwrap();
let second = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, second, &vec![0x22; ps]).await.unwrap();
assert!(
!seed.page_lease.is_empty(),
"bead_id={BEAD_ID} case=page_lease_test_must_exercise_batch_allocator"
);
let leased_lowest = *seed
.page_lease
.iter()
.min()
.expect("batch allocator should leave unused lease pages");
seed.commit(&cx).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&leased_lowest),
"bead_id={BEAD_ID} case=commit_retains_unused_lease_page_in_volatile_freelist"
);
}
let mut reuse = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let reused = reuse.allocate_page(&cx).await.unwrap();
assert_eq!(
reused, leased_lowest,
"bead_id={BEAD_ID} case=allocator_reuses_volatile_lease_page_before_eof_growth"
);
reuse.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_pending_commit_pages_ignore_beyond_db_size_freelist_entries() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/pending_commit_pages_ignore_beyond_db_size.db");
let pager = SimplePager::open(vfs, &path, PageSize::MIN).await.unwrap();
let cx = Cx::new();
let ps = PageSize::MIN.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x11; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x22; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
let overflow_freelist_pages = (ps / 4).saturating_sub(2) + 2;
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let mut abandoned_pages = Vec::with_capacity(overflow_freelist_pages);
for _ in 0..overflow_freelist_pages {
abandoned_pages.push(abandoned.allocate_page(&cx).await.unwrap());
}
abandoned.rollback(&cx).await.unwrap();
let first_beyond_db_size_page = abandoned_pages[0];
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
txn.free_page(&cx, p2).await.unwrap();
let predicted = txn.pending_commit_pages().unwrap();
assert!(
predicted.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=pending_commit_pages_still_include_page_one_rewrite"
);
assert!(
predicted.contains(&p2),
"bead_id={BEAD_ID} case=pending_commit_pages_include_real_durable_trunk"
);
assert!(
!predicted.contains(&first_beyond_db_size_page),
"bead_id={BEAD_ID} case=pending_commit_pages_ignore_eof_only_freelist_pages"
);
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_commit_keeps_newly_freed_pages_below_future_db_size_on_durable_freelist() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/commit_keeps_newly_freed_pages.db");
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x11; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x22; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
let (p4, p5, p6, p7) = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p4 = txn.allocate_page(&cx).await.unwrap();
let p5 = txn.allocate_page(&cx).await.unwrap();
let p6 = txn.allocate_page(&cx).await.unwrap();
let p7 = txn.allocate_page(&cx).await.unwrap();
txn.free_page(&cx, p4).await.unwrap();
txn.free_page(&cx, p5).await.unwrap();
txn.free_page(&cx, p6).await.unwrap();
txn.write_page(&cx, p7, &vec![0x77; ps]).await.unwrap();
let predicted = txn.pending_commit_pages().unwrap();
assert!(
predicted.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=future_db_size_commit_still_rewrites_page_one"
);
assert!(
predicted.contains(&p6),
"bead_id={BEAD_ID} case=future_db_size_commit_includes_descending_freelist_trunk"
);
assert!(
predicted.contains(&p7),
"bead_id={BEAD_ID} case=future_db_size_commit_includes_live_high_page"
);
txn.commit(&cx).await.unwrap();
(p4, p5, p6, p7)
};
let mut txn_ro = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw = txn_ro
.get_page(&cx, PageNumber::ONE)
.await
.unwrap()
.into_vec();
let hdr_bytes: [u8; DATABASE_HEADER_SIZE] =
raw[..DATABASE_HEADER_SIZE].try_into().unwrap();
let hdr = DatabaseHeader::from_bytes(&hdr_bytes).unwrap();
assert_eq!(
hdr.page_count,
p7.get(),
"bead_id={BEAD_ID} case=future_db_size_commit_advances_page_count_to_live_high_page"
);
assert_eq!(
hdr.freelist_count, 3,
"bead_id={BEAD_ID} case=future_db_size_commit_persists_newly_freed_pages"
);
assert_eq!(
hdr.freelist_trunk,
p6.get(),
"bead_id={BEAD_ID} case=future_db_size_commit_uses_descending_freelist_head_as_trunk"
);
txn_ro.commit(&cx).await.unwrap();
let reopened = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
{
let inner = reopened.inner.lock().unwrap();
assert_eq!(
inner.db_size,
p7.get(),
"bead_id={BEAD_ID} case=future_db_size_commit_reopen_keeps_page_count"
);
assert_eq!(
inner.freelist.len(),
3,
"bead_id={BEAD_ID} case=future_db_size_commit_reopen_restores_freelist_len"
);
assert!(
inner.freelist.contains(&p4)
&& inner.freelist.contains(&p5)
&& inner.freelist.contains(&p6),
"bead_id={BEAD_ID} case=future_db_size_commit_reopen_restores_newly_freed_pages"
);
}
let mut reuse = reopened
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
let mut reused = vec![
reuse.allocate_page(&cx).await.unwrap(),
reuse.allocate_page(&cx).await.unwrap(),
reuse.allocate_page(&cx).await.unwrap(),
];
reused.sort_unstable();
assert_eq!(
reused,
vec![p4, p5, p6],
"bead_id={BEAD_ID} case=future_db_size_commit_reuses_newly_freed_pages_after_reopen"
);
reuse.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_concurrent_allocate_reuses_beyond_db_size_freelist_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let p2 = abandoned.allocate_page(&cx).await.unwrap();
let p3 = abandoned.allocate_page(&cx).await.unwrap();
abandoned
.write_page(&cx, p2, &vec![0x33; ps])
.await
.unwrap();
abandoned
.write_page(&cx, p3, &vec![0x44; ps])
.await
.unwrap();
abandoned.rollback(&cx).await.unwrap();
let mut concurrent = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let reused1 = concurrent.allocate_page(&cx).await.unwrap();
let reused2 = concurrent.allocate_page(&cx).await.unwrap();
assert!(
reused1 != reused2 && [p2, p3].contains(&reused1) && [p2, p3].contains(&reused2),
"bead_id={BEAD_ID} case=concurrent_allocate_reuses_beyond_db_size_freelist_pages reused=({}, {}) expected=({}, {})",
reused1.get(),
reused2.get(),
p2.get(),
p3.get()
);
concurrent.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_journal_commit_detects_cross_connection_committed_freelist_reuse_alias() {
asupersync::test_utils::run_test(|| async {
// Two connections (their own `PagerInner` each) both snapshot the same
// committed freelist and both pop the same committed free page for
// different content. Reuse does not grow the file, so the am#152
// db-size growth check alone cannot see it. The second committer must
// abort with `BusySnapshot` (first-committer-wins) instead of writing
// its content over the page the first committer now owns.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/journal_freelist_alias.db");
let cx = Cx::new();
// Seed: grow the db, then free one page onto the COMMITTED freelist.
let p2 = {
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = txn.allocate_page(&cx).await.unwrap();
let p3 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p2, &sample_page(0x22)).await.unwrap();
txn.write_page(&cx, p3, &sample_page(0x33)).await.unwrap();
txn.commit(&cx).await.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.free_page(&cx, p2).await.unwrap();
txn.commit(&cx).await.unwrap();
p2
};
let pager_a = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let pager_b = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
// Both writers begin (snapshotting the committed freelist) BEFORE
// either commits, so both see p2 as free.
let mut txn_a = pager_a
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
let mut txn_b = pager_b
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
let a_page = txn_a.allocate_page(&cx).await.unwrap();
let b_page = txn_b.allocate_page(&cx).await.unwrap();
assert_eq!(
a_page, p2,
"premise: writer A reuses the committed free page"
);
assert_eq!(
b_page, p2,
"premise: writer B reuses the SAME committed free page from its own snapshot"
);
txn_a
.write_page(&cx, a_page, &sample_page(0xAA))
.await
.unwrap();
txn_b
.write_page(&cx, b_page, &sample_page(0xBB))
.await
.unwrap();
txn_a.commit(&cx).await.unwrap();
let err = txn_b
.commit(&cx)
.await
.expect_err("second committer reusing an already-claimed freelist page must abort");
assert!(
matches!(err, FrankenError::BusySnapshot { .. }),
"expected BusySnapshot first-committer-wins abort, got: {err}"
);
// The surviving on-disk content of the contested page is writer A's.
let verify = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let txn = verify.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let page = txn.get_page(&cx, p2).await.unwrap();
assert_eq!(
page.as_ref(),
&sample_page(0xAA)[..],
"contested page must hold the first committer's bytes"
);
});
}
#[test]
fn test_concurrent_allocate_reuses_committed_freelist_when_sole_current_snapshot() {
asupersync::test_utils::run_test(|| async {
// GH#302: default (concurrent) transactions must reuse committed
// freelist pages at/below db_size when this transaction is the
// only active one and its snapshot is current. Otherwise churn
// workloads grow the file at EOF without bound while a large
// committed freelist sits unused.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/gh302_concurrent_freelist_reuse.db");
let cx = Cx::new();
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
// Seed: grow the database, then durably free four pages.
let mut pages = Vec::new();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for fill in 0x21..0x29u8 {
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &sample_page(fill)).await.unwrap();
pages.push(p);
}
txn.commit(&cx).await.unwrap();
}
let freed: Vec<PageNumber> = pages[..4].to_vec();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for &p in &freed {
txn.free_page(&cx, p).await.unwrap();
}
txn.commit(&cx).await.unwrap();
}
let committed_size_after_free = pager.committed_snapshot().db_size;
// Churn: a sole concurrent transaction with a current snapshot
// must satisfy new allocations from the committed freelist, not
// at EOF.
{
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
for round in 0..4u8 {
let p = txn.allocate_page(&cx).await.unwrap();
assert!(
freed.contains(&p),
"case=gh302_sole_concurrent_txn_reuses_committed_free_page \
round={round} allocated={} freed={:?}",
p.get(),
freed.iter().map(|p| p.get()).collect::<Vec<_>>()
);
txn.write_page(&cx, p, &sample_page(0x90 + round))
.await
.unwrap();
}
txn.commit(&cx).await.unwrap();
}
assert_eq!(
pager.committed_snapshot().db_size,
committed_size_after_free,
"case=gh302_reuse_does_not_grow_the_database"
);
assert_eq!(
pager.committed_snapshot().freelist_count,
0,
"case=gh302_reused_pages_leave_the_committed_freelist"
);
// Reused pages carry the new content.
let mut txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for (round, &p) in freed.iter().enumerate() {
let got = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
got.as_ref(),
&sample_page(0x90 + u8::try_from(round).unwrap())[..],
"case=gh302_reused_page_holds_new_content page={}",
p.get()
);
}
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_concurrent_allocate_skips_committed_freelist_while_older_snapshot_active() {
asupersync::test_utils::run_test(|| async {
// GH#302 safety gate: while ANOTHER local transaction holds a
// snapshot, committed freelist pages at/below db_size stay
// snapshot-pinned and a concurrent writer must allocate at EOF
// instead of reusing them.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/gh302_concurrent_freelist_pinned.db");
let cx = Cx::new();
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut pages = Vec::new();
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for fill in 0x41..0x45u8 {
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &sample_page(fill)).await.unwrap();
pages.push(p);
}
txn.commit(&cx).await.unwrap();
}
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.free_page(&cx, pages[0]).await.unwrap();
txn.commit(&cx).await.unwrap();
}
let committed_size = pager.committed_snapshot().db_size;
// Hold a reader snapshot open, then allocate concurrently.
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
{
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
assert!(
p.get() > committed_size,
"case=gh302_active_reader_pins_committed_freelist allocated={} \
committed_size={committed_size}",
p.get()
);
txn.rollback(&cx).await.unwrap();
}
reader.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_journal_commit_preserves_peer_growth_for_disjoint_writer() {
asupersync::test_utils::run_test(|| async {
// Both pagers capture the same two-page image. Writer A grows the
// database, while writer B changes only a page that already existed
// in its snapshot. B has no newly allocated page in A's claimed
// range, but its synthesized page 1 still carries the old page
// count. Letting B commit that stale header would orphan A's page.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/journal_stale_page_one_after_growth.db");
let cx = Cx::new();
let existing_page = {
let seed = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = seed.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x22)).await.unwrap();
txn.commit(&cx).await.unwrap();
page
};
let pager_a = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let pager_b = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn_a = pager_a
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let mut txn_b = pager_b
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let grown_page = txn_a.allocate_page(&cx).await.unwrap();
assert_eq!(grown_page.get(), existing_page.get() + 1);
txn_a
.write_page(&cx, grown_page, &sample_page(0xAA))
.await
.unwrap();
txn_b
.write_page(&cx, existing_page, &sample_page(0xBB))
.await
.unwrap();
txn_a.commit(&cx).await.unwrap();
// B touched only a page that already existed in its snapshot, so it
// has no page-level conflict with A and must be allowed to commit.
// Aborting it here would be a spurious retry against the
// concurrent-writer contract. What must hold is that B's commit
// does not regress the extent: the page-1 header it publishes is
// re-read at commit time and already carries A's larger page count.
txn_b
.commit(&cx)
.await
.expect("a disjoint existing-page writer must still commit after peer growth");
let verify = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let txn = verify.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
txn.snapshot_db_size(),
grown_page.get(),
"committing the disjoint writer must preserve the peer-grown extent"
);
assert_eq!(
txn.get_page(&cx, grown_page).await.unwrap().as_ref(),
&sample_page(0xAA)[..],
"committing the disjoint writer must preserve the peer-grown page"
);
assert_eq!(
txn.get_page(&cx, existing_page).await.unwrap().as_ref(),
&sample_page(0xBB)[..],
"the disjoint writer's own page must be durable"
);
});
}
#[test]
fn test_wal_commit_preserves_peer_growth_for_disjoint_writer() {
asupersync::test_utils::run_test(|| async {
// WAL commit records carry the complete database size. If a stale
// existing-page-only writer publishes its snapshot size after a
// peer grows the file, the commit marker makes the peer's new page
// disappear even though the writers touched disjoint data pages.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_stale_db_size_after_growth.db");
let cx = Cx::new();
// Every participating pager must share one WAL frame log. Without an
// installed backend `set_journal_mode(Wal)` returns `Unsupported`
// (see the `has_wal_backend` gate in `set_journal_mode`), so the
// test would die in setup and never reach the commit behavior it
// claims to cover. Each `set_journal_mode` result is asserted so a
// future silent fallback to a rollback mode cannot pass unnoticed.
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let existing_page = {
let seed = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (seed_backend, _, _) =
MockWalBackend::with_shared_frames(StdArc::clone(&frames));
seed.set_wal_backend(Box::new(seed_backend)).unwrap();
assert_eq!(
seed.set_journal_mode(&cx, JournalMode::Wal).await.unwrap(),
JournalMode::Wal,
"seed pager must actually enter WAL mode"
);
let mut txn = seed.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x22)).await.unwrap();
txn.commit(&cx).await.unwrap();
page
};
let pager_a = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend_a, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
pager_a.set_wal_backend(Box::new(backend_a)).unwrap();
assert_eq!(
pager_a
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap(),
JournalMode::Wal,
"writer A must actually enter WAL mode"
);
let pager_b = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend_b, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
pager_b.set_wal_backend(Box::new(backend_b)).unwrap();
assert_eq!(
pager_b
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap(),
JournalMode::Wal,
"writer B must actually enter WAL mode"
);
let mut txn_a = pager_a
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let mut txn_b = pager_b
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let grown_page = txn_a.allocate_page(&cx).await.unwrap();
assert_eq!(grown_page.get(), existing_page.get() + 1);
txn_a
.write_page(&cx, grown_page, &sample_page(0xAA))
.await
.unwrap();
txn_b
.write_page(&cx, existing_page, &sample_page(0xBB))
.await
.unwrap();
txn_a.commit(&cx).await.unwrap();
// Same contract as the rollback-journal case: disjoint data pages
// are not a conflict, so B commits. The WAL commit marker must
// record the peer-grown extent rather than B's stale snapshot size.
txn_b
.commit(&cx)
.await
.expect("a disjoint existing-page writer must still commit after peer growth");
let verify = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let (verify_backend, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
verify.set_wal_backend(Box::new(verify_backend)).unwrap();
assert_eq!(
verify
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap(),
JournalMode::Wal,
"the verifying reader must read through the same WAL"
);
let txn = verify.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
txn.snapshot_db_size(),
grown_page.get(),
"the WAL commit marker must preserve the peer-grown extent"
);
assert!(
txn.snapshot_db_size() > existing_page.get(),
"B's stale snapshot size must not shrink the database below A's growth"
);
assert_eq!(
txn.get_page(&cx, grown_page).await.unwrap().as_ref(),
&sample_page(0xAA)[..],
"the WAL commit marker must preserve the peer-grown page"
);
assert_eq!(
txn.get_page(&cx, existing_page).await.unwrap().as_ref(),
&sample_page(0xBB)[..],
"the disjoint writer's own page must be durable"
);
});
}
#[cfg(all(feature = "native", unix))]
#[test]
fn file_begin_binds_the_refreshed_wal_snapshot_when_publication_advances() {
asupersync::test_utils::run_test(|| async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal_begin_publication_advance.db");
let cx = Cx::new();
let pager = SimplePager::open(UnixVfs::new(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, frames, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
assert_eq!(
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap(),
JournalMode::Wal
);
let seeded_page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &sample_page(0x4A))
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page
};
let (expected_commit_seq, expected_db_size, newer_update) = {
let inner = pager.inner.lock().unwrap();
(
inner.commit_seq,
inner.db_size,
PublishedPagerUpdate {
visible_commit_seq: inner.commit_seq.next(),
db_size: inner.db_size.saturating_add(1),
journal_mode: inner.journal_mode,
freelist_count: inner.freelist.len(),
checkpoint_active: inner.checkpoint_active,
},
)
};
let (backend, _, _) = MockWalBackend::with_shared_frames(frames);
let backend =
backend.with_publish_after_begin(Arc::clone(&pager.published), newer_update);
pager.set_wal_backend(Box::new(backend)).unwrap();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
pager.published.snapshot().visible_commit_seq,
newer_update.visible_commit_seq,
"the fixture must advance shared publication after the WAL read pin is captured"
);
assert_eq!(
reader.published_visible_commit_seq.get(),
expected_commit_seq,
"a file transaction must remain bound to its refreshed WAL snapshot"
);
assert_eq!(reader.snapshot_db_size(), expected_db_size);
assert_eq!(
reader.get_page(&cx, seeded_page).await.unwrap().as_ref(),
&sample_page(0x4A)[..]
);
reader.rollback(&cx).await.unwrap();
});
}
#[cfg(all(feature = "native", unix))]
#[test]
fn test_wal_fcw_keeps_each_transactions_begin_snapshot() {
asupersync::test_utils::run_test(|| async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal_begin_snapshot_fcw.db");
let cx = Cx::new();
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let pager = SimplePager::open(UnixVfs::new(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, begin_calls, _) =
MockWalBackend::with_shared_frames(StdArc::clone(&frames));
pager.set_wal_backend(Box::new(backend)).unwrap();
assert_eq!(
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap(),
JournalMode::Wal
);
let contested_page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &sample_page(0x22))
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page
};
let mut stale = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let stale_snapshot = stale
.wal_conflict_snapshot
.expect("WAL transaction must retain its BEGIN snapshot");
frames.lock().unwrap().push((
contested_page.get(),
sample_page(0xAA),
contested_page.get(),
));
let begin_calls_before_sibling = *begin_calls.lock().unwrap();
let mut sibling = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
*begin_calls.lock().unwrap(),
begin_calls_before_sibling,
"a sibling BEGIN must reuse, not replace, the active backend read pin"
);
let sibling_snapshot = sibling
.wal_conflict_snapshot
.expect("sibling must reuse the active pager snapshot");
assert_eq!(sibling_snapshot, stale_snapshot);
assert_eq!(
stale.wal_conflict_snapshot,
Some(stale_snapshot),
"a sibling BEGIN must not replace the stale transaction's FCW horizon"
);
assert_eq!(
stale.get_page(&cx, contested_page).await.unwrap().as_ref(),
&sample_page(0x22)[..],
"an uncached stale read must remain bound to the pager's active WAL pin"
);
stale
.write_page(&cx, contested_page, &sample_page(0xBB))
.await
.unwrap();
let error = stale
.commit(&cx)
.await
.expect_err("FCW must reject a page changed after this transaction began");
let FrankenError::BusySnapshot { conflicting_pages } = &error else {
panic!("FCW must reject with BusySnapshot: {error:?}");
};
let named = conflicting_pages
.split(',')
.filter_map(|raw| raw.trim().parse::<u32>().ok())
.collect::<Vec<_>>();
assert!(
named.contains(&contested_page.get()),
"BusySnapshot payload {conflicting_pages:?} must name page {}",
contested_page.get()
);
stale.rollback(&cx).await.unwrap();
sibling.rollback(&cx).await.unwrap();
});
}
#[cfg(all(feature = "native", unix))]
#[test]
fn test_wal_commit_rejects_missing_conflict_snapshot() {
asupersync::test_utils::run_test(|| async {
let dir = tempfile::tempdir().unwrap();
let path = dir.path().join("wal_missing_conflict_snapshot.db");
let cx = Cx::new();
let pager = SimplePager::open(UnixVfs::new(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, _, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
assert_eq!(
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap(),
JournalMode::Wal
);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x44)).await.unwrap();
txn.wal_conflict_snapshot = None;
assert!(matches!(
txn.commit(&cx).await,
Err(FrankenError::Unsupported)
));
assert!(
txn.pending_group_commit_attempt.is_none(),
"pre-admission rejection must synchronously clear the pending attempt"
);
assert_eq!(
txn.pager_commit_state(),
PagerCommitState::NotCommitted,
"pre-admission rejection must not leave the transaction in doubt"
);
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_concurrent_rollback_quarantines_peer_claimed_eof_page_until_refresh() {
asupersync::test_utils::run_test(|| async {
// Keep pager B non-quiescent so its retry cannot refresh durable
// metadata immediately after losing a first-committer-wins race.
// The losing EOF page must not become locally reusable while B's
// PagerInner still has the stale pre-growth db_size.
let vfs = MemoryVfs::new();
let path = PathBuf::from("/wal_rollback_eof_quarantine.db");
let cx = Cx::new();
// Every participating pager shares one WAL frame log; without an
// installed backend `set_journal_mode(Wal)` returns `Unsupported`
// and this test dies in setup before exercising EOF aliasing at all.
let frames: SharedFrames = StdArc::new(StdMutex::new(Vec::new()));
let seeded_page = {
let seed = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (seed_backend, _, _) =
MockWalBackend::with_shared_frames(StdArc::clone(&frames));
seed.set_wal_backend(Box::new(seed_backend)).unwrap();
assert_eq!(
seed.set_journal_mode(&cx, JournalMode::Wal).await.unwrap(),
JournalMode::Wal,
"seed pager must actually enter WAL mode"
);
let mut txn = seed.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x22)).await.unwrap();
txn.commit(&cx).await.unwrap();
page
};
let pager_a = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend_a, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
pager_a.set_wal_backend(Box::new(backend_a)).unwrap();
assert_eq!(
pager_a
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap(),
JournalMode::Wal,
"writer A must actually enter WAL mode"
);
let pager_b = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let (backend_b, _, _) = MockWalBackend::with_shared_frames(StdArc::clone(&frames));
pager_b.set_wal_backend(Box::new(backend_b)).unwrap();
assert_eq!(
pager_b
.set_journal_mode(&cx, JournalMode::Wal)
.await
.unwrap(),
JournalMode::Wal,
"writer B must actually enter WAL mode"
);
let mut reader_b = pager_b.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let mut txn_a = pager_a
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let mut txn_b = pager_b
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let contested_page = txn_a.allocate_page(&cx).await.unwrap();
assert_eq!(
contested_page.get(),
seeded_page.get() + 1,
"premise: A must allocate the page just past the seeded extent"
);
assert_eq!(
txn_b.allocate_page(&cx).await.unwrap(),
contested_page,
"premise: independent stale allocators choose the same EOF page"
);
txn_a
.write_page(&cx, contested_page, &sample_page(0xAA))
.await
.unwrap();
txn_b
.write_page(&cx, contested_page, &sample_page(0xBB))
.await
.unwrap();
txn_a.commit(&cx).await.unwrap();
let error = txn_b
.commit(&cx)
.await
.expect_err("the second writer of the same EOF page must lose");
// The payload must name the contested EOF page: `fsqlite_executor`'s
// `parse_conflicting_pages` reads this comma-separated list to drive
// retry decisions, so an empty or mis-scoped payload is a silent
// downgrade even though the error variant looks correct.
let FrankenError::BusySnapshot { conflicting_pages } = &error else {
panic!(
"the second writer of the same EOF page must lose with BusySnapshot: {error:?}"
);
};
let named: Vec<u32> = conflicting_pages
.split(',')
.filter_map(|raw| raw.trim().parse::<u32>().ok())
.collect();
assert!(
named.contains(&contested_page.get()),
"BusySnapshot payload {conflicting_pages:?} must name the contested EOF page {}",
contested_page.get()
);
txn_b.rollback(&cx).await.unwrap();
// A's committed image must survive B's loss: first-committer-wins
// means the loser's rollback cannot shrink the extent or overwrite
// the winner's bytes.
{
let mut winner = pager_a.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
winner.snapshot_db_size(),
contested_page.get(),
"the winner's growth must remain durable after the loser rolls back"
);
assert_eq!(
winner.get_page(&cx, contested_page).await.unwrap().as_ref(),
&sample_page(0xAA)[..],
"the loser's rollback must not overwrite the winner's page"
);
winner.commit(&cx).await.unwrap();
}
let mut retry_b = pager_b
.begin(&cx, TransactionMode::Concurrent)
.await
.unwrap();
let retry_page = retry_b.allocate_page(&cx).await.unwrap();
assert_ne!(
retry_page, contested_page,
"a peer-claimed EOF page must stay quarantined until a quiescent durable refresh"
);
retry_b.rollback(&cx).await.unwrap();
reader_b.commit(&cx).await.unwrap();
});
}
#[test]
fn test_concurrent_reuse_and_free_beyond_db_size_page_is_net_zero() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let p2 = abandoned.allocate_page(&cx).await.unwrap();
let p3 = abandoned.allocate_page(&cx).await.unwrap();
abandoned
.write_page(&cx, p2, &vec![0x33; ps])
.await
.unwrap();
abandoned
.write_page(&cx, p3, &vec![0x44; ps])
.await
.unwrap();
abandoned.rollback(&cx).await.unwrap();
let current_db_size = pager.published_snapshot().db_size;
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let reused = txn.allocate_page(&cx).await.unwrap();
assert!(
[p2, p3].contains(&reused),
"bead_id={BEAD_ID} case=concurrent_reuse_then_free_picks_eof_only_page reused={}",
reused.get()
);
txn.free_page(&cx, reused).await.unwrap();
let plan =
txn.classify_wal_page_one_write(current_db_size, txn.freelist_metadata_dirty());
assert_eq!(
plan,
WalPageOneWritePlan {
max_written: 0,
page_one_dirty: false,
freelist_metadata_dirty: false,
db_growth: false,
},
"bead_id={BEAD_ID} case=concurrent_reuse_then_free_has_no_wal_page_one_trigger"
);
assert!(
!txn.has_pending_writes(),
"bead_id={BEAD_ID} case=concurrent_reuse_then_free_has_no_pending_writes"
);
assert!(
txn.pending_commit_pages().unwrap().is_empty(),
"bead_id={BEAD_ID} case=concurrent_reuse_then_free_has_no_commit_pages"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_allocate_page_requires_page_one_conflict_tracking_skips_pure_concurrent_eof_allocate() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(
!txn.allocate_page_requires_page_one_conflict_tracking()
.unwrap(),
"bead_id={BEAD_ID} case=concurrent_eof_allocate_does_not_predeclare_page_one_conflict"
);
});
}
#[test]
fn test_allocate_page_requires_page_one_conflict_tracking_skips_beyond_db_size_reuse() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let reusable_pages = {
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page_one = abandoned.allocate_page(&cx).await.unwrap();
let page_two = abandoned.allocate_page(&cx).await.unwrap();
abandoned
.write_page(&cx, page_one, &vec![0x44; ps])
.await
.unwrap();
abandoned
.write_page(&cx, page_two, &vec![0x55; ps])
.await
.unwrap();
abandoned.rollback(&cx).await.unwrap();
[page_one, page_two]
};
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(
!txn.allocate_page_requires_page_one_conflict_tracking()
.unwrap(),
"bead_id={BEAD_ID} case=beyond_db_size_reuse_skips_page_one_conflict_tracking"
);
let reused = txn.allocate_page(&cx).await.unwrap();
assert!(
reusable_pages.contains(&reused),
"bead_id={BEAD_ID} case=allocator_page_one_hook_matches_actual_reuse reused={} expected=({}, {})",
reused.get(),
reusable_pages[0].get(),
reusable_pages[1].get()
);
});
}
#[test]
fn test_memory_db_allocator_skips_committed_freelist_and_page_one_conflicts() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let pager = SimplePager::open(vfs, Path::new("/:memory:"), PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let (page_two, page_three) = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_two = seed.allocate_page(&cx).await.unwrap();
let page_three = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page_two, &vec![0x11; ps])
.await
.unwrap();
seed.write_page(&cx, page_three, &vec![0x22; ps])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
(page_two, page_three)
};
{
let mut free_txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
free_txn.free_page(&cx, page_two).await.unwrap();
free_txn.commit(&cx).await.unwrap();
}
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
assert!(
!txn.allocate_page_requires_page_one_conflict_tracking()
.unwrap(),
"bead_id={BEAD_ID} case=memory_db_allocate_skips_page_one_conflict_tracking"
);
let allocated = txn.allocate_page(&cx).await.unwrap();
let expected = PageNumber::new(page_three.get() + 1).unwrap();
assert_eq!(
allocated,
expected,
"bead_id={BEAD_ID} case=memory_db_allocator_uses_bump_path allocated={} expected={}",
allocated.get(),
expected.get()
);
});
}
#[test]
fn test_memory_db_allocator_stays_bump_only_after_rollback() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let pager = SimplePager::open(vfs, Path::new("/:memory:"), PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
// Concurrent rollback keeps `next_page` advanced (see the comment at
// the "Concurrent: next_page is NOT reset" branch of rollback) so a
// later writer in bump-only mode cannot drift into the freelist. The
// test relies on that invariant specifically — Immediate rollback
// rewinds next_page, so it would be a weaker check.
//
// The second allocate in Concurrent mode batches
// PAGE_LEASE_BATCH_SIZE pages (one returned, the rest leased). After
// rollback the bump counter has advanced by exactly
// PAGE_LEASE_BATCH_SIZE past the first alloc's page, so the next
// alloc lands at page_one + PAGE_LEASE_BATCH_SIZE + 1 (the
// "already_allocated" branch also applied on the second call).
let abandoned_pages = {
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page_one = abandoned.allocate_page(&cx).await.unwrap();
let page_two = abandoned.allocate_page(&cx).await.unwrap();
abandoned.rollback(&cx).await.unwrap();
[page_one, page_two]
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
assert!(
!txn.allocate_page_requires_page_one_conflict_tracking()
.unwrap(),
"bead_id={BEAD_ID} case=memory_db_post_rollback_allocate_skips_page_one_conflict_tracking"
);
let allocated = txn.allocate_page(&cx).await.unwrap();
// next_page after rollback = page_one + 1 (first alloc, batch=1)
// + PAGE_LEASE_BATCH_SIZE (second alloc, batch=lease_size).
let expected_next_page = abandoned_pages[0].get() + 1 + PAGE_LEASE_BATCH_SIZE;
let expected = PageNumber::new(expected_next_page).unwrap();
assert!(
!abandoned_pages.contains(&allocated),
"bead_id={BEAD_ID} case=memory_db_allocator_skips_freelist_after_rollback allocated={} abandoned=({}, {})",
allocated.get(),
abandoned_pages[0].get(),
abandoned_pages[1].get()
);
assert_eq!(
allocated,
expected,
"bead_id={BEAD_ID} case=memory_db_allocator_keeps_bump_sequence allocated={} expected={}",
allocated.get(),
expected.get()
);
});
}
#[test]
fn test_live_db_size_excludes_unissued_page_lease_reservations() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let pager = SimplePager::open(vfs, Path::new("/:memory:"), PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let _page_one = txn.allocate_page(&cx).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
assert_eq!(
txn.live_db_size(),
page_two.get(),
"bead_id={BEAD_ID} case=live_db_size_counts_issued_pages_not_unissued_lease"
);
});
}
#[test]
fn test_free_page_requires_page_one_conflict_tracking_defers_concurrent_freelist_reconciliation()
{
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let durable_page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &[0xAB; 32]).await.unwrap();
seed.commit(&cx).await.unwrap();
page
};
let mut durable_txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(
!durable_txn
.free_page_requires_page_one_conflict_tracking(durable_page)
.unwrap(),
"bead_id={BEAD_ID} case=concurrent_durable_free_defers_page_one_conflict_tracking"
);
durable_txn.free_page(&cx, durable_page).await.unwrap();
let durable_predicted = durable_txn.pending_commit_pages().unwrap();
assert!(
durable_predicted.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=concurrent_durable_free_still_puts_page_one_in_pending_commit_surface"
);
durable_txn.rollback(&cx).await.unwrap();
let reusable_pages = {
let mut abandoned = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page_one = abandoned.allocate_page(&cx).await.unwrap();
let page_two = abandoned.allocate_page(&cx).await.unwrap();
abandoned
.write_page(&cx, page_one, &vec![0x55; ps])
.await
.unwrap();
abandoned
.write_page(&cx, page_two, &vec![0x66; ps])
.await
.unwrap();
abandoned.rollback(&cx).await.unwrap();
[page_one, page_two]
};
let mut net_zero_txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let reused = net_zero_txn.allocate_page(&cx).await.unwrap();
assert!(
reusable_pages.contains(&reused),
"bead_id={BEAD_ID} case=net_zero_free_reuses_abandoned_page reused={} expected=({}, {})",
reused.get(),
reusable_pages[0].get(),
reusable_pages[1].get()
);
assert!(
!net_zero_txn
.free_page_requires_page_one_conflict_tracking(reused)
.unwrap(),
"bead_id={BEAD_ID} case=net_zero_free_skips_page_one_conflict_tracking"
);
});
}
/// Regression test for beads_rust#138: concurrent freelist corruption.
///
/// Before the fix, two transactions committing concurrently could push
/// freed pages into the shared `inner.freelist` during Phase A, then
/// each serialize a different freelist snapshot into their write_set.
/// When the WAL flusher wrote both batches, the last writer's page 1
/// (with potentially stale freelist trunk/count) would overwrite the
/// first writer's, creating orphaned pages.
///
/// This test verifies that after two sequential commits that free
/// different pages, the inner.freelist is consistent (contains exactly
/// the freed pages) and the page 1 freelist metadata matches.
#[test]
fn test_concurrent_freelist_no_orphaned_pages_after_sequential_free_commits() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Seed: allocate pages 2..5, commit.
let (p2, p3, p4, p5) = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
let p4 = seed.allocate_page(&cx).await.unwrap();
let p5 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x22; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x33; ps]).await.unwrap();
seed.write_page(&cx, p4, &vec![0x44; ps]).await.unwrap();
seed.write_page(&cx, p5, &vec![0x55; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
(p2, p3, p4, p5)
};
// Transaction 1: free p2
{
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn1.free_page(&cx, p2).await.unwrap();
txn1.commit(&cx).await.unwrap();
}
// Verify p2 is in the freelist after T1 commits.
{
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&p2),
"bead_id={BEAD_ID} case=freed_page_promoted_to_freelist_after_commit p2={}",
p2.get()
);
}
// Transaction 2: free p3
{
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p3).await.unwrap();
txn2.commit(&cx).await.unwrap();
}
// Verify both p2 and p3 are in the freelist.
{
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&p2),
"bead_id={BEAD_ID} case=p2_still_in_freelist_after_second_commit p2={}",
p2.get()
);
assert!(
inner.freelist.contains(&p3),
"bead_id={BEAD_ID} case=p3_in_freelist_after_commit p3={}",
p3.get()
);
assert!(
!inner.freelist.contains(&p4),
"bead_id={BEAD_ID} case=p4_not_freed p4={}",
p4.get()
);
assert!(
!inner.freelist.contains(&p5),
"bead_id={BEAD_ID} case=p5_not_freed p5={}",
p5.get()
);
}
// Verify the pages can be re-allocated and read correctly.
{
let mut txn3 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused1 = txn3.allocate_page(&cx).await.unwrap();
let reused2 = txn3.allocate_page(&cx).await.unwrap();
// Should get p2 and p3 back (from freelist).
assert!(
[p2, p3].contains(&reused1) && [p2, p3].contains(&reused2),
"bead_id={BEAD_ID} case=freed_pages_reusable reused1={} reused2={}",
reused1.get(),
reused2.get()
);
txn3.rollback(&cx).await.unwrap();
}
});
}
/// Regression test for beads_rust#138: freed pages must not leak into
/// inner.freelist when commit fails.
///
/// Before the fix, freed pages were pushed into inner.freelist during
/// Phase A regardless of whether Phase B (WAL I/O) succeeded. If the
/// commit failed, those pages remained in the freelist and could be
/// allocated by subsequent transactions even though the free was never
/// committed to the WAL.
#[test]
fn test_freed_pages_not_in_freelist_before_commit_success() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Seed: allocate p2
let p2 = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0xAA; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
p2
};
// Begin a transaction that frees p2 but DON'T commit yet.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.free_page(&cx, p2).await.unwrap();
// Before commit, p2 should NOT be in inner.freelist
// (it's only in the transaction's local freed_pages).
{
let inner = pager.inner.lock().unwrap();
assert!(
!inner.freelist.contains(&p2),
"bead_id={BEAD_ID} case=freed_page_not_leaked_before_commit p2={}",
p2.get()
);
}
// Now commit — p2 should appear in freelist only after success.
txn.commit(&cx).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
assert!(
inner.freelist.contains(&p2),
"bead_id={BEAD_ID} case=freed_page_promoted_after_successful_commit p2={}",
p2.get()
);
}
});
}
#[test]
fn test_write_page_requires_page_one_conflict_tracking_defers_concurrent_growth_to_commit_surface()
{
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
assert!(
!txn.write_page_requires_page_one_conflict_tracking(page)
.unwrap(),
"bead_id={BEAD_ID} case=concurrent_growth_write_defers_page_one_conflict_tracking"
);
txn.write_page(&cx, page, &[0x7B; 32]).await.unwrap();
let predicted = txn.pending_commit_pages().unwrap();
assert!(
predicted.contains(&page),
"bead_id={BEAD_ID} case=concurrent_growth_pending_commit_surface_keeps_high_page"
);
assert!(
predicted.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=concurrent_growth_pending_commit_surface_still_contains_page_one"
);
});
}
#[test]
fn test_pending_conflict_pages_exclude_synthetic_page_one_for_concurrent_wal_growth() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &[0x7B; 32]).await.unwrap();
let pending_commit = txn.pending_commit_pages().unwrap();
let pending_conflict = txn.pending_conflict_pages().unwrap();
// D1-CRITICAL: Pure WAL growth does not put Page 1 in the pending
// transaction surface before commit preparation. The commit path may
// still inject a synthetic Page 1 header frame so WAL readers see the
// new page count, but that bookkeeping frame must stay out of the
// first-committer-wins conflict surface.
assert!(
!pending_commit.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=concurrent_wal_growth_commit_surface_excludes_page_one"
);
assert!(
pending_commit.contains(&page),
"bead_id={BEAD_ID} case=concurrent_wal_growth_commit_surface_keeps_data_page"
);
assert!(
!pending_conflict.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=concurrent_wal_growth_conflict_surface_excludes_synthetic_page_one"
);
assert!(
pending_conflict.contains(&page),
"bead_id={BEAD_ID} case=concurrent_wal_growth_conflict_surface_keeps_real_data_page"
);
});
}
#[test]
fn test_freelist_dirty_after_lease_return_short_circuits_clean_append_growth() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &[0x7D; 32]).await.unwrap();
let inner = txn.inner.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
assert!(
!txn.freelist_metadata_dirty_with_pending_free_pages(
&inner,
committed_db_size,
&[]
),
"bead_id={BEAD_ID} case=clean_append_growth_has_no_freelist_metadata_delta"
);
});
}
#[test]
fn test_freelist_dirty_after_lease_return_skips_volatile_eof_lease_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let first_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, first_page, &vec![0x71; ps])
.await
.unwrap();
let second_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, second_page, &vec![0x72; ps])
.await
.unwrap();
let inner_arc = Arc::clone(&txn.inner);
let inner = inner_arc.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
assert!(
txn.page_lease
.iter()
.all(|page| page.get() > committed_db_size),
"bead_id=bd-wee9a case=volatile_eof_lease_pages_must_be_above_commit_size"
);
let pending_returned_pages = txn.page_lease.clone();
let lease_return_affects_durable_freelist = pending_returned_pages
.iter()
.any(|page| page.get() <= committed_db_size);
assert!(
!lease_return_affects_durable_freelist,
"bead_id=bd-wee9a case=volatile_eof_lease_return_has_no_durable_freelist_effect"
);
assert!(
!txn.freelist_metadata_dirty_with_pending_free_pages(
&inner,
committed_db_size,
&pending_returned_pages,
),
"bead_id=bd-wee9a case=volatile_eof_lease_return_skips_freelist_dirty_work"
);
});
}
#[test]
fn test_freelist_dirty_after_lease_return_keeps_durable_lease_holes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let first_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, first_page, &vec![0x81; ps])
.await
.unwrap();
let second_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, second_page, &vec![0x82; ps])
.await
.unwrap();
let high_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, high_page, &vec![0x83; ps])
.await
.unwrap();
assert!(
!txn.page_lease.is_empty(),
"bead_id={BEAD_ID} case=lease_hole_test_must_exercise_batched_allocator"
);
let inner_arc = Arc::clone(&txn.inner);
let inner = inner_arc.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
let pending_returned_pages = txn.page_lease.clone();
let lease_return_affects_durable_freelist = pending_returned_pages
.iter()
.any(|page| page.get() <= committed_db_size);
assert!(
lease_return_affects_durable_freelist,
"bead_id={BEAD_ID} case=lease_hole_return_affects_durable_freelist"
);
assert!(
txn.freelist_metadata_dirty_with_pending_free_pages(
&inner,
committed_db_size,
&pending_returned_pages,
),
"bead_id={BEAD_ID} case=returned_lease_holes_remain_durable_freelist_delta"
);
});
}
#[test]
fn test_concurrent_commit_freelists_unstaged_allocated_eof_holes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let (unwritten_page, high_page) = {
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let first_page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, first_page, &vec![0x91; ps])
.await
.unwrap();
let unwritten_page = txn.allocate_page(&cx).await.unwrap();
let high_page = txn.allocate_page(&cx).await.unwrap();
assert!(
unwritten_page.get() < high_page.get(),
"bead_id={BEAD_ID} case=unstaged_allocation_test_needs_page_count_hole"
);
assert!(
!txn.write_set.contains_key(&unwritten_page),
"bead_id={BEAD_ID} case=unwritten_allocated_page_must_not_be_staged"
);
txn.write_page(&cx, high_page, &vec![0x92; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
(unwritten_page, high_page)
};
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.db_size,
high_page.get(),
"bead_id={BEAD_ID} case=high_page_advances_committed_db_size"
);
assert!(
inner.freelist.contains(&unwritten_page),
"bead_id={BEAD_ID} case=unstaged_allocated_page_reusable_in_memory page={}",
unwritten_page.get()
);
let page1 = inner
.read_committed_page_copy(&cx, &pager.cache, &pager.wal_backend, PageNumber::ONE)
.await
.unwrap();
let header_bytes: [u8; DATABASE_HEADER_SIZE] =
page1[..DATABASE_HEADER_SIZE].try_into().unwrap();
let header = DatabaseHeader::from_bytes(&header_bytes).unwrap();
let durable_freelist = load_freelist_from_committed_state(
&cx,
&inner,
&pager.cache,
&pager.wal_backend,
header.page_count,
header.freelist_trunk,
header.freelist_count,
)
.await
.unwrap();
assert!(
durable_freelist.contains(&unwritten_page),
"bead_id={BEAD_ID} case=unstaged_allocated_page_serialized_to_durable_freelist page={} freelist={durable_freelist:?}",
unwritten_page.get()
);
});
}
#[test]
fn test_commit_conflict_pages_exclude_synthetic_page_one_after_wal_growth_injection() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x7C; ps]).await.unwrap();
let wal_page1_plan = {
let inner = txn.inner.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
let freelist_dirty =
txn.freelist_metadata_dirty_with_inner(&inner, committed_db_size);
txn.classify_wal_page_one_write(inner.db_size, freelist_dirty)
};
assert!(
!wal_page1_plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=synthetic_wal_page1_plan_not_direct_page_one_write"
);
assert!(
wal_page1_plan.requires_page_count_advance(),
"bead_id={BEAD_ID} case=synthetic_wal_page1_plan_db_growth"
);
let synthetic_page_one = StagedPage::from_bytes(&txn.pool, &[0xA5; 32]).unwrap();
insert_staged_page(
&mut txn.write_set,
&mut txn.write_pages_sorted,
PageNumber::ONE,
synthetic_page_one,
);
assert!(
txn.write_set.contains_key(&PageNumber::ONE),
"bead_id={BEAD_ID} case=commit_prepare_injected_synthetic_page_one"
);
let conflict_pages = {
let inner = txn.inner.lock().unwrap();
txn.predicted_conflict_pages_for_wal_commit_with_inner(&inner, wal_page1_plan, &[])
};
assert!(
!conflict_pages.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=synthetic_wal_page1_not_cross_process_conflict"
);
assert!(
conflict_pages.contains(&page),
"bead_id={BEAD_ID} case=data_page_remains_cross_process_conflict"
);
});
}
/// bd-db300.3.7 / bd-db300.3.6: two logically disjoint concurrent growth
/// writers must not collide on page 1.
///
/// The bug's original evidence (2026-03-13) showed `allocate_page`/`free_page`
/// forcing `PageNumber::ONE` into the per-op MVCC conflict surface, so every
/// growing writer converged on page 1 before it ever reached commit. The fix
/// landed in two parts: commit `1eb0ed67` deferred per-op page-1 tracking to
/// the commit surface for concurrent transactions, and bd-3wop3.8
/// (D1-CRITICAL) made the commit surface treat a pure page-count advance as a
/// commutative synthetic update rather than a cross-writer conflict (only a
/// *direct* page-1 rewrite — schema ops — stays a conflict).
///
/// The existing tests cover each gate in isolation; this is the missing
/// multi-writer regression guard: two overlapping concurrent writers that
/// each grow the database on disjoint EOF pages must (a) keep page 1 out of
/// their predicted conflict surface and (b) both commit and persist.
#[test]
fn test_disjoint_concurrent_growth_writers_exclude_page_one_from_conflict_surface() {
asupersync::test_utils::run_test(|| async {
let (pager, _frames) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut writer_a = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let mut writer_b = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
// Neither pure-growth writer pre-declares page 1 in the per-op MVCC
// conflict surface (commit 1eb0ed67 deferral).
assert!(
!writer_a
.allocate_page_requires_page_one_conflict_tracking()
.unwrap(),
"bead_id=bd-db300.3.7 case=writer_a_growth_defers_page_one_conflict_tracking"
);
assert!(
!writer_b
.allocate_page_requires_page_one_conflict_tracking()
.unwrap(),
"bead_id=bd-db300.3.7 case=writer_b_growth_defers_page_one_conflict_tracking"
);
let page_a = writer_a.allocate_page(&cx).await.unwrap();
let page_b = writer_b.allocate_page(&cx).await.unwrap();
assert_ne!(
page_a,
page_b,
"bead_id=bd-db300.3.7 case=leased_allocator_gives_disjoint_eof_pages a={} b={}",
page_a.get(),
page_b.get()
);
assert_ne!(
page_a,
PageNumber::ONE,
"bead_id=bd-db300.3.7 case=growth_allocates_beyond_page_one"
);
assert_ne!(
page_b,
PageNumber::ONE,
"bead_id=bd-db300.3.7 case=growth_allocates_beyond_page_one"
);
writer_a
.write_page(&cx, page_a, &vec![0x3A; ps])
.await
.unwrap();
writer_b
.write_page(&cx, page_b, &vec![0x3B; ps])
.await
.unwrap();
// Each writer's predicted cross-process conflict surface excludes page 1
// (a pure page-count advance is not a direct page-1 rewrite) while still
// keeping its own disjoint data page.
let assert_excludes_page_one =
|label: &str, txn: &SimpleTransaction<MemoryVfs>, page: PageNumber| {
let inner = txn.inner.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
let freelist_dirty =
txn.freelist_metadata_dirty_with_inner(&inner, committed_db_size);
let wal_page1_plan =
txn.classify_wal_page_one_write(inner.db_size, freelist_dirty);
assert!(
!wal_page1_plan.requires_page_one_rewrite(),
"bead_id=bd-db300.3.7 case=writer_{label}_growth_is_not_direct_page_one_rewrite"
);
assert!(
wal_page1_plan.requires_page_count_advance(),
"bead_id=bd-db300.3.7 case=writer_{label}_growth_advances_page_count"
);
let conflict_pages = txn.predicted_conflict_pages_for_wal_commit_with_inner(
&inner,
wal_page1_plan,
&[],
);
assert!(
!conflict_pages.contains(&PageNumber::ONE),
"bead_id=bd-db300.3.7 case=writer_{label}_growth_excludes_page_one conflicts={conflict_pages:?}"
);
assert!(
conflict_pages.contains(&page),
"bead_id=bd-db300.3.7 case=writer_{label}_disjoint_data_page_remains_conflict page={}",
page.get()
);
};
assert_excludes_page_one("a", &writer_a, page_a);
assert_excludes_page_one("b", &writer_b, page_b);
// Both disjoint writers commit successfully — no false page-1 collision
// serializes the second committer behind the first.
writer_a.commit(&cx).await.unwrap();
writer_b.commit(&cx).await.unwrap();
// Both disjoint pages persist, and the committed db size covers both.
let inner = pager.inner.lock().unwrap();
assert!(
inner.db_size >= page_a.get().max(page_b.get()),
"bead_id=bd-db300.3.7 case=committed_db_size_covers_both_disjoint_pages db_size={} a={} b={}",
inner.db_size,
page_a.get(),
page_b.get()
);
let persisted_a = inner
.read_committed_page_copy(&cx, &pager.cache, &pager.wal_backend, page_a)
.await
.unwrap();
let persisted_b = inner
.read_committed_page_copy(&cx, &pager.cache, &pager.wal_backend, page_b)
.await
.unwrap();
assert_eq!(
persisted_a[0], 0x3A,
"bead_id=bd-db300.3.7 case=writer_a_disjoint_page_persisted"
);
assert_eq!(
persisted_b[0], 0x3B,
"bead_id=bd-db300.3.7 case=writer_b_disjoint_page_persisted"
);
});
}
#[test]
fn test_commit_conflict_pages_exclude_synthetic_page_one_after_freelist_serialization() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &vec![0x8D; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
page
};
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
txn.free_page(&cx, page).await.unwrap();
let (wal_page1_plan, pending_freed) = {
let mut inner = txn.inner.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
let freelist_dirty =
txn.freelist_metadata_dirty_with_inner(&inner, committed_db_size);
let wal_page1_plan = txn.classify_wal_page_one_write(inner.db_size, freelist_dirty);
assert!(
freelist_dirty,
"bead_id={BEAD_ID} case=freelist_serialization_regression_requires_dirty_freelist"
);
assert!(
!wal_page1_plan.requires_page_one_rewrite(),
"bead_id={BEAD_ID} case=freelist_serialization_plan_not_direct_page_one_write"
);
let pending_freed = std::mem::take(&mut txn.freed_pages);
serialize_freelist_to_write_set(
&cx,
&mut inner,
&txn.cache,
&txn.wal_backend,
&txn.pool,
&mut txn.write_set,
&mut txn.write_pages_sorted,
committed_db_size,
&pending_freed,
None,
)
.await
.unwrap();
(wal_page1_plan, pending_freed)
};
assert!(
txn.write_set.contains_key(&PageNumber::ONE),
"bead_id={BEAD_ID} case=freelist_serialization_injected_synthetic_page_one"
);
let conflict_pages = {
let inner = txn.inner.lock().unwrap();
txn.predicted_conflict_pages_for_wal_commit_with_inner(
&inner,
wal_page1_plan,
&pending_freed,
)
};
assert!(
!conflict_pages.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=freelist_synthetic_page1_not_cross_process_conflict"
);
assert!(
conflict_pages.contains(&page),
"bead_id={BEAD_ID} case=freelist_trunk_page_remains_cross_process_conflict"
);
});
}
#[test]
fn test_commit_conflict_pages_keep_non_trunk_freed_pages_after_drain() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let (first_page, second_page) = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let first_page = seed.allocate_page(&cx).await.unwrap();
let second_page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, first_page, &vec![0x91; ps])
.await
.unwrap();
seed.write_page(&cx, second_page, &vec![0x92; ps])
.await
.unwrap();
seed.commit(&cx).await.unwrap();
(first_page, second_page)
};
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
txn.free_page(&cx, first_page).await.unwrap();
txn.free_page(&cx, second_page).await.unwrap();
let (wal_page1_plan, pending_freed) = {
let mut inner = txn.inner.lock().unwrap();
let committed_db_size = txn.committed_db_size_with_inner(&inner);
let freelist_dirty =
txn.freelist_metadata_dirty_with_inner(&inner, committed_db_size);
let wal_page1_plan = txn.classify_wal_page_one_write(inner.db_size, freelist_dirty);
let pending_freed = std::mem::take(&mut txn.freed_pages);
serialize_freelist_to_write_set(
&cx,
&mut inner,
&txn.cache,
&txn.wal_backend,
&txn.pool,
&mut txn.write_set,
&mut txn.write_pages_sorted,
committed_db_size,
&pending_freed,
None,
)
.await
.unwrap();
(wal_page1_plan, pending_freed)
};
let non_trunk_freed_page = pending_freed
.iter()
.copied()
.find(|page| !txn.write_set.contains_key(page))
.expect("at least one freed page should be carried only as freelist metadata");
let conflict_pages = {
let inner = txn.inner.lock().unwrap();
txn.predicted_conflict_pages_for_wal_commit_with_inner(
&inner,
wal_page1_plan,
&pending_freed,
)
};
assert!(
conflict_pages.contains(&first_page),
"bead_id={BEAD_ID} case=first_freed_page_remains_cross_process_conflict"
);
assert!(
conflict_pages.contains(&second_page),
"bead_id={BEAD_ID} case=second_freed_page_remains_cross_process_conflict"
);
assert!(
conflict_pages.contains(&non_trunk_freed_page),
"bead_id={BEAD_ID} case=non_trunk_freed_page_survives_drain"
);
});
}
#[test]
fn test_pending_conflict_pages_keep_explicit_page_one_write_for_concurrent_wal() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
txn.write_page(&cx, PageNumber::ONE, &[0x5A; 32])
.await
.unwrap();
let pending_conflict = txn.pending_conflict_pages().unwrap();
assert!(
pending_conflict.contains(&PageNumber::ONE),
"bead_id={BEAD_ID} case=explicit_page_one_write_remains_in_conflict_surface"
);
});
}
#[test]
fn test_write_page_requires_page_one_conflict_tracking_skips_interior_page() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let durable_page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &[0xAB; 32]).await.unwrap();
seed.commit(&cx).await.unwrap();
page
};
let txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
assert!(
!txn.write_page_requires_page_one_conflict_tracking(durable_page)
.unwrap(),
"bead_id={BEAD_ID} case=interior_page_write_skips_page_one_conflict_tracking"
);
});
}
#[test]
fn test_concurrent_rollback_to_savepoint_reclaims_eof_allocations() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let base = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, base, &vec![0x11; ps]).await.unwrap();
txn.savepoint(&cx, "sp").unwrap();
let p3 = txn.allocate_page(&cx).await.unwrap();
let p4 = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p3, &vec![0x22; ps]).await.unwrap();
txn.write_page(&cx, p4, &vec![0x33; ps]).await.unwrap();
txn.rollback_to_savepoint(&cx, "sp").unwrap();
let reused1 = txn.allocate_page(&cx).await.unwrap();
let reused2 = txn.allocate_page(&cx).await.unwrap();
assert!(
reused1.get() == 3 && reused2.get() == 4,
"bead_id={BEAD_ID} case=concurrent_savepoint_reuses_eof_pages reused=({}, {}) expected=(3, 4)",
reused1.get(),
reused2.get()
);
txn.write_page(&cx, reused1, &vec![0x44; ps]).await.unwrap();
txn.write_page(&cx, reused2, &vec![0x55; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.db_size, 4,
"bead_id={BEAD_ID} case=concurrent_savepoint_rollback_does_not_skip_page_numbers"
);
});
}
#[test]
fn test_concurrent_drop_reclaims_eof_allocations_without_holes() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
{
let mut concurrent = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
let p2 = concurrent.allocate_page(&cx).await.unwrap();
let p3 = concurrent.allocate_page(&cx).await.unwrap();
concurrent
.write_page(&cx, p2, &vec![0x66; ps])
.await
.unwrap();
concurrent
.write_page(&cx, p3, &vec![0x77; ps])
.await
.unwrap();
}
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused1 = txn.allocate_page(&cx).await.unwrap();
let reused2 = txn.allocate_page(&cx).await.unwrap();
assert!(
reused1.get() <= 3 && reused2.get() <= 3 && reused1 != reused2,
"bead_id={BEAD_ID} case=concurrent_drop_reuses_abandoned_eof_pages reused=({}, {})",
reused1.get(),
reused2.get()
);
txn.write_page(&cx, reused1, &vec![0x88; ps]).await.unwrap();
txn.write_page(&cx, reused2, &vec![0x99; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.db_size, 3,
"bead_id={BEAD_ID} case=concurrent_drop_does_not_skip_page_numbers"
);
});
}
#[test]
fn test_freelist_persisted_and_reloaded_on_reopen() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/freelist_persist.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn1.allocate_page(&cx).await.unwrap();
txn1.write_page(&cx, p, &vec![0xAB; ps]).await.unwrap();
txn1.commit(&cx).await.unwrap();
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p).await.unwrap();
txn2.commit(&cx).await.unwrap();
let txn_ro = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let raw = txn_ro
.get_page(&cx, PageNumber::ONE)
.await
.unwrap()
.into_vec();
let hdr_bytes: [u8; DATABASE_HEADER_SIZE] =
raw[..DATABASE_HEADER_SIZE].try_into().unwrap();
let hdr = DatabaseHeader::from_bytes(&hdr_bytes).unwrap();
assert_eq!(hdr.freelist_count, 1);
assert_eq!(hdr.freelist_trunk, p.get());
let reopened = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
{
let inner = reopened.inner.lock().unwrap();
assert_eq!(inner.freelist.len(), 1);
assert_eq!(inner.freelist[0], p);
}
let mut txn3 = reopened
.begin(&cx, TransactionMode::Immediate)
.await
.unwrap();
let reused = txn3.allocate_page(&cx).await.unwrap();
assert_eq!(
reused, p,
"reopened pager should reuse persisted freelist page"
);
txn3.commit(&cx).await.unwrap();
});
}
#[test]
#[allow(clippy::similar_names, clippy::cast_possible_truncation)]
fn test_cache_eviction_under_pressure() {
asupersync::test_utils::run_test(|| async {
// Verify that SimplePager can handle more pages than the cache capacity.
// PageCache is initialized with 256 pages. We write 300 pages.
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
// Write 300 pages. This exceeds the 256-page cache capacity.
for i in 0..300u32 {
let p = txn.allocate_page(&cx).await.unwrap();
pages.push(p);
// Unique pattern per page to verify content.
let byte = (i % 256) as u8;
let data = vec![byte; ps];
txn.write_page(&cx, p, &data).await.unwrap();
}
txn.commit(&cx).await.unwrap();
// Read all pages back. Some will be cache misses, requiring eviction of others.
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for (i, &p) in pages.iter().enumerate() {
let data = txn.get_page(&cx, p).await.unwrap();
let expected_byte = (i % 256) as u8;
assert_eq!(
data.as_ref()[0],
expected_byte,
"bead_id={BEAD_ID} case=cache_pressure page={p}"
);
}
});
}
// ═══════════════════════════════════════════════════════════════════
// bd-2ttd8.2: Pager invariant suite — SimplePager correctness
// ═══════════════════════════════════════════════════════════════════
const BEAD_INV: &str = "bd-2ttd8.2";
#[test]
fn test_inv_write_set_not_in_freelist_during_txn() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Allocate, write, commit.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xAA; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
// Free the page.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p).await.unwrap();
// The freed page should be in freed_pages, not in write_set.
assert!(
txn2.freed_pages.contains(&p),
"bead_id={BEAD_INV} inv=freed_page_tracked"
);
assert!(
!txn2.write_set.contains_key(&p),
"bead_id={BEAD_INV} inv=freed_not_in_write_set"
);
txn2.commit(&cx).await.unwrap();
});
}
#[test]
fn test_inv_allocated_pages_sequential_and_nonzero() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for _ in 0..10 {
let p = txn.allocate_page(&cx).await.unwrap();
assert!(p.get() > 0, "bead_id={BEAD_INV} inv=page_nonzero");
// No duplicates.
assert!(
!pages.contains(&p),
"bead_id={BEAD_INV} inv=page_unique p={p}"
);
pages.push(p);
}
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_inv_writer_serialization_single_writer() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let _w1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
// Second immediate should fail (writer_active).
let err = pager.begin(&cx, TransactionMode::Immediate).await;
assert!(
err.is_err(),
"bead_id={BEAD_INV} inv=single_writer_enforced"
);
// Exclusive also fails.
let err2 = pager.begin(&cx, TransactionMode::Exclusive).await;
assert!(
err2.is_err(),
"bead_id={BEAD_INV} inv=exclusive_blocked_by_writer"
);
});
}
#[test]
fn test_inv_writer_released_on_commit() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut w1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
w1.commit(&cx).await.unwrap();
// Writer lock should be released; new writer should succeed.
let _w2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
});
}
#[test]
fn test_inv_writer_released_on_rollback() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let mut w1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
w1.rollback(&cx).await.unwrap();
let _w2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
});
}
#[test]
fn test_inv_writer_released_on_drop() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
{
let _w1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
// Drop without commit or rollback.
}
let _w2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
});
}
#[test]
fn test_inv_commit_persists_all_dirty_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for i in 0..5u8 {
let p = txn.allocate_page(&cx).await.unwrap();
let mut data = vec![0u8; ps];
data[0] = 0xD0 + i;
data[ps - 1] = i;
txn.write_page(&cx, p, &data).await.unwrap();
pages.push((p, 0xD0 + i, i));
}
txn.commit(&cx).await.unwrap();
// Read back in a new read-only transaction.
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for (p, first_byte, last_byte) in &pages {
let data = txn2.get_page(&cx, *p).await.unwrap();
assert_eq!(
data.as_ref()[0],
*first_byte,
"bead_id={BEAD_INV} inv=dirty_page_committed p={p}"
);
assert_eq!(
data.as_ref()[ps - 1],
*last_byte,
"bead_id={BEAD_INV} inv=dirty_page_last_byte p={p}"
);
}
});
}
#[test]
fn test_inv_rollback_discards_all_dirty_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Initial committed data.
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn1.allocate_page(&cx).await.unwrap();
txn1.write_page(&cx, p, &vec![0xAA; ps]).await.unwrap();
txn1.commit(&cx).await.unwrap();
// Overwrite and rollback.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.write_page(&cx, p, &vec![0xBB; ps]).await.unwrap();
txn2.rollback(&cx).await.unwrap();
// Verify original data survives.
let txn3 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn3.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0xAA,
"bead_id={BEAD_INV} inv=rollback_preserves_committed"
);
});
}
#[test]
fn test_inv_savepoint_nested_stack_order() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x01; ps]).await.unwrap();
txn.savepoint(&cx, "sp1").unwrap();
txn.write_page(&cx, p, &vec![0x02; ps]).await.unwrap();
txn.savepoint(&cx, "sp2").unwrap();
txn.write_page(&cx, p, &vec![0x03; ps]).await.unwrap();
txn.savepoint(&cx, "sp3").unwrap();
txn.write_page(&cx, p, &vec![0x04; ps]).await.unwrap();
// Rollback to sp2 → data should be 0x02.
txn.rollback_to_savepoint(&cx, "sp2").unwrap();
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x02,
"bead_id={BEAD_INV} inv=nested_rollback_sp2"
);
// sp3 should no longer exist.
let err = txn.rollback_to_savepoint(&cx, "sp3");
assert!(
err.is_err(),
"bead_id={BEAD_INV} inv=sp3_removed_after_rollback_to_sp2"
);
// sp1 should still exist.
txn.rollback_to_savepoint(&cx, "sp1").unwrap();
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x01,
"bead_id={BEAD_INV} inv=nested_rollback_sp1"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_inv_savepoint_release_merges_to_parent() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x10; ps]).await.unwrap();
txn.savepoint(&cx, "outer").unwrap();
txn.write_page(&cx, p, &vec![0x20; ps]).await.unwrap();
txn.savepoint(&cx, "inner").unwrap();
txn.write_page(&cx, p, &vec![0x30; ps]).await.unwrap();
// Release inner → changes kept.
txn.release_savepoint(&cx, "inner").unwrap();
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x30,
"bead_id={BEAD_INV} inv=release_keeps_changes"
);
// Rollback to outer → restores data from before inner.
txn.rollback_to_savepoint(&cx, "outer").unwrap();
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0x10,
"bead_id={BEAD_INV} inv=rollback_outer_after_release_inner"
);
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_inv_freelist_restored_on_rollback() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Allocate + commit.
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn1.allocate_page(&cx).await.unwrap();
txn1.write_page(&cx, p, &vec![0xAA; ps]).await.unwrap();
txn1.commit(&cx).await.unwrap();
// Free + commit → moves to freelist.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p).await.unwrap();
txn2.commit(&cx).await.unwrap();
let freelist_before = {
let inner = pager.inner.lock().unwrap();
inner.freelist.clone()
};
assert!(
freelist_before.contains(&p),
"bead_id={BEAD_INV} inv=freed_in_freelist"
);
// Allocate from freelist, then rollback → page returns to freelist.
let mut txn3 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused = txn3.allocate_page(&cx).await.unwrap();
assert_eq!(reused, p, "should reuse freed page");
txn3.rollback(&cx).await.unwrap();
let freelist_after = {
let inner = pager.inner.lock().unwrap();
inner.freelist.clone()
};
assert_eq!(
freelist_after, freelist_before,
"bead_id={BEAD_INV} inv=freelist_restored_after_rollback"
);
});
}
#[test]
fn test_inv_page_identity_read_before_write() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Allocate a page, write, commit.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xAA; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
// Read in new transaction → should see committed data.
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn2.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0xAA,
"bead_id={BEAD_INV} inv=committed_visible"
);
});
}
#[test]
fn test_inv_write_set_isolation() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Commit baseline data.
let mut txn1 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn1.allocate_page(&cx).await.unwrap();
txn1.write_page(&cx, p, &vec![0x11; ps]).await.unwrap();
txn1.commit(&cx).await.unwrap();
// Start a reader → sees committed.
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let r_data = reader.get_page(&cx, p).await.unwrap();
assert_eq!(r_data.as_ref()[0], 0x11);
// Writer modifies.
let mut writer = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
writer.write_page(&cx, p, &vec![0x22; ps]).await.unwrap();
// Reader still sees committed data (write-set is txn-private).
let r_data2 = reader.get_page(&cx, p).await.unwrap();
assert_eq!(
r_data2.as_ref()[0],
0x11,
"bead_id={BEAD_INV} inv=write_set_isolated_from_readers"
);
writer.commit(&cx).await.unwrap();
});
}
#[test]
fn test_inv_db_size_grows_on_allocate_commit() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let initial_size = {
let inner = pager.inner.lock().unwrap();
inner.db_size
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for _ in 0..5 {
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x00; ps]).await.unwrap();
}
txn.commit(&cx).await.unwrap();
let final_size = {
let inner = pager.inner.lock().unwrap();
inner.db_size
};
assert!(
final_size > initial_size,
"bead_id={BEAD_INV} inv=db_size_grows initial={initial_size} final={final_size}"
);
});
}
#[test]
fn test_inv_db_size_restored_on_rollback() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let size_before = {
let inner = pager.inner.lock().unwrap();
inner.db_size
};
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for _ in 0..5 {
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x00; ps]).await.unwrap();
}
txn.rollback(&cx).await.unwrap();
let size_after = {
let inner = pager.inner.lock().unwrap();
inner.db_size
};
assert_eq!(
size_after, size_before,
"bead_id={BEAD_INV} inv=db_size_restored_on_rollback"
);
});
}
#[test]
fn test_inv_active_transaction_count() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let count_before = {
let inner = pager.inner.lock().unwrap();
inner.active_transactions
};
assert_eq!(count_before, 0, "bead_id={BEAD_INV} inv=initial_zero_txns");
let r1 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let r2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.active_transactions, 2,
"bead_id={BEAD_INV} inv=two_active_txns"
);
}
drop(r1);
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.active_transactions, 1,
"bead_id={BEAD_INV} inv=one_after_drop"
);
}
drop(r2);
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.active_transactions, 0,
"bead_id={BEAD_INV} inv=zero_after_all_dropped"
);
}
});
}
#[test]
fn test_inv_journal_mode_default_delete() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
assert_eq!(
pager.journal_mode(),
JournalMode::Delete,
"bead_id={BEAD_INV} inv=default_journal_delete"
);
});
}
#[test]
fn test_inv_commit_seq_monotonic() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut prev_seq = {
let inner = pager.inner.lock().unwrap();
inner.commit_seq.get()
};
for _ in 0..5 {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x00; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let seq = {
let inner = pager.inner.lock().unwrap();
inner.commit_seq.get()
};
assert!(
seq >= prev_seq,
"bead_id={BEAD_INV} inv=commit_seq_monotonic seq={seq} prev={prev_seq}"
);
prev_seq = seq;
}
});
}
// ═══════════════════════════════════════════════════════════════════
// bd-2ttd8.3: Deterministic pager e2e scenarios with cache-pressure
// telemetry
// ═══════════════════════════════════════════════════════════════════
const BEAD_E2E: &str = "bd-2ttd8.3";
#[test]
fn test_e2e_sequential_write_read_with_metrics() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
pager.reset_cache_metrics().unwrap();
// Phase 1: Sequential write of 20 pages.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for i in 0..20u32 {
let p = txn.allocate_page(&cx).await.unwrap();
let mut data = vec![0u8; ps];
data[0] = (i & 0xFF) as u8;
data[1] = ((i >> 8) & 0xFF) as u8;
txn.write_page(&cx, p, &data).await.unwrap();
pages.push(p);
}
txn.commit(&cx).await.unwrap();
let post_write = pager.cache_metrics_snapshot().unwrap();
assert!(
post_write.admits > 0,
"bead_id={BEAD_E2E} case=seq_write_admits"
);
// Phase 2: Sequential read — all pages should be cached.
pager.reset_cache_metrics().unwrap();
let read_before = read_surface_snapshot(&pager);
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for (i, &p) in pages.iter().enumerate() {
let data = txn2.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
(i & 0xFF) as u8,
"bead_id={BEAD_E2E} case=seq_read_content page={p}"
);
}
let post_read = pager.cache_metrics_snapshot().unwrap();
let read_after = read_surface_snapshot(&pager);
println!(
"DEBUG_METRICS: hits={} misses={} admits={} evictions={} cached={}",
post_read.hits,
post_read.misses,
post_read.admits,
post_read.evictions,
post_read.cached_pages
);
let total_reads = observed_read_total(read_before, read_after);
assert!(
total_reads == 20,
"bead_id={BEAD_E2E} case=seq_read_accesses total={}",
total_reads
);
let hit_rate = observed_read_hit_rate_percent(read_before, read_after);
assert!(
hit_rate > 40.0,
"bead_id={BEAD_E2E} case=seq_read_hit_rate rate={}",
hit_rate
);
});
}
#[test]
fn test_cache_efficiency_snapshot_matches_raw_cache_metrics() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0xAB; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
pager.reset_cache_metrics().unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let _ = reader.get_page(&cx, page).await.unwrap();
let raw = pager.cache_metrics_snapshot().unwrap();
let efficiency = pager.cache_efficiency_snapshot().unwrap();
assert_eq!(
efficiency.hits, raw.hits,
"bead_id={BEAD_E2E} case=efficiency_snapshot_hits"
);
assert_eq!(
efficiency.misses, raw.misses,
"bead_id={BEAD_E2E} case=efficiency_snapshot_misses"
);
assert_eq!(
efficiency.evictions, raw.evictions,
"bead_id={BEAD_E2E} case=efficiency_snapshot_evictions"
);
assert_eq!(
efficiency.cached_pages, raw.cached_pages,
"bead_id={BEAD_E2E} case=efficiency_snapshot_cached_pages"
);
assert!(
(efficiency.hit_rate_percent() - raw.hit_rate_percent()).abs() < f64::EPSILON,
"bead_id={BEAD_E2E} case=efficiency_snapshot_hit_rate"
);
});
}
#[test]
fn test_e2e_cache_pressure_eviction_telemetry() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Pool capacity is 1024. Write 300 pages — all fit in cache.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for i in 0..300u32 {
let p = txn.allocate_page(&cx).await.unwrap();
let byte = (i % 256) as u8;
txn.write_page(&cx, p, &vec![byte; ps]).await.unwrap();
pages.push((p, byte));
}
txn.commit(&cx).await.unwrap();
pager.reset_cache_metrics().unwrap();
let read_before = read_surface_snapshot(&pager);
// Sequential read of all 300 pages.
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for (p, expected) in &pages {
let data = txn2.get_page(&cx, *p).await.unwrap();
assert_eq!(
data.as_ref()[0],
*expected,
"bead_id={BEAD_E2E} case=pressure_content page={p}"
);
}
let read_after = read_surface_snapshot(&pager);
let total = observed_read_total(read_before, read_after);
assert_eq!(
total, 300,
"bead_id={BEAD_E2E} case=pressure_total_accesses"
);
let hit_rate = observed_read_hit_rate_percent(read_before, read_after);
assert!(
hit_rate > 40.0,
"bead_id={BEAD_E2E} case=pressure_hit_rate rate={}",
hit_rate
);
});
}
#[test]
fn test_e2e_hot_cold_workload_hit_rate() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Write 50 pages.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for i in 0..50u32 {
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![(i % 256) as u8; ps])
.await
.unwrap();
pages.push(p);
}
txn.commit(&cx).await.unwrap();
// Define hot set (first 5 pages) and cold set (remaining 45).
let hot = &pages[..5];
pager.reset_cache_metrics().unwrap();
let read_before = read_surface_snapshot(&pager);
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let mut logical_reads = 0_u64;
let mut unique_pages = std::collections::BTreeSet::new();
for idx in 0..200usize {
// Five hot accesses plus two cold accesses per round.
for h in hot {
let _ = txn2.get_page(&cx, *h).await.unwrap();
logical_reads += 1;
unique_pages.insert(h.get());
}
// 2 cold accesses (rotating through cold pages).
let cold_idx = (idx * 2) % 45;
let first_cold = pages[5 + cold_idx];
let second_cold = pages[5 + (cold_idx + 1) % 45];
let _ = txn2.get_page(&cx, first_cold).await.unwrap();
let _ = txn2.get_page(&cx, second_cold).await.unwrap();
logical_reads += 2;
unique_pages.insert(first_cold.get());
unique_pages.insert(second_cold.get());
}
drop(txn2);
let read_after = read_surface_snapshot(&pager);
let shared_plane_reads = observed_read_total(read_before, read_after);
let expected_total_reads = u64::try_from((hot.len() + 2) * 200).unwrap();
assert_eq!(
logical_reads, expected_total_reads,
"bead_id={BEAD_E2E} case=hot_cold_logical_accesses"
);
assert_eq!(
unique_pages.len(),
pages.len(),
"bead_id={BEAD_E2E} case=hot_cold_workload_coverage"
);
assert_eq!(
shared_plane_reads,
u64::try_from(unique_pages.len()).unwrap(),
"bead_id={BEAD_E2E} case=hot_cold_unique_shared_accesses"
);
// Hot pages should achieve high hit rate after first access.
let hit_rate = observed_read_hit_rate_percent(read_before, read_after);
assert!(
hit_rate > 40.0,
"bead_id={BEAD_E2E} case=hot_cold_hit_rate rate={}",
hit_rate
);
});
}
#[test]
fn test_e2e_random_access_pattern_deterministic() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Write 100 pages.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for _ in 0..100u32 {
let p = txn.allocate_page(&cx).await.unwrap();
let mut data = vec![0u8; ps];
// Unique fingerprint: page number in first 4 bytes.
data[..4].copy_from_slice(&p.get().to_le_bytes());
txn.write_page(&cx, p, &data).await.unwrap();
pages.push(p);
}
txn.commit(&cx).await.unwrap();
// Deterministic "random" access via linear congruential generator.
// LCG: next = (a * prev + c) mod m, with a=13, c=7, m=100.
pager.reset_cache_metrics().unwrap();
let read_before = read_surface_snapshot(&pager);
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let mut idx: usize = 0;
let mut unique_pages = std::collections::BTreeSet::new();
for _ in 0..200 {
idx = (13 * idx + 7) % 100;
let p = pages[idx];
unique_pages.insert(p.get());
let data = txn2.get_page(&cx, p).await.unwrap();
let stored_pgno = u32::from_le_bytes(data.as_ref()[..4].try_into().unwrap());
assert_eq!(
stored_pgno,
p.get(),
"bead_id={BEAD_E2E} case=random_fingerprint page={p}"
);
}
let read_after = read_surface_snapshot(&pager);
let shared_plane_reads = observed_read_total(read_before, read_after);
assert_eq!(
unique_pages.len(),
20,
"bead_id={BEAD_E2E} case=random_lcg_cycle_coverage"
);
assert_eq!(
shared_plane_reads,
u64::try_from(unique_pages.len()).unwrap(),
"bead_id={BEAD_E2E} case=random_unique_shared_accesses"
);
// With 100 pages and 256-page cache, everything fits → high hit rate.
let hit_rate = observed_read_hit_rate_percent(read_before, read_after);
assert!(
hit_rate > 40.0,
"bead_id={BEAD_E2E} case=random_hit_rate rate={}",
hit_rate
);
});
}
#[test]
fn test_e2e_mixed_read_write_workload() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Phase 1: Seed 30 pages.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
for _ in 0..30u32 {
let p = txn.allocate_page(&cx).await.unwrap();
let mut data = vec![0u8; ps];
// Unique fingerprint: page number in first 4 bytes.
data[..4].copy_from_slice(&p.get().to_le_bytes());
txn.write_page(&cx, p, &data).await.unwrap();
pages.push(p);
}
txn.commit(&cx).await.unwrap();
// Phase 2: Mixed read/write in batches (deterministic).
let mut phase_two_reads = 0_u64;
for batch in 0..5u32 {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let batch_read_before = read_surface_snapshot(&pager);
// Read existing pages.
for i in 0..10 {
let idx = ((batch as usize * 3) + i) % pages.len();
let _ = txn.get_page(&cx, pages[idx]).await.unwrap();
}
let batch_read_after = read_surface_snapshot(&pager);
phase_two_reads = phase_two_reads
.saturating_add(observed_read_total(batch_read_before, batch_read_after));
// Write/overwrite some pages.
for i in 0..3 {
let idx = ((batch as usize * 5) + i) % pages.len();
let new_val = ((batch * 10 + i as u32) % 256) as u8;
txn.write_page(&cx, pages[idx], &vec![new_val; ps])
.await
.unwrap();
}
// Allocate a new page per batch.
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xF0 + batch as u8; ps])
.await
.unwrap();
pages.push(p);
txn.commit(&cx).await.unwrap();
}
// Phase 3: Verify final state.
assert!(
phase_two_reads == 50,
"bead_id={BEAD_E2E} case=mixed_total_accesses total={}",
phase_two_reads
);
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
// Verify the 5 newly allocated pages.
for batch in 0..5u32 {
let p = pages[30 + batch as usize];
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0xF0 + batch as u8,
"bead_id={BEAD_E2E} case=mixed_new_page batch={batch}"
);
}
});
}
#[test]
fn test_e2e_write_overwrite_verify_latest() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Allocate and commit.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x01; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
// Overwrite 10 times across separate transactions.
for version in 2..=11u8 {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, p, &vec![version; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
}
// Final read should see version 11.
let txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
11,
"bead_id={BEAD_E2E} case=overwrite_latest_version"
);
});
}
#[test]
fn test_e2e_savepoint_heavy_workload() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::new();
// Allocate 10 pages.
for i in 0..10u8 {
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![i; ps]).await.unwrap();
pages.push(p);
}
// Savepoint → more writes → rollback → verify.
txn.savepoint(&cx, "sp_heavy").unwrap();
for &p in &pages {
txn.write_page(&cx, p, &vec![0xFF; ps]).await.unwrap();
}
// All pages should read 0xFF before rollback.
for &p in &pages {
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(data.as_ref()[0], 0xFF);
}
txn.rollback_to_savepoint(&cx, "sp_heavy").unwrap();
// After rollback, original values restored.
for (i, &p) in pages.iter().enumerate() {
let data = txn.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
i as u8,
"bead_id={BEAD_E2E} case=savepoint_heavy_restored page={p}"
);
}
txn.commit(&cx).await.unwrap();
});
}
#[test]
fn test_e2e_alloc_free_cycle_no_leak() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let initial_db_size = {
let inner = pager.inner.lock().unwrap();
inner.db_size
};
// Cycle: allocate → commit → free → commit, 10 times.
let mut freed_pages = Vec::new();
for _ in 0..10 {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xCC; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.free_page(&cx, p).await.unwrap();
txn2.commit(&cx).await.unwrap();
freed_pages.push(p);
}
// Freelist should have pages available for reuse.
let freelist_len = {
let inner = pager.inner.lock().unwrap();
inner.freelist.len()
};
assert!(
freelist_len > 0,
"bead_id={BEAD_E2E} case=alloc_free_freelist_populated"
);
// Allocate again — should reuse freed pages.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let reused = txn.allocate_page(&cx).await.unwrap();
assert!(
freed_pages.contains(&reused),
"bead_id={BEAD_E2E} case=alloc_free_reuse reused={reused}"
);
txn.commit(&cx).await.unwrap();
let final_db_size = {
let inner = pager.inner.lock().unwrap();
inner.db_size
};
assert_eq!(
final_db_size,
initial_db_size + 1,
"DB size should only grow by 1 page (the one currently allocated)"
);
});
}
#[test]
fn test_e2e_metrics_monotonic_across_transactions() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut prev_total = 0u64;
for round in 0..5u32 {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![round as u8; ps])
.await
.unwrap();
txn.commit(&cx).await.unwrap();
// Read back.
let txn2 = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let _ = txn2.get_page(&cx, p).await.unwrap();
let snapshot = read_surface_snapshot(&pager);
let total = snapshot
.cache
.total_accesses()
.saturating_add(snapshot.published_hits);
assert!(
total >= prev_total,
"bead_id={BEAD_E2E} case=metrics_monotonic round={round} \
total={} prev={}",
total,
prev_total
);
prev_total = total;
}
});
}
#[test]
fn test_e2e_journal_recovery_after_crash_simulation() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/crash_sim.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
// Write committed data.
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xAA; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
// Start another write but DON'T commit → simulates crash mid-journal.
let mut txn2 = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn2.write_page(&cx, p, &vec![0xBB; ps]).await.unwrap();
// Drop without commit → implicit rollback.
drop(txn2);
drop(pager);
// Re-open: hot journal recovery should restore original data.
let pager2 = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let txn3 = pager2.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = txn3.get_page(&cx, p).await.unwrap();
assert_eq!(
data.as_ref()[0],
0xAA,
"bead_id={BEAD_E2E} case=journal_recovery_restores_committed"
);
});
}
#[test]
fn test_published_current_sequence_gen_tracks_snapshot_gen() {
asupersync::test_utils::run_test(|| async {
// The seqlock recheck in `SimpleTransaction::get_page` relies on
// `current_sequence_gen()` returning exactly the value that
// `snapshot()` would have produced for `snapshot_gen`. Drift between
// the two would mean a published-read fast path could either falsely
// pass (return a torn page) or falsely fail (force a needless retry).
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let before_snap = pager.published_snapshot();
assert_eq!(
pager.published.current_sequence_gen(),
before_snap.snapshot_gen,
"bead_id={BEAD_ID} case=current_sequence_gen_matches_snapshot_before_commit"
);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x5A; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let after_snap = pager.published_snapshot();
assert_eq!(
pager.published.current_sequence_gen(),
after_snap.snapshot_gen,
"bead_id={BEAD_ID} case=current_sequence_gen_matches_snapshot_after_commit"
);
assert!(
after_snap.snapshot_gen > before_snap.snapshot_gen,
"bead_id={BEAD_ID} case=current_sequence_gen_advances_after_publish"
);
assert!(
pager.published.current_sequence_gen() % 2 == 0,
"bead_id={BEAD_ID} case=current_sequence_gen_quiescent_value_is_even"
);
});
}
#[test]
fn test_published_snapshot_monotonic_after_commit() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let before = pager.published_snapshot();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x5A; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let after = pager.published_snapshot();
assert!(
after.snapshot_gen > before.snapshot_gen,
"bead_id={BEAD_ID} case=publication_snapshot_gen_monotonic"
);
assert!(
after.visible_commit_seq > before.visible_commit_seq,
"bead_id={BEAD_ID} case=publication_commit_seq_monotonic"
);
assert_eq!(
after.db_size,
p.get(),
"bead_id={BEAD_ID} case=publication_db_size_updates"
);
assert_eq!(
after.freelist_count, 0,
"bead_id={BEAD_ID} case=publication_freelist_count_updates"
);
assert!(
!after.checkpoint_active,
"bead_id={BEAD_ID} case=publication_checkpoint_inactive_after_commit"
);
});
}
#[test]
fn test_commit_batches_publication_writes() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let publication_writes_before_commit = pager.publication_write_count();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x5A; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
assert_eq!(
pager.publication_write_count(),
publication_writes_before_commit + 1,
"bead_id={BEAD_ID} case=commit_batches_publication_writes"
);
});
}
#[test]
fn test_single_connection_commit_elides_publication_writes() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(1));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let before = pager.published_snapshot();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let publication_writes_before_commit = pager.publication_write_count();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x6B; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let after = pager.published_snapshot();
assert_eq!(
pager.publication_write_count(),
publication_writes_before_commit,
"bead_id={BEAD_ID} case=single_connection_commit_skips_publication_write"
);
assert!(
after.visible_commit_seq > before.visible_commit_seq,
"bead_id={BEAD_ID} case=single_connection_commit_still_advances_visible_commit_seq"
);
assert_eq!(
after.db_size,
p.get(),
"bead_id={BEAD_ID} case=single_connection_commit_updates_published_db_size"
);
assert_eq!(
after.page_set_size, 0,
"bead_id={BEAD_ID} case=single_connection_commit_keeps_publication_plane_empty"
);
shared_connection_count.store(2, AtomicOrdering::Release);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let publication_writes_before_multi_connection_commit = pager.publication_write_count();
txn.write_page(&cx, p, &vec![0x7C; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
assert_eq!(
pager.publication_write_count(),
publication_writes_before_multi_connection_commit + 1,
"bead_id={BEAD_ID} case=multi_connection_commit_restores_publication_write"
);
});
}
#[test]
fn test_single_connection_commit_updates_transaction_published_snapshot_hint() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(1));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let before = pager.published_snapshot();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x6B; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let committed_hint = txn
.published_visible_commit_seq_hint()
.expect("transaction should expose a published commit-seq hint");
let after = pager.published_snapshot();
assert!(
after.visible_commit_seq > before.visible_commit_seq,
"bead_id={BEAD_ID} case=single_connection_commit_advances_visible_commit_seq"
);
assert_eq!(
committed_hint, after.visible_commit_seq,
"bead_id={BEAD_ID} case=single_connection_commit_refreshes_transaction_published_snapshot_hint"
);
});
}
#[test]
fn test_single_connection_metadata_only_commit_clears_stale_page_plane_before_later_publish() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(2));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let p = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x11; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
p
};
shared_connection_count.store(1, AtomicOrdering::Release);
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, p, &vec![0x22; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
}
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_one = txn.get_page(&cx, PageNumber::ONE).await.unwrap().into_vec();
txn.write_page(&cx, PageNumber::ONE, &page_one)
.await
.unwrap();
txn.commit(&cx).await.unwrap();
}
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, p).await.unwrap().as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=metadata_only_single_connection_commit_must_not_leave_stale_page_plane_bytes"
);
});
}
#[test]
fn test_single_connection_read_skips_publication_plane_population() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(1));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let p = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x4D; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
p
};
let published_before = pager.published_snapshot();
let published_hits_before = pager.published_page_hits();
assert_eq!(
published_before.page_set_size, 0,
"bead_id={BEAD_ID} case=single_connection_read_starts_with_empty_publication_plane"
);
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = reader.get_page(&cx, p).await.unwrap();
assert_eq!(data.as_ref()[0], 0x4D);
let published_after = pager.published_snapshot();
assert_eq!(
published_after.page_set_size, 0,
"bead_id={BEAD_ID} case=single_connection_read_does_not_publish_observed_pages"
);
assert_eq!(
pager.published_page_hits(),
published_hits_before,
"bead_id={BEAD_ID} case=single_connection_read_bypasses_publication_plane_hits"
);
});
}
#[test]
fn test_single_connection_commit_and_retain_skips_stale_publication_plane_reads() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(2));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let p = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x11; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
p
};
shared_connection_count.store(1, AtomicOrdering::Release);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let published_before_retain = pager.published_snapshot();
let publication_writes_before_commit = pager.publication_write_count();
txn.write_page(&cx, p, &vec![0x22; ps]).await.unwrap();
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_should_retain_writer"
);
assert_eq!(
pager.publication_write_count(),
publication_writes_before_commit,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_skips_publication_write"
);
let published_after_retain = pager.published_snapshot();
assert_eq!(
published_after_retain.page_set_size, 0,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_keeps_publication_plane_empty"
);
assert!(
published_after_retain.visible_commit_seq
> published_before_retain.visible_commit_seq,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_still_advances_visible_commit_seq"
);
assert!(
txn.txn_read_cache.borrow().contains_key(&p),
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_keeps_authoritative_txn_read_cache"
);
shared_connection_count.store(2, AtomicOrdering::Release);
let retained_snapshot = pager.committed_snapshot();
assert_eq!(
retained_snapshot.commit_seq, published_after_retain.visible_commit_seq,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_refreshes_committed_snapshot_seq"
);
assert_eq!(
retained_snapshot.db_size, published_after_retain.db_size,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_refreshes_committed_snapshot_db_size"
);
shared_connection_count.store(1, AtomicOrdering::Release);
let mut stale_buf = PageBuf::new(PageSize::DEFAULT);
stale_buf.fill(0x7F);
pager.cache.insert_buffer(p, stale_buf);
assert_eq!(
txn.get_page(&cx, p).await.unwrap().as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_must_not_read_stale_shared_cache_page"
);
txn.commit(&cx).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, p).await.unwrap().as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_release_preserves_committed_visibility"
);
});
}
#[test]
fn test_committed_snapshot_retained_reader_arc_survives_publication_swap() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(2));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let first_page = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x33; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
page
};
let retained = pager.committed_snapshot();
let retained_commit_seq = retained.commit_seq;
let retained_db_size = retained.db_size;
assert_eq!(
Arc::strong_count(&retained),
2,
"bead_id={DB300_E3_3_A_BEAD_ID} case=retained_snapshot_starts_shared_with_publication_slot"
);
let second_page = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x44; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
page
};
let latest = pager.committed_snapshot();
assert!(
latest.commit_seq > retained_commit_seq,
"bead_id={DB300_E3_3_A_BEAD_ID} case=latest_snapshot_advances_after_swap"
);
assert!(
latest.db_size > retained_db_size,
"bead_id={DB300_E3_3_A_BEAD_ID} case=latest_snapshot_reflects_second_allocation"
);
assert_eq!(
retained.commit_seq, retained_commit_seq,
"bead_id={DB300_E3_3_A_BEAD_ID} case=retained_snapshot_commit_seq_remains_stable"
);
assert_eq!(
retained.db_size, retained_db_size,
"bead_id={DB300_E3_3_A_BEAD_ID} case=retained_snapshot_db_size_remains_stable"
);
assert_eq!(
Arc::strong_count(&retained),
1,
"bead_id={DB300_E3_3_A_BEAD_ID} case=old_snapshot_slot_released_after_swap"
);
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, first_page).await.unwrap().as_ref()[0],
0x33,
"bead_id={DB300_E3_3_A_BEAD_ID} case=first_page_still_visible"
);
assert_eq!(
reader.get_page(&cx, second_page).await.unwrap().as_ref()[0],
0x44,
"bead_id={DB300_E3_3_A_BEAD_ID} case=second_page_visible_after_swap"
);
});
}
#[test]
fn test_committed_snapshot_reclamation_stress_bounds_stale_arcs() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(2)));
const HELD_READER_CLONES: usize = 8;
const PUBLISH_ITERATIONS: u64 = 256;
for generation in 1..=PUBLISH_ITERATIONS {
let held_snapshots = (0..HELD_READER_CLONES)
.map(|_| pager.committed_snapshot())
.collect::<Vec<_>>();
let old_snapshot = Arc::clone(&held_snapshots[0]);
let old_commit_seq = old_snapshot.commit_seq;
assert!(
held_snapshots
.iter()
.all(|snapshot| Arc::ptr_eq(snapshot, &old_snapshot)),
"bead_id={DB300_E3_3_BEAD_ID} case=reclamation_readers_share_pre_swap_slot"
);
assert_eq!(
Arc::strong_count(&old_snapshot),
HELD_READER_CLONES + 2,
"bead_id={DB300_E3_3_BEAD_ID} case=reclamation_pre_swap_slot_refcount"
);
{
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.commit_seq = CommitSeq::new(generation);
inner.db_size =
u32::try_from(generation).expect("test generation fits db_size");
pager.publish_committed_snapshot_from_inner(&inner);
}
let latest = pager.committed_snapshot();
assert!(
!Arc::ptr_eq(&old_snapshot, &latest),
"bead_id={DB300_E3_3_BEAD_ID} case=reclamation_publication_slot_swapped"
);
assert!(
latest.commit_seq > old_commit_seq,
"bead_id={DB300_E3_3_BEAD_ID} case=reclamation_latest_snapshot_advances"
);
assert_eq!(
Arc::strong_count(&old_snapshot),
HELD_READER_CLONES + 1,
"bead_id={DB300_E3_3_BEAD_ID} case=reclamation_old_slot_released_after_swap"
);
}
let latest = pager.committed_snapshot();
assert_eq!(
Arc::strong_count(&latest),
2,
"bead_id={DB300_E3_3_BEAD_ID} case=reclamation_no_stale_snapshots_after_readers_drop"
);
});
}
#[test]
fn test_committed_snapshot_writer_not_starved_by_reader_pressure() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let pager = StdArc::new(pager);
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(2)));
const READERS: usize = 8;
const PUBLISHES: u64 = 1_024;
let stop = StdArc::new(AtomicBool::new(false));
let barrier = StdArc::new(std::sync::Barrier::new(READERS + 1));
let mut reader_handles = Vec::with_capacity(READERS);
for _ in 0..READERS {
let pager = StdArc::clone(&pager);
let stop = StdArc::clone(&stop);
let barrier = StdArc::clone(&barrier);
reader_handles.push(std::thread::spawn(move || {
barrier.wait();
let mut reads = 0_u64;
while !stop.load(AtomicOrdering::Acquire) {
let snapshot = pager.committed_snapshot();
std::hint::black_box(snapshot.commit_seq);
reads += 1;
}
reads
}));
}
barrier.wait();
let started = Instant::now();
for generation in 1..=PUBLISHES {
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.commit_seq = CommitSeq::new(generation);
inner.db_size = u32::try_from(generation).expect("test generation fits db_size");
pager.publish_committed_snapshot_from_inner(&inner);
}
let elapsed = started.elapsed();
stop.store(true, AtomicOrdering::Release);
let total_reader_snapshots = reader_handles
.into_iter()
.map(|handle| handle.join().expect("reader joined"))
.sum::<u64>();
assert!(
total_reader_snapshots > 0,
"bead_id={DB300_E3_3_BEAD_ID} case=starvation_readers_exercised_snapshot_path"
);
assert!(
elapsed < Duration::from_secs(2),
"bead_id={DB300_E3_3_BEAD_ID} case=starvation_writer_completed elapsed={elapsed:?}"
);
let latest = pager.committed_snapshot();
assert_eq!(
latest.commit_seq,
CommitSeq::new(PUBLISHES),
"bead_id={DB300_E3_3_BEAD_ID} case=starvation_latest_publication_visible"
);
});
}
#[test]
fn test_single_connection_commit_and_retain_rollback_preserves_last_committed_page() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let shared_connection_count = Arc::new(AtomicUsize::new(1));
pager.bind_shared_connection_count(Arc::clone(&shared_connection_count));
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x31; ps]).await.unwrap();
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_should_retain_writer_before_rollback"
);
txn.write_page(&cx, p, &vec![0x7A; ps]).await.unwrap();
txn.rollback(&cx).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, p).await.unwrap().as_ref()[0],
0x31,
"bead_id={BEAD_ID} case=single_connection_commit_and_retain_rollback_keeps_last_committed_page"
);
});
}
#[test]
fn test_private_memory_commit_and_retain_batches_multiple_logical_commits_without_journal() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let pager = private_memory_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let journal_path = SimplePager::<MemoryVfs>::journal_path(&pager.db_path);
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(1)));
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_one = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_one, &vec![0x11; ps])
.await
.unwrap();
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={BEAD_ID} case=private_memory_first_logical_commit_retained"
);
assert!(
!pager
.vfs
.access(&cx, &journal_path, AccessFlags::EXISTS)
.unwrap(),
"bead_id={BEAD_ID} case=private_memory_first_logical_commit_skips_journal"
);
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.active_transactions, 1,
"bead_id={BEAD_ID} case=private_memory_retained_writer_keeps_single_active_txn"
);
assert!(
inner.writer_active,
"bead_id={BEAD_ID} case=private_memory_retained_writer_keeps_writer_slot"
);
}
txn.write_page(&cx, page_one, &vec![0x22; ps])
.await
.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_two, &vec![0x33; ps])
.await
.unwrap();
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={BEAD_ID} case=private_memory_second_logical_commit_retained"
);
assert!(
!pager
.vfs
.access(&cx, &journal_path, AccessFlags::EXISTS)
.unwrap(),
"bead_id={BEAD_ID} case=private_memory_second_logical_commit_skips_journal"
);
txn.commit(&cx).await.unwrap();
assert!(
!pager
.vfs
.access(&cx, &journal_path, AccessFlags::EXISTS)
.unwrap(),
"bead_id={BEAD_ID} case=private_memory_release_commit_skips_journal"
);
{
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.active_transactions, 0,
"bead_id={BEAD_ID} case=private_memory_release_commit_drops_active_txn_count"
);
assert!(
!inner.writer_active,
"bead_id={BEAD_ID} case=private_memory_release_commit_releases_writer_slot"
);
}
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_one).await.unwrap().as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=private_memory_batched_commits_keep_latest_page_one"
);
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().as_ref()[0],
0x33,
"bead_id={BEAD_ID} case=private_memory_batched_commits_keep_latest_page_two"
);
});
}
#[test]
fn test_dirty_bitmap_set_check() {
asupersync::test_utils::run_test(|| async {
let pager = private_memory_pager().await;
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(1)));
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x41; ps]).await.unwrap();
assert_eq!(
txn.write_pages_sorted,
vec![page],
"bead_id={TRACK_U_BEAD_ID} case=dirty_bitmap_tracks_single_page"
);
assert!(
txn.write_set.contains_key(&page),
"bead_id={TRACK_U_BEAD_ID} case=dirty_bitmap_marks_page_in_write_set"
);
assert!(
!txn.retained_memory_overlay_dirty_pages.contains(&page),
"bead_id={TRACK_U_BEAD_ID} case=retained_overlay_stays_empty_before_retain"
);
track_u_log_counts("dirty_bitmap_set_check", 1, 0, 0);
});
}
#[test]
fn test_dirty_bitmap_double_write_dedups_wal_entry() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let (backend, frames, _begin_calls, _batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let page = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &sample_page(0x11))
.await
.unwrap();
seed.commit(&cx).await.unwrap();
page
};
frames.lock().unwrap().clear();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x22)).await.unwrap();
let final_bytes = sample_page(0x33);
txn.write_page(&cx, page, &final_bytes).await.unwrap();
assert_eq!(
txn.write_pages_sorted,
vec![page],
"bead_id={TRACK_U_BEAD_ID} case=double_write_dedups_dirty_surface"
);
txn.commit(&cx).await.unwrap();
let written = frames.lock().unwrap().clone();
let page_frames = written
.iter()
.filter(|(page_no, _, _)| *page_no == page.get())
.collect::<Vec<_>>();
assert_eq!(
page_frames.len(),
1,
"bead_id={TRACK_U_BEAD_ID} case=double_write_emits_single_data_frame"
);
assert_eq!(
page_frames[0].1.as_slice(),
final_bytes.as_slice(),
"bead_id={TRACK_U_BEAD_ID} case=double_write_keeps_last_page_image"
);
// bd-3wop3.8 (0a3e90fb) narrowed the WAL-mode Page-1 write gate in
// SimpleTransaction::commit to
// requires_page_one_rewrite() || requires_page_count_advance()
// — Page 1 is only emitted when the txn explicitly dirties it
// (schema change, VACUUM) OR causes db_growth. This test's main
// transaction double-writes the SAME pre-seeded page with no
// allocation and no schema change, so neither gate fires and Page 1
// must not appear in the WAL frame stream.
assert_eq!(
written
.iter()
.filter(|(page_no, _, _)| *page_no == PageNumber::ONE.get())
.count(),
0,
"bead_id={TRACK_U_BEAD_ID} case=double_write_emits_no_synthetic_page_one_frame"
);
track_u_log_counts("dirty_bitmap_double_write", 1, 1, page_frames.len());
});
}
#[test]
fn test_dirty_bitmap_commit_flushes_all_dirty_pages() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let (backend, frames, _begin_calls, _batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let pages = {
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::with_capacity(100);
for seed_byte in 0_u8..100 {
let page = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, page, &sample_page(seed_byte))
.await
.unwrap();
pages.push(page);
}
seed.commit(&cx).await.unwrap();
pages
};
frames.lock().unwrap().clear();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for (idx, page) in pages.iter().copied().enumerate() {
let seed_byte = u8::try_from(idx).expect("100-page test index fits u8");
txn.write_page(&cx, page, &sample_page(seed_byte.wrapping_add(100)))
.await
.unwrap();
}
assert_eq!(
txn.write_pages_sorted.len(),
pages.len(),
"bead_id={TRACK_U_BEAD_ID} case=dirty_surface_tracks_all_100_pages"
);
txn.commit(&cx).await.unwrap();
let written = frames.lock().unwrap().clone();
let flushed_pages = written
.iter()
.filter(|(page_no, _, _)| *page_no != PageNumber::ONE.get())
.map(|(page_no, _, _)| *page_no)
.collect::<Vec<_>>();
let expected_pages = pages.iter().map(|page| page.get()).collect::<Vec<_>>();
assert_eq!(
flushed_pages, expected_pages,
"bead_id={TRACK_U_BEAD_ID} case=commit_flushes_every_dirty_page_once"
);
// See sibling test for the bd-3wop3.8 (0a3e90fb) background: in WAL
// mode Page 1 is only written when the commit explicitly dirties it
// or grows the db. The measured commit writes 100 pre-seeded pages
// (no allocation → no db_growth) with no schema change, so Page 1
// must not appear in the WAL frame stream.
assert_eq!(
written
.iter()
.filter(|(page_no, _, _)| *page_no == PageNumber::ONE.get())
.count(),
0,
"bead_id={TRACK_U_BEAD_ID} case=commit_flush_emits_no_synthetic_page_one_frame"
);
track_u_log_counts(
"dirty_bitmap_commit_flushes_all",
pages.len(),
0,
flushed_pages.len(),
);
});
}
#[test]
fn test_dirty_bitmap_rollback_clears_retained_overlay() {
asupersync::test_utils::run_test(|| async {
let pager = private_memory_pager().await;
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(1)));
let cx = Cx::new();
let original_one = sample_page(0x31);
let original_two = sample_page(0x32);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_one = txn.allocate_page(&cx).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_one, &original_one).await.unwrap();
txn.write_page(&cx, page_two, &original_two).await.unwrap();
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={TRACK_U_BEAD_ID} case=rollback_clear_requires_retained_commit"
);
assert_eq!(
txn.retained_memory_overlay_dirty_pages.len(),
2,
"bead_id={TRACK_U_BEAD_ID} case=retained_overlay_tracks_committed_pages"
);
txn.write_page(&cx, page_one, &sample_page(0x7A))
.await
.unwrap();
txn.rollback(&cx).await.unwrap();
assert!(
txn.retained_memory_overlay_dirty_pages.is_empty(),
"bead_id={TRACK_U_BEAD_ID} case=rollback_clears_retained_overlay_bitmap"
);
assert!(
txn.write_set.is_empty() && txn.write_pages_sorted.is_empty(),
"bead_id={TRACK_U_BEAD_ID} case=rollback_clears_live_dirty_surface"
);
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_one).await.unwrap().into_vec(),
original_one,
"bead_id={TRACK_U_BEAD_ID} case=rollback_restores_last_committed_page_one"
);
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
original_two,
"bead_id={TRACK_U_BEAD_ID} case=rollback_restores_last_committed_page_two"
);
track_u_log_counts("dirty_bitmap_rollback_clears", 2, 0, 2);
});
}
#[test]
fn test_dirty_bitmap_large_10k_pages() {
asupersync::test_utils::run_test(|| async {
let pager = private_memory_pager().await;
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(1)));
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::with_capacity(10_000);
for idx in 0..10_000_usize {
let page = txn.allocate_page(&cx).await.unwrap();
let fill = u8::try_from(idx % 251).expect("modulo fits u8");
txn.write_page(&cx, page, &vec![fill; ps]).await.unwrap();
pages.push((page, fill));
}
assert_eq!(
txn.write_pages_sorted.len(),
pages.len(),
"bead_id={TRACK_U_BEAD_ID} case=large_dirty_surface_tracks_10k_pages"
);
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={TRACK_U_BEAD_ID} case=large_dirty_surface_retain_succeeds"
);
assert_eq!(
txn.retained_memory_overlay_dirty_pages.len(),
pages.len(),
"bead_id={TRACK_U_BEAD_ID} case=large_retained_overlay_tracks_10k_pages"
);
txn.rollback(&cx).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
for &(sample_idx, expected_fill) in &[
(0_usize, pages[0].1),
(4_999_usize, pages[4_999].1),
(9_999_usize, pages[9_999].1),
] {
let page = pages[sample_idx].0;
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_ref()[0],
expected_fill,
"bead_id={TRACK_U_BEAD_ID} case=large_dirty_surface_preserves_sample sample_idx={sample_idx}"
);
}
track_u_log_counts("dirty_bitmap_large", pages.len(), 0, pages.len());
});
}
#[test]
fn test_dirty_bitmap_crash_recovery_restores_original_pages() {
asupersync::test_utils::run_test(|| async {
let path = PathBuf::from("/track_u_dirty_bitmap_crash_recovery.db");
let journal_path = SimplePager::<DbWriteFailOnceVfs>::journal_path(&path);
let vfs = DbWriteFailOnceVfs::new(path.clone());
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let original_pages = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let mut pages = Vec::with_capacity(32);
for seed_byte in 1_u8..=32 {
let page = txn.allocate_page(&cx).await.unwrap();
let original = sample_page(seed_byte);
txn.write_page(&cx, page, &original).await.unwrap();
pages.push((page, original));
}
txn.commit(&cx).await.unwrap();
pages
};
vfs.arm_after_db_writes(8);
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
for (idx, (page, _)) in original_pages.iter().enumerate() {
let seed_byte = u8::try_from(idx).expect("32-page test index fits u8");
txn.write_page(&cx, *page, &sample_page(seed_byte.wrapping_add(101)))
.await
.unwrap();
}
let err = txn.commit(&cx).await.unwrap_err();
assert!(
matches!(err, FrankenError::Io(_)),
"bead_id={TRACK_U_BEAD_ID} case=crash_recovery_surfaces_partial_commit_io_error"
);
}
drop(pager);
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
let reader = reopened
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
for (page, original) in &original_pages {
assert_eq!(
reader.get_page(&cx, *page).await.unwrap().as_ref(),
original.as_slice(),
"bead_id={TRACK_U_BEAD_ID} case=crash_recovery_restores_original_page page={}",
page.get()
);
}
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={TRACK_U_BEAD_ID} case=crash_recovery_clears_abandoned_journal"
);
track_u_log_counts("dirty_bitmap_crash_recovery", original_pages.len(), 0, 0);
});
}
#[test]
fn test_dirty_bitmap_deferred_flush_materializes_retained_overlay_without_duplicates() {
asupersync::test_utils::run_test(|| async {
let pager = private_memory_pager().await;
pager.bind_shared_connection_count(Arc::new(AtomicUsize::new(1)));
let cx = Cx::new();
let original_one = sample_page(0x41);
let original_two = sample_page(0x42);
let updated_one = sample_page(0x51);
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_one = txn.allocate_page(&cx).await.unwrap();
let page_two = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_one, &original_one).await.unwrap();
txn.write_page(&cx, page_two, &original_two).await.unwrap();
assert!(
txn.commit_and_retain(&cx).await.unwrap(),
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_requires_retained_commit"
);
assert_eq!(
txn.retained_memory_overlay_dirty_pages.len(),
2,
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_tracks_retained_overlay"
);
txn.write_page(&cx, page_one, &updated_one).await.unwrap();
assert_eq!(
txn.write_set_page_numbers(),
vec![page_one],
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_starts_with_live_dirty_page"
);
txn.materialize_retained_memory_overlay_into_write_set()
.unwrap();
assert_eq!(
txn.write_set_page_numbers(),
vec![page_one, page_two],
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_backfills_only_missing_overlay_page"
);
assert_eq!(
txn.write_set.len(),
2,
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_avoids_duplicate_dirty_entries"
);
assert_eq!(
txn.write_set
.get(&page_one)
.expect("page_one staged entry")
.as_page_bytes(),
updated_one.as_slice(),
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_preserves_newest_page_image"
);
assert_eq!(
txn.write_set
.get(&page_two)
.expect("page_two staged entry")
.as_page_bytes(),
original_two.as_slice(),
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_restores_untouched_overlay_page"
);
txn.commit(&cx).await.unwrap();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page_one).await.unwrap().into_vec(),
updated_one,
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_commits_latest_page_one"
);
assert_eq!(
reader.get_page(&cx, page_two).await.unwrap().into_vec(),
original_two,
"bead_id={TRACK_U_BEAD_ID} case=deferred_flush_materialization_commits_restored_overlay_page_two"
);
track_u_log_counts("dirty_bitmap_deferred_flush_materialization", 2, 1, 2);
});
}
#[test]
fn test_dirty_bitmap_double_write_crash_recovery_restores_original_page() {
asupersync::test_utils::run_test(|| async {
let path = PathBuf::from("/track_u_dirty_bitmap_double_write_crash.db");
let journal_path = SimplePager::<DbWriteFailOnceVfs>::journal_path(&path);
let vfs = DbWriteFailOnceVfs::new(path.clone());
let pager = vfs.open_file_backed_pager(&path).await.unwrap();
let cx = Cx::new();
let (page, original) = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
let original = sample_page(0x2A);
txn.write_page(&cx, page, &original).await.unwrap();
txn.commit(&cx).await.unwrap();
(page, original)
};
vfs.arm_after_db_writes(1);
{
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x55)).await.unwrap();
txn.write_page(&cx, page, &sample_page(0x66)).await.unwrap();
let err = txn.commit(&cx).await.unwrap_err();
assert!(
matches!(err, FrankenError::Io(_)),
"bead_id={TRACK_U_BEAD_ID} case=double_write_crash_recovery_surfaces_commit_io_error"
);
}
drop(pager);
let reopened = vfs.open_file_backed_pager(&path).await.unwrap();
let reader = reopened
.begin(&cx, TransactionMode::ReadOnly)
.await
.unwrap();
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_ref(),
original.as_slice(),
"bead_id={TRACK_U_BEAD_ID} case=double_write_crash_recovery_restores_original_page"
);
assert!(
!vfs.access(&cx, &journal_path, AccessFlags::EXISTS).unwrap(),
"bead_id={TRACK_U_BEAD_ID} case=double_write_crash_recovery_clears_abandoned_journal"
);
track_u_log_counts("dirty_bitmap_double_write_crash_recovery", 1, 1, 0);
});
}
#[test]
fn test_wal_commit_skips_post_commit_cache_admission() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let cx = Cx::new();
let vfs = MemoryVfs::new();
let db_path = PathBuf::from("/wal_commit_skips_post_commit_cache_admission.db");
let pager = SimplePager::open(vfs, &db_path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, _frames, _begin_calls, _batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let cached_pages_before_commit = pager.cache_metrics_snapshot().unwrap().cached_pages;
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xAB; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let cache_after_commit = pager.cache_metrics_snapshot().unwrap();
assert_eq!(
cache_after_commit.cached_pages,
cached_pages_before_commit + 1,
"bead_id={BEAD_ID} case=wal_commit_avoids_post_commit_cache_fanout"
);
let read_before = read_surface_snapshot(&pager);
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, p).await.unwrap().as_ref()[0],
0xAB,
"bead_id={BEAD_ID} case=wal_commit_publication_keeps_read_visibility"
);
let read_after = read_surface_snapshot(&pager);
assert_eq!(
read_after.cache, read_before.cache,
"bead_id={BEAD_ID} case=wal_post_commit_read_skips_cache"
);
assert_eq!(
read_after.published_hits,
read_before.published_hits + 1,
"bead_id={BEAD_ID} case=wal_post_commit_read_hits_publication"
);
});
}
#[test]
fn test_named_memory_vfs_wal_commit_records_physical_main_file_size() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let db_path = PathBuf::from("/named_memory_wal_retain_file_size.db");
let pager = SimplePager::open(vfs, &db_path, PageSize::DEFAULT)
.await
.unwrap();
let (backend, _frames, _begin_calls, _batch_calls) = MockWalBackend::new();
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let initial_main_file_size = {
let db_file = {
let inner = pager.inner.lock().unwrap();
Arc::clone(&inner.db_file)
};
shared_db_file_read(&db_file, &cx)
.await
.unwrap()
.file_size(&cx)
.unwrap()
};
let ps = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page, &vec![0x6D; ps]).await.unwrap();
assert!(
!txn.commit_and_retain(&cx).await.unwrap(),
"WAL commit_and_retain must finish so the next transaction gets a coherent pin"
);
let (recorded_main_file_size, logical_db_size, db_file) = {
let inner = pager.inner.lock().unwrap();
(
inner.committed_db_file_size_bytes,
inner.db_size,
Arc::clone(&inner.db_file),
)
};
let physical_main_file_size = shared_db_file_read(&db_file, &cx)
.await
.unwrap()
.file_size(&cx)
.unwrap();
assert_eq!(
physical_main_file_size, initial_main_file_size,
"WAL append must not grow a named MemoryVfs main database file"
);
assert_eq!(
recorded_main_file_size, physical_main_file_size,
"WAL commit must record the named database's physical main-file size"
);
assert_ne!(
recorded_main_file_size,
u64::from(logical_db_size) * u64::from(PageSize::DEFAULT.get()),
"named MemoryVfs must not use the private :memory: synthetic size rule"
);
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
reader.get_page(&cx, page).await.unwrap().as_ref()[0],
0x6D,
"commit must keep its WAL-published page visible after finalization"
);
});
}
#[test]
fn test_published_read_hit_does_not_touch_cache_metrics() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let (pager, _) = test_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let p = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0xAB; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
p
};
let cache_before = pager.cache_metrics_snapshot().unwrap();
let published_hits_before = pager.published_page_hits();
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let data = reader.get_page(&cx, p).await.unwrap();
assert_eq!(data.as_ref()[0], 0xAB);
let cache_after = pager.cache_metrics_snapshot().unwrap();
assert_eq!(
cache_after, cache_before,
"bead_id={BEAD_ID} case=publication_hit_skips_cache_metrics"
);
assert_eq!(
pager.published_page_hits(),
published_hits_before + 1,
"bead_id={BEAD_ID} case=publication_hit_counter"
);
});
}
#[test]
fn test_published_pages_len_tracks_insert_remove_clear() {
let published_pages = PublishedPages::new(0);
let page_two = PageNumber::new(2).unwrap();
let page_three = PageNumber::new(3).unwrap();
assert_eq!(published_pages.len(), 0);
assert!(published_pages.insert(page_two, PageData::from_vec(sample_page(0x22))));
assert_eq!(published_pages.len(), 1);
assert!(!published_pages.insert(page_two, PageData::from_vec(sample_page(0x33))));
assert_eq!(
published_pages.len(),
1,
"bead_id=bd-qrss1 case=replacement_must_not_increment_page_count"
);
assert!(published_pages.insert(page_three, PageData::from_vec(sample_page(0x44))));
assert_eq!(published_pages.len(), 2);
assert!(published_pages.remove(page_two));
assert_eq!(published_pages.len(), 1);
assert!(!published_pages.remove(page_two));
assert_eq!(published_pages.len(), 1);
published_pages.clear();
assert_eq!(published_pages.len(), 0);
}
#[test]
fn test_published_pages_overflow_insert_remove_clear_tracks_count() {
let published_pages = PublishedPages::new(0);
let overflow_page = PageNumber::new(70_000).unwrap();
assert_eq!(published_pages.len(), 0);
assert!(published_pages.insert(overflow_page, PageData::from_vec(sample_page(0x70))));
assert_eq!(
published_pages.get(overflow_page),
Some(PageData::from_vec(sample_page(0x70))),
"bead_id=bd-uvdnk case=overflow_insert_makes_page_visible"
);
assert_eq!(published_pages.len(), 1);
assert!(!published_pages.insert(overflow_page, PageData::from_vec(sample_page(0x71))));
assert_eq!(
published_pages.get(overflow_page),
Some(PageData::from_vec(sample_page(0x71))),
"bead_id=bd-uvdnk case=overflow_replace_keeps_latest_page"
);
assert_eq!(
published_pages.len(),
1,
"bead_id=bd-uvdnk case=overflow_replace_does_not_increment_count"
);
assert!(published_pages.remove(overflow_page));
assert!(published_pages.get(overflow_page).is_none());
assert_eq!(published_pages.len(), 0);
assert!(published_pages.insert(overflow_page, PageData::from_vec(sample_page(0x72))));
published_pages.clear();
assert!(
published_pages.get(overflow_page).is_none(),
"bead_id=bd-uvdnk case=overflow_clear_removes_page"
);
assert_eq!(published_pages.len(), 0);
}
/// Shape: `PublishedPagerState::publish_clear_if` and the metadata-only
/// single-connection publish path call `PublishedPages::clear()` under
/// `publish_lock` on every transaction commit / rollback / metadata-only
/// republish. The atomic plane already short-circuits on `page_count == 0`;
/// the overflow plane previously walked every DashMap shard via
/// `pages.clear()` even when nothing had ever been admitted to the overflow
/// (the dominant case at MT8, where bench page numbers stay well below
/// the 65535-page direct-slot limit).
///
/// Force `page_count = 1` to defeat the new short-circuit on the baseline
/// pass; both passes operate on an empty DashMap so the only difference is
/// the per-shard sweep `DashMap::clear` performs unconditionally. Pairing
/// baseline and optimized in one binary avoids cross-build timing noise.
///
/// Run via:
/// cargo test -p fsqlite-pager --lib --profile release-perf --
/// --ignored --nocapture
/// bench_concurrent_published_pages_clear_empty_overflow_microbench
#[test]
#[ignore = "microbench, run with --ignored --nocapture"]
fn bench_concurrent_published_pages_clear_empty_overflow_microbench() {
use std::time::Instant;
const ITERS: usize = 1_000_000;
let overflow = ConcurrentPublishedPages::new();
// Warm up the optimized fast path.
for _ in 0..ITERS / 10 {
overflow.clear();
}
// Optimized: page_count starts at 0, the new short-circuit returns
// immediately without acquiring any DashMap shard locks.
let start = Instant::now();
for _ in 0..ITERS {
overflow.clear();
}
let elapsed_opt = start.elapsed();
let per_call_opt = elapsed_opt / u32::try_from(ITERS).unwrap();
// Baseline: force page_count to 1 at the top of every iteration so the
// short-circuit fails and we fall through into `pages.clear()` —
// matching the previous unconditional shard sweep on every publish.
let start = Instant::now();
for _ in 0..ITERS {
overflow.page_count.store(1, AtomicOrdering::Release);
overflow.clear();
}
let elapsed_base = start.elapsed();
let per_call_base = elapsed_base / u32::try_from(ITERS).unwrap();
let speedup = per_call_base.as_nanos() as f64 / per_call_opt.as_nanos().max(1) as f64;
eprintln!(
"bench_concurrent_published_pages_clear_empty_overflow_microbench: ITERS={ITERS} \
baseline={per_call_base:?} optimized={per_call_opt:?} \
speedup={speedup:.2}x"
);
}
#[test]
fn test_published_pages_insert_batch_and_retain_track_page_count() {
let published_pages = PublishedPages::new(0);
let page_two = PageNumber::new(2).unwrap();
let page_sixty_five = PageNumber::new(65).unwrap();
let page_seventy_thousand = PageNumber::new(70_000).unwrap();
published_pages.insert_batch([
(page_two, PageData::from_vec(sample_page(0x02))),
(page_sixty_five, PageData::from_vec(sample_page(0x41))),
(page_seventy_thousand, PageData::from_vec(sample_page(0x81))),
(page_two, PageData::from_vec(sample_page(0xFF))),
]);
assert_eq!(
published_pages.len(),
3,
"bead_id=bd-qrss1 case=batch_insert_counts_only_new_pages"
);
assert_eq!(
published_pages.get(page_two),
Some(PageData::from_vec(sample_page(0xFF))),
"bead_id=bd-qrss1 case=batch_insert_replaces_existing_page"
);
published_pages.retain(|page_no| page_no.get() >= 65);
assert_eq!(
published_pages.len(),
2,
"bead_id=bd-qrss1 case=retain_updates_atomic_page_count"
);
assert!(published_pages.get(page_two).is_none());
assert!(published_pages.get(page_sixty_five).is_some());
assert!(published_pages.get(page_seventy_thousand).is_some());
}
#[test]
fn test_published_pages_direct_slots_use_small_database_floor() {
let published_pages = PublishedPages::new(1);
assert_eq!(
published_pages.atomic_slot_count(),
ATOMIC_PUBLISHED_MIN_SLOT_COUNT,
"small databases should not allocate the full direct-slot plane"
);
let floor_page =
u32::try_from(ATOMIC_PUBLISHED_MIN_SLOT_COUNT).expect("slot floor fits in u32");
let direct_edge = PageNumber::new(floor_page).unwrap();
let overflow_edge = PageNumber::new(floor_page + 1).unwrap();
assert!(published_pages.insert(direct_edge, PageData::from_vec(sample_page(0xA1))));
assert!(published_pages.insert(overflow_edge, PageData::from_vec(sample_page(0xA2))));
assert_eq!(
published_pages.get(direct_edge),
Some(PageData::from_vec(sample_page(0xA1)))
);
assert_eq!(
published_pages.get(overflow_edge),
Some(PageData::from_vec(sample_page(0xA2)))
);
assert_eq!(published_pages.len(), 2);
}
#[test]
fn test_published_pages_direct_active_slots_track_live_pages() {
let published_pages = PublishedPages::new(1);
let page_one = PageNumber::new(1).unwrap();
let page_two = PageNumber::new(2).unwrap();
assert!(published_pages.insert(page_one, PageData::from_vec(sample_page(0xB1))));
assert_eq!(published_pages.atomic_active_slot_count(), 1);
assert!(!published_pages.insert(page_one, PageData::from_vec(sample_page(0xB2))));
assert_eq!(
published_pages.atomic_active_slot_count(),
1,
"direct replacement must not duplicate the active slot index"
);
assert!(published_pages.insert(page_two, PageData::from_vec(sample_page(0xB3))));
assert_eq!(published_pages.atomic_active_slot_count(), 2);
assert!(published_pages.remove(page_one));
assert_eq!(
published_pages.atomic_active_slot_count(),
1,
"direct remove should drop its active slot index"
);
published_pages.clear();
assert_eq!(published_pages.atomic_active_slot_count(), 0);
assert_eq!(published_pages.len(), 0);
assert!(published_pages.get(page_two).is_none());
}
#[test]
fn test_atomic_published_pages_remove_middle_and_tail_are_consistent() {
// Exercise the back-pointer: remove from head, middle, and tail
// positions and assert the structure stays consistent afterwards.
let published_pages = PublishedPages::new(16);
let page = |n: u32| PageNumber::new(n).unwrap();
for n in 1..=5 {
assert!(
published_pages.insert(page(n), PageData::from_vec(sample_page(n as u8))),
"fresh insert should succeed for page {n}"
);
}
assert_eq!(published_pages.atomic_active_slot_count(), 5);
// Middle
assert!(published_pages.remove(page(3)));
assert_eq!(published_pages.atomic_active_slot_count(), 4);
assert!(published_pages.get(page(3)).is_none());
assert!(published_pages.get(page(5)).is_some());
// Head (post-middle-removal the head is still page 1)
assert!(published_pages.remove(page(1)));
assert_eq!(published_pages.atomic_active_slot_count(), 3);
assert!(published_pages.get(page(1)).is_none());
assert!(published_pages.get(page(4)).is_some());
// Tail (before the above two removes, 5 was the tail — after the
// two swap_removes its back-pointer must still resolve)
assert!(published_pages.remove(page(5)));
assert_eq!(published_pages.atomic_active_slot_count(), 2);
assert!(published_pages.get(page(5)).is_none());
// Remaining two pages still reachable
assert!(published_pages.get(page(2)).is_some());
assert!(published_pages.get(page(4)).is_some());
// Removing the remaining pages also works
assert!(published_pages.remove(page(2)));
assert!(published_pages.remove(page(4)));
assert_eq!(published_pages.atomic_active_slot_count(), 0);
assert_eq!(published_pages.len(), 0);
}
#[test]
fn test_atomic_published_pages_remove_is_o1_vs_linear_scan() {
// Micro-benchmark gate: compares the new O(1) slot.active_pos
// back-pointer path in AtomicPublishedPages::remove against a
// reference O(n) implementation that simulates the prior
// `iter().position()` scan. Both implementations run the same
// insert/remove workload over the same input.
//
// The gate only fails if the O(1) path regresses below the O(n)
// reference. On CI the delta is hardware-dependent, so we print
// both timings with tracing and use a generous lower bound. The
// interesting number — the ratio — is captured in the commit
// body.
use std::time::Instant;
const N: u32 = 4_000;
let slot_cap = u32::try_from(PublishedPages::new(N).atomic_slot_count())
.expect("direct slot count fits in u32");
assert!(
N <= slot_cap,
"N must stay within the direct-slot plane for apples-to-apples timing"
);
let pages: Vec<PageNumber> = (1..=N).map(|n| PageNumber::new(n).unwrap()).collect();
let payload = PageData::from_vec(sample_page(0xE4));
// Warm caches / allocator before timing (alloc churn otherwise
// biases the first run).
{
let warm = PublishedPages::new(N);
for &p in &pages {
let _ = warm.insert(p, payload.clone());
}
for &p in &pages {
let _ = warm.remove(p);
}
}
// Timed run: O(1) back-pointer path (the actual production
// implementation after this commit).
let published = PublishedPages::new(N);
for &p in &pages {
assert!(published.insert(p, payload.clone()));
}
let t0 = Instant::now();
for &p in &pages {
assert!(published.remove(p));
}
let o1_elapsed = t0.elapsed();
assert_eq!(published.atomic_active_slot_count(), 0);
// Reference run: simulate the old O(n) path with a local
// Vec<usize> and iter().position(). This is a pure
// data-structure microbenchmark (no slot I/O), so it only
// reflects the search cost we removed — it is strictly
// conservative toward the O(1) path (which also pays the
// per-slot lock cost the reference skips).
let mut active_indices: Vec<usize> = (0..N as usize).collect();
let t0 = Instant::now();
for &p in &pages {
let idx = (p.get() - 1) as usize;
let pos = active_indices
.iter()
.position(|&i| i == idx)
.expect("active index must be present");
active_indices.swap_remove(pos);
}
let linear_elapsed = t0.elapsed();
assert!(active_indices.is_empty());
eprintln!(
"atomic_published_pages_remove: O(1) back-pointer = {o1_elapsed:?}, \
reference O(n) linear scan = {linear_elapsed:?} for N={N}"
);
// Sanity gate: the O(1) remove must not regress beyond the O(n)
// scan on this micro-workload. The scan has zero slot overhead
// (just Vec ops), so `linear_elapsed * 4` is still a very loose
// upper bound — we mainly care about catching accidental
// super-linear regressions.
assert!(
o1_elapsed <= linear_elapsed.saturating_mul(4),
"O(1) remove regressed vs pure O(n) reference: o1={o1_elapsed:?} scan={linear_elapsed:?}"
);
}
#[test]
fn test_published_pager_state_new_construction_drop_cost() {
// Direct construction+drop timing for PublishedPagerState on a
// small-DB (initial_db_size = 1 → clamped to
// ATOMIC_PUBLISHED_MIN_SLOT_COUNT slots). This is the hot path
// that mt_mvcc_bench 8t hit on every worker open and close —
// 2026-04-23 profile attributed ~9% self-time to the pair
// PublishedPagerState::new (4.65%) + Arc<...>::drop_slow
// (4.24%) with a 4096-slot floor. After lowering the floor to
// 512 the per-instance cost should drop proportionally.
//
// The gate is lenient (<= 20 ms for 8 construct+drop pairs) —
// the test prints the observed timings so the commit body can
// capture the measured delta without risking a flake on loaded
// CI hardware.
use std::time::Instant;
// Warm allocator / mutex caches.
for _ in 0..4 {
let _ = PublishedPagerState::new(1, CommitSeq::new(0), JournalMode::Wal, 0);
}
const ITERS: usize = 8;
let t0 = Instant::now();
for _ in 0..ITERS {
let state = PublishedPagerState::new(1, CommitSeq::new(0), JournalMode::Wal, 0);
// Consume and drop inside the measured window so we capture
// both construction AND drop_slow cost, matching the
// Connection::open → Connection::drop lifecycle.
drop(state);
}
let elapsed = t0.elapsed();
let per_pair = elapsed / ITERS as u32;
eprintln!(
"PublishedPagerState::new + drop (small-DB, floor={}): \
{ITERS}x = {elapsed:?} ({per_pair:?} per pair)",
ATOMIC_PUBLISHED_MIN_SLOT_COUNT,
);
// Very loose ceiling — mainly guards against a regression that
// silently un-lowers the floor.
assert!(
elapsed < std::time::Duration::from_millis(20 * ITERS as u64),
"PublishedPagerState::new+drop ran much slower than expected: {elapsed:?} for {ITERS} pairs"
);
}
#[test]
fn test_wal_frame_count_read_lock_does_not_block_behind_writer() {
// Regression gate for the `wal_frame_count()` read-lock fix.
//
// The real scenario: one thread is mid-`append_prepared_frames`
// holding the `SharedWalBackend` in `write()` mode (a ≥100 µs
// WAL-append + fsync on this host). A concurrent commit path
// fires `wal_frame_count()` via the checkpoint advisor.
// Pre-fix, that probe ALSO took `write()`, so it serialized
// behind the in-flight appender for the whole append-duration
// and then re-exported the same serialization window to every
// subsequent commit on the same pager. Post-fix it takes
// `read()` — std RwLock still blocks reads behind an active
// writer, BUT the probe itself does not block PEERS, so N
// concurrent commits each pay one write-holder-wait instead of
// N write-holder-waits stacking serially.
//
// This test encodes the latter invariant directly: while a
// single thread holds the SharedWalBackend-analogue in write()
// for a fixed hold duration, N "reader" threads take read()
// concurrently and must all finish inside roughly one
// hold-duration, not N stacked holds.
use std::sync::Barrier;
use std::sync::RwLock;
use std::sync::atomic::AtomicUsize;
use std::time::{Duration, Instant};
const READERS: usize = 8;
const HOLD: Duration = Duration::from_millis(20);
let shared = StdArc::new(RwLock::new(AtomicUsize::new(42)));
let barrier = StdArc::new(Barrier::new(READERS + 1));
// Writer: grab write(), pin for HOLD, release.
let writer = {
let shared = StdArc::clone(&shared);
let barrier = StdArc::clone(&barrier);
std::thread::spawn(move || {
barrier.wait();
let _g = shared.write().unwrap();
std::thread::sleep(HOLD);
})
};
// Readers: wait for barrier, then race to take read() and
// immediately return once unblocked.
let started = Instant::now();
let readers: Vec<_> = (0..READERS)
.map(|_| {
let shared = StdArc::clone(&shared);
let barrier = StdArc::clone(&barrier);
std::thread::spawn(move || {
barrier.wait();
let g = shared.read().unwrap();
std::hint::black_box(g.load(std::sync::atomic::Ordering::Relaxed));
})
})
.collect();
for r in readers {
r.join().expect("reader join");
}
let readers_elapsed = started.elapsed();
writer.join().expect("writer join");
eprintln!(
"wal_frame_count contention: {READERS} concurrent read-lock probes finished \
in {readers_elapsed:?} behind a {HOLD:?} write-hold. \
Stacked-write upper bound would be {READERS}×{HOLD:?} = {:?}.",
HOLD * READERS as u32
);
// Gate: readers finish in roughly one HOLD, not N × HOLD.
// Before the fix, each `wal_frame_count()` call would have
// taken write(), stacking serially behind the in-flight
// appender AND serially behind each other. 3 × HOLD is a very
// loose ceiling that still catches an accidental revert.
assert!(
readers_elapsed < HOLD * 3,
"read-lock probes should not stack serially: elapsed={readers_elapsed:?} HOLD={HOLD:?}"
);
}
#[test]
fn test_published_snapshot_page_set_size_tracks_concurrent_page_count() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(7), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_sixty_five = PageNumber::new(65).unwrap();
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(7),
db_size: 65,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_two,
PageData::from_vec(sample_page(0x22)),
);
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 65,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_sixty_five,
PageData::from_vec(sample_page(0x65)),
);
assert_eq!(
published.snapshot().page_set_size,
2,
"bead_id=bd-qrss1 case=publication_plane_page_count_after_inserts"
);
published.publish_remove_page(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(9),
db_size: 65,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_two,
);
assert_eq!(
published.snapshot().page_set_size,
1,
"bead_id=bd-qrss1 case=publication_plane_page_count_after_remove"
);
published.publish_clear_if(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(10),
db_size: 0,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
true,
);
assert_eq!(
published.snapshot().page_set_size,
0,
"bead_id=bd-qrss1 case=publication_plane_page_count_after_clear"
);
}
#[test]
fn test_stale_smaller_commit_publish_does_not_mutate_newer_snapshot() {
init_publication_test_tracing();
let published = PublishedPagerState::new(8, CommitSeq::new(8), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_seven = PageNumber::new(7).unwrap();
let original_page_two = PageData::from_vec(sample_page(0x22));
let original_page_seven = PageData::from_vec(sample_page(0x77));
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_two,
original_page_two.clone(),
);
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_seven,
original_page_seven.clone(),
);
let refreshed_page_two = PageData::from_vec(sample_page(0x99));
let write_set = HashMap::from([(
page_two,
StagedPage::from_page_data(refreshed_page_two.clone()),
)]);
published.publish_commit(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(7),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
&write_set,
);
assert_eq!(
published.try_get_page(page_two),
Some(original_page_two),
"bead_id=bd-wwqen.3 case=stale_smaller_commit_must_not_overwrite_newer_page_bytes"
);
assert_eq!(
published.try_get_page(page_seven),
Some(original_page_seven),
"bead_id=bd-wwqen.3 case=stale_smaller_commit_must_not_evict_newer_pages"
);
assert_eq!(
published.snapshot().db_size,
8,
"bead_id=bd-wwqen.3 case=stale_smaller_commit_preserves_published_db_size"
);
}
#[test]
fn test_publish_commit_sweeps_pages_but_keeps_db_size_monotonic() {
init_publication_test_tracing();
// Commit 49d0b194 ("keep db_size strictly monotonic across cross-
// process peers") changed `publish_commit` so it uses `fetch_max` on
// db_size rather than `store`. A shrink update at this layer still
// sweeps pages above the new smaller size (so readers of a lower
// snapshot can't reach stale pages) but the published db_size stays
// at the high-water mark. The authoritative shrink is delivered by
// `publish_truncate_checkpoint`, not `publish_commit`.
let published = PublishedPagerState::new(8, CommitSeq::new(8), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_seven = PageNumber::new(7).unwrap();
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_seven,
PageData::from_vec(sample_page(0x77)),
);
let refreshed_page_two = PageData::from_vec(sample_page(0x22));
let write_set = HashMap::from([(
page_two,
StagedPage::from_page_data(refreshed_page_two.clone()),
)]);
published.publish_commit(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(9),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
&write_set,
);
assert_eq!(
published.try_get_page(page_two),
Some(refreshed_page_two),
"bead_id=bd-wwqen.3 case=shrink_commit_keeps_written_page_inside_new_db_size"
);
assert!(
published.try_get_page(page_seven).is_none(),
"bead_id=bd-wwqen.3 case=shrink_commit_evicts_pages_above_new_db_size"
);
assert_eq!(
published.snapshot().db_size,
8,
"bead_id=bd-wwqen.3 case=shrink_commit_leaves_published_db_size_monotonic"
);
}
#[test]
fn test_read_page_copy_zero_fills_pages_beyond_db_size_even_if_cache_is_stale() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let page_seven = PageNumber::new(7).unwrap();
let mut stale_buf = PageBuf::new(PageSize::DEFAULT);
stale_buf.fill(0x7A);
pager.cache.insert_buffer(page_seven, stale_buf);
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.db_size = 4;
let bytes = inner
.read_page_copy(&cx, &pager.cache, &pager.wal_backend, page_seven)
.await
.expect("read_page_copy should succeed for pages beyond db_size");
assert!(
bytes.iter().all(|byte| *byte == 0),
"bead_id=bd-wwqen.3 case=stale_cached_page_above_db_size_must_zero_fill"
);
});
}
#[test]
fn test_publish_truncate_checkpoint_updates_published_db_size_on_same_commit_seq() {
init_publication_test_tracing();
let published = PublishedPagerState::new(8, CommitSeq::new(8), JournalMode::Wal, 0);
let cx = Cx::new();
let page_seven = PageNumber::new(7).unwrap();
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_seven,
PageData::from_vec(sample_page(0x77)),
);
published.publish_truncate_checkpoint(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
4,
);
assert!(
published.try_get_page(page_seven).is_none(),
"bead_id=bd-wwqen.3 case=truncate_checkpoint_evicts_pages_above_new_db_size"
);
assert_eq!(
published.snapshot().db_size,
4,
"bead_id=bd-wwqen.3 case=truncate_checkpoint_updates_published_db_size"
);
}
#[test]
fn test_stale_larger_commit_publish_does_not_reopen_newer_shrunken_snapshot() {
// Authoritative shrinks go through publish_truncate_checkpoint — it is
// the only publisher permitted to reduce db_size (see 49d0b194). A
// subsequent stale publish_commit must not regress visible_commit_seq
// and must not re-open the shrunken snapshot.
init_publication_test_tracing();
let published = PublishedPagerState::new(8, CommitSeq::new(8), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_seven = PageNumber::new(7).unwrap();
// Seed a page above the upcoming shrink boundary so we can verify the
// truncate removes it and a stale publish_commit does not restore it.
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_seven,
PageData::from_vec(sample_page(0x77)),
);
let current_page_two = PageData::from_vec(sample_page(0x22));
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(9),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_two,
current_page_two.clone(),
);
// Truncate-checkpoint is the shrink publisher. It retains pages
// below max_page (so page_two at offset 2 survives) and evicts
// page 1 + everything above max_page (so page_seven at offset 7 is
// swept). db_size is `store`-set to 4, not `fetch_max`'d.
published.publish_truncate_checkpoint(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(9),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
4,
);
let stale_page_two = PageData::from_vec(sample_page(0x99));
let stale_page_seven = PageData::from_vec(sample_page(0x77));
let stale_write_set = HashMap::from([
(page_two, StagedPage::from_page_data(stale_page_two)),
(page_seven, StagedPage::from_page_data(stale_page_seven)),
]);
published.publish_commit(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
&stale_write_set,
);
let after = published.snapshot();
assert_eq!(
after.visible_commit_seq,
CommitSeq::new(9),
"bead_id=bd-wwqen.3 case=stale_larger_commit_must_not_regress_visible_commit_seq"
);
assert_eq!(
after.db_size, 4,
"bead_id=bd-wwqen.3 case=stale_larger_commit_must_not_reopen_shrunken_db_size"
);
assert_eq!(
published.try_get_page(page_two),
Some(current_page_two),
"bead_id=bd-wwqen.3 case=stale_larger_commit_must_not_overwrite_newer_page_bytes"
);
assert!(
published.try_get_page(page_seven).is_none(),
"bead_id=bd-wwqen.3 case=stale_larger_commit_must_not_restore_truncated_pages"
);
}
#[test]
fn test_metadata_only_publish_without_page_clear_advances_snapshot_generation() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(7), JournalMode::Wal, 0);
let cx = Cx::new();
let before = published.snapshot();
let writes_before = published.publication_write_count();
published.publish_single_connection_metadata_update(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 2,
checkpoint_active: true,
},
false,
);
let after = published.snapshot();
assert!(
after.snapshot_gen > before.snapshot_gen,
"bead_id=bd-wwqen.3 case=metadata_only_publish_must_advance_snapshot_generation"
);
assert_eq!(
after.visible_commit_seq,
CommitSeq::new(8),
"bead_id=bd-wwqen.3 case=metadata_only_publish_updates_visible_commit_seq"
);
assert_eq!(
after.freelist_count, 2,
"bead_id=bd-wwqen.3 case=metadata_only_publish_updates_freelist_count"
);
assert!(
after.checkpoint_active,
"bead_id=bd-wwqen.3 case=metadata_only_publish_updates_checkpoint_flag"
);
assert_eq!(
after.page_set_size, 0,
"bead_id=bd-wwqen.3 case=metadata_only_publish_keeps_page_plane_empty"
);
assert_eq!(
published.page_plane_visible_commit_seq(),
before.visible_commit_seq,
"bead_id=bd-wwqen.3 case=metadata_only_publish_keeps_page_plane_horizon_stale"
);
assert_eq!(
published.publication_write_count(),
writes_before,
"bead_id=bd-wwqen.3 case=metadata_only_publish_still_elides_page_plane_write_count"
);
}
#[test]
fn test_metadata_only_publish_without_page_clear_preserves_newer_visible_snapshot() {
init_publication_test_tracing();
let published = PublishedPagerState::new(8, CommitSeq::new(8), JournalMode::Wal, 0);
let cx = Cx::new();
published.publish_single_connection_metadata_update(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 8,
journal_mode: JournalMode::Wal,
freelist_count: 3,
checkpoint_active: true,
},
false,
);
let before = published.snapshot();
published.publish_single_connection_metadata_update(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(7),
db_size: 4,
journal_mode: JournalMode::Delete,
freelist_count: 0,
checkpoint_active: false,
},
false,
);
let after = published.snapshot();
assert_eq!(
after.visible_commit_seq, before.visible_commit_seq,
"bead_id=bd-wwqen.3 case=stale_metadata_only_publish_must_not_regress_visible_commit_seq"
);
assert_eq!(
after.db_size, before.db_size,
"bead_id=bd-wwqen.3 case=stale_metadata_only_publish_must_not_shrink_newer_db_size"
);
assert_eq!(
after.snapshot_gen, before.snapshot_gen,
"bead_id=bd-wwqen.3 case=stale_metadata_only_publish_must_be_a_noop"
);
assert_eq!(
after.journal_mode, before.journal_mode,
"bead_id=bd-wwqen.3 case=stale_metadata_only_publish_must_not_regress_journal_mode"
);
assert_eq!(
after.freelist_count, before.freelist_count,
"bead_id=bd-wwqen.3 case=stale_metadata_only_publish_must_not_regress_freelist_count"
);
assert_eq!(
after.checkpoint_active, before.checkpoint_active,
"bead_id=bd-wwqen.3 case=stale_metadata_only_publish_must_not_regress_checkpoint_flag"
);
}
#[test]
fn test_publish_commit_draining_write_set_publishes_and_empties_staging() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(4), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_three = PageNumber::new(3).unwrap();
let page_two_data = PageData::from_vec(sample_page(0x22));
let page_three_data = PageData::from_vec(sample_page(0x33));
let mut write_set = HashMap::from([
(page_two, StagedPage::from_page_data(page_two_data.clone())),
(
page_three,
StagedPage::from_page_data(page_three_data.clone()),
),
]);
published.publish_commit_draining_write_set(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(5),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
&mut write_set,
);
assert!(
write_set.is_empty(),
"bead_id=bd-autocommit-publish-drain case=publish_drain_consumes_write_set"
);
assert_eq!(
published.try_get_page(page_two),
Some(page_two_data),
"bead_id=bd-autocommit-publish-drain case=publish_drain_keeps_page_two_visible"
);
assert_eq!(
published.try_get_page(page_three),
Some(page_three_data),
"bead_id=bd-autocommit-publish-drain case=publish_drain_keeps_page_three_visible"
);
}
#[test]
fn test_publish_commit_draining_write_set_single_page_publishes_and_empties_staging() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(4), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_two_data = PageData::from_vec(sample_page(0x24));
let mut write_set =
HashMap::from([(page_two, StagedPage::from_page_data(page_two_data.clone()))]);
published.publish_commit_draining_write_set(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(5),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
&mut write_set,
);
assert!(
write_set.is_empty(),
"bead_id=bd-autocommit-publish-drain case=single_page_publish_drain_consumes_write_set"
);
assert_eq!(
published.try_get_page(page_two),
Some(page_two_data),
"bead_id=bd-autocommit-publish-drain case=single_page_publish_drain_keeps_page_visible"
);
}
#[test]
fn test_publish_commit_staged_pages_publishes_drained_pages() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(4), JournalMode::Wal, 0);
let cx = Cx::new();
let page_two = PageNumber::new(2).unwrap();
let page_three = PageNumber::new(3).unwrap();
let staged_pages = vec![
(
page_two,
StagedPage::from_page_data(PageData::from_vec(sample_page(0x42))),
),
(
page_three,
StagedPage::from_page_data(PageData::from_vec(sample_page(0x53))),
),
];
published.publish_commit_staged_pages(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(5),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
staged_pages,
);
assert_eq!(
published.try_get_page(page_two),
Some(PageData::from_vec(sample_page(0x42))),
"bead_id=bd-autocommit-publish-drain case=publish_staged_pages_keeps_page_two_visible"
);
assert_eq!(
published.try_get_page(page_three),
Some(PageData::from_vec(sample_page(0x53))),
"bead_id=bd-autocommit-publish-drain case=publish_staged_pages_keeps_page_three_visible"
);
}
#[test]
fn test_observed_page_publication_populates_snapshot_plane() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(7), JournalMode::Wal, 0);
let cx = Cx::new();
let page_no = PageNumber::new(2).unwrap();
let page = PageData::from_vec(vec![0xAB; PageSize::DEFAULT.as_usize()]);
let published_page = published.publish_observed_page(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(7),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_no,
page.clone(),
);
assert!(
published_page,
"bead_id={BEAD_ID} case=observed_page_publication_applies"
);
assert_eq!(
published.try_get_page(page_no),
Some(page),
"bead_id={BEAD_ID} case=observed_page_visible"
);
assert_eq!(
published.snapshot().visible_commit_seq,
CommitSeq::new(7),
"bead_id={BEAD_ID} case=observed_page_keeps_commit_seq"
);
}
#[test]
fn observed_page_publication_drops_pages_from_an_older_plane_only_once() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(7), JournalMode::Wal, 0);
let cx = Cx::new();
let stale_page_no = PageNumber::new(2).unwrap();
let first_current_page_no = PageNumber::new(3).unwrap();
let second_current_page_no = PageNumber::new(4).unwrap();
let stale_page = PageData::from_vec(sample_page(0x27));
let first_current_page = PageData::from_vec(sample_page(0x38));
let second_current_page = PageData::from_vec(sample_page(0x48));
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(7),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
stale_page_no,
stale_page,
);
published.publish_metadata_only(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
);
assert_eq!(
published.page_plane_visible_commit_seq(),
CommitSeq::new(7),
"metadata-only publication must leave the old page-plane horizon explicit"
);
assert!(published.publish_observed_page(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
first_current_page_no,
first_current_page.clone(),
));
assert_eq!(
published.try_get_page(stale_page_no),
None,
"the first observed page at a newer horizon must not relabel older resident pages"
);
assert_eq!(
published.try_get_page(first_current_page_no),
Some(first_current_page.clone())
);
assert!(published.publish_observed_page(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
second_current_page_no,
second_current_page.clone(),
));
assert_eq!(
published.try_get_page(first_current_page_no),
Some(first_current_page),
"same-horizon observations must accumulate instead of clearing each other"
);
assert_eq!(
published.try_get_page(second_current_page_no),
Some(second_current_page)
);
assert_eq!(published.page_plane_visible_commit_seq(), CommitSeq::new(8));
}
#[test]
fn test_observed_page_publication_skips_stale_commit_regression() {
init_publication_test_tracing();
let published = PublishedPagerState::new(4, CommitSeq::new(7), JournalMode::Wal, 0);
let cx = Cx::new();
let page_no = PageNumber::new(2).unwrap();
let current_page = PageData::from_vec(vec![0xCC; PageSize::DEFAULT.as_usize()]);
// D1-CRITICAL Change 3: Use sharded publish_insert_single (test-only method).
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(8),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_no,
current_page.clone(),
);
let stale_publish_applied = published.publish_observed_page(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(7),
db_size: 4,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_no,
PageData::from_vec(vec![0x11; PageSize::DEFAULT.as_usize()]),
);
assert!(
!stale_publish_applied,
"bead_id={BEAD_ID} case=stale_observed_page_publication_skipped"
);
assert_eq!(
published.snapshot().visible_commit_seq,
CommitSeq::new(8),
"bead_id={BEAD_ID} case=stale_publication_preserves_commit_seq"
);
assert_eq!(
published.try_get_page(page_no),
Some(current_page),
"bead_id={BEAD_ID} case=stale_publication_preserves_page"
);
}
#[test]
fn test_staged_page_owned_publication_reuses_shared_snapshot() {
let expected = sample_page(0x42);
let staged = StagedPage::from_page_data(PageData::from_vec(expected.clone()));
let first = staged.published_page();
let second = staged.published_page();
assert_eq!(
first.as_bytes(),
expected.as_slice(),
"bead_id=bd-db300.10.6 case=owned_publication_keeps_bytes"
);
assert_eq!(
first.as_bytes().as_ptr(),
second.as_bytes().as_ptr(),
"bead_id=bd-db300.10.6 case=owned_publication_reuses_shared_snapshot"
);
}
#[test]
fn test_staged_page_buffered_publication_reuses_shared_snapshot() {
let mut buf = PageBuf::new(PageSize::DEFAULT);
buf.as_mut_slice().fill(0x53);
let staged = StagedPage::from_buf(buf);
let first = staged.published_page();
let second = staged.published_page();
assert!(
first.as_bytes().iter().all(|byte| *byte == 0x53),
"bead_id=bd-db300.10.6 case=buffered_publication_keeps_bytes"
);
assert_eq!(
first.as_bytes().as_ptr(),
second.as_bytes().as_ptr(),
"bead_id=bd-db300.10.6 case=buffered_publication_reuses_shared_snapshot"
);
}
#[test]
fn test_staged_page_unpublishes_before_mutation() {
let pool = PageBufPool::new(PageSize::DEFAULT, 2);
let cache = ShardedPageCache::with_pool(pool.clone(), PageSize::DEFAULT);
let mut original = sample_page(0x42);
original[24] = 0x11;
let staged = StagedPage::from_page_data(PageData::from_vec(original));
let old_snapshot = staged.published_page();
let mut staged = staged;
staged
.make_unpublished_for_mutation(&pool, &cache, "test_page_mutation")
.unwrap();
staged.as_page_bytes_mut()[24] = 0x99;
let final_page = staged.into_published_page();
assert_eq!(
old_snapshot.as_bytes()[24],
0x11,
"bead_id=bd-page1-staged-invariant case=old_snapshot_keeps_original_byte"
);
assert_eq!(
final_page.as_bytes()[24],
0x99,
"bead_id=bd-page1-staged-invariant case=final_publication_uses_mutated_byte"
);
}
#[test]
fn gh_131_published_page_one_mutation_reclaims_clean_cache_buffer() {
let pool = PageBufPool::new(PageSize::DEFAULT, 1);
let cache = ShardedPageCache::with_pool(pool.clone(), PageSize::DEFAULT);
let cached_page = PageNumber::new(2).unwrap();
cache.insert_buffer(cached_page, pool.acquire().unwrap());
assert_eq!(pool.available(), 0, "pool must start saturated");
let mut page_one = sample_page(0x61);
page_one[32] = 0x12;
let staged = StagedPage::from_page_data(PageData::from_vec(page_one));
let published = staged.published_page();
let mut staged = staged;
staged
.make_unpublished_for_mutation(&pool, &cache, "test_page_one_commit_mutation")
.expect("published page-one mutation must reclaim a clean cache buffer");
staged.as_page_bytes_mut()[32] = 0x34;
assert!(!cache.contains(cached_page));
assert_eq!(pool.total_buffers(), 1);
assert_eq!(published.as_bytes()[32], 0x12);
assert_eq!(staged.as_page_bytes()[32], 0x34);
assert!(matches!(
staged.backing,
StagedPageBacking::Buffered(ref buffer) if buffer.returns_to_pool(&pool)
));
}
#[test]
fn gh_131_failed_published_page_one_detach_preserves_staged_state() {
let pool = PageBufPool::new(PageSize::DEFAULT, 1);
let cache = ShardedPageCache::with_pool(pool.clone(), PageSize::DEFAULT);
let dirty_page = PageNumber::new(2).unwrap();
cache
.insert_fresh(dirty_page, |bytes| bytes[0] = 0xD1)
.unwrap();
assert_eq!(pool.available(), 0, "pool must start saturated");
let mut page_one = sample_page(0x71);
page_one[32] = 0x45;
let mut staged = StagedPage::from_page_data(PageData::from_vec(page_one.clone()));
let published_before = staged.published_page();
let error = staged
.make_unpublished_for_mutation(&pool, &cache, "test_page_one_commit_mutation_failure")
.unwrap_err();
assert!(matches!(
error,
FrankenError::PageBufferCapacityExhausted { .. }
));
let published_after = staged.published_page();
assert_eq!(staged.as_page_bytes(), page_one);
assert_eq!(
published_after.as_bytes().as_ptr(),
published_before.as_bytes().as_ptr()
);
assert!(matches!(staged.backing, StagedPageBacking::Owned(_)));
assert!(cache.contains(dirty_page));
assert!(
cache
.page_snapshots()
.iter()
.any(|snapshot| snapshot.page_no == dirty_page && snapshot.dirty)
);
assert_eq!(pool.total_buffers(), pool.capacity());
}
#[test]
fn test_write_page_data_short_buffer_is_zero_filled_to_page_size() {
asupersync::test_utils::run_test(|| async {
let (pager, _path) = test_pager().await;
let cx = Cx::new();
let page_size = PageSize::DEFAULT.as_usize();
let page_no = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
txn.write_page_data(&cx, page_no, PageData::from_vec(vec![0xAB; 32]))
.await
.unwrap();
txn.commit(&cx).await.unwrap();
page_no
};
let reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let page = reader.get_page(&cx, page_no).await.unwrap();
assert_eq!(
page.len(),
page_size,
"bead_id={BEAD_ID} case=short_owned_page_write_preserves_page_size"
);
assert!(
page.as_bytes()[..32].iter().all(|byte| *byte == 0xAB),
"bead_id={BEAD_ID} case=short_owned_page_write_keeps_prefix"
);
assert!(
page.as_bytes()[32..].iter().all(|byte| *byte == 0),
"bead_id={BEAD_ID} case=short_owned_page_write_zero_fills_tail"
);
});
}
#[test]
fn test_external_refresh_clears_stale_published_pages() {
asupersync::test_utils::run_test(|| async {
init_publication_test_tracing();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/published_refresh.db");
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let pager1 = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let pager2 = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let p = {
let mut txn = pager1.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, p, &vec![0x11; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
p
};
let reader = pager1.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(reader.get_page(&cx, p).await.unwrap().as_ref()[0], 0x11);
drop(reader);
let published_before = pager1.published_snapshot();
assert!(
published_before.page_set_size > 0,
"bead_id={BEAD_ID} case=publication_plane_populated_before_refresh"
);
let mut txn = pager2.begin(&cx, TransactionMode::Immediate).await.unwrap();
txn.write_page(&cx, p, &vec![0x22; ps]).await.unwrap();
txn.commit(&cx).await.unwrap();
let refreshed_reader = pager1.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let published_after_refresh = pager1.published_snapshot();
assert!(
published_after_refresh.snapshot_gen > published_before.snapshot_gen,
"bead_id={BEAD_ID} case=publication_gen_advances_on_refresh"
);
assert_eq!(
published_after_refresh.visible_commit_seq,
pager2.published_snapshot().visible_commit_seq,
"bead_id={BEAD_ID} case=publication_visible_seq_tracks_external_commit"
);
assert_eq!(
published_after_refresh.page_set_size, 0,
"bead_id={BEAD_ID} case=publication_clears_stale_pages_on_refresh"
);
assert_eq!(
refreshed_reader.get_page(&cx, p).await.unwrap().as_ref()[0],
0x22,
"bead_id={BEAD_ID} case=publication_refresh_reads_latest_committed_page"
);
});
}
#[test]
fn external_page_size_change_fails_closed_until_reopen() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/external_page_size_drift.db");
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let replacement_vfs = MemoryVfs::new();
let replacement_path = PathBuf::from("/replacement_page_size.db");
let replacement_page_size = PageSize::new(8192).unwrap();
drop(
SimplePager::open(
replacement_vfs.clone(),
&replacement_path,
replacement_page_size,
)
.await
.unwrap(),
);
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut replacement, _) = replacement_vfs
.open(&cx, Some(&replacement_path), flags)
.unwrap();
let mut replacement_bytes = vec![0_u8; replacement_page_size.as_usize()];
replacement
.read(&cx, &mut replacement_bytes, 0)
.await
.unwrap();
replacement.close(&cx).unwrap();
let (mut live_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
live_file.write(&cx, &replacement_bytes, 0).await.unwrap();
live_file.close(&cx).unwrap();
let error = pager.refresh_published_snapshot(&cx).await.unwrap_err();
assert!(
matches!(error, FrankenError::DatabaseCorrupt { ref detail }
if detail.contains("page size changed") && detail.contains("reopen")),
"unexpected page-size drift error: {error}"
);
assert_eq!(pager.page_size(), PageSize::DEFAULT);
assert!(matches!(
pager.begin(&cx, TransactionMode::ReadOnly).await,
Err(FrankenError::DatabaseCorrupt { .. })
));
let reopened = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
assert_eq!(reopened.page_size(), replacement_page_size);
});
}
#[test]
fn zero_frame_wal_refresh_observes_same_size_main_header_replacement() {
asupersync::test_utils::run_test(|| async {
use crate::traits::{WalBackend, WalPublicationSnapshot};
struct ZeroFrameWalBackend {
snapshot: WalPublicationSnapshot,
}
impl WalBackend for ZeroFrameWalBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn pinned_read_snapshot(&self) -> Option<WalPublicationSnapshot> {
Some(self.snapshot)
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
0
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let cx = Cx::new();
let vfs = MemoryVfs::new();
let path = PathBuf::from("/same_size_zero_frame_wal_refresh.db");
let pager = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
let generation = WalGenerationIdentity {
checkpoint_seq: 1,
salts: fsqlite_wal::WalSalts {
salt1: 0x1111_2222,
salt2: 0x3333_4444,
},
};
pager
.set_wal_backend(Box::new(ZeroFrameWalBackend {
snapshot: WalPublicationSnapshot {
publication_seq: 0,
generation,
last_commit_frame: None,
commit_count: 0,
latest_frame_entries: 0,
index_is_partial: false,
},
}))
.unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let before = pager.refresh_published_snapshot(&cx).await.unwrap();
let flags = VfsOpenFlags::READWRITE | VfsOpenFlags::MAIN_DB;
let (mut db_file, _) = vfs.open(&cx, Some(&path), flags).unwrap();
let mut page_one = vec![0_u8; PageSize::DEFAULT.as_usize()];
db_file.read(&cx, &mut page_one, 0).await.unwrap();
let mut header_bytes = [0_u8; DATABASE_HEADER_SIZE];
header_bytes.copy_from_slice(&page_one[..DATABASE_HEADER_SIZE]);
let mut header = DatabaseHeader::from_bytes(&header_bytes).unwrap();
header.change_counter = header.change_counter.wrapping_add(1).max(1);
header.version_valid_for = header.change_counter;
page_one[..DATABASE_HEADER_SIZE].copy_from_slice(&header.to_bytes().unwrap());
db_file.write(&cx, &page_one, 0).await.unwrap();
db_file.close(&cx).unwrap();
let after = pager
.refresh_published_snapshot_for_clean_wal_read(&cx)
.await
.unwrap();
assert!(after.visible_commit_seq > before.visible_commit_seq);
let inner = pager.inner.lock().unwrap();
assert_eq!(
inner.committed_db_change_counter,
u64::from(header.change_counter)
);
assert_eq!(inner.committed_wal_visible_commit_count, 0);
assert_eq!(inner.committed_wal_generation, Some(generation));
});
}
#[test]
fn logical_pinned_wal_horizon_must_match_the_reader_snapshot() {
asupersync::test_utils::run_test(|| async {
use crate::traits::{WalBackend, WalLogicalReadSnapshot, WalPublicationSnapshot};
struct LogicalPinnedHorizonWalBackend {
snapshot: WalPublicationSnapshot,
logical_snapshot: WalLogicalReadSnapshot,
}
impl WalBackend for LogicalPinnedHorizonWalBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn pinned_read_snapshot(&self) -> Option<WalPublicationSnapshot> {
Some(self.snapshot)
}
fn pinned_logical_read_snapshot<'a>(
&'a self,
_cx: &'a Cx,
) -> WalFuture<'a, Option<WalLogicalReadSnapshot>> {
Box::pin(async { Ok(Some(self.logical_snapshot)) })
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
1
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 1,
frames_backfilled: 0,
completed: false,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let cx = Cx::new();
let generation = WalGenerationIdentity {
checkpoint_seq: 9,
salts: fsqlite_wal::WalSalts {
salt1: 0x1111_2222,
salt2: 0x3333_4444,
},
};
let physical_snapshot = WalPublicationSnapshot {
publication_seq: 1,
generation,
last_commit_frame: Some(0),
commit_count: 1,
latest_frame_entries: 1,
index_is_partial: false,
};
let logical_snapshot = WalLogicalReadSnapshot {
generation,
last_commit_frame: Some(0),
visible_commit_seq: CommitSeq::new(42),
};
let (pager, _) = test_pager().await;
pager
.set_wal_backend(Box::new(LogicalPinnedHorizonWalBackend {
snapshot: physical_snapshot,
logical_snapshot,
}))
.expect("install matching logical-horizon WAL backend");
pager
.set_journal_mode(&cx, JournalMode::Wal)
.await
.expect("enable WAL mode");
let inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let probe = inner
.probe_visible_commit_seq(&cx, &pager.wal_backend)
.await
.expect("matching logical horizon is accepted");
assert_eq!(
probe.visible_commit_seq,
CommitSeq::new(42),
"the logical group horizon must not collapse to one physical marker"
);
drop(inner);
let mismatched_generation = WalGenerationIdentity {
checkpoint_seq: generation.checkpoint_seq.saturating_add(1),
salts: generation.salts,
};
let (mismatched_pager, _) = test_pager().await;
mismatched_pager
.set_wal_backend(Box::new(LogicalPinnedHorizonWalBackend {
snapshot: physical_snapshot,
logical_snapshot: WalLogicalReadSnapshot {
generation: mismatched_generation,
..logical_snapshot
},
}))
.expect("install mismatched logical-horizon WAL backend");
mismatched_pager
.set_journal_mode(&cx, JournalMode::Wal)
.await
.expect("enable WAL mode");
let inner = mismatched_pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let err = inner
.probe_visible_commit_seq(&cx, &mismatched_pager.wal_backend)
.await
.expect_err("logical horizon from another WAL generation is rejected");
assert!(matches!(err, FrankenError::WalCorrupt { .. }));
});
}
#[test]
fn test_refresh_committed_state_restores_probe_identity_on_page1_error() {
asupersync::test_utils::run_test(|| async {
use crate::traits::{WalBackend, WalPublicationSnapshot};
struct PageOneReadFailWalBackend {
snapshot: WalPublicationSnapshot,
}
impl WalBackend for PageOneReadFailWalBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn pinned_read_snapshot(&self) -> Option<WalPublicationSnapshot> {
Some(self.snapshot)
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async {
Err(FrankenError::internal(
"forced committed page-1 materialization failure",
))
})
}
fn sync(&mut self, _cx: &Cx) -> Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
1
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 1,
frames_backfilled: 0,
completed: false,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
let (pager, _) = test_pager().await;
let cx = Cx::new();
let old_generation = WalGenerationIdentity {
checkpoint_seq: 3,
salts: fsqlite_wal::WalSalts {
salt1: 0x1111_2222,
salt2: 0x3333_4444,
},
};
let new_generation = WalGenerationIdentity {
checkpoint_seq: 4,
salts: fsqlite_wal::WalSalts {
salt1: 0x5555_6666,
salt2: 0x7777_8888,
},
};
pager
.set_wal_backend(Box::new(PageOneReadFailWalBackend {
snapshot: WalPublicationSnapshot {
publication_seq: 1,
generation: new_generation,
last_commit_frame: Some(0),
commit_count: 1,
latest_frame_entries: 1,
index_is_partial: false,
},
}))
.expect("install page-1 failing WAL backend");
pager
.set_journal_mode(&cx, JournalMode::Wal)
.await
.expect("enable WAL mode");
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.committed_db_change_counter = 9_999;
inner.committed_wal_generation = Some(old_generation);
inner.committed_wal_visible_commit_count = 77;
let previous_db_change_counter = inner.committed_db_change_counter;
let previous_wal_generation = inner.committed_wal_generation;
let previous_wal_visible_commit_count = inner.committed_wal_visible_commit_count;
let err = inner
.refresh_committed_state(&cx, &pager.cache, &pager.wal_backend)
.await
.expect_err("forced page-1 materialization failure should surface");
assert!(
format!("{err}").contains("forced committed page-1 materialization failure"),
"bead_id={BEAD_ID} case=refresh_page1_error_surfaces_original_error err={err}"
);
assert_eq!(
inner.committed_db_change_counter, previous_db_change_counter,
"bead_id={BEAD_ID} case=refresh_error_restores_base_change_counter"
);
assert_eq!(
inner.committed_wal_generation, previous_wal_generation,
"bead_id={BEAD_ID} case=refresh_error_restores_wal_generation"
);
assert_eq!(
inner.committed_wal_visible_commit_count, previous_wal_visible_commit_count,
"bead_id={BEAD_ID} case=refresh_error_restores_visible_wal_count"
);
});
}
#[test]
fn test_stale_main_header_recovery_ignores_uncommitted_wal_page1_tail() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let db_path = PathBuf::from("/stale-main-header-uncommitted-tail.db");
let wal_path = PathBuf::from("/stale-main-header-uncommitted-tail.db-wal");
let page_size = PageSize::DEFAULT;
let valid_header = DatabaseHeader {
page_size,
page_count: 1,
..DatabaseHeader::default()
};
let mut committed_page1 = vec![0_u8; page_size.as_usize()];
committed_page1[..DATABASE_HEADER_SIZE]
.copy_from_slice(&valid_header.to_bytes().expect("valid page-1 header"));
let mut stale_header_bytes = valid_header.to_bytes().expect("base header bytes");
stale_header_bytes[44..48].fill(0);
let stale_error = DatabaseHeader::from_bytes(&stale_header_bytes)
.expect_err("schema format 0 must be treated as a stale main-file header");
let mut uncommitted_page1 = committed_page1.clone();
uncommitted_page1[..DATABASE_HEADER_SIZE].copy_from_slice(&stale_header_bytes);
let open_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::CREATE | VfsOpenFlags::WAL;
let (file, _) = vfs.open(&cx, Some(&wal_path), open_flags).unwrap();
let mut wal = fsqlite_wal::WalFile::create(
&cx,
file,
page_size.get(),
0,
fsqlite_wal::WalSalts::default(),
)
.await
.unwrap();
wal.append_frame(&cx, PageNumber::ONE.get(), &committed_page1, 1)
.await
.expect("append committed page-1 frame");
wal.append_frame(&cx, PageNumber::ONE.get(), &uncommitted_page1, 0)
.await
.expect("append uncommitted page-1 tail");
wal.close(&cx).unwrap();
let recoverable = stale_main_header_can_be_recovered_from_live_wal(
&cx,
&vfs,
&db_path,
&stale_header_bytes,
&stale_error,
false,
)
.await
.expect("probe stale-header recovery");
assert!(
recoverable,
"recovery must use the committed WAL horizon, not an uncommitted tail frame"
);
});
}
#[test]
fn test_stale_main_header_recovery_never_falls_back_to_readonly_wal_probe() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = WalReadonlyFallbackProbeVfs::new();
let db_path = PathBuf::from("/stale-main-header-readonly-fallback.db");
let wal_path = PathBuf::from("/stale-main-header-readonly-fallback.db-wal");
let page_size = PageSize::DEFAULT;
let valid_header = DatabaseHeader {
page_size,
page_count: 1,
..DatabaseHeader::default()
};
let mut stale_header_bytes = valid_header.to_bytes().expect("base header bytes");
stale_header_bytes[44..48].fill(0);
let stale_error = DatabaseHeader::from_bytes(&stale_header_bytes)
.expect_err("schema format 0 must be treated as a stale main-file header");
let open_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::CREATE | VfsOpenFlags::WAL;
let (mut wal_file, _) = vfs.inner.open(&cx, Some(&wal_path), open_flags).unwrap();
wal_file.close(&cx).unwrap();
let recoverable = stale_main_header_can_be_recovered_from_live_wal(
&cx,
&vfs,
&db_path,
&stale_header_bytes,
&stale_error,
false,
)
.await
.expect("probe stale-header recovery");
assert!(
!recoverable,
"readwrite probe failure must stop recovery instead of retrying READONLY"
);
assert!(
!vfs.readonly_wal_open_attempted(),
"bead_id={BEAD_ID} case=stale_header_recovery_must_not_probe_wal_readonly"
);
});
}
#[test]
fn test_readonly_stale_main_header_recovery_allows_readonly_wal_probe() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = WalReadonlyFallbackProbeVfs::new();
let db_path = PathBuf::from("/readonly-stale-main-header-fallback.db");
let wal_path = PathBuf::from("/readonly-stale-main-header-fallback.db-wal");
let page_size = PageSize::DEFAULT;
let valid_header = DatabaseHeader {
page_size,
page_count: 1,
..DatabaseHeader::default()
};
let mut committed_page1 = vec![0_u8; page_size.as_usize()];
committed_page1[..DATABASE_HEADER_SIZE]
.copy_from_slice(&valid_header.to_bytes().expect("valid page-1 header"));
let mut stale_header_bytes = valid_header.to_bytes().expect("base header bytes");
stale_header_bytes[44..48].fill(0);
let stale_error = DatabaseHeader::from_bytes(&stale_header_bytes)
.expect_err("schema format 0 must be treated as a stale main-file header");
let open_flags = VfsOpenFlags::READWRITE | VfsOpenFlags::CREATE | VfsOpenFlags::WAL;
let (file, _) = vfs.inner.open(&cx, Some(&wal_path), open_flags).unwrap();
let mut wal = fsqlite_wal::WalFile::create(
&cx,
file,
page_size.get(),
0,
fsqlite_wal::WalSalts::default(),
)
.await
.unwrap();
wal.append_frame(&cx, PageNumber::ONE.get(), &committed_page1, 1)
.await
.expect("append committed page-1 frame");
wal.close(&cx).unwrap();
let recoverable = stale_main_header_can_be_recovered_from_live_wal(
&cx,
&vfs,
&db_path,
&stale_header_bytes,
&stale_error,
true,
)
.await
.expect("probe stale-header recovery");
assert!(
recoverable,
"read-only startup may validate a live WAL through a READONLY sidecar handle"
);
assert!(
vfs.readonly_wal_open_attempted(),
"read-only recovery should fall back to a READONLY WAL probe"
);
});
}
#[test]
fn test_set_wal_backend_owned_returns_backend_when_install_is_rejected() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let dropped = Arc::new(Mutex::new(false));
{
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.checkpoint_active = true;
}
let backend = DropAwareWalBackend {
dropped: Arc::clone(&dropped),
};
let (err, backend) = pager
.set_wal_backend_owned(backend)
.expect_err("checkpoint-active pager must reject WAL backend install");
assert!(
matches!(err, FrankenError::Busy),
"unexpected error: {err:?}"
);
assert!(
!*dropped
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
"rejected install must return the backend instead of dropping it"
);
assert!(
!has_wal_backend(&pager.wal_backend).unwrap(),
"failed install must not publish a backend"
);
drop(backend);
assert!(
*dropped
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
"caller should own backend cleanup after a rejected install"
);
});
}
#[test]
fn test_set_wal_backend_owned_rejects_active_transaction() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let (initial_backend, _, _, _) = MockWalBackend::new();
pager.set_wal_backend(Box::new(initial_backend)).unwrap();
let cx = Cx::new();
let mut reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let dropped = Arc::new(Mutex::new(false));
let replacement = DropAwareWalBackend {
dropped: Arc::clone(&dropped),
};
let (error, replacement) = pager
.set_wal_backend_owned(replacement)
.expect_err("an active transaction must fence WAL backend replacement");
assert!(matches!(error, FrankenError::Busy));
assert!(
!*dropped
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
"rejected replacement must remain caller-owned"
);
reader.rollback(&cx).await.unwrap();
drop(replacement);
assert!(
*dropped
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner),
"caller must be able to clean up the rejected replacement"
);
});
}
#[test]
fn test_set_journal_mode_wal_requires_backend_even_when_inner_mode_is_wal() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let (pager, _) = test_pager().await;
{
let mut inner = pager
.inner
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
inner.journal_mode = JournalMode::Wal;
}
let err = pager
.set_journal_mode(&cx, JournalMode::Wal)
.await
.expect_err("WAL mode must require an installed WAL backend");
assert!(
matches!(err, FrankenError::Unsupported),
"unexpected error: {err:?}"
);
assert!(
!has_wal_backend(&pager.wal_backend).unwrap(),
"failed WAL mode confirmation must not publish a backend"
);
});
}
#[test]
fn test_set_journal_mode_same_rollback_mode_allowed_during_active_transaction() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let (pager, _) = test_pager().await;
let mut txn = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
assert_eq!(
pager
.set_journal_mode(&cx, JournalMode::Delete)
.await
.unwrap(),
JournalMode::Delete,
"idempotent rollback journal-mode confirmation should not require a mode switch"
);
txn.rollback(&cx).await.unwrap();
});
}
#[test]
fn test_conservative_conflict_pages_include_free_only_freelist_surface() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = wal_pager().await;
let cx = Cx::new();
let ps = PageSize::DEFAULT.as_usize();
let mut seed = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let p2 = seed.allocate_page(&cx).await.unwrap();
let p3 = seed.allocate_page(&cx).await.unwrap();
let p4 = seed.allocate_page(&cx).await.unwrap();
seed.write_page(&cx, p2, &vec![0x22; ps]).await.unwrap();
seed.write_page(&cx, p3, &vec![0x33; ps]).await.unwrap();
seed.write_page(&cx, p4, &vec![0x44; ps]).await.unwrap();
seed.commit(&cx).await.unwrap();
let mut txn = pager.begin(&cx, TransactionMode::Concurrent).await.unwrap();
txn.free_page(&cx, p2).await.unwrap();
txn.free_page(&cx, p3).await.unwrap();
let precise = txn.pending_conflict_pages().unwrap();
let conservative = txn.pending_conflict_pages_conservative();
assert!(precise.contains(&p2) && precise.contains(&p3));
assert_eq!(
conservative,
vec![PageNumber::ONE, p2, p3],
"free-only FCW planning must include both freed pages and the shared page-1 freelist token; precise={precise:?}"
);
txn.rollback(&cx).await.unwrap();
});
}
async fn read_all_vfs_bytes<V: Vfs>(vfs: &V, cx: &Cx, path: &Path) -> Vec<u8> {
let flags = VfsOpenFlags::MAIN_DB | VfsOpenFlags::READWRITE;
let (mut file, _) = vfs.open(cx, Some(path), flags).unwrap();
let size = usize::try_from(file.file_size(cx).unwrap()).unwrap();
let mut out = vec![0_u8; size];
let read = file.read(cx, &mut out, 0).await.unwrap();
assert_eq!(read, size);
file.close(cx).unwrap();
out
}
#[test]
fn test_copy_database_to_copies_main_db_via_vfs() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let source_path = PathBuf::from("/copy_source.db");
let target_path = PathBuf::from("/copy_target.db");
let pager = SimplePager::open(vfs.clone(), &source_path, PageSize::DEFAULT)
.await
.unwrap();
let page_size = PageSize::DEFAULT.as_usize();
let page_no = {
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
let mut page = vec![0xA5; page_size];
page[0] = 0x5A;
page[page_size - 1] = 0xC3;
txn.write_page(&cx, page_no, &page).await.unwrap();
txn.commit(&cx).await.unwrap();
page_no
};
pager.copy_database_to(&cx, &target_path).await.unwrap();
let source_bytes = read_all_vfs_bytes(&vfs, &cx, &source_path).await;
let target_bytes = read_all_vfs_bytes(&vfs, &cx, &target_path).await;
assert_eq!(
target_bytes, source_bytes,
"bead_id={BEAD_ID} case=copy_database_to_byte_identical_copy"
);
let copied = SimplePager::open(vfs, &target_path, PageSize::DEFAULT)
.await
.unwrap();
let reader = copied.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let page = reader.get_page(&cx, page_no).await.unwrap();
assert_eq!(
page.as_ref()[0],
0x5A,
"bead_id={BEAD_ID} case=copy_database_to_reopen_reads_committed_page"
);
assert_eq!(
page.as_ref()[page_size - 1],
0xC3,
"bead_id={BEAD_ID} case=copy_database_to_preserves_page_tail"
);
});
}
#[test]
fn test_copy_database_to_rejects_existing_target() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let source_path = PathBuf::from("/copy_existing_source.db");
let target_path = PathBuf::from("/copy_existing_target.db");
let pager = SimplePager::open(vfs.clone(), &source_path, PageSize::DEFAULT)
.await
.unwrap();
let _target = SimplePager::open(vfs, &target_path, PageSize::DEFAULT)
.await
.unwrap();
let err = pager.copy_database_to(&cx, &target_path).await.unwrap_err();
assert!(
matches!(err, FrankenError::CannotOpen { .. }),
"bead_id={BEAD_ID} case=copy_database_to_existing_target_err={err:?}"
);
});
}
#[test]
fn test_copy_database_to_requires_quiescent_pager() {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
let vfs = MemoryVfs::new();
let source_path = PathBuf::from("/copy_busy_source.db");
let target_path = PathBuf::from("/copy_busy_target.db");
let pager = SimplePager::open(vfs, &source_path, PageSize::DEFAULT)
.await
.unwrap();
let _reader = pager.begin(&cx, TransactionMode::ReadOnly).await.unwrap();
let err = pager.copy_database_to(&cx, &target_path).await.unwrap_err();
assert!(
matches!(err, FrankenError::Busy),
"bead_id={BEAD_ID} case=copy_database_to_rejects_active_transactions err={err:?}"
);
});
}
#[test]
fn test_published_snapshot_retries_during_inflight_publication() {
init_publication_test_tracing();
let published = Arc::new(PublishedPagerState::new(
1,
CommitSeq::new(1),
JournalMode::Delete,
0,
));
published.sequence.store(3, AtomicOrdering::Release);
let reader_plane = Arc::clone(&published);
let handle = std::thread::spawn(move || reader_plane.snapshot());
for _ in 0..10_000 {
if published.read_retry_count() > 0 {
break;
}
std::thread::yield_now();
}
assert!(
published.read_retry_count() > 0,
"bead_id={BEAD_ID} case=publication_retry_counter_increments"
);
published
.visible_commit_seq
.store(2, AtomicOrdering::Release);
published.db_size.store(2, AtomicOrdering::Release);
published.journal_mode.store(
encode_journal_mode(JournalMode::Wal),
AtomicOrdering::Release,
);
published.freelist_count.store(1, AtomicOrdering::Release);
published
.checkpoint_active
.store(true, AtomicOrdering::Release);
published.page_set_size.store(0, AtomicOrdering::Release);
published.sequence.store(4, AtomicOrdering::Release);
let snapshot = handle.join().unwrap();
assert_eq!(
snapshot.snapshot_gen, 4,
"bead_id={BEAD_ID} case=publication_retry_returns_new_snapshot"
);
assert_eq!(
snapshot.visible_commit_seq,
CommitSeq::new(2),
"bead_id={BEAD_ID} case=publication_retry_visible_commit_seq"
);
assert_eq!(
snapshot.freelist_count, 1,
"bead_id={BEAD_ID} case=publication_retry_freelist_count"
);
assert!(
snapshot.checkpoint_active,
"bead_id={BEAD_ID} case=publication_retry_checkpoint_flag"
);
}
#[test]
fn test_published_sequence_waiters_wake_on_targeted_transitions() {
init_publication_test_tracing();
let published = Arc::new(PublishedPagerState::new(
1,
CommitSeq::new(1),
JournalMode::Delete,
0,
));
published.sequence.store(4, AtomicOrdering::Release);
let begin_plane = Arc::clone(&published);
let (begin_ready_tx, begin_ready_rx) = std::sync::mpsc::channel();
let (begin_done_tx, begin_done_rx) = std::sync::mpsc::channel();
let begin_waiter = std::thread::spawn(move || {
begin_ready_tx
.send(())
.expect("begin waiter should signal readiness");
begin_plane.wait_for_sequence_change(4, Duration::from_secs(1));
begin_done_tx
.send(())
.expect("begin waiter should signal completion");
});
begin_ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("begin waiter should start");
for _ in 0..10_000 {
if published.sequence_waiters.has_slot(4) {
break;
}
std::thread::yield_now();
}
assert!(
published.sequence_waiters.has_slot(4),
"bead_id={BEAD_ID} case=publication_begin_waiter_registers_targeted_slot"
);
assert!(
begin_done_rx
.recv_timeout(Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=publication_begin_waiter_stays_parked_until_targeted_signal"
);
let begin_sequence = published.sequence.fetch_add(1, AtomicOrdering::AcqRel);
assert_eq!(begin_sequence, 4);
published.signal_sequence_waiters(begin_sequence, "test_begin");
begin_done_rx
.recv_timeout(Duration::from_millis(100))
.expect("publish-begin signal should wake matching waiter");
begin_waiter.join().unwrap();
let complete_plane = Arc::clone(&published);
let (complete_ready_tx, complete_ready_rx) = std::sync::mpsc::channel();
let (complete_done_tx, complete_done_rx) = std::sync::mpsc::channel();
let complete_waiter = std::thread::spawn(move || {
complete_ready_tx
.send(())
.expect("complete waiter should signal readiness");
complete_plane.wait_for_sequence_change(5, Duration::from_secs(1));
complete_done_tx
.send(())
.expect("complete waiter should signal completion");
});
complete_ready_rx
.recv_timeout(Duration::from_secs(1))
.expect("complete waiter should start");
for _ in 0..10_000 {
if published.sequence_waiters.has_slot(5) {
break;
}
std::thread::yield_now();
}
assert!(
published.sequence_waiters.has_slot(5),
"bead_id={BEAD_ID} case=publication_complete_waiter_registers_targeted_slot"
);
assert!(
complete_done_rx
.recv_timeout(Duration::from_millis(20))
.is_err(),
"bead_id={BEAD_ID} case=publication_complete_waiter_stays_parked_until_targeted_signal"
);
let complete_sequence = published.sequence.fetch_add(1, AtomicOrdering::AcqRel);
assert_eq!(complete_sequence, 5);
published.signal_sequence_waiters(complete_sequence, "test_complete");
complete_done_rx
.recv_timeout(Duration::from_millis(100))
.expect("publish-complete signal should wake matching waiter");
complete_waiter.join().unwrap();
}
fn run_parallel_counter_benchmark<F>(
thread_count: usize,
increments_per_thread: usize,
increment: F,
) -> u64
where
F: Fn() + Send + Sync + 'static,
{
let start_barrier = Arc::new(std::sync::Barrier::new(thread_count + 1));
let increment = Arc::new(increment);
let handles: Vec<_> = (0..thread_count)
.map(|_| {
let start_barrier = Arc::clone(&start_barrier);
let increment = Arc::clone(&increment);
std::thread::spawn(move || {
start_barrier.wait();
for _ in 0..increments_per_thread {
increment();
}
})
})
.collect();
start_barrier.wait();
let started = Instant::now();
for handle in handles {
handle.join().expect("counter worker should finish");
}
u64::try_from(started.elapsed().as_nanos()).unwrap_or(u64::MAX)
}
#[test]
fn test_published_counter_striping_tracks_parallel_increments() {
let counter = Arc::new(StripedCounter64::new());
let thread_count = 8;
let increments_per_thread = 5_000;
let expected_total =
u64::try_from(thread_count * increments_per_thread).unwrap_or(u64::MAX);
let counter_for_bench = Arc::clone(&counter);
let elapsed_ns =
run_parallel_counter_benchmark(thread_count, increments_per_thread, move || {
counter_for_bench.increment();
});
assert!(
elapsed_ns > 0,
"bead_id={BEAD_ID} case=publication_counter_parallel_elapsed"
);
assert_eq!(
counter.load(),
expected_total,
"bead_id={BEAD_ID} case=publication_counter_parallel_total"
);
}
#[test]
#[ignore = "manual perf evidence for bd-db300.2.3.2"]
fn bench_bd_db300_2_3_2_publication_counter_striping() {
#[derive(Debug)]
struct BaselineCounter(AtomicU64);
impl BaselineCounter {
fn increment(&self) {
self.0.fetch_add(1, AtomicOrdering::Relaxed);
}
fn load(&self) -> u64 {
self.0.load(AtomicOrdering::Acquire)
}
}
let thread_count = std::thread::available_parallelism()
.map_or(4, |parallelism| parallelism.get().clamp(2, 16));
let increments_per_thread = 200_000;
let expected_total =
u64::try_from(thread_count * increments_per_thread).unwrap_or(u64::MAX);
let baseline = Arc::new(BaselineCounter(AtomicU64::new(0)));
let baseline_counter = Arc::clone(&baseline);
let baseline_ns =
run_parallel_counter_benchmark(thread_count, increments_per_thread, move || {
baseline_counter.increment();
});
assert_eq!(
baseline.load(),
expected_total,
"bead_id={BEAD_ID} case=publication_counter_baseline_total"
);
let striped = Arc::new(StripedCounter64::new());
let striped_counter = Arc::clone(&striped);
let striped_ns =
run_parallel_counter_benchmark(thread_count, increments_per_thread, move || {
striped_counter.increment();
});
assert_eq!(
striped.load(),
expected_total,
"bead_id={BEAD_ID} case=publication_counter_striped_total"
);
let speedup_milli = if striped_ns == 0 {
0_u64
} else {
u64::try_from((u128::from(baseline_ns)).saturating_mul(1_000) / u128::from(striped_ns))
.unwrap_or(u64::MAX)
};
println!("BEGIN_BD_DB300_2_3_2_REPORT");
println!(
"{{\"threads\":{thread_count},\"increments_per_thread\":{increments_per_thread},\"baseline_ns\":{baseline_ns},\"striped_ns\":{striped_ns},\"speedup_milli\":{speedup_milli}}}"
);
println!("END_BD_DB300_2_3_2_REPORT");
}
// ── bd-db300.3.8.7: targeted regression tests for lock-scope narrowing ──
#[test]
fn test_read_page_from_wal_backend_falls_back_to_write_lock_when_pinned_reads_unsupported() {
asupersync::test_utils::run_test(|| async {
// Verify that read_page_from_wal_backend falls back to the write-lock
// path when supports_pinned_reads() returns false.
let wal_backend: SharedWalBackend = new_shared_wal_backend();
let cx = Cx::new();
let page_no = PageNumber::new(1).unwrap();
// With no backend installed, read should return an error.
let result = read_page_from_wal_backend(&wal_backend, &cx, page_no).await;
assert!(
result.is_err(),
"bead_id=bd-db300.3.8.7 read_page_from_wal_backend should error without backend"
);
// Install a mock backend that does NOT support pinned reads.
let (mock, _frames, _begin, _batch) = MockWalBackend::new();
*wal_backend.write().unwrap() = Some(Arc::new(AsyncRwLock::with_name(
"wal_backend",
Box::new(mock) as Box<dyn crate::traits::WalBackend>,
)));
// Verify it falls back to write-lock path (default read_page).
let result = read_page_from_wal_backend(&wal_backend, &cx, page_no).await;
assert!(
result.is_ok(),
"bead_id=bd-db300.3.8.7 fallback to write-lock read_page should succeed"
);
// The mock returns None for unwritten pages.
assert_eq!(
result.unwrap(),
None,
"bead_id=bd-db300.3.8.7 unwritten page should return None"
);
});
}
#[test]
fn test_read_page_from_wal_backend_uses_pinned_read_without_write_fallback() {
asupersync::test_utils::run_test(|| async {
use crate::traits::WalBackend;
struct PinnedReadBackend {
pinned_calls: Arc<Mutex<usize>>,
fallback_calls: Arc<Mutex<usize>>,
response: Vec<u8>,
}
impl WalBackend for PinnedReadBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async move {
*self.fallback_calls.lock().unwrap() += 1;
Ok(Some(vec![0xEE]))
})
}
fn read_page_pinned<'a>(
&'a self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async move {
*self.pinned_calls.lock().unwrap() += 1;
Ok(Some(self.response.clone()))
})
}
fn supports_pinned_reads(&self) -> bool {
true
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
0
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
let pinned_calls = Arc::new(Mutex::new(0_usize));
let fallback_calls = Arc::new(Mutex::new(0_usize));
let expected = vec![0xAB, 0xCD, 0xEF];
let wal_backend: SharedWalBackend = new_shared_wal_backend();
*wal_backend.write().unwrap() = Some(Arc::new(AsyncRwLock::with_name(
"wal_backend",
Box::new(PinnedReadBackend {
pinned_calls: Arc::clone(&pinned_calls),
fallback_calls: Arc::clone(&fallback_calls),
response: expected.clone(),
}) as Box<dyn crate::traits::WalBackend>,
)));
let cx = Cx::new();
let page_no = PageNumber::new(7).unwrap();
let result = read_page_from_wal_backend(&wal_backend, &cx, page_no)
.await
.unwrap();
assert_eq!(
result,
Some(expected),
"bead_id=bd-db300.3.8.7 case=wal_read_scope_pinned_read_returns_data_without_fallback"
);
assert_eq!(
*pinned_calls.lock().unwrap(),
1,
"bead_id=bd-db300.3.8.7 case=wal_read_scope_pinned_read_call_count"
);
assert_eq!(
*fallback_calls.lock().unwrap(),
0,
"bead_id=bd-db300.3.8.7 case=wal_read_scope_pinned_read_must_not_take_write_lock_fallback"
);
});
}
#[test]
fn test_read_page_pinned_error_does_not_fall_back() {
asupersync::test_utils::run_test(|| async {
// Verify that a REAL error from read_page_pinned propagates
// instead of silently falling back to the write-lock path.
use crate::traits::WalBackend;
/// A WAL backend that supports pinned reads but always returns an
/// error from read_page_pinned to simulate corruption.
struct CorruptPinnedReadBackend;
impl WalBackend for CorruptPinnedReadBackend {
fn begin_transaction<'a>(&'a mut self, _cx: &'a Cx) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async {
// This should NEVER be called if pinned reads are supported
// and the pinned read fails with a real error.
panic!(
"bead_id=bd-db300.3.8.7 MUST NOT fall back to read_page \
when read_page_pinned returns a real error"
);
})
}
fn read_page_pinned<'a>(
&'a self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async {
Err(fsqlite_error::FrankenError::WalCorrupt {
detail: "simulated corruption in pinned read".to_owned(),
})
})
}
fn supports_pinned_reads(&self) -> bool {
true
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
0
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
_mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: _mode,
effective_mode: _mode,
})
})
}
}
let wal_backend: SharedWalBackend = new_shared_wal_backend();
*wal_backend.write().unwrap() = Some(Arc::new(AsyncRwLock::with_name(
"wal_backend",
Box::new(CorruptPinnedReadBackend) as Box<dyn crate::traits::WalBackend>,
)));
let cx = Cx::new();
let page_no = PageNumber::new(1).unwrap();
let result = read_page_from_wal_backend(&wal_backend, &cx, page_no).await;
assert!(
result.is_err(),
"bead_id=bd-db300.3.8.7 real read_page_pinned error must propagate, not fall back"
);
let err = result.unwrap_err();
assert!(
format!("{err}").contains("corruption"),
"bead_id=bd-db300.3.8.7 error should be the corruption error, got: {err}"
);
});
}
/// bd-db300.3.8.6: Prove fused batch assembly preserves cross-batch frame
/// order while collapsing multiple per-transaction commit markers into a
/// single trailing commit frame that carries the max db_size for the group.
#[test]
fn test_fused_batch_assembly_preserves_order_and_db_size() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
// Three batches simulating three concurrent transactions.
let batches = vec![
TransactionFrameBatch::new(vec![
FrameSubmission {
page_number: 2,
page_data: vec![0xAA; 4096],
db_size_if_commit: 0, // non-commit frame
},
FrameSubmission {
page_number: 3,
page_data: vec![0xBB; 4096],
db_size_if_commit: 10, // commit: db has 10 pages
},
]),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 5,
page_data: vec![0xCC; 4096],
db_size_if_commit: 12, // commit: db has 12 pages
}]),
TransactionFrameBatch::new(vec![
FrameSubmission {
page_number: 7,
page_data: vec![0xDD; 4096],
db_size_if_commit: 0, // non-commit
},
FrameSubmission {
page_number: 8,
page_data: vec![0xEE; 4096],
db_size_if_commit: 8, // commit: db has 8 pages (smaller)
},
]),
];
let current_db_size: u32 = 5;
let (frame_refs, final_db_size) = flatten_group_commit_batches(current_db_size, &batches);
// 1. Frame order: batch-by-batch, frame-by-frame.
let page_numbers: Vec<u32> = frame_refs.iter().map(|f| f.page_number).collect();
assert_eq!(
page_numbers,
vec![2, 3, 5, 7, 8],
"bd-db300.3.8.6: fused assembly must preserve cross-batch frame order"
);
// 2. Total frame count.
assert_eq!(frame_refs.len(), 5, "should flatten all 5 frames");
// 3. final_db_size = max(current_db_size, max positive db_size_if_commit).
// max(5, 0, 10, 12, 0, 8) = 12
assert_eq!(
final_db_size, 12,
"bd-db300.3.8.6: final_db_size must be max commit size across group"
);
// 4. Group commit must publish exactly one trailing commit marker so the
// WAL-visible db_size cannot regress to a smaller earlier transaction.
let commit_sizes: Vec<u32> = frame_refs
.iter()
.map(|frame| frame.db_size_if_commit)
.collect();
assert_eq!(
commit_sizes,
vec![0, 0, 0, 0, 12],
"only the final frame should carry the consolidated commit db_size"
);
// 5. Pre-sized capacity: no reallocation should have occurred.
assert!(
frame_refs.capacity() >= 5,
"Vec should have been pre-sized to avoid realloc"
);
}
#[test]
fn test_group_commit_page_one_headers_promote_to_consolidated_db_size() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
fn page_one_with_page_count(fill: u8, page_count: u32) -> Vec<u8> {
let mut page = vec![fill; 4096];
page[28..32].copy_from_slice(&page_count.to_be_bytes());
page
}
fn page_count(page: &[u8]) -> u32 {
let mut bytes = [0_u8; 4];
bytes.copy_from_slice(&page[28..32]);
u32::from_be_bytes(bytes)
}
let mut batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: page_one_with_page_count(0xAA, 12),
db_size_if_commit: 12,
}]),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: page_one_with_page_count(0xBA, 8),
db_size_if_commit: 8,
}]),
];
let final_db_size = group_commit_final_db_size(5, &batches);
assert_eq!(
final_db_size, 12,
"group final db_size should be the max commit marker"
);
assert!(
promote_group_commit_page_one_headers(&mut batches, final_db_size),
"a smaller trailing Page 1 header must be promoted before WAL append"
);
assert_eq!(page_count(&batches[0].frames[0].page_data), 12);
assert_eq!(page_count(&batches[1].frames[0].page_data), 12);
let (frame_refs, flattened_db_size) = flatten_group_commit_batches(5, &batches);
assert_eq!(flattened_db_size, 12);
assert_eq!(
frame_refs
.iter()
.map(|frame| frame.db_size_if_commit)
.collect::<Vec<_>>(),
vec![0, 12],
"the final WAL commit marker and promoted Page 1 header must agree on group db_size"
);
}
#[test]
fn test_group_commit_page_one_header_promotion_never_shrinks_existing_count() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
let mut page = vec![0xCC; 4096];
page[28..32].copy_from_slice(&20_u32.to_be_bytes());
let mut batches = vec![TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: page,
db_size_if_commit: 12,
}])];
assert!(
!promote_group_commit_page_one_headers(&mut batches, 12),
"Page 1 promotion should not shrink a header that already advertises a larger database"
);
let mut bytes = [0_u8; 4];
bytes.copy_from_slice(&batches[0].frames[0].page_data[28..32]);
assert_eq!(u32::from_be_bytes(bytes), 20);
}
#[test]
fn test_group_commit_conflict_detection_reports_only_cross_batch_page_overlaps() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
let batches = vec![
TransactionFrameBatch::new(vec![
FrameSubmission {
page_number: 1,
page_data: vec![0xA0; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 2,
page_data: vec![0xAA; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 2,
page_data: vec![0xAB; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 3,
page_data: vec![0xAC; 4096],
db_size_if_commit: 10,
},
]),
TransactionFrameBatch::new(vec![
FrameSubmission {
page_number: 1,
page_data: vec![0xB0; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 4,
page_data: vec![0xBA; 4096],
db_size_if_commit: 11,
},
]),
TransactionFrameBatch::new(vec![
FrameSubmission {
page_number: 1,
page_data: vec![0xC0; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 3,
page_data: vec![0xCA; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 4,
page_data: vec![0xCB; 4096],
db_size_if_commit: 12,
},
]),
];
assert_eq!(
conflicting_pages_across_group_commit_batches(&batches),
vec![3, 4],
"only pages written by multiple distinct transaction batches should force an epoch retry"
);
}
#[test]
fn test_group_commit_conflict_detection_uses_semantic_conflict_metadata() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
let batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 2,
page_data: vec![0xAA; 4096],
db_size_if_commit: 10,
}])
.with_conflict_snapshot(vec![2, 50, 60], None),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 3,
page_data: vec![0xBA; 4096],
db_size_if_commit: 11,
}])
.with_conflict_snapshot(vec![50], None),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 60,
page_data: vec![0xCA; 4096],
db_size_if_commit: 12,
}]),
];
assert_eq!(
conflicting_pages_across_group_commit_batches(&batches),
vec![50, 60],
"group-commit overlap checks must include conflict-only pages such as pending freed pages, not just emitted WAL frames"
);
}
#[test]
fn test_group_commit_conflict_detection_keeps_synthetic_page_one_batching_safe() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
let synthetic_page_one_batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: vec![0xAA; 4096],
db_size_if_commit: 10,
}]),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: vec![0xBA; 4096],
db_size_if_commit: 11,
}]),
];
assert_eq!(
conflicting_pages_across_group_commit_batches(&synthetic_page_one_batches),
Vec::<u32>::new(),
"synthetic Page 1 header/count frames remain batchable when no transaction explicitly rewrote Page 1"
);
let explicit_page_one_with_synthetic = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: vec![0xCA; 4096],
db_size_if_commit: 10,
}])
.with_conflict_snapshot(vec![1], None),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: 1,
page_data: vec![0xDA; 4096],
db_size_if_commit: 11,
}]),
];
assert_eq!(
conflicting_pages_across_group_commit_batches(&explicit_page_one_with_synthetic),
vec![1],
"an explicit Page 1 rewrite must not batch with another transaction's synthetic Page 1 frame because the synthetic bytes could otherwise overwrite schema/header changes"
);
}
/// bd-db300.3.8.6: Edge case — all db_size_if_commit are zero (no commits
/// in the batch, only non-commit frames). final_db_size must fall back to
/// current_db_size.
#[test]
fn test_fused_batch_assembly_all_zero_db_size() {
use fsqlite_wal::group_commit::{FrameSubmission, TransactionFrameBatch};
let batches = vec![TransactionFrameBatch::new(vec![
FrameSubmission {
page_number: 2,
page_data: vec![0; 4096],
db_size_if_commit: 0,
},
FrameSubmission {
page_number: 3,
page_data: vec![0; 4096],
db_size_if_commit: 0,
},
])];
let current_db_size: u32 = 7;
let (frame_refs, final_db_size) = flatten_group_commit_batches(current_db_size, &batches);
assert_eq!(
final_db_size, 7,
"bd-db300.3.8.6: all-zero db_size_if_commit must preserve current_db_size"
);
assert!(
frame_refs.iter().all(|frame| frame.db_size_if_commit == 0),
"all-zero inputs must stay non-commit after flattening"
);
}
/// Repeated writes to the same page within a single transaction must
/// reuse the existing `StagedPage` buffer in place instead of allocating
/// a fresh `PageBuf` from the pool and dropping the old one.
///
/// This test exercises both the allocation-count path (pool metrics) and
/// the custom steal counter to ensure the fast path is actually taken.
#[test]
fn test_same_page_write_steals_existing_buffer() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let page_size = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
// First write into an empty slot — always allocates a fresh buffer.
let initial = vec![0x11_u8; page_size];
txn.write_page(&cx, page_no, &initial).await.unwrap();
// Capture baselines AFTER the first write so we isolate the repeated
// same-page writes. Per-pool metrics are isolated from other tests
// running in parallel; the global steal counter is not, so we only
// assert a lower bound on its delta.
let pool_before = txn.pool.metrics_snapshot();
let steals_before = staged_page_overwrite_steals_total();
// Drive many same-page rewrites. Each one should go through the
// steal fast path.
let iterations = 10_000_u64;
for i in 0..iterations {
let mut data = vec![0_u8; page_size];
data[0] = u8::try_from(i & 0xFF).expect("mask fits u8");
data[1] = u8::try_from((i >> 8) & 0xFF).expect("mask fits u8");
txn.write_page(&cx, page_no, &data).await.unwrap();
}
let pool_after = txn.pool.metrics_snapshot();
let steals_after = staged_page_overwrite_steals_total();
assert!(
steals_after.saturating_sub(steals_before) >= iterations,
"bead_id=bd-steal-same-page case=all_repeated_writes_stole \
steals_before={steals_before} steals_after={steals_after} \
iterations={iterations}"
);
assert_eq!(
pool_after.page_buffer_pool_misses, pool_before.page_buffer_pool_misses,
"bead_id=bd-steal-same-page case=no_new_pool_misses \
before={} after={}",
pool_before.page_buffer_pool_misses, pool_after.page_buffer_pool_misses
);
assert_eq!(
pool_after.page_buffer_pool_hits, pool_before.page_buffer_pool_hits,
"bead_id=bd-steal-same-page case=no_new_pool_hits \
before={} after={}",
pool_before.page_buffer_pool_hits, pool_after.page_buffer_pool_hits
);
// Final read must observe the last write.
let final_mask = (iterations - 1) & 0xFF;
let read_back = txn.get_page(&cx, page_no).await.unwrap();
assert_eq!(
u64::from(read_back.as_ref()[0]),
final_mask,
"bead_id=bd-steal-same-page case=last_write_wins_byte0"
);
});
}
/// Once the staged page has been published as a shared snapshot (via
/// `StagedPage::published`), subsequent writes MUST fall back to the
/// allocate-new-buffer path to preserve the shared view. This guards
/// against the fast path silently mutating bytes an MVCC reader already
/// sees.
#[test]
fn test_same_page_write_after_publish_does_not_steal() {
asupersync::test_utils::run_test(|| async {
let (pager, _) = test_pager().await;
let cx = Cx::new();
let page_size = PageSize::DEFAULT.as_usize();
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
let initial = vec![0xA5_u8; page_size];
txn.write_page(&cx, page_no, &initial).await.unwrap();
// Force a shared-snapshot publication via a read. `get_page` returns
// a `PageData` clone whose existence populates the `published`
// OnceLock on the underlying `StagedPage`, forbidding in-place
// overwrite afterward.
let snapshot = txn.get_page(&cx, page_no).await.unwrap();
assert_eq!(
snapshot.as_ref()[0],
0xA5,
"bead_id=bd-steal-same-page case=pre_publish_read"
);
// After a publish, another in-place steal on this specific page
// must not happen, so the per-pool allocator must take a new hit or
// miss. Other tests may still be running and incrementing the
// global steal counter concurrently, so the single-write delta
// cannot be asserted against the global. Instead, verify via the
// per-pool metrics (isolated to this pager) that a fresh buffer
// was acquired for the post-publish write.
let pool_before = txn.pool.metrics_snapshot();
let updated = vec![0x5A_u8; page_size];
txn.write_page(&cx, page_no, &updated).await.unwrap();
let pool_after = txn.pool.metrics_snapshot();
let new_hits = pool_after.page_buffer_pool_hits - pool_before.page_buffer_pool_hits;
let new_misses =
pool_after.page_buffer_pool_misses - pool_before.page_buffer_pool_misses;
assert!(
new_hits + new_misses >= 1,
"bead_id=bd-steal-same-page case=published_forbids_steal \
new_hits={new_hits} new_misses={new_misses}"
);
// Post-write state must reflect the NEW bytes but the previously
// captured snapshot MUST still see the old ones.
let fresh = txn.get_page(&cx, page_no).await.unwrap();
assert_eq!(
fresh.as_ref()[0],
0x5A,
"bead_id=bd-steal-same-page case=post_write_reads_new"
);
assert_eq!(
snapshot.as_ref()[0],
0xA5,
"bead_id=bd-steal-same-page case=captured_snapshot_unchanged"
);
});
}
// =========================================================================
// bd-3wop3.8 acceptance tests: Split inner lock + group commit + shard publish
// =========================================================================
#[test]
fn test_wal_append_does_not_hold_pager_inner() {
asupersync::test_utils::run_test(|| async {
const BEAD: &str = "bd-3wop3.8";
struct InnerLockProbeWalBackend {
inner: Arc<Mutex<PagerInner<MemoryFile>>>,
append_probes: Arc<AtomicUsize>,
all_append_probes_unlocked: Arc<AtomicBool>,
total_frames: Arc<AtomicUsize>,
}
impl InnerLockProbeWalBackend {
fn record_append_probe(&self) {
self.append_probes.fetch_add(1, AtomicOrdering::AcqRel);
match self.inner.try_lock() {
Ok(guard) => drop(guard),
Err(_) => self
.all_append_probes_unlocked
.store(false, AtomicOrdering::Release),
}
}
}
impl crate::traits::WalBackend for InnerLockProbeWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.record_append_probe();
self.total_frames.fetch_add(1, AtomicOrdering::AcqRel);
Ok(())
})
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.record_append_probe();
self.total_frames
.fetch_add(frames.len(), AtomicOrdering::AcqRel);
Ok(())
})
}
fn prepare_append_frames(
&self,
_frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>>
{
// Force the raw physical append callback, which is the
// deterministic observation point for the Phase-A mutex.
Ok(None)
}
fn append_prepared_frames<'a>(
&'a mut self,
_cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
self.record_append_probe();
self.total_frames
.fetch_add(prepared.frame_count(), AtomicOrdering::AcqRel);
Ok(())
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
self.total_frames.load(AtomicOrdering::Acquire)
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let vfs = MemoryVfs::new();
let path = PathBuf::from("/split_lock_parallel_prepare.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let append_probes = Arc::new(AtomicUsize::new(0));
let all_append_probes_unlocked = Arc::new(AtomicBool::new(true));
let backend = InnerLockProbeWalBackend {
inner: Arc::clone(&pager.inner),
append_probes: Arc::clone(&append_probes),
all_append_probes_unlocked: Arc::clone(&all_append_probes_unlocked),
total_frames: Arc::new(AtomicUsize::new(0)),
};
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
// Exercise the real transaction commit path. The backend callback
// runs at physical WAL append, after production Phase A must have
// released PagerInner and before Phase C reacquires it.
let mut txn = pager.begin(&cx, TransactionMode::Immediate).await.unwrap();
let page_no = txn.allocate_page(&cx).await.unwrap();
txn.write_page(&cx, page_no, &sample_page(0xA7))
.await
.unwrap();
let receipt = Arc::new(CommitFastPathLockReceipt::default());
let _receipt_scope = CommitFastPathLockReceipt::scope(&receipt);
txn.commit(&cx).await.unwrap();
assert!(
append_probes.load(AtomicOrdering::Acquire) > 0,
"bead_id={BEAD} case=phase_b_append_probe_non_vacuous"
);
assert!(
all_append_probes_unlocked.load(AtomicOrdering::Acquire),
"bead_id={BEAD} case=phase_b_append_releases_phase_a_mutex"
);
assert_eq!(
receipt.count(CommitFastPathLockClass::ProcessGlobalRegistry),
0,
"bead_id={BEAD} case=normal_wal_commit_avoids_process_global_registry"
);
for lock_class in [
CommitFastPathLockClass::PagerInner,
CommitFastPathLockClass::QueueConsolidator,
CommitFastPathLockClass::QueueEpochState,
CommitFastPathLockClass::ExactHandleCoordination,
CommitFastPathLockClass::WalBackendSlot,
CommitFastPathLockClass::WalBackendRead,
CommitFastPathLockClass::WalBackendWrite,
CommitFastPathLockClass::PublishedPagerState,
] {
assert!(
receipt.count(lock_class) > 0,
"bead_id={BEAD} case=expected_local_wal_commit_lock_not_observed class={lock_class:?}"
);
}
});
}
#[test]
fn test_group_commit_batches_frames_fewer_io_ops_than_commits() {
asupersync::test_utils::run_test(|| async {
const BEAD: &str = "bd-3wop3.8";
struct CountingWalBackend {
append_calls: SharedCounter,
total_frames: SharedCounter,
}
impl crate::traits::WalBackend for CountingWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.append_calls.lock().unwrap() += 1;
*self.total_frames.lock().unwrap() += frames.len();
Ok(())
})
}
fn prepare_append_frames(
&self,
frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>>
{
if frames.is_empty() {
return Ok(None);
}
let frame_size =
fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE + frames[0].page_data.len();
let mut frame_bytes = Vec::with_capacity(frame_size * frames.len());
let mut frame_metas = Vec::with_capacity(frames.len());
let mut checksum_transforms = Vec::with_capacity(frames.len());
let mut last_commit: Option<usize> = None;
for (i, frame) in frames.iter().enumerate() {
frame_metas.push(crate::traits::PreparedWalFrameMeta {
page_number: frame.page_number,
db_size_if_commit: frame.db_size_if_commit,
});
checksum_transforms.push(crate::traits::PreparedWalChecksumTransform {
a11: 0,
a12: 0,
a21: 0,
a22: 0,
c1: 0,
c2: 0,
});
let mut header = [0_u8; fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE];
header[0..4].copy_from_slice(&frame.page_number.to_be_bytes());
header[4..8].copy_from_slice(&frame.db_size_if_commit.to_be_bytes());
frame_bytes.extend_from_slice(&header);
frame_bytes.extend_from_slice(frame.page_data);
if frame.db_size_if_commit != 0 {
last_commit = Some(i);
}
}
Ok(Some(crate::traits::PreparedWalFrameBatch {
frame_size,
page_data_offset: fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE,
big_endian_checksum: false,
frame_metas,
checksum_transforms,
frame_bytes,
last_commit_frame_offset: last_commit,
finalized_for: None,
finalized_running_checksum: None,
}))
}
fn append_prepared_frames<'a>(
&'a mut self,
_cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
*self.append_calls.lock().unwrap() += 1;
*self.total_frames.lock().unwrap() += prepared.frame_count();
Ok(())
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
*self.total_frames.lock().unwrap()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let vfs = MemoryVfs::new();
let path = PathBuf::from("/group_commit_batches_frames.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let append_calls: SharedCounter = StdArc::new(StdMutex::new(0));
let total_frames: SharedCounter = StdArc::new(StdMutex::new(0));
let backend = CountingWalBackend {
append_calls: StdArc::clone(&append_calls),
total_frames: StdArc::clone(&total_frames),
};
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface::default(),
));
let pool = pager.pool.clone();
const WORKERS: u32 = 8;
const ITERS: u32 = 20;
let barrier = StdArc::new(std::sync::Barrier::new(WORKERS as usize));
let mut handles = Vec::with_capacity(WORKERS as usize);
for worker_id in 0..WORKERS {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
let barrier = StdArc::clone(&barrier);
let handle = std::thread::spawn(move || {
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
barrier.wait();
for iter in 0..ITERS {
let page_no = PageNumber::new(2 + worker_id * ITERS + iter).unwrap();
let mut write_set = HashMap::new();
write_set.insert(
page_no,
StagedPage::from_bytes(&pool, &sample_page(worker_id as u8))
.unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_no],
&[],
&queue,
)
.await
.expect("group commit succeeded");
}
});
});
handles.push(handle);
}
for handle in handles {
handle.join().expect("worker joined");
}
let total_commits = (WORKERS * ITERS) as usize;
let io_ops = *append_calls.lock().unwrap();
let frames_written = *total_frames.lock().unwrap();
assert_eq!(
frames_written, total_commits,
"bead_id={BEAD} case=batched_frame_count — every commit frame must be written"
);
assert!(
io_ops < total_commits,
"bead_id={BEAD} case=batched_io_ops \
io_ops={io_ops} total_commits={total_commits} — \
group commit should batch multiple commits into fewer I/O calls"
);
eprintln!(
"INFO bead_id={BEAD} case=batched_io_ops \
io_ops={io_ops} total_commits={total_commits} \
batch_ratio={:.2}x",
total_commits as f64 / io_ops as f64
);
});
}
#[test]
fn test_sharded_publish_no_reader_blocking() {
const BEAD: &str = "bd-3wop3.8";
let pages = PublishedPages::new(0);
let page_data = |seed: u8| -> PageData {
let buf = vec![seed; PageSize::DEFAULT.as_usize()];
PageData::from_vec(buf)
};
// Insert pages across the full range — low pages go to atomic slots,
// high pages go to sharded overflow.
for i in 1..=128_u32 {
let pn = PageNumber::new(i).unwrap();
pages.insert(pn, page_data(i as u8));
}
// Concurrent reads and writes on disjoint page ranges must not block.
let pages = StdArc::new(pages);
let barrier = StdArc::new(std::sync::Barrier::new(3));
let reader_done = StdArc::new(AtomicBool::new(false));
let writer_done = StdArc::new(AtomicBool::new(false));
let pages_r = StdArc::clone(&pages);
let barrier_r = StdArc::clone(&barrier);
let reader_done_w = StdArc::clone(&reader_done);
let reader = std::thread::spawn(move || {
barrier_r.wait();
let mut reads = 0_u64;
for _ in 0..1000 {
for i in 1..=64_u32 {
let pn = PageNumber::new(i).unwrap();
if pages_r.get(pn).is_some() {
reads += 1;
}
}
}
reader_done_w.store(true, AtomicOrdering::Release);
reads
});
let pages_w = StdArc::clone(&pages);
let barrier_w = StdArc::clone(&barrier);
let writer_done_w = StdArc::clone(&writer_done);
let writer = std::thread::spawn(move || {
barrier_w.wait();
let mut writes = 0_u64;
for round in 0_u8..200 {
for i in 65..=128_u32 {
let pn = PageNumber::new(i).unwrap();
pages_w.insert(pn, page_data(round.wrapping_add(i as u8)));
writes += 1;
}
}
writer_done_w.store(true, AtomicOrdering::Release);
writes
});
barrier.wait();
let reads = reader.join().expect("reader joined");
let writes = writer.join().expect("writer joined");
assert!(
reads > 0 && writes > 0,
"bead_id={BEAD} case=sharded_publish_concurrent reads={reads} writes={writes}"
);
}
#[test]
fn test_no_data_loss_under_batched_commit() {
asupersync::test_utils::run_test(|| async {
const BEAD: &str = "bd-3wop3.8";
struct RecordingWalBackend {
committed_pages: StdArc<StdMutex<HashMap<u32, Vec<u8>>>>,
}
impl crate::traits::WalBackend for RecordingWalBackend {
fn append_frame<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
_page_data: &'a [u8],
_db_size_if_commit: u32,
) -> WalFuture<'a, ()> {
Box::pin(async { Ok(()) })
}
fn append_frames<'a>(
&'a mut self,
_cx: &'a Cx,
frames: &'a [crate::traits::WalFrameRef<'a>],
) -> WalFuture<'a, ()> {
Box::pin(async move {
let mut committed = self.committed_pages.lock().unwrap();
for frame in frames {
committed.insert(frame.page_number, frame.page_data.to_vec());
}
Ok(())
})
}
fn prepare_append_frames(
&self,
frames: &[crate::traits::WalFrameRef<'_>],
) -> fsqlite_error::Result<Option<crate::traits::PreparedWalFrameBatch>>
{
if frames.is_empty() {
return Ok(None);
}
let frame_size =
fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE + frames[0].page_data.len();
let mut frame_bytes = Vec::with_capacity(frame_size * frames.len());
let mut frame_metas = Vec::with_capacity(frames.len());
let mut checksum_transforms = Vec::with_capacity(frames.len());
let mut last_commit: Option<usize> = None;
for (i, frame) in frames.iter().enumerate() {
frame_metas.push(crate::traits::PreparedWalFrameMeta {
page_number: frame.page_number,
db_size_if_commit: frame.db_size_if_commit,
});
checksum_transforms.push(crate::traits::PreparedWalChecksumTransform {
a11: 0,
a12: 0,
a21: 0,
a22: 0,
c1: 0,
c2: 0,
});
let mut header = [0_u8; fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE];
header[0..4].copy_from_slice(&frame.page_number.to_be_bytes());
header[4..8].copy_from_slice(&frame.db_size_if_commit.to_be_bytes());
frame_bytes.extend_from_slice(&header);
frame_bytes.extend_from_slice(frame.page_data);
if frame.db_size_if_commit != 0 {
last_commit = Some(i);
}
}
Ok(Some(crate::traits::PreparedWalFrameBatch {
frame_size,
page_data_offset: fsqlite_wal::checksum::WAL_FRAME_HEADER_SIZE,
big_endian_checksum: false,
frame_metas,
checksum_transforms,
frame_bytes,
last_commit_frame_offset: last_commit,
finalized_for: None,
finalized_running_checksum: None,
}))
}
fn append_prepared_frames<'a>(
&'a mut self,
_cx: &'a Cx,
prepared: &'a mut crate::traits::PreparedWalFrameBatch,
) -> WalFuture<'a, ()> {
Box::pin(async move {
let mut committed = self.committed_pages.lock().unwrap();
for i in 0..prepared.frame_count() {
let meta = &prepared.frame_metas[i];
let data = prepared.page_data(i).to_vec();
committed.insert(meta.page_number, data);
}
Ok(())
})
}
fn read_page<'a>(
&'a mut self,
_cx: &'a Cx,
_page_number: u32,
) -> WalFuture<'a, Option<Vec<u8>>> {
Box::pin(async { Ok(None) })
}
fn sync(&mut self, _cx: &Cx) -> fsqlite_error::Result<()> {
Ok(())
}
fn frame_count(&self) -> usize {
self.committed_pages.lock().unwrap().len()
}
fn checkpoint<'a>(
&'a mut self,
_cx: &'a Cx,
mode: crate::traits::CheckpointMode,
_writer: &'a mut dyn crate::traits::CheckpointPageWriter,
_backfilled_frames: u32,
_oldest_reader_frame: Option<u32>,
) -> WalFuture<'a, crate::traits::CheckpointResult> {
Box::pin(async move {
Ok(crate::traits::CheckpointResult {
total_frames: 0,
frames_backfilled: 0,
completed: true,
wal_was_reset: false,
requested_mode: mode,
effective_mode: mode,
})
})
}
}
let _guard = PARALLEL_WAL_LANE_TEST_LOCK
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let vfs = MemoryVfs::new();
let path = PathBuf::from("/no_data_loss_batched.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
let cx = Cx::new();
let committed_pages: StdArc<StdMutex<HashMap<u32, Vec<u8>>>> =
StdArc::new(StdMutex::new(HashMap::new()));
let backend = RecordingWalBackend {
committed_pages: StdArc::clone(&committed_pages),
};
pager.set_wal_backend(Box::new(backend)).unwrap();
pager.set_journal_mode(&cx, JournalMode::Wal).await.unwrap();
let inner = Arc::clone(&pager.inner);
let wal_backend = Arc::clone(&pager.wal_backend);
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface::default(),
));
let pool = pager.pool.clone();
const WORKERS: u32 = 8;
let barrier = StdArc::new(std::sync::Barrier::new(WORKERS as usize));
let mut handles = Vec::with_capacity(WORKERS as usize);
for worker_id in 0..WORKERS {
let inner = Arc::clone(&inner);
let wal_backend = Arc::clone(&wal_backend);
let queue = Arc::clone(&queue);
let pool = pool.clone();
let barrier = StdArc::clone(&barrier);
let handle = std::thread::spawn(move || {
let mut outcome = None;
asupersync::test_utils::run_test(|| async {
let cx = Cx::new();
barrier.wait();
let page_no = PageNumber::new(2 + worker_id).unwrap();
let fill = worker_id as u8;
let mut write_set = HashMap::new();
write_set.insert(
page_no,
StagedPage::from_bytes(&pool, &sample_page(fill)).unwrap(),
);
SimpleTransaction::<MemoryVfs>::commit_wal_group_commit(
&cx,
&wal_backend,
&inner,
&write_set,
&[page_no],
&[],
&queue,
)
.await
.expect("commit succeeded");
outcome = Some((page_no.get(), fill));
});
outcome.expect("worker must record its committed page")
});
handles.push(handle);
}
let expected: Vec<(u32, u8)> = handles
.into_iter()
.map(|h| h.join().expect("worker joined"))
.collect();
let committed = committed_pages.lock().unwrap();
for (page_no, fill) in &expected {
let data = committed.get(page_no).unwrap_or_else(|| {
panic!(
"bead_id={BEAD} case=data_loss page_no={page_no} — \
page missing from WAL after batched commit"
)
});
assert_eq!(
data[0], *fill,
"bead_id={BEAD} case=data_integrity page_no={page_no} — \
page content mismatch"
);
}
assert!(
committed.len() >= expected.len(),
"bead_id={BEAD} case=committed_count \
committed={} expected={}",
committed.len(),
expected.len()
);
});
}
#[test]
fn test_combiner_wired_in_production() {
use fsqlite_mvcc::TxnManager;
const BEAD: &str = "bd-3wop3.8";
const THREADS: usize = 8;
const ALLOCS_PER_THREAD: usize = 50;
let mgr = TxnManager::new(1, 1);
let mgr = StdArc::new(mgr);
let barrier = StdArc::new(std::sync::Barrier::new(THREADS));
let mut handles = Vec::with_capacity(THREADS);
for _ in 0..THREADS {
let mgr = StdArc::clone(&mgr);
let barrier = StdArc::clone(&barrier);
let handle = std::thread::spawn(move || {
barrier.wait();
let mut seqs = Vec::with_capacity(ALLOCS_PER_THREAD);
for _ in 0..ALLOCS_PER_THREAD {
seqs.push(mgr.alloc_commit_seq().get());
}
seqs
});
handles.push(handle);
}
let mut all_seqs: Vec<u64> = Vec::with_capacity(THREADS * ALLOCS_PER_THREAD);
for handle in handles {
all_seqs.extend(handle.join().expect("thread joined"));
}
// Uniqueness: every allocated sequence must be distinct.
all_seqs.sort_unstable();
for window in all_seqs.windows(2) {
assert_ne!(
window[0], window[1],
"bead_id={BEAD} case=combiner_uniqueness — \
duplicate commit seq {}",
window[0]
);
}
// Contiguity: sequences form a dense range with no gaps.
let min = all_seqs[0];
let max = *all_seqs.last().unwrap();
let expected_count = (THREADS * ALLOCS_PER_THREAD) as u64;
assert_eq!(
max - min + 1,
expected_count,
"bead_id={BEAD} case=combiner_contiguous \
min={min} max={max} count={expected_count} — \
sequences must form a dense range"
);
}
#[test]
fn test_group_commit_epoch_maps_bounded_under_sustained_load() {
const BEAD: &str = "bd-vn2ea";
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface::default(),
));
let total_epochs: u64 = 500;
for epoch in 1..=total_epochs {
let error = FrankenError::Busy;
queue.publish_failed_epoch(epoch, &error, false);
queue.publish_completed_epoch(epoch, false);
}
let failed_len = queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len();
let persisted_len = queue
.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len();
let consumer_len = queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.len();
assert_eq!(
(failed_len, persisted_len, consumer_len),
(0, 0, 0),
"bead_id={BEAD} case=unowned_epoch_evidence_reclaimed"
);
eprintln!(
"INFO bead_id={BEAD} case=owner_safe_epoch_gc \
failed_epochs_len={failed_len} persisted_epochs_len={persisted_len} \
consumer_epochs_len={consumer_len} total_epochs={total_epochs}"
);
}
#[test]
fn test_group_commit_epoch_evidence_survives_more_than_128_later_epochs() {
const BEAD: &str = "bd-vn2ea";
let queue = Arc::new(GroupCommitQueue::with_parallel_wal_control(
GroupCommitConfig::default(),
ParallelWalControlSurface::default(),
));
let persisted_owner_a = queue.register_epoch_consumer(1);
let persisted_owner_b = queue.register_epoch_consumer(1);
let failed_owner = queue.register_epoch_consumer(2);
let authorization = publication_authorization_for_test(false);
queue
.persisted_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.insert(
1,
PersistedGroupCommitEpoch {
members: HashSet::from([authorization.batch_id]),
frames_start: 1,
frames_end: 1,
fsync_seq: 1,
durability_receipt: authorization.durability_receipt,
},
);
queue.publish_completed_epoch(1, false);
queue.publish_failed_epoch(
2,
&FrankenError::internal("owner-retained group commit failure"),
false,
);
for epoch in 3..=260 {
queue.publish_completed_epoch(epoch, false);
}
assert!(
queue.persisted_epoch_for(1).is_some(),
"bead_id={BEAD} case=persisted_evidence_outlives_128_later_epochs"
);
let guard = queue
.consolidator
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner);
let error = queue.wait_for_epoch_outcome(guard, 2).unwrap_err();
assert!(
error
.to_string()
.contains("owner-retained group commit failure"),
"bead_id={BEAD} case=failed_evidence_outlives_128_later_epochs error={error}"
);
drop(persisted_owner_a);
assert!(
queue.persisted_epoch_for(1).is_some(),
"bead_id={BEAD} case=first_owner_cannot_reclaim_shared_evidence"
);
drop(persisted_owner_b);
assert!(
queue.persisted_epoch_for(1).is_none(),
"bead_id={BEAD} case=final_persisted_owner_reclaims"
);
drop(failed_owner);
assert!(
!queue
.failed_epochs
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.contains_key(&2),
"bead_id={BEAD} case=final_failed_owner_reclaims"
);
assert!(
queue
.epoch_consumer_counts
.lock()
.unwrap_or_else(std::sync::PoisonError::into_inner)
.is_empty(),
"bead_id={BEAD} case=all_epoch_consumer_counts_released"
);
}
#[test]
#[ignore = "release blocker: current-source 16-thread cross-engine evidence must come from the e2e benchmark gate"]
fn test_16t_throughput_exceeds_sqlite() {
// Acceptance test bd-3wop3.8 #4: Verify that 16-thread DML
// throughput on FrankenSQLite exceeds C SQLite (via rusqlite).
//
// This test is intentionally #[ignore] because it requires:
// 1. A real file-backed database (not :memory:)
// 2. rusqlite for reference timing
// 3. Multiple iterations for statistical significance
// 4. The release-perf profile for meaningful numbers
//
// The exact ignored invocation is deliberately fail-closed. It must
// not turn the absence of the external evidence into a passing test.
//
// The e2e benchmark harness (fsqlite-e2e) provides the canonical
// evidence for this acceptance criterion.
panic!(
"bd-3wop3.8: pager-local code cannot prove the 16-thread cross-engine throughput gate; require current-source and running-binary-bound e2e evidence with correctness oracles, a declared statistical threshold, and independent artifact verification"
);
}
#[test]
fn test_simple_pager_open_defaults_to_s3_fifo_eviction() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/s3fifo_default_rw.db");
let pager = SimplePager::open(vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
assert!(
matches!(
pager.cache.eviction_policy(),
crate::page_cache::PageCacheEvictionPolicy::S3Fifo(_)
),
"read-write pager must default to S3-FIFO eviction"
);
});
}
#[test]
fn test_simple_pager_open_readonly_defaults_to_s3_fifo_eviction() {
asupersync::test_utils::run_test(|| async {
let vfs = MemoryVfs::new();
let path = PathBuf::from("/s3fifo_default_ro.db");
{
let _rw = SimplePager::open(vfs.clone(), &path, PageSize::DEFAULT)
.await
.unwrap();
}
let cx = Cx::new();
let pager = SimplePager::open_readonly_with_cx(&cx, vfs, &path, PageSize::DEFAULT)
.await
.unwrap();
assert!(
matches!(
pager.cache.eviction_policy(),
crate::page_cache::PageCacheEvictionPolicy::S3Fifo(_)
),
"read-only pager must default to S3-FIFO eviction"
);
});
}
#[test]
fn pager_commit_profile_snapshot_default_and_debug() {
let def = PagerCommitProfileSnapshot::default();
assert_eq!(def.commit_calls, 0);
assert_eq!(def.phase_a_time_ns, 0);
assert_eq!(def.wal_commit_time_ns, 0);
let copied = def;
assert_eq!(copied, def);
let dbg = format!("{def:?}");
assert!(dbg.contains("PagerCommitProfileSnapshot"));
}
#[test]
fn wal_commit_sync_policy_variants_eq() {
let deferred = WalCommitSyncPolicy::Deferred;
let per_commit = WalCommitSyncPolicy::PerCommit;
assert_ne!(deferred, per_commit);
assert!(!deferred.should_sync_on_commit());
assert!(per_commit.should_sync_on_commit());
let copied = deferred;
assert_eq!(copied, deferred);
let dbg = format!("{per_commit:?}");
assert!(dbg.contains("PerCommit"));
}
#[test]
fn pager_metadata_publication_class_all_variants() {
let variants = [
PagerMetadataPublicationClass::SnapshotSummary,
PagerMetadataPublicationClass::PagePlaneResidency,
PagerMetadataPublicationClass::CertificateDerivedIntent,
];
for (i, v) in variants.iter().enumerate() {
let copied = *v;
assert_eq!(copied, *v);
for (j, w) in variants.iter().enumerate() {
assert_eq!(i == j, v == w);
}
}
assert_eq!(PAGER_METADATA_PUBLICATION_CONTRACTS.len(), 3);
assert_eq!(
PAGER_METADATA_PUBLICATION_CONTRACTS[0].class,
PagerMetadataPublicationClass::SnapshotSummary
);
}
#[test]
fn pager_published_snapshot_debug_clone_copy_eq() {
let snap = PagerPublishedSnapshot {
snapshot_gen: 4,
visible_commit_seq: CommitSeq::new(10),
db_size: 100,
journal_mode: JournalMode::Wal,
freelist_count: 5,
checkpoint_active: false,
page_set_size: 42,
};
let copied = snap;
assert_eq!(copied, snap);
let other = PagerPublishedSnapshot {
db_size: 200,
..snap
};
assert_ne!(snap, other);
let dbg = format!("{snap:?}");
assert!(dbg.contains("PagerPublishedSnapshot"));
}
#[test]
fn pager_committed_snapshot_copy_eq_debug() {
let snap = PagerCommittedSnapshot {
commit_seq: CommitSeq::new(5),
db_size: 100,
journal_mode: JournalMode::Wal,
freelist_count: 3,
checkpoint_active: false,
writer_active: true,
db_file_size_bytes: 409_600,
};
let copied = snap;
assert_eq!(copied, snap);
let other = PagerCommittedSnapshot {
db_size: 200,
..snap
};
assert_ne!(snap, other);
let dbg = format!("{snap:?}");
assert!(dbg.contains("PagerCommittedSnapshot"));
}
#[test]
fn parallel_wal_publication_intent_copy_eq_debug() {
let intent = ParallelWalPublicationIntent {
certificate_epoch: 7,
visible_commit_seq: CommitSeq::new(10),
page_plane_visible_commit_seq: CommitSeq::new(9),
db_size: 50,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
page_set_size: 12,
};
let copied = intent;
assert_eq!(copied, intent);
let other = ParallelWalPublicationIntent {
certificate_epoch: 8,
..intent
};
assert_ne!(intent, other);
let dbg = format!("{intent:?}");
assert!(dbg.contains("ParallelWalPublicationIntent"));
}
fn publication_authorization_for_test(
checkpoint_active: bool,
) -> ParallelWalPublicationAuthorization {
let combiner = ParallelWalDurabilityCombiner::default();
let receipt = combiner
.certify_and_publish(
ParallelWalDurabilityRequest {
trace_id: 11,
scenario_id: PARALLEL_WAL_PUBLICATION_SCENARIO_ID.to_owned(),
certificate_epoch: 1,
durable_segment_epoch: 1,
batch_size: 2,
batch_ids: vec![41, 42],
lane_record_counts: vec![1, 1],
db_size_pages: 23,
page_set_size: 2,
control_mode: ParallelWalOperatingMode::Auto,
fallback_reason: None,
checkpoint_active,
wal_frame_payload_digest: [0xA5; 32],
},
|_| Ok(()),
)
.expect("test certificate should publish");
ParallelWalPublicationAuthorization {
assigned_commit_seq: receipt
.commit_seq_for_batch(41)
.expect("test batch should have a commit sequence"),
durability_receipt: receipt,
batch_id: 41,
}
}
#[test]
fn certificate_authorization_builds_bounded_publication_intent() {
let authorization = publication_authorization_for_test(false);
let intent =
parallel_wal_publication_intent(&authorization, 20, JournalMode::Wal, 3, false)
.expect("certificate-backed publication should validate");
assert_eq!(intent.certificate_epoch, 1);
assert_eq!(intent.visible_commit_seq, CommitSeq::new(2));
assert_eq!(intent.page_plane_visible_commit_seq, CommitSeq::new(2));
assert_eq!(intent.db_size, 23);
assert_eq!(intent.page_set_size, 2);
}
#[test]
fn parallel_wal_publication_gap_evicts_uncovered_resident_pages() {
let cx = Cx::new();
let published = PublishedPagerState::new(3, CommitSeq::new(1), JournalMode::Wal, 0);
let stale_page = PageNumber::new(2).unwrap();
let current_group_page = PageNumber::new(3).unwrap();
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(1),
db_size: 3,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
stale_page,
PageData::from_vec(sample_page(0x11)),
);
let current_page = PageData::from_vec(sample_page(0x33));
published.publish_prepared_parallel_wal_group(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(3),
db_size: 3,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
HashMap::from([(current_group_page, current_page.clone())]),
CommitSeq::new(3),
);
assert_eq!(published.try_get_page(stale_page), None);
assert_eq!(
published.try_get_page(current_group_page),
Some(current_page)
);
assert_eq!(published.page_plane_visible_commit_seq(), CommitSeq::new(3));
}
#[test]
fn published_page_fast_path_rejects_newer_metadata_snapshot() {
let cx = Cx::new();
let published = PublishedPagerState::new(3, CommitSeq::new(1), JournalMode::Wal, 0);
assert!(
published
.snapshot_for_page_plane(CommitSeq::new(1))
.is_some()
);
published.publish_single_connection_metadata_update(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(2),
db_size: 3,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
false,
);
assert_eq!(
published.page_plane_visible_commit_seq(),
CommitSeq::new(1),
"metadata-only publication deliberately leaves the resident page plane behind"
);
assert!(
published
.snapshot_for_page_plane(CommitSeq::new(1))
.is_none(),
"the fast path must not pair an old page plane with newer publication metadata"
);
}
#[test]
fn parallel_wal_group_commit_publication_replaces_stale_pages_before_out_of_order_phase_c() {
init_publication_test_tracing();
let cx = Cx::new();
let published = PublishedPagerState::new(3, CommitSeq::ZERO, JournalMode::Wal, 0);
let page_two = PageNumber::new(2).expect("page two");
let page_three = PageNumber::new(3).expect("page three");
let stale_page_two = PageData::from_vec(sample_page(0x20));
let committed_page_two = PageData::from_vec(sample_page(0xA2));
let committed_page_three = PageData::from_vec(sample_page(0xA3));
published.publish_insert_single(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::ZERO,
db_size: 3,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
page_two,
stale_page_two,
);
let batches = vec![
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: page_two.get(),
page_data: committed_page_two.as_bytes().to_vec(),
db_size_if_commit: 3,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 41,
lane_id: 0,
staged_frame_count: 1,
staging_elapsed_ns: 0,
}),
TransactionFrameBatch::new(vec![FrameSubmission {
page_number: page_three.get(),
page_data: committed_page_three.as_bytes().to_vec(),
db_size_if_commit: 3,
}])
.with_context(TransactionFrameBatchContext {
batch_id: 42,
lane_id: 1,
staged_frame_count: 1,
staging_elapsed_ns: 0,
}),
];
published
.publish_parallel_wal_group(
&cx,
PublishedPagerUpdate {
visible_commit_seq: CommitSeq::new(2),
db_size: 3,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
},
&batches,
)
.expect("publish complete certificate group");
let group_snapshot = published.snapshot();
assert_eq!(group_snapshot.visible_commit_seq, CommitSeq::new(2));
assert_eq!(
published.page_plane_visible_commit_seq(),
CommitSeq::new(2),
"the complete group page plane must reach the certificate high-water mark before waiters run"
);
assert_eq!(
published.try_get_page(page_two),
Some(committed_page_two.clone())
);
assert_eq!(
published.try_get_page(page_three),
Some(committed_page_three.clone())
);
// Simulate the current group binding first, then a delayed callback
// from an older certificate. Neither callback may reveal the stale
// pre-group page or regress the contiguous page-plane horizon.
let higher_member_intent = ParallelWalPublicationIntent {
certificate_epoch: 1,
visible_commit_seq: CommitSeq::new(2),
page_plane_visible_commit_seq: CommitSeq::new(2),
db_size: 3,
journal_mode: JournalMode::Wal,
freelist_count: 0,
checkpoint_active: false,
page_set_size: 2,
};
published.bind_parallel_wal_publication(higher_member_intent);
let lower_member_intent = ParallelWalPublicationIntent {
certificate_epoch: 0,
visible_commit_seq: CommitSeq::new(1),
page_plane_visible_commit_seq: CommitSeq::new(1),
db_size: 2,
page_set_size: 1,
..higher_member_intent
};
published.bind_parallel_wal_publication(lower_member_intent);
assert_eq!(published.try_get_page(page_two), Some(committed_page_two));
assert_eq!(
published.try_get_page(page_three),
Some(committed_page_three)
);
assert_eq!(
published.page_plane_visible_commit_seq(),
CommitSeq::new(2),
"out-of-order Phase C callbacks must preserve the complete group horizon"
);
}
#[test]
fn certificate_authorization_rejects_checksum_drift_and_checkpoint_overlap() {
let mut damaged = publication_authorization_for_test(false);
damaged.durability_receipt.certificate.certificate_crc32c ^= 1;
assert!(matches!(
parallel_wal_publication_intent(&damaged, 23, JournalMode::Wal, 0, false),
Err(FrankenError::Internal(_))
));
let no_checkpoint_fallback = publication_authorization_for_test(false);
assert!(matches!(
parallel_wal_publication_intent(&no_checkpoint_fallback, 23, JournalMode::Wal, 0, true,),
Err(FrankenError::Busy)
));
let checkpoint_fallback = publication_authorization_for_test(true);
assert!(
parallel_wal_publication_intent(&checkpoint_fallback, 23, JournalMode::Wal, 0, true,)
.is_ok()
);
}
#[test]
fn pager_metadata_publication_contract_copy_eq() {
let c = PAGER_METADATA_PUBLICATION_CONTRACTS[0];
assert_eq!(c.class, PagerMetadataPublicationClass::SnapshotSummary);
assert!(!c.touchpoint.is_empty());
assert!(!c.current_primitive.is_empty());
let copied = c;
assert_eq!(copied, c);
}
#[test]
fn pager_metadata_publication_contracts_cover_all_classes() {
let classes: Vec<_> = PAGER_METADATA_PUBLICATION_CONTRACTS
.iter()
.map(|c| c.class)
.collect();
assert!(classes.contains(&PagerMetadataPublicationClass::SnapshotSummary));
assert!(classes.contains(&PagerMetadataPublicationClass::PagePlaneResidency));
assert!(classes.contains(&PagerMetadataPublicationClass::CertificateDerivedIntent));
assert_eq!(classes.len(), 3);
}
}