powercut 0.1.1

Prove an acknowledged write survives a power cut or process crash, in cargo test. A database-agnostic crash-injection harness that derives the durable set from the fsyncs that actually happened.
Documentation
//! `powercut` against **rusqlite**, and the honest story about an engine that
//! syncs where you cannot see it.
//!
//! ```console
//! cargo run --example sqlite_durable_ack
//! ```
//!
//! libsqlite3 calls `fsync` itself, through libc, inside FFI. `powercut` 0.1
//! has no interposition backend, so it cannot observe those calls — and the
//! one thing it must never do is guess. This example shows all three honest
//! ways that plays out:
//!
//! | case | pragma | what powercut can see | verdict |
//! |---|---|---|---|
//! | `sqlite/synchronous=OFF` | `OFF` | everything: SQLite syncs nothing | **lost** |
//! | `sqlite/durable-ack` | `OFF` + we fsync at the ack | everything, by construction | **survived** |
//! | `sqlite/synchronous=FULL` | `FULL` | nothing: SQLite syncs internally | **not-measured** |
//!
//! The third row is the important one. SQLite at `synchronous=FULL` almost
//! certainly *does* survive — and `powercut` still refuses to say it does,
//! because it did not measure it. It equally refuses to say it does not.
//!
//! The route to a fully measured result on an FFI engine is the second row:
//! turn the engine's own syncing **off** and do it yourself at the ack. That
//! is not a testing trick — it is the `durable_ack` pattern applications use
//! when they will not take the engine's word for it, and now it is testable.

use std::path::{Path, PathBuf};

use powercut::{
    Checker, Coverage, DurabilityWitness, Report, Scenario, ScratchDir, SyncJournal, Tier, Verdict,
    Violation,
};
use rusqlite::Connection;

const DB: &str = "app.db";

fn main() {
    let reports = vec![
        synchronous_off(),
        durable_ack(),
        synchronous_full(),
        read_fork_close_tear(),
    ];
    let expected = [
        Verdict::Lost,
        Verdict::Survived,
        Verdict::NotMeasured,
        Verdict::Lost,
    ];

    for report in &reports {
        println!("{}", report.to_text());
    }
    println!("{}", powercut::to_markdown(&reports));

    for (report, expected) in reports.iter().zip(expected) {
        assert_eq!(
            report.verdict(),
            expected,
            "`{}` was expected to be {expected}\n{}",
            report.label(),
            report.to_text()
        );
    }
    println!("all {} cases behaved as documented", reports.len());
}

// ── The three honest cases ──────────────────────────────────────────────

/// `synchronous=OFF`: `COMMIT` returns `Ok` and nothing is durable. The
/// classic Hacker News argument, settled in one run.
fn synchronous_off() -> Report {
    let store = ScratchDir::new("sqlite-off").unwrap();
    let dir = store.path().to_path_buf();
    let journal = SyncJournal::complete();
    provision(&dir, "OFF", &journal);

    // The connection is opened OUTSIDE the workload and stays open across the
    // capture, because that is where a serving process acks: mid-life, handle
    // still held. Closing it is itself a durability event — see the last case.
    let conn = open(&dir, "OFF");

    Scenario::new(&dir, journal.clone())
        .label("sqlite/synchronous=OFF")
        .tier(Tier::PowerCut)
        .ack(|| {
            conn.execute("INSERT INTO ledger(id, amount) VALUES (7, 100)", [])
                .unwrap();
            (7_i64, 100_i64)
        })
        .unwrap()
        .with_teeth_control(|| durable_ack_now(&dir, &journal))
        .unwrap()
        .check(check_ledger)
}

/// The fix, and the reason this crate exists: one `fsync` per file at the ack.
/// Coverage is complete **by construction** — SQLite is syncing nothing, so
/// every sync that touches the store is one of ours.
fn durable_ack() -> Report {
    let store = ScratchDir::new("sqlite-ack").unwrap();
    let dir = store.path().to_path_buf();
    let journal = SyncJournal::complete();
    provision(&dir, "OFF", &journal);
    let conn = open(&dir, "OFF");

    Scenario::new(&dir, journal.clone())
        .label("sqlite/durable-ack")
        .tier(Tier::PowerCut)
        .ack(|| {
            conn.execute("INSERT INTO ledger(id, amount) VALUES (7, 100)", [])
                .unwrap();
            durable_ack_now(&dir, &journal).unwrap(); // ← delete this line and it goes red
            (7_i64, 100_i64)
        })
        .unwrap()
        .check(check_ledger)
}

/// `synchronous=FULL`: SQLite does the right thing, in a place `powercut`
/// cannot see. The witness says so, and the verdict is `NotMeasured` — not a
/// pass, and emphatically not an accusation.
fn synchronous_full() -> Report {
    let store = ScratchDir::new("sqlite-full").unwrap();
    let dir = store.path().to_path_buf();
    let journal = SyncJournal::with_coverage(Coverage::partial(
        "libsqlite3 issues its own fsync/F_FULLFSYNC through libc at synchronous=FULL, \
         and powercut 0.1 has no syscall-interposition backend to observe them",
    ));
    provision(&dir, "FULL", &journal);
    let conn = open(&dir, "FULL");

    Scenario::new(&dir, journal.clone())
        .label("sqlite/synchronous=FULL")
        .tier(Tier::PowerCut)
        .ack(|| {
            conn.execute("INSERT INTO ledger(id, amount) VALUES (7, 100)", [])
                .unwrap();
            (7_i64, 100_i64)
        })
        .unwrap()
        .check(check_ledger)
}

/// **The flagship.** A connection that never writes a byte destroys the
/// durability of data that was already committed, acked and `fsync`'d.
///
/// The sequence, and every step of it is ordinary application behaviour:
///
/// 1. writer `A` opens; reader `B` opens;
/// 2. `A` inserts and durable-acks — the row is genuinely on stable storage;
/// 3. `A` closes. `B` is still open, so SQLite does **not** checkpoint;
/// 4. `B` — a pure reader — closes. It is now the last handle, so SQLite
///    checkpoints: the WAL is folded into `app.db` and deleted.
///
/// `app.db` was rewritten in place and nobody `fsync`'d it; the WAL that held
/// the durability warrant is gone. The row is still there if you look — and it
/// is no longer covered by anything.
fn read_fork_close_tear() -> Report {
    let store = ScratchDir::new("sqlite-read-fork").unwrap();
    let dir = store.path().to_path_buf();
    let journal = SyncJournal::complete();
    provision(&dir, "OFF", &journal);

    let writer = open(&dir, "OFF");
    let reader = open(&dir, "OFF");

    writer
        .execute("INSERT INTO ledger(id, amount) VALUES (7, 100)", [])
        .unwrap();
    durable_ack_now(&dir, &journal).unwrap();
    drop(writer);

    // Right here the row IS durable. Prove it, so the tear below cannot be
    // dismissed as "it was never durable in the first place".
    let before = Scenario::new(&dir, journal.clone())
        .label("sqlite/read-fork — before the reader closes")
        .tier(Tier::PowerCut)
        .ack(|| (7_i64, 100_i64))
        .unwrap()
        .check(check_ledger);
    assert_eq!(
        before.verdict(),
        Verdict::Survived,
        "the premise of this case is that the row starts out durable\n{}",
        before.to_text()
    );

    Scenario::new(&dir, journal.clone())
        .label("sqlite/read-fork-close-tear")
        .tier(Tier::PowerCut)
        .ack(|| {
            let count: i64 = reader
                .query_row("SELECT count(*) FROM ledger", [], |r| r.get(0))
                .unwrap();
            assert_eq!(count, 1, "the reader really does read the acked row");
            drop(reader); // ← the last handle closes; SQLite checkpoints
            (7_i64, 100_i64)
        })
        .unwrap()
        .check(check_ledger)
}

// ── The four small things a store has to supply ─────────────────────────

/// Open a connection in WAL mode at the requested `synchronous` level.
fn open(dir: &Path, synchronous: &str) -> Connection {
    let conn = Connection::open(dir.join(DB)).expect("open sqlite");
    conn.pragma_update(None, "journal_mode", "WAL")
        .expect("WAL mode");
    conn.pragma_update(None, "synchronous", synchronous)
        .expect("set synchronous");
    conn
}

/// Create the schema and make it durable **before** the measured workload.
///
/// This is what makes `Checker::baseline` meaningful: anything missing after
/// the crash is unambiguously the measured write, not a store that was never
/// on disk to begin with.
fn provision(dir: &Path, synchronous: &str, journal: &SyncJournal) {
    let conn = open(dir, synchronous);
    conn.execute_batch("CREATE TABLE ledger(id INTEGER PRIMARY KEY, amount INTEGER);")
        .expect("create schema");
    conn.execute_batch("PRAGMA wal_checkpoint(FULL);")
        .expect("fold the schema into the main file");
    drop(conn);
    durable_ack_now(dir, journal).expect("make the provisioned schema durable");
}

/// The durable-ack: `fsync` every file SQLite keeps state in, plus the
/// directory, and record each one. `-shm` is deliberately absent — it is a
/// shared-memory index SQLite rebuilds from the WAL, not durable state.
fn durable_ack_now(dir: &Path, journal: &SyncJournal) -> std::io::Result<()> {
    journal.sync_path_if_present(dir.join(DB))?;
    journal.sync_path_if_present(dir.join(format!("{DB}-wal")))?;
    journal.sync_dir(dir)?;
    Ok(())
}

/// Reboot from the captured bytes and read the ack back.
fn check_ledger(crash: &powercut::CrashState, acked: &(i64, i64)) -> Vec<Violation> {
    let mut c = Checker::new();
    let path: PathBuf = crash.dir().join(DB);

    // A fresh instance, booted from nothing but what survived.
    let conn = Connection::open(&path).expect("open the crash copy");

    // Baseline FIRST: if the schema itself did not survive, the copy is not a
    // usable database and nothing below is a durability measurement.
    let schema: i64 = conn
        .query_row(
            "SELECT count(*) FROM sqlite_master WHERE type='table' AND name='ledger'",
            [],
            |r| r.get(0),
        )
        .unwrap_or(0);
    c.baseline("the `ledger` table exists", 1_i64, schema);

    let amount: Option<i64> = conn
        .query_row("SELECT amount FROM ledger WHERE id = ?1", [acked.0], |r| {
            r.get(0)
        })
        .ok();
    c.survives("ledger row 7", Some(acked.1), amount);
    c.finish()
}

/// Not called by `main` — it exists so the file also documents the shape of a
/// custom [`DurabilityWitness`] for readers who want one.
#[allow(dead_code)]
fn coverage_of(journal: &SyncJournal) -> Coverage {
    journal.coverage()
}