slate_flash_calib/
slate_flash_calib.rs

1//! Paper measurement: cost of a single `FileFlash::program` under each
2//! `Durability` mode, measured at the HAL level with no engine above it.
3//!
4//! Why this exists. In `slate_throughput.rs` the two `Durability` modes come
5//! out indistinguishable, which contradicts the expectation that `OsCache`
6//! (documented as benchmark-only, i.e. cheaper) should be much faster than
7//! `Full` (`F_FULLFSYNC`). Rather than explain that away, this isolates one
8//! `program` call so the barrier cost is measured directly and the throughput
9//! numbers can be attributed to a page count times a per-page cost.
10//!
11//! It also times a raw `pwrite` with no barrier at all, which bounds how much
12//! of the cost is the durability barrier versus the surrounding work
13//! (`FileFlash::program` reads the target range back first to enforce
14//! program-once-per-page, so every program is a read plus a write plus a
15//! flush).
16//!
17//! PLATFORM: file-backed emulation on this host's filesystem. Not NOR flash.
18//!
19//! `cargo run --release -p slate-kv --example slate_flash_calib`
20
21use slate_kv::file_flash::{Durability, FileFlash};
22use slate_kv_hal::Flash;
23use std::fs::OpenOptions;
24use std::time::Instant;
25
26const CAPACITY: u32 = 4 * 1024 * 1024;
27const PAGE: usize = 256;
28const BLOCK: usize = 4096;
29const N: usize = 300;
30
31fn pct(sorted_ns: &[u64], q: f64) -> f64 {
32    let idx = ((q * sorted_ns.len() as f64).ceil() as usize).clamp(1, sorted_ns.len()) - 1;
33    sorted_ns[idx] as f64 / 1000.0
34}
35
36fn summarize(label: &str, mut ns: Vec<u64>) {
37    ns.sort_unstable();
38    let n = ns.len();
39    let mean = ns.iter().map(|&x| x as u128).sum::<u128>() as f64 / n as f64 / 1000.0;
40    println!(
41        "{label},{n},{mean:.3},{:.3},{:.3},{:.3},{:.3}",
42        pct(&ns, 0.50),
43        pct(&ns, 0.90),
44        pct(&ns, 0.99),
45        ns[n - 1] as f64 / 1000.0
46    );
47}
48
49fn time_flash_programs(dir: &std::path::Path, label: &str, dur: Durability) {
50    let path = dir.join(format!("calib_{label}.bin"));
51    let file = OpenOptions::new()
52        .read(true)
53        .write(true)
54        .create(true)
55        .truncate(false)
56        .open(&path)
57        .unwrap();
58    let mut flash = FileFlash::new(file, CAPACITY, PAGE, BLOCK, dur).unwrap();
59    let buf = [0xA5u8; PAGE];
60    let mut ns = Vec::with_capacity(N);
61    // Each page may be programmed once, so every iteration targets a fresh
62    // page rather than re-erasing (an erase is a different cost).
63    for i in 0..N {
64        let addr = (i * PAGE) as u32;
65        let t0 = Instant::now();
66        flash.program(addr, &buf).unwrap();
67        ns.push(t0.elapsed().as_nanos() as u64);
68    }
69    summarize(&format!("FileFlash::program,{label}"), ns);
70    drop(flash);
71    let _ = std::fs::remove_file(&path);
72}
73
74fn main() {
75    let dir = std::env::temp_dir().join(format!("slate_calib_{}", std::process::id()));
76    std::fs::create_dir_all(&dir).unwrap();
77
78    println!("# SLATE paper measurement: per-page flash barrier cost, HAL level, no engine");
79    println!(
80        "# platform=host os={} arch={} backend=FileFlash(file-backed emulation) NOT_real_NOR_flash",
81        std::env::consts::OS,
82        std::env::consts::ARCH
83    );
84    println!("# geometry: capacity={CAPACITY} page={PAGE} block={BLOCK} n_programs_per_mode={N}");
85    println!(
86        "# Durability::Full = fcntl(F_FULLFSYNC); Durability::OsCache = std File::sync_data(). \
87         raw_pwrite_no_barrier programs the same page size with no flush at all, as a floor."
88    );
89    println!("operation,mode,n,mean_us,p50_us,p90_us,p99_us,max_us");
90
91    time_flash_programs(&dir, "Full", Durability::Full);
92    time_flash_programs(&dir, "OsCache", Durability::OsCache);
93
94    // Barrier primitives in isolation. `Durability::OsCache` calls
95    // `File::sync_data()` expecting it to be the cheap barrier, so whether it
96    // actually is cheaper than `F_FULLFSYNC` on this platform is measured here
97    // rather than assumed from the flag's name.
98    {
99        use std::os::unix::fs::FileExt;
100        use std::os::unix::io::AsRawFd;
101        let path = dir.join("calib_barrier.bin");
102        let file = OpenOptions::new()
103            .read(true)
104            .write(true)
105            .create(true)
106            .truncate(true)
107            .open(&path)
108            .unwrap();
109        file.set_len(CAPACITY as u64).unwrap();
110        let buf = [0x5Au8; PAGE];
111        let fd = file.as_raw_fd();
112
113        let mut rust_sync_data = Vec::with_capacity(N);
114        let mut libc_fsync = Vec::with_capacity(N);
115        let mut libc_fullfsync = Vec::with_capacity(N);
116        for i in 0..N {
117            let off = ((i % 1024) * PAGE) as u64;
118            file.write_all_at(&buf, off).unwrap();
119            let t0 = Instant::now();
120            file.sync_data().unwrap();
121            rust_sync_data.push(t0.elapsed().as_nanos() as u64);
122
123            file.write_all_at(&buf, off).unwrap();
124            let t0 = Instant::now();
125            let rc = unsafe { libc::fsync(fd) };
126            libc_fsync.push(t0.elapsed().as_nanos() as u64);
127            assert_eq!(rc, 0);
128
129            // `F_FULLFSYNC` is a Darwin-only fcntl: on macOS `fsync` only pushes
130            // to the drive cache, so this is the strongest available barrier and
131            // the one `Durability::Full` uses (see file_flash.rs, which gates it
132            // the same way). Linux has no equivalent — `fsync` there is already
133            // the full barrier — so the strongest-barrier row is `fdatasync`.
134            file.write_all_at(&buf, off).unwrap();
135            let t0 = Instant::now();
136            #[cfg(target_os = "macos")]
137            let rc = unsafe { libc::fcntl(fd, libc::F_FULLFSYNC) };
138            #[cfg(not(target_os = "macos"))]
139            let rc = unsafe { libc::fdatasync(fd) };
140            libc_fullfsync.push(t0.elapsed().as_nanos() as u64);
141            assert_ne!(rc, -1);
142        }
143        summarize("barrier_only,rust_File::sync_data", rust_sync_data);
144        summarize("barrier_only,libc_fsync", libc_fsync);
145        #[cfg(target_os = "macos")]
146        summarize("barrier_only,libc_fcntl_F_FULLFSYNC", libc_fullfsync);
147        #[cfg(not(target_os = "macos"))]
148        summarize("barrier_only,libc_fdatasync", libc_fullfsync);
149        drop(file);
150        let _ = std::fs::remove_file(&path);
151    }
152
153    // Barrier-free floor: same page size, same file API, no flush.
154    {
155        use std::os::unix::fs::FileExt;
156        let path = dir.join("calib_raw.bin");
157        let file = OpenOptions::new()
158            .read(true)
159            .write(true)
160            .create(true)
161            .truncate(true)
162            .open(&path)
163            .unwrap();
164        file.set_len(CAPACITY as u64).unwrap();
165        let buf = [0xA5u8; PAGE];
166        let mut ns = Vec::with_capacity(N);
167        for i in 0..N {
168            let t0 = Instant::now();
169            file.write_all_at(&buf, (i * PAGE) as u64).unwrap();
170            ns.push(t0.elapsed().as_nanos() as u64);
171        }
172        summarize("raw_pwrite,no_barrier", ns);
173        drop(file);
174        let _ = std::fs::remove_file(&path);
175    }
176
177    let _ = std::fs::remove_dir_all(&dir);
178}
slate_flash_calib/slate_flash_calib.rs

slate_flash_calib/
slate_flash_calib.rs
