#![allow(clippy::missing_errors_doc)]
use std::fs::{File, OpenOptions};
use std::io;
use std::path::Path;
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
use memmap2::{MmapMut, MmapOptions};
pub const LATCH_ARENA_MAGIC: u64 = 0x464C_4C41_0000_0001;
pub const LATCH_CELL_SIZE: usize = 64;
pub const RESULT_BYTES: usize = LATCH_CELL_SIZE - 8;
pub const UNSET: u8 = 0;
pub const SET: u8 = 1;
pub const ERR: u8 = 2;
#[repr(C, align(64))]
pub struct LatchArenaHeader {
pub magic: u64,
pub capacity: u64,
pub next_free: AtomicU64,
pub _reserved: [u8; 40],
}
#[repr(C, align(64))]
pub struct LatchCell {
pub state: AtomicU8,
pub result_len: u8,
pub _pad: [u8; 6],
pub result_bytes: [u8; RESULT_BYTES],
}
pub const fn arena_file_size(capacity: usize) -> usize {
std::mem::size_of::<LatchArenaHeader>() + capacity * LATCH_CELL_SIZE
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum LatchError {
NotResetable(u32),
BadOffset(u32),
PayloadTooLarge,
}
pub struct MmfLatchArena {
_file: File,
mmap: MmapMut,
capacity: usize,
}
unsafe impl Send for MmfLatchArena {}
unsafe impl Sync for MmfLatchArena {}
impl MmfLatchArena {
pub fn create<P: AsRef<Path>>(path: P, capacity: usize) -> io::Result<Self> {
let capacity = capacity.max(2).next_power_of_two();
let size = arena_file_size(capacity);
let file = OpenOptions::new()
.read(true)
.write(true)
.create(true)
.truncate(true)
.open(path.as_ref())?;
file.set_len(size as u64)?;
let mut mmap = unsafe { MmapOptions::new().len(size).map_mut(&file)? };
let header_ptr = mmap.as_mut_ptr() as *mut LatchArenaHeader;
unsafe {
(*header_ptr).magic = LATCH_ARENA_MAGIC;
(*header_ptr).capacity = capacity as u64;
(*header_ptr).next_free = AtomicU64::new(0);
std::ptr::write_bytes((*header_ptr)._reserved.as_mut_ptr(), 0, 40);
}
let cells_start = std::mem::size_of::<LatchArenaHeader>();
for i in 0..capacity {
let off = cells_start + i * LATCH_CELL_SIZE;
let cell_ptr = unsafe { mmap.as_mut_ptr().add(off) as *mut LatchCell };
unsafe {
(*cell_ptr).state = AtomicU8::new(UNSET);
(*cell_ptr).result_len = 0;
std::ptr::write_bytes((*cell_ptr)._pad.as_mut_ptr(), 0, 6);
std::ptr::write_bytes(
(*cell_ptr).result_bytes.as_mut_ptr(),
0,
RESULT_BYTES,
);
}
}
mmap.flush()?;
Ok(Self {
_file: file,
mmap,
capacity,
})
}
pub fn open<P: AsRef<Path>>(path: P) -> io::Result<Self> {
let file = OpenOptions::new().read(true).write(true).open(path.as_ref())?;
let size = file.metadata()?.len() as usize;
if size < std::mem::size_of::<LatchArenaHeader>() {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"latch arena file too small to contain header",
));
}
let mmap = unsafe { MmapOptions::new().len(size).map_mut(&file)? };
let header_ptr = mmap.as_ptr() as *const LatchArenaHeader;
let (magic, capacity) = unsafe {
((*header_ptr).magic, (*header_ptr).capacity as usize)
};
if magic != LATCH_ARENA_MAGIC {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("latch arena magic mismatch: got {magic:#x}, want {LATCH_ARENA_MAGIC:#x}"),
));
}
if !capacity.is_power_of_two() || capacity < 2 {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("latch arena capacity {capacity} is not a power of two >= 2"),
));
}
if size < arena_file_size(capacity) {
return Err(io::Error::new(
io::ErrorKind::InvalidData,
format!("latch arena file size {size} below expected {}", arena_file_size(capacity)),
));
}
Ok(Self {
_file: file,
mmap,
capacity,
})
}
pub fn capacity(&self) -> usize {
self.capacity
}
fn header(&self) -> &LatchArenaHeader {
unsafe { &*(self.mmap.as_ptr() as *const LatchArenaHeader) }
}
fn cell(&self, idx: usize) -> &LatchCell {
let off = std::mem::size_of::<LatchArenaHeader>() + idx * LATCH_CELL_SIZE;
unsafe { &*(self.mmap.as_ptr().add(off) as *const LatchCell) }
}
pub fn alloc(&self) -> u32 {
let n = self.header().next_free.fetch_add(1, Ordering::Relaxed);
let idx = (n & (self.capacity as u64 - 1)) as usize;
let cell = self.cell(idx);
cell.state.store(UNSET, Ordering::Release);
unsafe {
let cell_ptr = cell as *const _ as *mut LatchCell;
(*cell_ptr).result_len = 0;
}
(std::mem::size_of::<LatchArenaHeader>() + idx * LATCH_CELL_SIZE) as u32
}
fn offset_to_idx(&self, offset: u32) -> Result<usize, LatchError> {
let header_size = std::mem::size_of::<LatchArenaHeader>();
if (offset as usize) < header_size {
return Err(LatchError::BadOffset(offset));
}
let rel = offset as usize - header_size;
if !rel.is_multiple_of(LATCH_CELL_SIZE) {
return Err(LatchError::BadOffset(offset));
}
let idx = rel / LATCH_CELL_SIZE;
if idx >= self.capacity {
return Err(LatchError::BadOffset(offset));
}
Ok(idx)
}
pub fn is_set(&self, offset: u32) -> Result<bool, LatchError> {
let idx = self.offset_to_idx(offset)?;
let s = self.cell(idx).state.load(Ordering::Acquire);
Ok(s != UNSET)
}
pub(crate) fn state_line(&self, offset: u32) -> Result<*const u8, LatchError> {
let idx = self.offset_to_idx(offset)?;
Ok(self.cell(idx).state.as_ptr().cast_const())
}
pub fn publish(&self, offset: u32, payload: &[u8]) -> Result<(), LatchError> {
if payload.len() > RESULT_BYTES {
return Err(LatchError::PayloadTooLarge);
}
let idx = self.offset_to_idx(offset)?;
let cell = self.cell(idx);
unsafe {
let cell_ptr = cell as *const _ as *mut LatchCell;
std::ptr::write_bytes(
(*cell_ptr).result_bytes.as_mut_ptr(),
0,
RESULT_BYTES,
);
std::ptr::copy_nonoverlapping(
payload.as_ptr(),
(*cell_ptr).result_bytes.as_mut_ptr(),
payload.len(),
);
(*cell_ptr).result_len = payload.len() as u8;
}
cell.state.store(SET, Ordering::Release);
Ok(())
}
pub fn publish_err(&self, offset: u32, diagnostic: &[u8]) -> Result<(), LatchError> {
if diagnostic.len() > RESULT_BYTES {
return Err(LatchError::PayloadTooLarge);
}
let idx = self.offset_to_idx(offset)?;
let cell = self.cell(idx);
unsafe {
let cell_ptr = cell as *const _ as *mut LatchCell;
std::ptr::write_bytes(
(*cell_ptr).result_bytes.as_mut_ptr(),
0,
RESULT_BYTES,
);
std::ptr::copy_nonoverlapping(
diagnostic.as_ptr(),
(*cell_ptr).result_bytes.as_mut_ptr(),
diagnostic.len(),
);
(*cell_ptr).result_len = diagnostic.len() as u8;
}
cell.state.store(ERR, Ordering::Release);
Ok(())
}
pub fn read_result(&self, offset: u32, dst: &mut Vec<u8>) -> Result<u8, LatchError> {
let idx = self.offset_to_idx(offset)?;
let cell = self.cell(idx);
let state = cell.state.load(Ordering::Acquire);
let len = cell.result_len as usize;
dst.clear();
dst.resize(len, 0);
unsafe {
std::ptr::copy_nonoverlapping(
cell.result_bytes.as_ptr(),
dst.as_mut_ptr(),
len,
);
}
Ok(state)
}
pub fn reset(&self, offset: u32) -> Result<(), LatchError> {
let idx = self.offset_to_idx(offset)?;
self.cell(idx).state.store(UNSET, Ordering::Release);
Ok(())
}
pub fn flush(&self) -> io::Result<()> {
self.mmap.flush()
}
}
pub(crate) fn spin_phase<T, E>(
arena: &MmfLatchArena,
offset: u32,
polls: u32,
monitor: bool,
line_error: impl FnOnce(LatchError) -> E,
mut poll: impl FnMut() -> Result<Option<T>, E>,
) -> Result<Option<T>, E> {
let polls = polls as usize;
let use_monitor = monitor && crate::sched::sleep::monitor_wait_available();
if !use_monitor {
for _ in 0..polls {
if let Some(done) = poll()? {
return Ok(Some(done));
}
std::hint::spin_loop();
}
return Ok(None);
}
let floor = polls.min(crate::sched::latch::MONITOR_SPIN_FLOOR);
let floor_start = crate::sched::sleep::cycles_now();
for _ in 0..floor {
if let Some(done) = poll()? {
return Ok(Some(done));
}
std::hint::spin_loop();
}
if floor == polls {
return Ok(None);
}
let per_poll = crate::sched::sleep::cycles_now().saturating_sub(floor_start) / floor as u64;
if per_poll == 0 || per_poll > crate::sched::latch::MAX_MEASURED_POLL_CYCLES {
for _ in floor..polls {
if let Some(done) = poll()? {
return Ok(Some(done));
}
std::hint::spin_loop();
}
return Ok(None);
}
let span = ((polls - floor) as u64).saturating_mul(per_poll);
let line = arena.state_line(offset).map_err(line_error)?;
let state = line.cast::<AtomicU8>();
let set_now = || {
unsafe { &*state }.load(Ordering::Acquire) != UNSET
};
crate::sched::sleep::note_backend_monitor_wait();
let start = crate::sched::sleep::cycles_now();
loop {
if let Some(done) = poll()? {
return Ok(Some(done));
}
let left = span.saturating_sub(crate::sched::sleep::cycles_now().saturating_sub(start));
if left == 0 {
return Ok(None);
}
crate::sched::sleep::note_backend_monitor_arm();
let waited = unsafe { crate::sched::sleep::monitor_wait_once(line, left, set_now) };
if !waited {
while crate::sched::sleep::cycles_now().saturating_sub(start) < span {
if let Some(done) = poll()? {
return Ok(Some(done));
}
std::hint::spin_loop();
}
return Ok(None);
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Arc;
use std::thread;
fn remove(path: &std::path::Path) {
if let Err(e) = std::fs::remove_file(path) {
eprintln!("could not remove {}: {e}", path.display());
}
}
fn poll_cell(arena: &MmfLatchArena, off: u32) -> Result<Option<Vec<u8>>, LatchError> {
if !arena.is_set(off)? {
return Ok(None);
}
let mut out = Vec::new();
let state = arena.read_result(off, &mut out)?;
assert_eq!(state, SET, "the tests publish only successes");
Ok(Some(out))
}
#[test]
fn the_spin_phase_answers_a_publish_on_both_arms() {
for monitor in [false, true] {
let path = temp_path(if monitor { "spin_monitor" } else { "spin_plain" });
let arena = Arc::new(MmfLatchArena::create(&path, 4).expect("create"));
let off = arena.alloc();
let publisher = {
let arena = Arc::clone(&arena);
thread::spawn(move || {
thread::sleep(std::time::Duration::from_millis(5));
arena.publish(off, b"done").expect("publish");
})
};
let got = spin_phase(&arena, off, u32::MAX, monitor, |e| e, || poll_cell(&arena, off))
.expect("the cell reads");
publisher.join().expect("publisher joined");
assert_eq!(got.as_deref(), Some(&b"done"[..]), "monitor = {monitor}");
remove(&path);
}
}
#[test]
fn the_spin_phase_ends_unset_when_nothing_publishes() {
for monitor in [false, true] {
let path = temp_path(if monitor { "spin_monitor_none" } else { "spin_plain_none" });
let arena = MmfLatchArena::create(&path, 4).expect("create");
let off = arena.alloc();
let got = spin_phase(&arena, off, 4096, monitor, |e| e, || poll_cell(&arena, off))
.expect("the cell reads");
assert!(got.is_none(), "monitor = {monitor}: an unpublished cell ends the phase unset");
remove(&path);
}
}
#[test]
fn the_spin_phase_reports_a_bad_offset_on_both_arms() {
let path = temp_path("spin_bad_offset");
let arena = MmfLatchArena::create(&path, 4).expect("create");
for monitor in [false, true] {
let r = spin_phase(&arena, 8, 4096, monitor, |e| e, || poll_cell(&arena, 8));
assert!(matches!(r, Err(LatchError::BadOffset(8))), "monitor = {monitor}");
}
remove(&path);
}
fn temp_path(name: &str) -> std::path::PathBuf {
let mut p = std::env::temp_dir();
let pid = std::process::id();
let nonce = std::time::SystemTime::now()
.duration_since(std::time::UNIX_EPOCH)
.map(|d| d.as_nanos())
.unwrap_or(0);
p.push(format!("flynnel_latch_arena_{pid}_{nonce}_{name}.bin"));
p
}
#[test]
fn create_then_open_validates_header() {
let path = temp_path("create_open");
let _arena = MmfLatchArena::create(&path, 16).expect("create");
let reopen = MmfLatchArena::open(&path).expect("open");
assert_eq!(reopen.capacity(), 16);
std::fs::remove_file(&path).ok();
}
#[test]
fn open_rejects_bad_magic() {
let path = temp_path("bad_magic");
std::fs::write(&path, vec![0u8; 8192]).expect("seed");
let r = MmfLatchArena::open(&path);
assert!(r.is_err(), "open must reject bad-magic file");
std::fs::remove_file(&path).ok();
}
#[test]
fn alloc_publish_read_round_trip() {
let path = temp_path("alloc_publish_read");
let arena = MmfLatchArena::create(&path, 4).expect("create");
let off = arena.alloc();
assert!(!arena.is_set(off).expect("is_set"));
let payload = b"hello-latch";
arena.publish(off, payload).expect("publish");
assert!(arena.is_set(off).expect("is_set after publish"));
let mut out = Vec::new();
let s = arena.read_result(off, &mut out).expect("read");
assert_eq!(s, SET);
assert_eq!(&out[..], payload);
std::fs::remove_file(&path).ok();
}
#[test]
fn publish_err_round_trips() {
let path = temp_path("publish_err");
let arena = MmfLatchArena::create(&path, 4).expect("create");
let off = arena.alloc();
arena.publish_err(off, b"oops").expect("publish_err");
let mut out = Vec::new();
let s = arena.read_result(off, &mut out).expect("read");
assert_eq!(s, ERR);
assert_eq!(&out[..], b"oops");
std::fs::remove_file(&path).ok();
}
#[test]
fn reset_returns_cell_to_unset() {
let path = temp_path("reset");
let arena = MmfLatchArena::create(&path, 4).expect("create");
let off = arena.alloc();
arena.publish(off, b"x").expect("publish");
arena.reset(off).expect("reset");
assert!(!arena.is_set(off).expect("is_set after reset"));
std::fs::remove_file(&path).ok();
}
#[test]
fn bad_offset_rejected() {
let path = temp_path("bad_offset");
let arena = MmfLatchArena::create(&path, 4).expect("create");
let r = arena.is_set(8);
assert!(matches!(r, Err(LatchError::BadOffset(_))));
let r = arena.is_set((std::mem::size_of::<LatchArenaHeader>() + 1) as u32);
assert!(matches!(r, Err(LatchError::BadOffset(_))));
std::fs::remove_file(&path).ok();
}
#[test]
fn payload_too_large_rejected() {
let path = temp_path("oversize");
let arena = MmfLatchArena::create(&path, 2).expect("create");
let off = arena.alloc();
let big = vec![0u8; RESULT_BYTES + 1];
let err = arena.publish(off, &big).expect_err("expected oversize");
assert_eq!(err, LatchError::PayloadTooLarge);
std::fs::remove_file(&path).ok();
}
#[test]
fn alloc_wraps_modulo_capacity() {
let path = temp_path("wrap");
let arena = MmfLatchArena::create(&path, 4).expect("create");
let header_size = std::mem::size_of::<LatchArenaHeader>() as u32;
let mut seen = Vec::new();
for _ in 0..8 {
seen.push(arena.alloc());
}
assert_eq!(seen[0], header_size);
assert_eq!(seen[4], header_size);
assert_eq!(seen[1], header_size + LATCH_CELL_SIZE as u32);
assert_eq!(seen[5], header_size + LATCH_CELL_SIZE as u32);
std::fs::remove_file(&path).ok();
}
#[test]
fn two_threads_publish_and_observe() {
let path = temp_path("two_threads");
let n = 200usize;
let capacity = (n * 2).next_power_of_two();
let arena = Arc::new(MmfLatchArena::create(&path, capacity).expect("create"));
let mut offsets = Vec::with_capacity(n);
for _ in 0..n {
offsets.push(arena.alloc());
}
let pub_arena = Arc::clone(&arena);
let pub_offsets = offsets.clone();
let pub_handle = thread::spawn(move || {
for (i, off) in pub_offsets.iter().enumerate() {
let bytes = (i as u32).to_le_bytes();
pub_arena.publish(*off, &bytes).expect("publish");
}
});
let mut buf = Vec::new();
for (i, off) in offsets.iter().enumerate() {
loop {
if arena.is_set(*off).expect("is_set") {
break;
}
std::thread::yield_now();
}
let s = arena.read_result(*off, &mut buf).expect("read");
assert_eq!(s, SET);
let mut arr = [0u8; 4];
arr.copy_from_slice(&buf[..4]);
assert_eq!(u32::from_le_bytes(arr) as usize, i, "value mismatch at i={i}");
}
pub_handle.join().expect("publisher joined");
std::fs::remove_file(&path).ok();
}
}