1use std::fs::{File, OpenOptions};
48use std::marker::PhantomData;
49use std::mem::{align_of, size_of};
50use std::path::Path;
51use std::sync::atomic::{AtomicU32, AtomicU64, Ordering};
52
53use memmap2::{MmapMut, MmapOptions};
54
55pub const HANDLE_TABLE_MAGIC: u64 = 0x4150_4D46_5354_424C;
56
57pub const SLOT_PAYLOAD_BYTES: usize = 48;
58
59pub const NIL_SLOT: u32 = u32::MAX;
61
62#[repr(C, align(64))]
63pub struct HandleHeader {
64 pub magic: u64,
65 pub capacity: u32,
66 pub slot_size: u32,
67 pub free_list_head: AtomicU64,
68 pub live_count: AtomicU64,
69 _pad: [u8; 32],
70}
71
72#[repr(C, align(64))]
73pub struct SharedSlot {
74 pub generation: AtomicU32,
75 pub occupied: AtomicU32,
76 pub next_free: AtomicU32,
77 _pad: u32,
78 pub payload: [u8; SLOT_PAYLOAD_BYTES],
79}
80
81#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Default)]
83pub struct Handle(u64);
84
85impl Handle {
86 pub const NULL: Self = Self(0);
87
88 #[inline]
89 pub const fn from_parts(generation: u32, slot: u32) -> Self {
90 Self(((generation as u64) << 32) | (slot as u64))
91 }
92
93 #[inline]
94 pub const fn generation(self) -> u32 { (self.0 >> 32) as u32 }
95
96 #[inline]
97 pub const fn slot(self) -> u32 { self.0 as u32 }
98
99 #[inline]
100 pub const fn is_null(self) -> bool { self.0 == 0 }
101
102 #[inline]
103 pub const fn raw(self) -> u64 { self.0 }
104}
105
106pub const fn slot_offset() -> usize { size_of::<HandleHeader>() }
107
108pub const fn handle_table_file_size(capacity: usize) -> usize {
109 slot_offset() + capacity * size_of::<SharedSlot>()
110}
111
112#[derive(Debug, Clone, Copy, PartialEq, Eq)]
113pub enum HandleTableError {
114 LayoutMismatch,
115 PayloadTooLarge,
116 Full,
117 IoError(std::io::ErrorKind),
118}
119
120impl From<std::io::Error> for HandleTableError {
121 fn from(e: std::io::Error) -> Self { Self::IoError(e.kind()) }
122}
123
124pub struct SharedHandleTable<T: Copy + 'static> {
125 _file: File,
126 mmap: MmapMut,
127 capacity: usize,
128 _phantom: PhantomData<T>,
129 header_sidecar: subetha_core::HandshakeHeader,
130 ring_sidecar: Box<subetha_core::ObservationRing>,
131}
132
133unsafe impl<T: Copy + Send + 'static> Send for SharedHandleTable<T> {}
134unsafe impl<T: Copy + Sync + 'static> Sync for SharedHandleTable<T> {}
135
136impl<T: Copy + Send + Sync + 'static> subetha_sidecar::AdaptiveInstance for SharedHandleTable<T> {
137 fn header(&self) -> &subetha_core::HandshakeHeader { &self.header_sidecar }
138 fn ring(&self) -> &subetha_core::ObservationRing { &self.ring_sidecar }
139 fn make_policy(&self) -> Box<dyn subetha_sidecar::Policy> {
140 Box::new(subetha_sidecar::NoMigrationPolicy)
141 }
142}
143
144#[inline]
145fn pack_head(counter: u32, slot: u32) -> u64 {
146 ((counter as u64) << 32) | (slot as u64)
147}
148
149#[inline]
150fn unpack_head(v: u64) -> (u32, u32) {
151 ((v >> 32) as u32, v as u32)
152}
153
154impl<T: Copy + 'static> SharedHandleTable<T> {
155 pub fn create(path: impl AsRef<Path>, capacity: usize) -> Result<Self, HandleTableError> {
161 Self::check_layout()?;
162 assert!(capacity >= 1 && capacity <= (u32::MAX - 1) as usize);
163 let total = handle_table_file_size(capacity);
164 let (file, mmap) = crate::mmf_attach::create_or_attach(
165 path.as_ref(),
166 total,
167 |ptr| unsafe { Self::init_region(ptr, capacity) },
168 |ptr| unsafe { (*(ptr as *const HandleHeader)).magic == HANDLE_TABLE_MAGIC },
169 )?;
170 Self::from_region(file, mmap, capacity)
171 }
172
173 pub fn reset(path: impl AsRef<Path>, capacity: usize) -> Result<Self, HandleTableError> {
177 Self::check_layout()?;
178 assert!(capacity >= 1 && capacity <= (u32::MAX - 1) as usize);
179 let total = handle_table_file_size(capacity);
180 let (file, mmap) = crate::mmf_attach::reset(path.as_ref(), total, |ptr| unsafe {
181 Self::init_region(ptr, capacity)
182 })?;
183 Self::from_region(file, mmap, capacity)
184 }
185
186 unsafe fn init_region(ptr: *mut u8, capacity: usize) {
196 let hdr = ptr as *mut HandleHeader;
197 unsafe {
198 (*hdr).capacity = capacity as u32;
199 (*hdr).slot_size = size_of::<T>() as u32;
200 std::ptr::write(
201 &raw mut (*hdr).free_list_head,
202 AtomicU64::new(pack_head(0, 0)),
203 );
204 let slots_base = ptr.add(slot_offset());
205 for i in 0..capacity {
206 let slot_ptr = slots_base.add(i * size_of::<SharedSlot>()) as *mut SharedSlot;
207 let next = if i + 1 < capacity { (i + 1) as u32 } else { NIL_SLOT };
208 std::ptr::write(&raw mut (*slot_ptr).next_free, AtomicU32::new(next));
209 }
210 std::ptr::write_volatile(&raw mut (*hdr).magic, HANDLE_TABLE_MAGIC);
211 }
212 }
213
214 fn from_region(
217 file: File,
218 mmap: MmapMut,
219 capacity: usize,
220 ) -> Result<Self, HandleTableError> {
221 let hdr = unsafe { &*(mmap.as_ptr() as *const HandleHeader) };
222 if hdr.magic != HANDLE_TABLE_MAGIC
223 || hdr.capacity != capacity as u32
224 || hdr.slot_size as usize != size_of::<T>()
225 {
226 return Err(HandleTableError::LayoutMismatch);
227 }
228 Ok(Self {
229 _file: file, mmap, capacity, _phantom: PhantomData,
230 header_sidecar: subetha_core::HandshakeHeader::new(),
231 ring_sidecar: Box::new(subetha_core::ObservationRing::new()),
232 })
233 }
234
235 pub fn open(path: impl AsRef<Path>, expected_capacity: usize) -> Result<Self, HandleTableError> {
236 Self::check_layout()?;
237 let file = OpenOptions::new().read(true).write(true).open(path.as_ref())?;
238 let total = handle_table_file_size(expected_capacity);
239 if file.metadata()?.len() < total as u64 {
240 return Err(HandleTableError::LayoutMismatch);
241 }
242 let mmap = unsafe { MmapOptions::new().len(total).map_mut(&file)? };
243 Self::from_region(file, mmap, expected_capacity)
244 }
245
246 fn check_layout() -> Result<(), HandleTableError> {
247 if size_of::<T>() > SLOT_PAYLOAD_BYTES {
248 return Err(HandleTableError::PayloadTooLarge);
249 }
250 if align_of::<T>() > 8 {
251 return Err(HandleTableError::PayloadTooLarge);
252 }
253 Ok(())
254 }
255
256 #[inline]
257 pub fn capacity(&self) -> usize { self.capacity }
258
259 #[inline]
260 pub fn header(&self) -> &HandleHeader {
261 unsafe { &*(self.mmap.as_ptr() as *const HandleHeader) }
262 }
263
264 #[inline]
265 fn slot(&self, idx: u32) -> &SharedSlot {
266 debug_assert!((idx as usize) < self.capacity);
267 let base = unsafe { self.mmap.as_ptr().add(slot_offset()) };
268 unsafe { &*(base.add((idx as usize) * size_of::<SharedSlot>()) as *const SharedSlot) }
269 }
270
271 fn pop_free(&self) -> Option<u32> {
274 let header = self.header();
275 loop {
276 let head = header.free_list_head.load(Ordering::Acquire);
277 let (cnt, idx) = unpack_head(head);
278 if idx == NIL_SLOT { return None; }
279 let next = self.slot(idx).next_free.load(Ordering::Acquire);
280 let new_head = pack_head(cnt.wrapping_add(1), next);
281 if header.free_list_head.compare_exchange_weak(
282 head, new_head, Ordering::AcqRel, Ordering::Acquire,
283 ).is_ok() {
284 return Some(idx);
285 }
286 std::hint::spin_loop();
287 }
288 }
289
290 fn push_free(&self, slot_idx: u32) {
292 let header = self.header();
293 loop {
294 let head = header.free_list_head.load(Ordering::Acquire);
295 let (cnt, head_idx) = unpack_head(head);
296 self.slot(slot_idx).next_free.store(head_idx, Ordering::Release);
297 let new_head = pack_head(cnt.wrapping_add(1), slot_idx);
298 if header.free_list_head.compare_exchange_weak(
299 head, new_head, Ordering::AcqRel, Ordering::Acquire,
300 ).is_ok() {
301 return;
302 }
303 std::hint::spin_loop();
304 }
305 }
306
307 pub fn insert(&self, value: T) -> Result<Handle, HandleTableError> {
310 let slot_idx = match self.pop_free() {
311 Some(i) => i,
312 None => {
313 self.ring_sidecar
314 .push_op(crate::sidecar_ops::ownership::OP_ACQUIRE, 1);
315 return Err(HandleTableError::Full);
316 }
317 };
318 let slot = self.slot(slot_idx);
319 let new_gen = slot.generation.fetch_add(1, Ordering::AcqRel)
321 .wrapping_add(1)
322 .max(1); unsafe {
326 let dst = slot.payload.as_ptr() as *mut T;
327 std::ptr::write_unaligned(dst, value);
328 }
329 slot.occupied.store(1, Ordering::Release);
330 self.header().live_count.fetch_add(1, Ordering::AcqRel);
331 self.ring_sidecar
332 .push_op(crate::sidecar_ops::ownership::OP_ACQUIRE, 0);
333 Ok(Handle::from_parts(new_gen, slot_idx))
334 }
335
336 pub fn get(&self, h: Handle) -> Option<T> {
339 let r = self.get_inner(h);
340 self.ring_sidecar.push_op(
341 crate::sidecar_ops::ownership::OP_GET,
342 if r.is_none() { 2 } else { 0 },
343 );
344 r
345 }
346
347 fn get_inner(&self, h: Handle) -> Option<T> {
348 if h.is_null() { return None; }
349 let slot_idx = h.slot();
350 if (slot_idx as usize) >= self.capacity { return None; }
351 let slot = self.slot(slot_idx);
352 loop {
355 let gen1 = slot.generation.load(Ordering::Acquire);
356 if gen1 != h.generation() { return None; }
357 if slot.occupied.load(Ordering::Acquire) == 0 { return None; }
358 let value: T = unsafe {
359 let src = slot.payload.as_ptr() as *const T;
360 std::ptr::read_unaligned(src)
361 };
362 let gen2 = slot.generation.load(Ordering::Acquire);
363 if gen1 == gen2 {
364 return Some(value);
365 }
366 std::hint::spin_loop();
368 }
369 }
370
371 pub fn contains(&self, h: Handle) -> bool {
373 if h.is_null() { return false; }
374 let slot_idx = h.slot();
375 if (slot_idx as usize) >= self.capacity { return false; }
376 let slot = self.slot(slot_idx);
377 slot.generation.load(Ordering::Acquire) == h.generation()
378 && slot.occupied.load(Ordering::Acquire) == 1
379 }
380
381 pub fn remove(&self, h: Handle) -> Option<T> {
384 let r = self.remove_inner(h);
385 self.ring_sidecar.push_op(
386 crate::sidecar_ops::ownership::OP_RELEASE,
387 if r.is_none() { 2 } else { 0 },
388 );
389 r
390 }
391
392 fn remove_inner(&self, h: Handle) -> Option<T> {
393 if h.is_null() { return None; }
394 let slot_idx = h.slot();
395 if (slot_idx as usize) >= self.capacity { return None; }
396 let slot = self.slot(slot_idx);
397 if slot.occupied.compare_exchange(
399 1, 0, Ordering::AcqRel, Ordering::Acquire,
400 ).is_err() {
401 return None;
402 }
403 let cur_gen = slot.generation.load(Ordering::Acquire);
407 if cur_gen != h.generation() {
408 slot.occupied.store(1, Ordering::Release);
410 return None;
411 }
412 let value: T = unsafe {
413 let src = slot.payload.as_ptr() as *const T;
414 std::ptr::read_unaligned(src)
415 };
416 slot.generation.fetch_add(1, Ordering::AcqRel);
418 self.header().live_count.fetch_sub(1, Ordering::AcqRel);
419 self.push_free(slot_idx);
421 Some(value)
422 }
423
424 pub fn len(&self) -> usize {
425 self.header().live_count.load(Ordering::Acquire) as usize
426 }
427
428 pub fn is_empty(&self) -> bool { self.len() == 0 }
429
430 pub fn flush_async(&self) -> Result<(), HandleTableError> {
434 self.mmap.flush_async()?;
435 Ok(())
436 }
437
438 pub fn flush(&self) -> Result<(), HandleTableError> {
439 self.mmap.flush()?;
440 Ok(())
441 }
442}
443
444#[cfg(test)]
445mod tests {
446 use super::*;
447 use std::thread;
448 use std::sync::Arc;
449
450 fn tmp(name: &str) -> std::path::PathBuf {
451 let mut p = std::env::temp_dir();
452 let pid = std::process::id();
453 p.push(format!("subetha-handle-{name}-{pid}.bin"));
454 p
455 }
456
457 #[test]
460 fn second_create_attaches_and_keeps_handles() {
461 let p = tmp("attach");
462 std::fs::remove_file(&p).ok();
463 let t: SharedHandleTable<u64> = SharedHandleTable::create(&p, 16).unwrap();
464 let h = t.insert(777).unwrap();
465
466 let t2: SharedHandleTable<u64> = SharedHandleTable::create(&p, 16).unwrap();
467 assert_eq!(t2.get(h), Some(777), "attach lost a live handle");
468 assert!(matches!(
469 SharedHandleTable::<u64>::create(&p, 8),
470 Err(HandleTableError::LayoutMismatch),
471 ));
472
473 drop(t);
476 drop(t2);
477 let fresh: SharedHandleTable<u64> = SharedHandleTable::reset(&p, 16).unwrap();
478 assert_eq!(fresh.get(h), None, "reset left a handle live");
479 assert_eq!(fresh.len(), 0);
480 drop(fresh);
481 std::fs::remove_file(&p).ok();
482 }
483
484 #[test]
485 fn insert_get_remove_round_trip() {
486 let p = tmp("rt");
487 let t: SharedHandleTable<u64> = SharedHandleTable::create(&p, 16).unwrap();
488 let h = t.insert(42).unwrap();
489 assert!(t.contains(h));
490 assert_eq!(t.get(h), Some(42));
491 assert_eq!(t.len(), 1);
492 let v = t.remove(h);
493 assert_eq!(v, Some(42));
494 assert!(!t.contains(h));
495 assert_eq!(t.get(h), None);
496 assert_eq!(t.len(), 0);
497 std::fs::remove_file(&p).ok();
498 }
499
500 #[test]
501 fn stale_handle_after_remove_returns_none() {
502 let p = tmp("stale");
503 let t: SharedHandleTable<u64> = SharedHandleTable::create(&p, 4).unwrap();
504 let h1 = t.insert(100).unwrap();
505 assert_eq!(t.remove(h1), Some(100));
506 let h2 = t.insert(200).unwrap();
508 assert_eq!(h2.slot(), h1.slot(), "free-list LIFO reused the slot");
509 assert_ne!(h2.generation(), h1.generation(), "generation differs");
510 assert_eq!(t.get(h1), None, "stale h1 must not see h2's value");
511 assert_eq!(t.get(h2), Some(200));
512 std::fs::remove_file(&p).ok();
513 }
514
515 #[test]
516 fn full_table_insert_returns_error() {
517 let p = tmp("full");
518 let t: SharedHandleTable<u64> = SharedHandleTable::create(&p, 2).unwrap();
519 let _val = t.insert(1).unwrap();
520 let _val = t.insert(2).unwrap();
521 assert_eq!(t.insert(3), Err(HandleTableError::Full));
522 std::fs::remove_file(&p).ok();
523 }
524
525 #[test]
526 fn cross_handle_visibility() {
527 let p = tmp("cross-handle");
528 let writer: SharedHandleTable<u64> = SharedHandleTable::create(&p, 16).unwrap();
529 let reader: SharedHandleTable<u64> = SharedHandleTable::open(&p, 16).unwrap();
530 let h = writer.insert(777).unwrap();
531 assert_eq!(reader.get(h), Some(777));
532 assert!(reader.contains(h));
533 assert_eq!(writer.remove(h), Some(777));
535 assert_eq!(reader.get(h), None);
536 std::fs::remove_file(&p).ok();
537 }
538
539 #[test]
540 fn concurrent_inserts_and_removes_preserve_count() {
541 let p = tmp("concurrent");
542 let t: Arc<SharedHandleTable<u64>>
543 = Arc::new(SharedHandleTable::create(&p, 256).unwrap());
544 let n_threads = 4usize;
545 let per_thread = 50usize;
546 let mut handles = vec![];
547 for tid in 0..n_threads {
548 let t = t.clone();
549 handles.push(thread::spawn(move || {
550 let mut owned = Vec::with_capacity(per_thread);
551 for i in 0..per_thread {
552 let v = (tid * per_thread + i) as u64;
553 loop {
554 match t.insert(v) {
555 Ok(h) => { owned.push((h, v)); break; }
556 Err(HandleTableError::Full) => {
557 std::thread::yield_now();
558 }
559 Err(e) => panic!("unexpected error: {e:?}"),
560 }
561 }
562 }
563 for &(h, v) in &owned {
565 assert_eq!(t.get(h), Some(v));
566 }
567 for (h, v) in owned {
569 assert_eq!(t.remove(h), Some(v));
570 }
571 }));
572 }
573 for h in handles { h.join().unwrap(); }
574 assert_eq!(t.len(), 0);
575 std::fs::remove_file(&p).ok();
576 }
577
578 #[test]
579 fn disk_persistence_survives_reopen() {
580 let p = tmp("disk-persist");
581 let h1;
582 let h2;
583 {
584 let t: SharedHandleTable<u64> = SharedHandleTable::create(&p, 8).unwrap();
585 h1 = t.insert(11).unwrap();
586 h2 = t.insert(22).unwrap();
587 t.flush().unwrap();
588 }
589 let t2: SharedHandleTable<u64> = SharedHandleTable::open(&p, 8).unwrap();
590 assert_eq!(t2.get(h1), Some(11));
591 assert_eq!(t2.get(h2), Some(22));
592 assert_eq!(t2.len(), 2);
593 std::fs::remove_file(&p).ok();
594 }
595
596 #[test]
597 fn null_handle_returns_none() {
598 let p = tmp("null-handle");
599 let t: SharedHandleTable<u64> = SharedHandleTable::create(&p, 4).unwrap();
600 assert_eq!(t.get(Handle::NULL), None);
601 assert!(!t.contains(Handle::NULL));
602 assert_eq!(t.remove(Handle::NULL), None);
603 std::fs::remove_file(&p).ok();
604 }
605
606 #[test]
607 fn handle_packing_round_trips() {
608 let h = Handle::from_parts(42, 1337);
609 assert_eq!(h.generation(), 42);
610 assert_eq!(h.slot(), 1337);
611 assert!(!h.is_null());
612 assert!(Handle::NULL.is_null());
613 assert_eq!(Handle::NULL.generation(), 0);
614 assert_eq!(Handle::NULL.slot(), 0);
615 }
616
617 #[test]
618 fn struct_payload_round_trip() {
619 #[derive(Clone, Copy, Debug, PartialEq)]
620 #[repr(C)]
621 struct Item { id: u32, weight: f32, flags: u32 }
622 let p = tmp("struct");
623 let t: SharedHandleTable<Item> = SharedHandleTable::create(&p, 8).unwrap();
624 let item = Item { id: 42, weight: 1.5, flags: 0xFF };
625 let h = t.insert(item).unwrap();
626 assert_eq!(t.get(h), Some(item));
627 std::fs::remove_file(&p).ok();
628 }
629
630 #[test]
631 fn payload_too_large_at_create() {
632 #[allow(dead_code)] struct Big([u8; SLOT_PAYLOAD_BYTES + 1]);
634 impl Copy for Big {}
635 impl Clone for Big { fn clone(&self) -> Self { *self } }
636 let p = tmp("too-large");
637 match SharedHandleTable::<Big>::create(&p, 4) {
638 Err(HandleTableError::PayloadTooLarge) => {}
639 other => panic!("expected PayloadTooLarge, got {:?}", other.as_ref().err()),
640 }
641 std::fs::remove_file(&p).ok();
642 }
643}