userspace_pagefault/
lib.rs

1#![warn(static_mut_refs)]
2
3use std::collections::BTreeMap;
4use std::num::NonZeroUsize;
5use std::os::fd::{AsRawFd, BorrowedFd, IntoRawFd, RawFd};
6use std::ptr::NonNull;
7use std::sync::Arc;
8use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering};
9
10pub use nix::libc;
11use nix::libc::c_void;
12use nix::sys::mman::MapFlags;
13pub use nix::sys::mman::ProtFlags;
14use nix::sys::signal;
15use nix::unistd;
16use parking_lot::Mutex;
17
18mod machdep;
19
20#[derive(Debug, PartialEq, Eq)]
21pub enum Error {
22    NullBase,
23    ZeroSize,
24    BaseNotAligned,
25    SizeNotAligned,
26    PageSizeUnavail,
27    Unsupported,
28    SegmentOverlap,
29    SegmentOutOfBound,
30    UnixError(nix::errno::Errno),
31}
32
33#[derive(Debug, PartialEq, Eq)]
34pub enum AccessType {
35    Read,
36    Write,
37}
38
39/// Backend store for the user-space paging. The store is a collection of pages for their
40/// persistent contents.
41///
42/// Because an access to the memory could wait for [PageStore::page_fault] to load the page, to
43/// avoid dead-lock, make sure the resources the store will acquire are not held by the code
44/// accessing the paged memory.
45///
46/// For example, using `println!("{}", mem.as_slice()[0])` could dead-lock the system if the
47/// implementation of [PageStore] also involves some `println`: the I/O is first locked before the
48/// dereference of `mem[0]` that induces a page fault, then because `page_fault` handler contains
49/// another `println`, it gets stuck.
50///
51/// As a good practice, always make sure [PageStore] grabs the least resources that do not overlap
52/// with the paged memory user.
53///
54/// Some platforms may have limited support of precisely telling the [AccessType] of a page fault.
55/// In such a case, [AccessType::Write] is used conservatively. In other words, a write can never
56/// be incorrectly categorized as [AccessType::Read], whereas a read could be categorized as a
57/// write.
58pub trait PageStore {
59    /// Callback that is triggered upon a page fault. Return `Some()` if the page needs to be
60    /// filled with the given data.
61    ///
62    /// The returned iterator will iterate through all chunks that together make up the entire data
63    /// for the page, in their order of concatenation. This avoids unnecessary copies if the page's
64    /// data partially come from different sources.
65    fn page_fault(
66        &mut self, offset: usize, length: usize, access: AccessType,
67    ) -> Option<Box<dyn Iterator<Item = Box<dyn AsRef<[u8]> + '_>> + '_>>;
68}
69
70/// A segment of OS memory.
71pub struct Segment {
72    base: AtomicPtr<u8>,
73    size: usize,
74    owned: bool,
75    shared: Mutex<Vec<SharedMemory>>,
76}
77
78impl Segment {
79    /// Create a new managed segment of memory.
80    ///
81    /// When `base` is None, new OS memory will be allocated. Otherwise, the given pointer will be
82    /// used, and [Segment] will not assume ownership of the underlying memory. `page_size` must be
83    /// a power of 2.
84    pub fn new(base: Option<*mut u8>, mut size: usize, page_size: usize, flags: ProtFlags) -> Result<Self, Error> {
85        let rem = size & (page_size - 1);
86        match base {
87            Some(base) => {
88                if (base as usize) & (page_size - 1) != 0 {
89                    return Err(Error::BaseNotAligned);
90                }
91                if rem != 0 {
92                    return Err(Error::SizeNotAligned);
93                }
94            }
95            None => {
96                if rem != 0 {
97                    // Round up to a whole page size.
98                    size += page_size - rem
99                }
100            }
101        }
102
103        let (base_ptr, map_flags) = match base {
104            Some(ptr) => (
105                Some(NonZeroUsize::new(ptr as usize).ok_or(Error::NullBase)?),
106                MapFlags::MAP_FIXED,
107            ),
108            None => (None, MapFlags::empty()),
109        };
110
111        let new_base = unsafe {
112            nix::sys::mman::mmap_anonymous(
113                base_ptr,
114                NonZeroUsize::new(size).ok_or(Error::ZeroSize)?,
115                flags,
116                map_flags | MapFlags::MAP_PRIVATE,
117            )
118            .map_err(Error::UnixError)?
119            .cast::<u8>()
120        };
121
122        if let Some(base) = base {
123            if base != new_base.as_ptr() {
124                return Err(Error::Unsupported);
125            }
126        }
127
128        Ok(Self {
129            base: AtomicPtr::new(new_base.as_ptr()),
130            size,
131            owned: base.is_none(),
132            shared: Mutex::new(Vec::new()),
133        })
134    }
135
136    /// Base pointer to the segment.
137    #[inline(always)]
138    pub fn base(&self) -> *mut u8 {
139        unsafe { *self.base.as_ptr() }
140    }
141
142    /// Access the segment as a byte slice.
143    #[inline(always)]
144    pub fn as_slice(&self) -> &mut [u8] {
145        unsafe { std::slice::from_raw_parts_mut(self.base(), self.size) }
146    }
147
148    /// Patch a [SharedMemory] to a part of this memory segment. The given `shm` will start with an
149    /// `offset` from the beginning of [Segment], and cannot overrun the end of the segment. The
150    /// given [SharedMemory] outlives this object once patched (even if it is shadowed by subsequent patching).
151    pub fn make_shared(&self, offset: usize, shm: &SharedMemory, flags: ProtFlags) -> Result<(), Error> {
152        let size = shm.0.size;
153        if offset + size >= self.size {
154            return Err(Error::SegmentOutOfBound);
155        }
156        unsafe {
157            nix::sys::mman::mmap(
158                Some(NonZeroUsize::new(self.base().add(offset) as usize).ok_or(Error::NullBase)?),
159                NonZeroUsize::new(size).ok_or(Error::ZeroSize)?,
160                flags,
161                MapFlags::MAP_FIXED | MapFlags::MAP_SHARED,
162                &shm.0.fd,
163                0,
164            )
165            .map_err(Error::UnixError)?;
166        }
167        // Keep a reference to the shared memory so it is not deallocated.
168        self.shared.lock().push(shm.clone());
169        Ok(())
170    }
171}
172
173impl Drop for Segment {
174    fn drop(&mut self) {
175        if self.owned {
176            unsafe {
177                if let Some(ptr) = NonNull::new(self.base() as *mut c_void) {
178                    if let Err(e) = nix::sys::mman::munmap(ptr, self.size) {
179                        eprintln!("Segment: Failed to munmap: {e:?}.");
180                    }
181                }
182            }
183        }
184    }
185}
186
187type SignalHandler = extern "C" fn(libc::c_int, *mut libc::siginfo_t, *mut c_void);
188
189static HANDLER_SPIN: AtomicBool = AtomicBool::new(false);
190static mut TO_HANDLER: (RawFd, RawFd) = (0, 1);
191static mut FROM_HANDLER: (RawFd, RawFd) = (0, 1);
192static mut FALLBACK_SIGSEGV_HANDLER: Option<SignalHandler> = None;
193static mut FALLBACK_SIGBUS_HANDLER: Option<SignalHandler> = None;
194
195static MANAGER: Mutex<PagedSegmentManager> = Mutex::new(PagedSegmentManager {
196    entries: BTreeMap::new(),
197});
198static MANAGER_THREAD: Mutex<Option<std::thread::JoinHandle<()>>> = Mutex::new(None);
199static INITIALIZED: AtomicBool = AtomicBool::new(false);
200const ADDR_SIZE: usize = std::mem::size_of::<usize>();
201
202#[inline]
203fn handle_page_fault_(info: *mut libc::siginfo_t, ctx: *mut c_void) -> bool {
204    // NOTE: this function should be SIGBUS/SIGSEGV-free, another signal can't be raised during the
205    // handling of the signal.
206    let (tx, rx, addr, ctx) = unsafe {
207        let (rx, _) = TO_HANDLER;
208        let (_, tx) = FROM_HANDLER;
209        (tx, rx, (*info).si_addr() as usize, &mut *(ctx as *mut libc::ucontext_t))
210    };
211    let flag = machdep::check_page_fault_rw_flag_from_context(*ctx);
212    let mut buff = [0; ADDR_SIZE + 1];
213    buff[..ADDR_SIZE].copy_from_slice(&addr.to_le_bytes());
214    buff[ADDR_SIZE] = flag;
215    // Use a spin lock to avoid ABA (another thread could interfere in-between read and write).
216    while HANDLER_SPIN.swap(true, Ordering::Acquire) {
217        std::thread::yield_now();
218    }
219    if unistd::write(unsafe { BorrowedFd::borrow_raw(tx) }, &buff).is_err() {
220        HANDLER_SPIN.swap(false, Ordering::Release);
221        return true;
222    }
223    // Wait until manager gives a response.
224    let _ = unistd::read(unsafe { BorrowedFd::borrow_raw(rx) }, &mut buff[..1]);
225    HANDLER_SPIN.swap(false, Ordering::Release);
226    // The first byte indicates if the request address is managed. (If so, it's already valid in
227    // memory.)
228    buff[0] == 1
229}
230
231extern "C" fn handle_page_fault(signum: libc::c_int, info: *mut libc::siginfo_t, ctx: *mut c_void) {
232    if !handle_page_fault_(info, ctx) {
233        return;
234    }
235    // Otherwise, not hitting a managed memory location, invoke the fallback handler.
236    unsafe {
237        let sig = signal::Signal::try_from(signum).expect("Invalid signum.");
238        let fallback_handler = match sig {
239            signal::SIGSEGV => FALLBACK_SIGSEGV_HANDLER,
240            signal::SIGBUS => FALLBACK_SIGBUS_HANDLER,
241            _ => panic!("Unknown signal: {}.", sig),
242        };
243
244        if let Some(handler) = fallback_handler {
245            // Delegate to the fallback handler
246            handler(signum, info, ctx);
247        } else {
248            // No fallback handler (was SIG_DFL or SIG_IGN), reset to default and raise
249            let sig_action = signal::SigAction::new(
250                signal::SigHandler::SigDfl,
251                signal::SaFlags::empty(),
252                signal::SigSet::empty(),
253            );
254            signal::sigaction(sig, &sig_action).expect("Fail to reset signal handler.");
255            signal::raise(sig).expect("Fail to raise SIG_DFL.");
256            unreachable!("SIG_DFL should have terminated the process");
257        }
258    }
259}
260
261unsafe fn register_signal_handlers(handler: SignalHandler) {
262    let register = |fallback_handler: *mut Option<SignalHandler>, sig: signal::Signal| {
263        // The flags here are relatively careful, and they are...
264        //
265        // SA_SIGINFO gives us access to information like the program counter from where the fault
266        // happened.
267        //
268        // SA_ONSTACK allows us to handle signals on an alternate stack, so that the handler can
269        // run in response to running out of stack space on the main stack. Rust installs an
270        // alternate stack with sigaltstack, so we rely on that.
271        //
272        // SA_NODEFER allows us to reenter the signal handler if we crash while handling the signal,
273        // instead of getting stuck.
274        let sig_action = signal::SigAction::new(
275            signal::SigHandler::SigAction(handler),
276            signal::SaFlags::SA_NODEFER | signal::SaFlags::SA_SIGINFO | signal::SaFlags::SA_ONSTACK,
277            signal::SigSet::empty(),
278        );
279
280        // Extract and save the fallback handler function pointer if it's a SigAction with SA_SIGINFO.
281        unsafe {
282            let sig = signal::sigaction(sig, &sig_action).expect("Fail to register signal handler.");
283            *fallback_handler = match sig.handler() {
284                signal::SigHandler::SigAction(h)
285                    if sig.flags() & signal::SaFlags::SA_SIGINFO == signal::SaFlags::SA_SIGINFO =>
286                {
287                    Some(h)
288                }
289                _ => None,
290            };
291        }
292    };
293
294    register(&raw mut FALLBACK_SIGSEGV_HANDLER, signal::SIGSEGV);
295    register(&raw mut FALLBACK_SIGBUS_HANDLER, signal::SIGBUS);
296}
297
298struct PagedSegmentEntry {
299    mem: Arc<Segment>,
300    store: Box<dyn PageStore + Send + 'static>,
301    start: usize,
302    len: usize,
303    page_size: usize,
304}
305
306struct PagedSegmentManager {
307    entries: BTreeMap<usize, PagedSegmentEntry>,
308}
309
310impl PagedSegmentManager {
311    fn insert(&mut self, entry: PagedSegmentEntry) -> bool {
312        if let Some((start, e)) = self.entries.range(..=entry.start).next_back() {
313            if start == &entry.start || start + e.len > entry.start {
314                return false;
315            }
316        }
317        assert!(self.entries.insert(entry.start, entry).is_none()); // Each key must be unique.
318        true
319    }
320
321    fn remove(&mut self, start: usize, len: usize) {
322        use std::collections::btree_map::Entry;
323        if let Entry::Occupied(e) = self.entries.entry(start) {
324            if e.get().len == len {
325                e.remove();
326                return;
327            }
328        }
329        panic!(
330            "Failed to locate PagedSegmentEntry (start = 0x{:x}, end = 0x{:x}).",
331            start,
332            start + len
333        )
334    }
335
336    fn hit(&mut self, addr: usize) -> Option<&mut PagedSegmentEntry> {
337        if let Some((start, e)) = self.entries.range_mut(..=addr).next_back() {
338            assert!(start <= &addr);
339            if start + e.len > addr {
340                return Some(e);
341            }
342        }
343        None
344    }
345}
346
347fn init() {
348    let (to_read, to_write) = nix::unistd::pipe().expect("Fail to create pipe to the handler.");
349    let (from_read, from_write) = nix::unistd::pipe().expect("Fail to create pipe from the handler.");
350    let from_handler = unsafe { BorrowedFd::borrow_raw(from_read.as_raw_fd()) };
351    let to_handler = unsafe { BorrowedFd::borrow_raw(to_write.as_raw_fd()) };
352    unsafe {
353        TO_HANDLER = (to_read.into_raw_fd(), to_write.into_raw_fd());
354        FROM_HANDLER = (from_read.into_raw_fd(), from_write.into_raw_fd());
355        register_signal_handlers(handle_page_fault);
356    }
357
358    std::sync::atomic::fence(Ordering::SeqCst);
359
360    let handle = std::thread::spawn(move || {
361        let mut buff = [0; ADDR_SIZE + 1];
362        loop {
363            if unistd::read(&from_handler, &mut buff).is_err() {
364                // Pipe closed, exit thread
365                break;
366            }
367            let addr = usize::from_le_bytes(buff[..ADDR_SIZE].try_into().unwrap());
368            let (access_type, mprotect_flag) = match buff[ADDR_SIZE] {
369                0 => (AccessType::Read, ProtFlags::PROT_READ),
370                _ => (AccessType::Write, ProtFlags::PROT_READ | ProtFlags::PROT_WRITE),
371            };
372            let mut mgr = MANAGER.lock();
373            let mut fallback = 1;
374            if let Some(entry) = mgr.hit(addr) {
375                let page_mask = usize::MAX ^ (entry.page_size - 1);
376                let page_addr = addr & page_mask;
377                let page_ptr = unsafe { NonNull::new_unchecked(page_addr as *mut c_void) };
378                // Load the page data.
379                let slice = entry.mem.as_slice();
380                let base = slice.as_ptr() as usize;
381                let page_offset = page_addr - base;
382                if let Some(page) = entry.store.page_fault(page_offset, entry.page_size, access_type) {
383                    unsafe {
384                        nix::sys::mman::mprotect(
385                            page_ptr,
386                            entry.page_size,
387                            ProtFlags::PROT_READ | ProtFlags::PROT_WRITE,
388                        )
389                        .expect("Failed to mprotect.");
390                    }
391                    let target = &mut slice[page_offset..page_offset + entry.page_size];
392                    let mut base = 0;
393                    for chunk in page {
394                        let chunk = (*chunk).as_ref();
395                        let chunk_len = chunk.len();
396                        target[base..base + chunk_len].copy_from_slice(&chunk);
397                        base += chunk_len;
398                    }
399                }
400                // Mark as readable/writable.
401                unsafe {
402                    nix::sys::mman::mprotect(page_ptr, entry.page_size, mprotect_flag).expect("Failed to mprotect.");
403                }
404                fallback = 0;
405            }
406            // Otherwise this SIGSEGV falls through and we don't do anything about it.
407            if unistd::write(&to_handler, &[fallback]).is_err() {
408                // Pipe closed, exit thread
409                break;
410            }
411        }
412    });
413    *MANAGER_THREAD.lock() = Some(handle);
414}
415
416/// Memory segment that allows customized page fault handling in user space.
417pub struct PagedSegment<'a> {
418    mem: Arc<Segment>,
419    page_size: usize,
420    _phantom: std::marker::PhantomData<&'a ()>,
421}
422
423impl<'a> PagedSegment<'a> {
424    /// Make the user-space paged segement at an existing memory location.
425    pub unsafe fn from_raw<S: PageStore + Send + 'static>(
426        base: *mut u8, size: usize, store: S, page_size: Option<usize>,
427    ) -> Result<PagedSegment<'static>, Error> {
428        let mem: &'static mut [u8] = unsafe { std::slice::from_raw_parts_mut(base, size) };
429        Self::new_(Some(mem.as_ptr() as *mut u8), mem.len(), store, page_size)
430    }
431
432    /// Allocate a new segment to be user-space paged.
433    pub fn new<S: PageStore + Send + 'static>(
434        length: usize, store: S, page_size: Option<usize>,
435    ) -> Result<PagedSegment<'static>, Error> {
436        Self::new_(None, length, store, page_size)
437    }
438
439    fn new_<'b, S: PageStore + Send + 'static>(
440        base: Option<*mut u8>, length: usize, store: S, page_size: Option<usize>,
441    ) -> Result<PagedSegment<'b>, Error> {
442        // Lazy initialization of the process-level manager and its state.
443        if !INITIALIZED.swap(true, Ordering::AcqRel) {
444            init();
445        }
446        let page_size = match page_size {
447            Some(s) => s,
448            None => get_page_size()?,
449        };
450        let mem = std::sync::Arc::new(Segment::new(base, length, page_size, ProtFlags::PROT_NONE)?);
451        let mut mgr = MANAGER.lock();
452        if !mgr.insert(PagedSegmentEntry {
453            mem: mem.clone(),
454            store: Box::new(store),
455            start: mem.base() as usize,
456            len: length,
457            page_size,
458        }) {
459            return Err(Error::SegmentOverlap);
460        }
461
462        Ok(PagedSegment {
463            mem,
464            page_size,
465            _phantom: std::marker::PhantomData,
466        })
467    }
468
469    pub fn as_slice_mut(&mut self) -> &mut [u8] {
470        self.mem.as_slice()
471    }
472
473    pub fn as_slice(&self) -> &[u8] {
474        self.mem.as_slice()
475    }
476
477    pub fn as_raw_parts(&self) -> (*mut u8, usize) {
478        let s = self.mem.as_slice();
479        (s.as_mut_ptr(), s.len())
480    }
481
482    /// Return the page size in use.
483    pub fn page_size(&self) -> usize {
484        self.page_size
485    }
486
487    /// Mark the entire [PagedSegment] to be aware of all subsequent writes, this will trigger
488    /// write-access page faults again when write operation is made in the future, even though the
489    /// location was previously written.
490    pub fn reset_write_detection(&self, offset: usize, size: usize) -> Result<(), Error> {
491        assert!(offset + size <= self.mem.size);
492        unsafe {
493            let ptr = NonNull::new_unchecked(self.mem.base().add(offset) as *mut c_void);
494            nix::sys::mman::mprotect(ptr, size, ProtFlags::PROT_READ).map_err(Error::UnixError)?;
495        }
496        Ok(())
497    }
498
499    /// Release the OS page with its content loaded from [PageStore]. The next access to an address
500    /// within this page will trigger a page fault.
501    ///
502    /// This operation punches "holes" on the managed segment to free the pages back to the OS.
503    /// `page_offset` must be aligned to the beginning of a page. It should be one of the offsets
504    /// historically passed to [PageStore::page_fault].
505    pub fn release_page(&self, page_offset: usize) -> Result<(), Error> {
506        if page_offset & (self.page_size - 1) != 0 {
507            return Err(Error::BaseNotAligned);
508        }
509        if page_offset >= self.mem.size {
510            return Err(Error::SegmentOutOfBound);
511        }
512        let page_addr = self.mem.base() as usize + page_offset;
513        unsafe {
514            nix::sys::mman::mmap_anonymous(
515                Some(NonZeroUsize::new(page_addr).ok_or(Error::NullBase)?),
516                NonZeroUsize::new(self.page_size).ok_or(Error::ZeroSize)?,
517                ProtFlags::PROT_NONE,
518                MapFlags::MAP_FIXED | MapFlags::MAP_PRIVATE,
519            )
520            .map_err(Error::UnixError)?;
521        }
522        Ok(())
523    }
524
525    /// Release all OS pages with their contents in this [PagedSegment].
526    pub fn release_all_pages(&self) -> Result<(), Error> {
527        unsafe {
528            nix::sys::mman::mmap_anonymous(
529                Some(NonZeroUsize::new(self.mem.base() as usize).ok_or(Error::NullBase)?),
530                NonZeroUsize::new(self.mem.size).ok_or(Error::ZeroSize)?,
531                ProtFlags::PROT_NONE,
532                MapFlags::MAP_FIXED | MapFlags::MAP_PRIVATE,
533            )
534            .map_err(Error::UnixError)?;
535        }
536        self.mem.shared.lock().clear();
537        Ok(())
538    }
539
540    /// Patch a [SharedMemory] to a part of this memory segment. The given `shm` will start with an
541    /// `offset` from the beginning of [PagedSegment], and cannot overrun the end of the segment. The
542    /// given [SharedMemory] outlives this object once patched (even if it is shadowed by subsequent patches).
543    pub fn make_shared(&self, offset: usize, shm: &SharedMemory) -> Result<(), Error> {
544        self.mem.make_shared(offset, shm, ProtFlags::PROT_NONE)
545    }
546}
547
548impl<'a> Drop for PagedSegment<'a> {
549    fn drop(&mut self) {
550        let mut mgr = MANAGER.lock();
551        mgr.remove(self.mem.base() as usize, self.mem.size);
552    }
553}
554
555/// Reference used to share memory among [Segment] (or [PagedSegment]).
556///
557/// For the reference created by [SharedMemory::new], the patched portion of [Segment] or
558/// [PagedSegment] will be mapped to share the same memory content. Cloning [SharedMemory] only
559/// duplicates the reference, not the underlying memory.
560#[derive(Clone)]
561pub struct SharedMemory(Arc<SharedMemoryInner>);
562
563struct SharedMemoryInner {
564    fd: std::os::fd::OwnedFd,
565    size: usize,
566}
567
568impl SharedMemory {
569    pub fn new(size: usize) -> Result<Self, Error> {
570        let fd = machdep::get_shared_memory()?;
571        nix::unistd::ftruncate(&fd, size as libc::off_t).map_err(Error::UnixError)?;
572        Ok(Self(Arc::new(SharedMemoryInner { fd, size })))
573    }
574}
575
576/// Obtain the page size of the current platform.
577pub fn get_page_size() -> Result<usize, Error> {
578    Ok(unistd::sysconf(unistd::SysconfVar::PAGE_SIZE)
579        .map_err(Error::UnixError)?
580        .ok_or(Error::PageSizeUnavail)? as usize)
581}
582
583pub struct VecPageStore(Vec<u8>);
584
585impl VecPageStore {
586    pub fn new(vec: Vec<u8>) -> Self {
587        Self(vec)
588    }
589}
590
591impl PageStore for VecPageStore {
592    fn page_fault(
593        &mut self, offset: usize, length: usize, _access: AccessType,
594    ) -> Option<Box<dyn Iterator<Item = Box<dyn AsRef<[u8]> + '_>> + '_>> {
595        #[cfg(debug_assertions)]
596        println!(
597            "{:?} loading page at 0x{:x} access={:?}",
598            self as *mut Self, offset, _access,
599        );
600        Some(Box::new(std::iter::once(
601            Box::new(&self.0[offset..offset + length]) as Box<dyn AsRef<[u8]>>
602        )))
603    }
604}
605
606#[cfg(test)]
607mod tests {
608    use super::*;
609    use lazy_static::lazy_static;
610    use parking_lot::Mutex;
611
612    lazy_static! {
613        static ref PAGE_SIZE: usize = unistd::sysconf(unistd::SysconfVar::PAGE_SIZE).unwrap().unwrap() as usize;
614    }
615
616    static TEST_MUTEX: Mutex<()> = Mutex::new(());
617
618    #[test]
619    fn test1() {
620        let _guard = TEST_MUTEX.lock();
621        for _ in 0..100 {
622            let mut v = Vec::new();
623            v.resize(*PAGE_SIZE * 100, 0);
624            v[0] = 42;
625            v[*PAGE_SIZE * 10 + 1] = 43;
626            v[*PAGE_SIZE * 20 + 1] = 44;
627
628            let pm = PagedSegment::new(*PAGE_SIZE * 100, VecPageStore::new(v), None).unwrap();
629            let m = pm.as_slice();
630            assert_eq!(m[0], 42);
631            assert_eq!(m[*PAGE_SIZE * 10 + 1], 43);
632            assert_eq!(m[*PAGE_SIZE * 20 + 1], 44);
633        }
634    }
635
636    #[test]
637    fn test2() {
638        let _guard = TEST_MUTEX.lock();
639        for _ in 0..100 {
640            let mut v = Vec::new();
641            v.resize(*PAGE_SIZE * 100, 0);
642            v[0] = 1;
643            v[*PAGE_SIZE * 10 + 1] = 2;
644            v[*PAGE_SIZE * 20 + 1] = 3;
645
646            let pm1 = PagedSegment::new(*PAGE_SIZE * 100, VecPageStore::new(v), None).unwrap();
647
648            let mut v = Vec::new();
649            v.resize(*PAGE_SIZE * 100, 0);
650            for (i, v) in v.iter_mut().enumerate() {
651                *v = i as u8;
652            }
653            let mut pm2 = PagedSegment::new(*PAGE_SIZE * 100, VecPageStore::new(v), None).unwrap();
654
655            let m2 = pm2.as_slice_mut();
656            let m1 = pm1.as_slice();
657
658            assert_eq!(m2[100], 100);
659            m2[100] = 0;
660            assert_eq!(m2[100], 0);
661
662            assert_eq!(m1[0], 1);
663            assert_eq!(m1[*PAGE_SIZE * 10 + 1], 2);
664            assert_eq!(m1[*PAGE_SIZE * 20 + 1], 3);
665        }
666    }
667
668    #[test]
669    fn test_shared_memory() {
670        let _guard = TEST_MUTEX.lock();
671        let mut v = Vec::new();
672        v.resize(*PAGE_SIZE * 100, 0);
673        v[0] = 42;
674        v[*PAGE_SIZE * 10 + 1] = 43;
675        v[*PAGE_SIZE * 20 + 1] = 44;
676
677        let shm = SharedMemory::new(*PAGE_SIZE).unwrap();
678        let mut pm1 = PagedSegment::new(*PAGE_SIZE * 100, VecPageStore::new(v.clone()), None).unwrap();
679        let pm2 = PagedSegment::new(*PAGE_SIZE * 100, VecPageStore::new(v), None).unwrap();
680        pm1.make_shared(*PAGE_SIZE * 10, &shm).unwrap();
681        pm2.make_shared(*PAGE_SIZE * 10, &shm).unwrap();
682
683        assert_eq!(pm1.as_slice()[*PAGE_SIZE * 10 + 1], 43);
684        assert_eq!(pm2.as_slice()[*PAGE_SIZE * 10 + 1], 43);
685        pm1.as_slice_mut()[*PAGE_SIZE * 10 + 1] = 99;
686        assert_eq!(pm2.as_slice()[*PAGE_SIZE * 10 + 1], 99);
687        assert_eq!(pm1.as_slice()[*PAGE_SIZE * 10 + 1], 99);
688
689        let m = pm1.as_slice();
690        assert_eq!(m[0], 42);
691        assert_eq!(m[*PAGE_SIZE * 20 + 1], 44);
692    }
693
694    #[test]
695    fn test_release_page() {
696        let _guard = TEST_MUTEX.lock();
697        let mut v = Vec::new();
698        v.resize(*PAGE_SIZE * 20, 0);
699        v[0] = 42;
700        v[*PAGE_SIZE * 10 + 1] = 43;
701
702        let pm = PagedSegment::new(*PAGE_SIZE * 100, VecPageStore::new(v), None).unwrap();
703        let m = pm.as_slice();
704        assert_eq!(m[0], 42);
705        assert_eq!(m[*PAGE_SIZE * 10 + 1], 43);
706        for _ in 0..5 {
707            pm.release_page(0).unwrap();
708            pm.release_page(*PAGE_SIZE * 10).unwrap();
709            assert_eq!(m[0], 42);
710            assert_eq!(m[*PAGE_SIZE * 10 + 1], 43);
711        }
712    }
713
714    #[test]
715    fn out_of_order_scan() {
716        let _guard = TEST_MUTEX.lock();
717        let mut v = Vec::new();
718        v.resize(*PAGE_SIZE * 100, 0);
719        for (i, v) in v.iter_mut().enumerate() {
720            *v = i as u8;
721        }
722        let store = VecPageStore::new(v);
723        let pm = PagedSegment::new(*PAGE_SIZE * 100, store, None).unwrap();
724        use rand::{SeedableRng, seq::SliceRandom};
725        use rand_chacha::ChaChaRng;
726        let seed = [0; 32];
727        let mut rng = ChaChaRng::from_seed(seed);
728
729        let m = pm.as_slice();
730        let mut idxes = Vec::new();
731        for i in 0..m.len() {
732            idxes.push(i);
733        }
734        idxes.shuffle(&mut rng);
735        for i in idxes.into_iter() {
736            #[cfg(debug_assertions)]
737            {
738                let x = m[i];
739                println!("m[0x{:08x}] = {}", i, x);
740            }
741            assert_eq!(m[i], i as u8);
742        }
743    }
744
745    use signal::{SaFlags, SigAction, SigHandler, SigSet, Signal};
746
747    /// Reset the handler state for testing
748    unsafe fn handler_reset_init() {
749        unsafe {
750            // Close pipe file descriptors to cause the handler thread to exit
751            let (to_read, to_write) = TO_HANDLER;
752            let (from_read, from_write) = FROM_HANDLER;
753
754            if to_read != 0 {
755                let _ = nix::unistd::close(to_read);
756            }
757            if to_write != 1 {
758                let _ = nix::unistd::close(to_write);
759            }
760            if from_read != 0 {
761                let _ = nix::unistd::close(from_read);
762            }
763            if from_write != 1 {
764                let _ = nix::unistd::close(from_write);
765            }
766
767            // Wait for the handler thread to exit
768            if let Some(handle) = MANAGER_THREAD.lock().take() {
769                let _ = handle.join();
770            }
771
772            // Reset signal handlers to SIG_DFL so next init sees default handlers
773            let sig_dfl = SigAction::new(SigHandler::SigDfl, SaFlags::empty(), SigSet::empty());
774            let _ = signal::sigaction(Signal::SIGSEGV, &sig_dfl);
775            let _ = signal::sigaction(Signal::SIGBUS, &sig_dfl);
776
777            // Clear fallback handlers
778            FALLBACK_SIGSEGV_HANDLER = None;
779            FALLBACK_SIGBUS_HANDLER = None;
780
781            // Reset pipe fds to initial values
782            TO_HANDLER = (0, 1);
783            FROM_HANDLER = (0, 1);
784
785            // Reset the init flag so init() can run again
786            INITIALIZED.store(false, Ordering::Release);
787        }
788    }
789
790    static SIGSEGV_CALLED: AtomicBool = AtomicBool::new(false);
791    static SIGBUS_CALLED: AtomicBool = AtomicBool::new(false);
792
793    // Make the memory accessible so the instruction can succeed on retry
794    fn make_test_mem_valid(info: *mut libc::siginfo_t) {
795        unsafe {
796            let addr = (*info).si_addr();
797            let page_size = nix::unistd::sysconf(nix::unistd::SysconfVar::PAGE_SIZE)
798                .unwrap()
799                .unwrap() as usize;
800            let page_addr = (addr as usize) & !(page_size - 1);
801            nix::sys::mman::mprotect(
802                NonNull::new_unchecked(page_addr as *mut c_void),
803                page_size,
804                ProtFlags::PROT_READ | ProtFlags::PROT_WRITE,
805            )
806            .expect("mprotect failed in handler");
807        }
808    }
809
810    extern "C" fn test_sigsegv_handler(_signum: libc::c_int, info: *mut libc::siginfo_t, _ctx: *mut c_void) {
811        SIGSEGV_CALLED.store(true, Ordering::SeqCst);
812        make_test_mem_valid(info);
813    }
814
815    extern "C" fn test_sigbus_handler(_signum: libc::c_int, info: *mut libc::siginfo_t, _ctx: *mut c_void) {
816        SIGBUS_CALLED.store(true, Ordering::SeqCst);
817        make_test_mem_valid(info);
818    }
819
820    #[test]
821    fn test_fallback_handlers_set_and_called() {
822        let _guard = TEST_MUTEX.lock();
823
824        unsafe {
825            // Reset handler state before test
826            handler_reset_init();
827
828            // Register the fallback SIGSEGV handler
829            let sigsegv_action = SigAction::new(
830                SigHandler::SigAction(test_sigsegv_handler),
831                SaFlags::SA_SIGINFO | SaFlags::SA_NODEFER,
832                SigSet::empty(),
833            );
834            signal::sigaction(Signal::SIGSEGV, &sigsegv_action).expect("failed to set SIGSEGV handler");
835
836            // Register the fallback SIGBUS handler
837            let sigbus_action = SigAction::new(
838                SigHandler::SigAction(test_sigbus_handler),
839                SaFlags::SA_SIGINFO | SaFlags::SA_NODEFER,
840                SigSet::empty(),
841            );
842            signal::sigaction(Signal::SIGBUS, &sigbus_action).expect("failed to set SIGBUS handler");
843
844            // Create a PagedSegment - this will trigger init() which considers the fallback
845            // handlers.
846            let _pm1 = PagedSegment::new(*PAGE_SIZE, VecPageStore::new(vec![0u8; *PAGE_SIZE]), None).unwrap();
847
848            // Save the handler pointers to verify they don't change
849            let saved_sigsegv = FALLBACK_SIGSEGV_HANDLER.map(|f| f as usize);
850            let saved_sigbus = FALLBACK_SIGBUS_HANDLER.map(|f| f as usize);
851
852            // Verify that the fallback handlers were saved
853            assert!(saved_sigsegv.is_some(), "SIGSEGV fallback handler should be saved");
854            assert!(saved_sigbus.is_some(), "SIGBUS fallback handler should be saved");
855
856            // Create another PagedSegment - init () should NOT run again due to Once guard
857            let _pm2 = PagedSegment::new(*PAGE_SIZE, VecPageStore::new(vec![0u8; *PAGE_SIZE]), None).unwrap();
858
859            // Verify the handler pointers haven't changed (Once guard prevented re-registration)
860            let current_sigsegv = FALLBACK_SIGSEGV_HANDLER.map(|f| f as usize);
861            let current_sigbus = FALLBACK_SIGBUS_HANDLER.map(|f| f as usize);
862            assert_eq!(
863                current_sigsegv, saved_sigsegv,
864                "SIGSEGV fallback handler should not change"
865            );
866            assert_eq!(
867                current_sigbus, saved_sigbus,
868                "SIGBUS fallback handler should not change"
869            );
870
871            // Test SIGSEGV/SIGBUS fallback by accessing memory protected with PROT_NONE
872            SIGSEGV_CALLED.store(false, Ordering::SeqCst);
873            SIGBUS_CALLED.store(false, Ordering::SeqCst);
874
875            // Allocate memory with PROT_NONE to trigger SIGSEGV or SIGBUS when accessed
876            let test_mem = nix::sys::mman::mmap_anonymous(
877                None,
878                NonZeroUsize::new(*PAGE_SIZE).unwrap(),
879                ProtFlags::PROT_NONE,
880                MapFlags::MAP_PRIVATE | MapFlags::MAP_ANONYMOUS,
881            )
882            .expect("mmap failed");
883            // Access the protected memory - this triggers SIGSEGV or SIGBUS, handler makes it accessible
884            std::ptr::write_volatile(test_mem.cast::<u8>().as_ptr(), 42);
885            // Verify at least one fallback handler was called (platform dependent)
886            assert!(
887                SIGSEGV_CALLED.load(Ordering::SeqCst) || SIGBUS_CALLED.load(Ordering::SeqCst),
888                "SIGSEGV or SIGBUS fallback handler should have been called"
889            );
890            // Clean up
891            nix::sys::mman::munmap(test_mem.cast(), *PAGE_SIZE).expect("munmap failed");
892            // Reset handler state after test
893            handler_reset_init();
894        }
895    }
896}
userspace_pagefault/lib.rs

userspace_pagefault/
lib.rs
