starry_kernel/mm/aspace/
objects.rs

1//! Typed page ownership and reverse-mapping records.
2
3use alloc::{sync::Arc, vec::Vec};
4use core::{
5    any::Any,
6    fmt,
7    hash::{Hash, Hasher},
8    sync::atomic::{AtomicBool, AtomicU8, Ordering},
9};
10
11use ax_memory_addr::{MemoryAddr, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange};
12use ax_runtime::hal::paging::HugeSplitDeposit;
13
14use super::{AddressSpaceId, MappingId, PageOrder, RssKind};
15use crate::sync::IrqMutex;
16
17#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
18pub struct PageId(u64);
19
20impl PageId {
21    pub fn allocate() -> Self {
22        static NEXT_ID: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(1);
23        Self(NEXT_ID.fetch_add(1, Ordering::Relaxed))
24    }
25
26    pub const fn new(value: u64) -> Self {
27        Self(value)
28    }
29
30    pub const fn get(self) -> u64 {
31        self.0
32    }
33}
34
35/// Shared allocation owner behind one or more bounded frame leases.
36///
37/// A huge allocation may be split into base-page PageObjects without asking
38/// the buddy allocator to retag the block: each sublease keeps this allocation
39/// alive, and only the last sublease releases the original range.
40struct FrameAllocation {
41    base: PhysAddr,
42    bytes: usize,
43    /// `None` is used for a frame borrowed from platform firmware or a test
44    /// fixture.  Allocated frames carry their allocator alignment and are
45    /// released exactly once by this token's destructor.
46    release_align: Option<usize>,
47    /// The default release path is the user/data-frame allocator.  Page-table
48    /// detached tokens use a different allocator domain, so the lease carries
49    /// an explicit function pointer rather than making callers smuggle a raw
50    /// physical address through the ownership API.
51    release: fn(PhysAddr, usize),
52    /// External allocations (DMA/device/cache pages) retain their provider in
53    /// the same capability that names the physical range.  The MM layer never
54    /// has to keep a parallel owner beside a bare address list.
55    _anchor: Option<Arc<dyn Any + Send + Sync>>,
56}
57
58impl Drop for FrameAllocation {
59    fn drop(&mut self) {
60        if let Some(align) = self.release_align.take() {
61            (self.release)(self.base, align);
62        }
63    }
64}
65
66/// A bounded capability for a physical frame range owned by a PageObject.
67/// Allocation/deallocation policy remains in the architecture allocator; the
68/// MM layer never treats a bare `PhysAddr` as ownership.
69#[derive(Clone)]
70pub struct FrameLease {
71    paddr: PhysAddr,
72    bytes: usize,
73    allocation: Arc<FrameAllocation>,
74}
75
76impl fmt::Debug for FrameLease {
77    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
78        formatter
79            .debug_struct("FrameLease")
80            .field("paddr", &self.paddr)
81            .field("bytes", &self.bytes)
82            .field("owned", &self.allocation.release_align.is_some())
83            .field("anchored", &self.allocation._anchor.is_some())
84            .finish()
85    }
86}
87
88fn release_data_frame(paddr: PhysAddr, align: usize) {
89    super::backend::dealloc_frame(paddr, align);
90}
91
92impl FrameLease {
93    pub fn new(paddr: PhysAddr) -> Self {
94        Self {
95            paddr,
96            bytes: PAGE_SIZE_4K,
97            allocation: Arc::new(FrameAllocation {
98                base: paddr,
99                bytes: PAGE_SIZE_4K,
100                release_align: None,
101                release: release_data_frame,
102                _anchor: None,
103            }),
104        }
105    }
106
107    /// Creates a non-releasing lease whose provider remains alive for as long
108    /// as any PageObject or sublease can name this physical range.
109    pub fn borrowed(
110        paddr: PhysAddr,
111        bytes: usize,
112        anchor: Option<Arc<dyn Any + Send + Sync>>,
113    ) -> Option<Self> {
114        if bytes == 0 {
115            return None;
116        }
117        Some(Self {
118            paddr,
119            bytes,
120            allocation: Arc::new(FrameAllocation {
121                base: paddr,
122                bytes,
123                release_align: None,
124                release: release_data_frame,
125                _anchor: anchor,
126            }),
127        })
128    }
129
130    /// Creates an owning lease for a frame allocated by the Starry MM backend.
131    /// The alignment is the same order/size passed to `alloc_frame`.
132    ///
133    /// Ownership cannot be fabricated from an arbitrary physical address:
134    /// ```compile_fail
135    /// fn fabricate_owner() -> starry_kernel::FrameLease {
136    ///     starry_kernel::FrameLease::owned(ax_memory_addr::PhysAddr::from(0x1000usize), 4096)
137    /// }
138    /// ```
139    ///
140    /// # Safety
141    ///
142    /// `paddr` must be the unique, still-owned result of the Starry MM frame
143    /// allocator for exactly `align` bytes/alignment. The caller transfers
144    /// that ownership and must neither free it nor construct another owner.
145    pub unsafe fn owned(paddr: PhysAddr, align: usize) -> Self {
146        Self {
147            paddr,
148            bytes: align,
149            allocation: Arc::new(FrameAllocation {
150                base: paddr,
151                bytes: align,
152                release_align: Some(align),
153                release: release_data_frame,
154                _anchor: None,
155            }),
156        }
157    }
158
159    /// Creates an owning lease whose release function belongs to another
160    /// allocator domain (for example page-table frames).  The function is a
161    /// static capability, not a closure, so dropping a lease remains IRQ-safe
162    /// and cannot allocate.
163    pub fn owned_with_releaser(
164        paddr: PhysAddr,
165        align: usize,
166        release: fn(PhysAddr, usize),
167    ) -> Self {
168        Self {
169            paddr,
170            bytes: align,
171            allocation: Arc::new(FrameAllocation {
172                base: paddr,
173                bytes: align,
174                release_align: Some(align),
175                release,
176                _anchor: None,
177            }),
178        }
179    }
180
181    pub const fn paddr(&self) -> PhysAddr {
182        self.paddr
183    }
184
185    pub const fn size(&self) -> usize {
186        self.bytes
187    }
188
189    /// Derives a bounded lease for a subrange of this allocation. The returned
190    /// lease shares only the final-release owner; its physical identity and
191    /// bounds are independent and can back a separate PageObject/rmap set.
192    pub fn sublease(&self, offset: usize, bytes: usize) -> Option<Self> {
193        if bytes == 0 {
194            return None;
195        }
196        let end = offset.checked_add(bytes)?;
197        if end > self.bytes {
198            return None;
199        }
200        let paddr = self.paddr.as_usize().checked_add(offset)?;
201        let allocation_offset = paddr.checked_sub(self.allocation.base.as_usize())?;
202        if allocation_offset.checked_add(bytes)? > self.allocation.bytes {
203            return None;
204        }
205        Some(Self {
206            paddr: PhysAddr::from_usize(paddr),
207            bytes,
208            allocation: self.allocation.clone(),
209        })
210    }
211}
212
213#[repr(u8)]
214#[derive(Debug, Clone, Copy, PartialEq, Eq)]
215pub enum PageState {
216    Reserved  = 0,
217    Present   = 1,
218    /// Anonymous page carrying the Linux `MADV_FREE` lazy-reclaim mark.
219    LazyFree  = 2,
220    Evicting  = 3,
221    Writeback = 4,
222    Retired   = 5,
223}
224
225impl PageState {
226    fn from_u8(value: u8) -> Self {
227        match value {
228            0 => Self::Reserved,
229            1 => Self::Present,
230            2 => Self::LazyFree,
231            3 => Self::Evicting,
232            4 => Self::Writeback,
233            _ => Self::Retired,
234        }
235    }
236}
237
238/// Reverse mappings are explicit records rather than an implicit scan of all
239/// VMAs.  A slot remains in this set until its PTE invalidation is acknowledged.
240#[derive(Debug, Default)]
241pub struct RmapSet {
242    entries: IrqMutex<RmapEntries>,
243}
244
245#[derive(Debug, Default)]
246struct RmapEntries {
247    keys: Vec<MappingSlotKey>,
248    /// Capacity owned by prepared transactions, not yet installed mappings.
249    reserved: usize,
250}
251
252#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
253pub struct MappingSlotKey {
254    pub space_id: AddressSpaceId,
255    pub va: VirtAddr,
256}
257
258/// An exclusive claim on rmap capacity prepared outside IRQ-saving locks.
259///
260/// Preparation installs any larger backing store before returning this token.
261/// Other publishers reserve their own space, so committing this claim cannot
262/// lose its capacity to a concurrent fork. Displaced storage remains owned by
263/// the token until all caller-held PTE and graph guards have been released.
264pub(crate) struct MappingGraphReservation<'a> {
265    owner: &'a RmapSet,
266    additional: usize,
267    displaced: Vec<MappingSlotKey>,
268}
269
270impl Drop for MappingGraphReservation<'_> {
271    fn drop(&mut self) {
272        if self.additional != 0 {
273            let mut entries = self.owner.entries.lock();
274            entries.reserved -= self.additional;
275        }
276        // Free backing storage only after the cancellation guard has gone away.
277        drop(core::mem::take(&mut self.displaced));
278    }
279}
280
281impl Hash for MappingSlotKey {
282    fn hash<H: Hasher>(&self, state: &mut H) {
283        self.space_id.hash(state);
284        self.va.as_usize().hash(state);
285    }
286}
287
288impl RmapSet {
289    fn prepare_replace(
290        &self,
291        additional: usize,
292    ) -> Result<MappingGraphReservation<'_>, MappingGraphError> {
293        let mut replacement = Vec::new();
294        loop {
295            let mut entries = self.entries.lock();
296            let reserved = entries
297                .reserved
298                .checked_add(additional)
299                .ok_or(MappingGraphError::ResourceExhausted)?;
300            let required = entries
301                .keys
302                .len()
303                .checked_add(reserved)
304                .ok_or(MappingGraphError::ResourceExhausted)?;
305            if entries.keys.capacity() < required {
306                if replacement.capacity() < required {
307                    drop(entries);
308                    replacement
309                        .try_reserve_exact(required)
310                        .map_err(|_| MappingGraphError::ResourceExhausted)?;
311                    continue;
312                }
313                replacement.extend_from_slice(&entries.keys);
314                core::mem::swap(&mut entries.keys, &mut replacement);
315            }
316            entries.reserved = reserved;
317            return Ok(MappingGraphReservation {
318                owner: self,
319                additional,
320                displaced: replacement,
321            });
322        }
323    }
324
325    /// Commits a capacity claim without allocating or freeing backing storage.
326    /// Validation precedes both key changes and consumption of the claim.
327    fn replace_reserved(
328        &self,
329        old: &[MappingSlotKey],
330        new: &[MappingSlotKey],
331        reservation: &mut MappingGraphReservation<'_>,
332    ) -> Result<(), MappingGraphError> {
333        if !core::ptr::eq(reservation.owner, self) {
334            return Err(MappingGraphError::SlotIdentityMismatch);
335        }
336        if new.len().saturating_sub(old.len()) > reservation.additional {
337            return Err(MappingGraphError::SlotStateConflict);
338        }
339        let mut entries = self.entries.lock();
340        for (index, key) in old.iter().enumerate() {
341            if old[..index].contains(key) || !entries.keys.contains(key) {
342                return Err(MappingGraphError::MissingOldSlot);
343            }
344        }
345        for (index, key) in new.iter().enumerate() {
346            if new[..index].contains(key)
347                || (entries.keys.contains(key) && !old.contains(key))
348            {
349                return Err(MappingGraphError::DuplicateNewSlot);
350            }
351        }
352        for key in old {
353            let index = entries
354                .keys
355                .iter()
356                .position(|entry| entry == key)
357                .ok_or(MappingGraphError::MissingOldSlot)?;
358            entries.keys.swap_remove(index);
359        }
360        entries.keys.extend_from_slice(new);
361        entries.reserved -= reservation.additional;
362        reservation.additional = 0;
363        Ok(())
364    }
365
366    pub fn try_snapshot(&self) -> Result<Vec<MappingSlotKey>, MappingGraphError> {
367        let mut snapshot = Vec::new();
368        loop {
369            let required = self.entries.lock().keys.len();
370            if snapshot.capacity() < required {
371                snapshot
372                    .try_reserve_exact(required.saturating_sub(snapshot.len()))
373                    .map_err(|_| MappingGraphError::ResourceExhausted)?;
374            }
375            let entries = self.entries.lock();
376            if entries.keys.len() > snapshot.capacity() {
377                drop(entries);
378                continue;
379            }
380            snapshot.clear();
381            snapshot.extend_from_slice(&entries.keys);
382            return Ok(snapshot);
383        }
384    }
385
386    #[cfg(test)]
387    pub fn snapshot(&self) -> Vec<MappingSlotKey> {
388        self.try_snapshot()
389            .expect("test reverse-mapping snapshot allocation must succeed")
390    }
391
392    fn all_mappings_belong_to(&self, mm_id: AddressSpaceId, expected: u32) -> bool {
393        let entries = self.entries.lock();
394        usize::try_from(expected).is_ok_and(|expected| entries.keys.len() == expected)
395            && entries.keys.iter().all(|entry| entry.space_id == mm_id)
396    }
397
398    pub fn is_empty(&self) -> bool {
399        self.entries.lock().keys.is_empty()
400    }
401}
402
403pub struct PageObject {
404    pub id: PageId,
405    frame: FrameLease,
406    state: AtomicU8,
407    /// Origin of the resident contents.  Mapping-specific policy (for example
408    /// whether a cached file page is reported as File or Shmem) still belongs
409    /// to MappingSlot, but private COW replacement changes the underlying
410    /// object itself from File to Anon.
411    resident_kind: AtomicU8,
412    /// Number of installed mapping slots referring to this page.  The frame
413    /// lease is released only after the last slot is detached and the registry
414    /// drops its own reference.
415    mapping_refs: core::sync::atomic::AtomicU32,
416    writeback_generation: core::sync::atomic::AtomicU64,
417    /// Set only after every TLB obligation for the most recently detached
418    /// reverse mapping has completed.  An evicting page cannot resume its
419    /// cache eviction while this flag is false.
420    eviction_tlb_ready: AtomicBool,
421    /// Serializes the page-state check with mapping-ref and rmap publication.
422    /// This is the Rust ownership boundary corresponding to Linux's page/rmap
423    /// locking layer; it is deliberately independent from VMA publication and
424    /// PTE stripe locks.
425    mapping_graph: IrqMutex<()>,
426    pub rmap: RmapSet,
427}
428
429impl PageObject {
430    pub fn new(id: PageId, frame: FrameLease) -> Arc<Self> {
431        Self::new_with_resident_kind(id, frame, None)
432    }
433
434    pub fn new_with_resident_kind(
435        id: PageId,
436        frame: FrameLease,
437        resident_kind: Option<RssKind>,
438    ) -> Arc<Self> {
439        Arc::new(Self {
440            id,
441            frame,
442            state: AtomicU8::new(PageState::Reserved as u8),
443            resident_kind: AtomicU8::new(RssKind::slot_value(resident_kind)),
444            mapping_refs: core::sync::atomic::AtomicU32::new(0),
445            writeback_generation: core::sync::atomic::AtomicU64::new(0),
446            eviction_tlb_ready: AtomicBool::new(false),
447            mapping_graph: IrqMutex::new(()),
448            rmap: RmapSet::default(),
449        })
450    }
451
452    pub fn new_present(id: PageId, frame: FrameLease) -> Arc<Self> {
453        Self::new_present_with_resident_kind(id, frame, None)
454    }
455
456    pub fn new_present_with_resident_kind(
457        id: PageId,
458        frame: FrameLease,
459        resident_kind: Option<RssKind>,
460    ) -> Arc<Self> {
461        let page = Self::new_with_resident_kind(id, frame, resident_kind);
462        // A freshly allocated frame is reserved until its first PTE is ready.
463        // This transition cannot fail for a new object; retaining the explicit
464        // check keeps the invariant visible to callers and tests.
465        let _ = page.transition(PageState::Reserved, PageState::Present);
466        page
467    }
468
469    pub fn state(&self) -> PageState {
470        PageState::from_u8(self.state.load(Ordering::Acquire))
471    }
472
473    pub fn frame(&self) -> &FrameLease {
474        &self.frame
475    }
476
477    pub fn resident_kind(&self) -> Option<RssKind> {
478        RssKind::from_slot_value(self.resident_kind.load(Ordering::Acquire))
479    }
480
481    pub(crate) fn set_resident_kind(&self, kind: Option<RssKind>) {
482        self.resident_kind
483            .store(RssKind::slot_value(kind), Ordering::Release);
484    }
485
486    pub fn mapping_refs(&self) -> u32 {
487        self.mapping_refs.load(Ordering::Acquire)
488    }
489
490    /// Returns whether all installed mappings are owned by one address space.
491    ///
492    /// This mirrors Linux's large-anonymous-folio `mm_id` reuse test: splitting
493    /// one PMD into 512 PTEs raises the mapcount, but does not by itself make
494    /// the folio shared.  A write may reuse the subpage when every rmap still
495    /// belongs to this MM; fork introduces another MM identity and therefore
496    /// forces a base-page COW copy.
497    pub(crate) fn exclusively_mapped_by(&self, mm_id: AddressSpaceId) -> bool {
498        let _graph = self.mapping_graph.lock();
499        if !matches!(self.state(), PageState::Present | PageState::LazyFree) {
500            return false;
501        }
502        let mappings = self.mapping_refs.load(Ordering::Acquire);
503        mappings != 0 && self.rmap.all_mappings_belong_to(mm_id, mappings)
504    }
505
506    fn publish_slot_graph(&self, key: MappingSlotKey) -> bool {
507        let Ok(mut reservation) = self.rmap.prepare_replace(1) else {
508            return false;
509        };
510        let published = {
511            let _graph = self.mapping_graph.lock();
512            if !matches!(self.state(), PageState::Present | PageState::LazyFree) {
513                false
514            } else {
515                let current = self.mapping_refs.load(Ordering::Acquire);
516                let Some(next) = current.checked_add(1) else {
517                    return false;
518                };
519                match self.rmap.replace_reserved(&[], &[key], &mut reservation) {
520                    Err(_) => false,
521                    Ok(()) if !matches!(self.state(), PageState::Present | PageState::LazyFree) => {
522                        let _ = self.rmap.replace_reserved(&[key], &[], &mut reservation);
523                        false
524                    }
525                    Ok(()) => {
526                        self.mapping_refs.store(next, Ordering::Release);
527                        true
528                    }
529                }
530            }
531        };
532        // Preparation may leave the old vector allocation in the token.
533        // Release it only after the IRQ-saving graph guard has gone away.
534        drop(reservation);
535        published
536    }
537
538    fn detach_slot_graph(&self, key: MappingSlotKey) -> bool {
539        let Ok(mut reservation) = self.rmap.prepare_replace(0) else {
540            return false;
541        };
542        let (detached, became_exclusive) = {
543            let _graph = self.mapping_graph.lock();
544            let current = self.mapping_refs.load(Ordering::Acquire);
545            let Some(next) = current.checked_sub(1) else {
546                return false;
547            };
548            if self
549                .rmap
550                .replace_reserved(&[key], &[], &mut reservation)
551                .is_err()
552            {
553                (false, false)
554            } else {
555                self.mapping_refs.store(next, Ordering::Release);
556                (true, current > 1 && next == 1)
557            }
558        };
559        drop(reservation);
560        if became_exclusive && self.state() == PageState::LazyFree {
561            // A fork-shared lazy-free page was ineligible during the original
562            // MADV_FREE pass. Its last foreign mapping disappearing is a new
563            // reclaimability edge, analogous to Linux putting a newly eligible
564            // lazy-free folio back on reclaimable LRU state.
565            super::lifecycle::request_lazy_free_reclaim();
566        }
567        detached
568    }
569
570    pub(crate) fn prepare_mapping_graph_replace(
571        &self,
572        old: &[MappingSlotKey],
573        new: &[MappingSlotKey],
574    ) -> Result<MappingGraphReservation<'_>, MappingGraphError> {
575        self.rmap
576            .prepare_replace(new.len().saturating_sub(old.len()))
577    }
578
579    pub(crate) fn replace_mapping_graph_reserved(
580        &self,
581        old: &[MappingSlotKey],
582        new: &[MappingSlotKey],
583        reservation: &mut MappingGraphReservation<'_>,
584    ) -> Result<(), MappingGraphError> {
585        let became_exclusive = {
586            let _graph = self.mapping_graph.lock();
587            if !matches!(self.state(), PageState::Present | PageState::LazyFree) {
588                return Err(MappingGraphError::PageNotPresent);
589            }
590            let current = self.mapping_refs.load(Ordering::Acquire);
591            let next = if new.len() >= old.len() {
592                let additional = u32::try_from(new.len() - old.len())
593                    .map_err(|_| MappingGraphError::RefOverflow)?;
594                current
595                    .checked_add(additional)
596                    .ok_or(MappingGraphError::RefOverflow)?
597            } else {
598                let removed = u32::try_from(old.len() - new.len())
599                    .map_err(|_| MappingGraphError::RefUnderflow)?;
600                current
601                    .checked_sub(removed)
602                    .ok_or(MappingGraphError::RefUnderflow)?
603            };
604            self.rmap.replace_reserved(old, new, reservation)?;
605            self.mapping_refs.store(next, Ordering::Release);
606            current > 1 && next == 1
607        };
608        if became_exclusive && self.state() == PageState::LazyFree {
609            super::lifecycle::request_lazy_free_reclaim();
610        }
611        Ok(())
612    }
613
614    /// Atomically replaces a set of reverse mappings and adjusts the installed
615    /// PTE reference count by the same cardinality delta.  All fallible rmap
616    /// reservation and validation completes before either fact is changed.
617    pub(crate) fn replace_mapping_graph(
618        &self,
619        old: &[MappingSlotKey],
620        new: &[MappingSlotKey],
621    ) -> Result<(), MappingGraphError> {
622        let mut reservation = self.prepare_mapping_graph_replace(old, new)?;
623        let result = self.replace_mapping_graph_reserved(old, new, &mut reservation);
624        drop(reservation);
625        result
626    }
627
628    pub(crate) fn transition(&self, expected: PageState, next: PageState) -> bool {
629        let valid = matches!(
630            (expected, next),
631            (PageState::Reserved, PageState::Present)
632                | (PageState::Present, PageState::LazyFree)
633                | (PageState::LazyFree, PageState::Present)
634                | (PageState::LazyFree, PageState::Evicting)
635                | (PageState::LazyFree, PageState::Retired)
636                | (PageState::Present, PageState::Evicting)
637                | (PageState::Present, PageState::Writeback)
638                | (PageState::Evicting, PageState::Present)
639                | (PageState::Evicting, PageState::Retired)
640                | (PageState::Writeback, PageState::Present)
641                | (PageState::Writeback, PageState::Retired)
642        );
643        valid
644            && self
645                .state
646                .compare_exchange(
647                    expected as u8,
648                    next as u8,
649                    Ordering::AcqRel,
650                    Ordering::Acquire,
651                )
652                .is_ok()
653    }
654
655    pub(crate) fn mark_lazy_free(&self) -> bool {
656        self.transition(PageState::Present, PageState::LazyFree)
657    }
658
659    pub(crate) fn clear_lazy_free(&self) -> bool {
660        self.transition(PageState::LazyFree, PageState::Present)
661    }
662
663    /// Pins a resident page for rmap-driven eviction.  The lease must either
664    /// be cancelled or completed; dropping it intentionally leaves the page
665    /// in `Evicting`, so a failed caller cannot accidentally make a page
666    /// reclaimable while a stale PTE still exists.
667    pub(crate) fn eviction_lease(self: &Arc<Self>) -> Result<EvictionLease, EvictionError> {
668        let _graph = self.mapping_graph.lock();
669        if !self.transition(PageState::Present, PageState::Evicting) {
670            return Err(EvictionError::NotPresent);
671        }
672        self.eviction_tlb_ready.store(false, Ordering::Release);
673        Ok(EvictionLease { page: self.clone() })
674    }
675
676    /// Marks the current eviction safe to resume after its detached PTEs have
677    /// been acknowledged by every target CPU.
678    pub(crate) fn complete_eviction_tlb(&self) {
679        if self.state() == PageState::Evicting {
680            self.eviction_tlb_ready.store(true, Ordering::Release);
681        }
682    }
683
684    /// Reacquires ownership of an eviction that previously stopped at a TLB
685    /// quarantine boundary.  The readiness bit is consumed so at most one
686    /// reclaimer can continue that state transition.
687    pub(crate) fn resume_eviction_lease(self: &Arc<Self>) -> Result<EvictionLease, EvictionError> {
688        if self.state() != PageState::Evicting
689            || self
690                .eviction_tlb_ready
691                .compare_exchange(true, false, Ordering::AcqRel, Ordering::Acquire)
692                .is_err()
693        {
694            return Err(EvictionError::Busy);
695        }
696        Ok(EvictionLease { page: self.clone() })
697    }
698
699    /// Begins one writeback protection generation.  While this lease exists a
700    /// new mapping slot cannot be published and eviction cannot retire the
701    /// frame; the caller must protect every rmap entry before completing it.
702    pub(crate) fn writeback_lease(self: &Arc<Self>) -> Result<WritebackLease, WritebackError> {
703        let _graph = self.mapping_graph.lock();
704        if !self.transition(PageState::Present, PageState::Writeback) {
705            return Err(WritebackError::Busy);
706        }
707        let generation = match self.writeback_generation.try_update(
708            Ordering::AcqRel,
709            Ordering::Acquire,
710            |current| current.checked_add(1),
711        ) {
712            Ok(previous) => previous + 1,
713            Err(_) => {
714                let _ = self.transition(PageState::Writeback, PageState::Present);
715                return Err(WritebackError::GenerationExhausted);
716            }
717        };
718        Ok(WritebackLease {
719            page: self.clone(),
720            generation,
721        })
722    }
723}
724
725#[derive(Debug, Clone, Copy, PartialEq, Eq)]
726pub enum MappingGraphError {
727    PageNotPresent,
728    MissingOldSlot,
729    DuplicateNewSlot,
730    ResourceExhausted,
731    RefOverflow,
732    RefUnderflow,
733    SlotStateConflict,
734    SlotIdentityMismatch,
735    RollbackFailed,
736}
737
738#[derive(Debug, Clone, Copy, PartialEq, Eq)]
739pub enum EvictionError {
740    NotPresent,
741    Busy,
742}
743
744#[derive(Debug, Clone, Copy, PartialEq, Eq)]
745pub enum WritebackError {
746    Busy,
747    GenerationExhausted,
748}
749
750/// Ownership token held while a page's reverse mappings are being revoked.
751pub struct EvictionLease {
752    page: Arc<PageObject>,
753}
754
755impl EvictionLease {
756    pub fn page(&self) -> &Arc<PageObject> {
757        &self.page
758    }
759
760    pub(crate) fn cancel(self) -> bool {
761        self.page.eviction_tlb_ready.store(false, Ordering::Release);
762        self.page
763            .transition(PageState::Evicting, PageState::Present)
764    }
765
766    pub(crate) fn retire(self) -> Result<(), (Self, EvictionError)> {
767        if !self.page.rmap.is_empty() || self.page.mapping_refs() != 0 {
768            return Err((self, EvictionError::Busy));
769        }
770        if self
771            .page
772            .transition(PageState::Evicting, PageState::Retired)
773        {
774            Ok(())
775        } else {
776            Err((self, EvictionError::Busy))
777        }
778    }
779}
780
781/// Pins a PageObject while one dirty-generation snapshot is protected.
782pub struct WritebackLease {
783    page: Arc<PageObject>,
784    generation: u64,
785}
786
787impl WritebackLease {
788    pub const fn generation(&self) -> u64 {
789        self.generation
790    }
791
792    pub(crate) fn cancel(self) -> bool {
793        self.page
794            .transition(PageState::Writeback, PageState::Present)
795    }
796
797    pub(crate) fn complete(self) -> Result<u64, (Self, WritebackError)> {
798        if self
799            .page
800            .transition(PageState::Writeback, PageState::Present)
801        {
802            Ok(self.generation)
803        } else {
804            Err((self, WritebackError::Busy))
805        }
806    }
807}
808
809#[repr(u8)]
810#[derive(Debug, Clone, Copy, PartialEq, Eq)]
811pub enum SlotState {
812    Reserved = 0,
813    Present  = 1,
814    Detached = 2,
815}
816
817impl SlotState {
818    fn from_u8(value: u8) -> Self {
819        match value {
820            0 => Self::Reserved,
821            1 => Self::Present,
822            _ => Self::Detached,
823        }
824    }
825}
826
827/// Ownership record for one installed PTE.
828pub struct MappingSlot {
829    pub mapping: MappingId,
830    pub mm_id: AddressSpaceId,
831    pub va: VirtAddr,
832    pub page_order: PageOrder,
833    pub page: Arc<PageObject>,
834    /// Byte offset of this materialized leaf inside `page.frame()`.
835    ///
836    /// A PMD split keeps one large PageObject, matching Linux's large-folio
837    /// ownership, while each base PTE names a different physical subrange.
838    /// Recording that subrange here lets later mutation/rmap code validate
839    /// exact ownership without rediscovering a PageObject from a raw PFN.
840    frame_offset: usize,
841    /// Linux resident category owned by this installed mapping.  It belongs
842    /// to the slot rather than the VMA or PageObject because a file-private
843    /// page can change from File to Anon at one VA while other mappings of the
844    /// same logical object keep their own classification.
845    resident_kind: AtomicU8,
846    /// Preallocated child table bound to this slot's huge leaf.  It is never
847    /// visible to hardware until a typed split consumes it.  Dropping a whole
848    /// huge mapping therefore releases the unpublished deposit without a TLB
849    /// obligation, matching Linux's deposited-PTE-page ownership.
850    huge_split_deposit: IrqMutex<Option<HugeSplitDeposit>>,
851    state: AtomicU8,
852}
853
854impl MappingSlot {
855    #[cfg(test)]
856    pub(crate) fn new(
857        mapping: MappingId,
858        mm_id: AddressSpaceId,
859        va: VirtAddr,
860        page_order: PageOrder,
861        page: Arc<PageObject>,
862        resident_kind: Option<RssKind>,
863    ) -> Self {
864        Self {
865            mapping,
866            mm_id,
867            va,
868            page_order,
869            page,
870            frame_offset: 0,
871            resident_kind: AtomicU8::new(RssKind::slot_value(resident_kind)),
872            huge_split_deposit: IrqMutex::new(None),
873            state: AtomicU8::new(SlotState::Reserved as u8),
874        }
875    }
876
877    pub(crate) fn new_with_frame_offset(
878        mapping: MappingId,
879        mm_id: AddressSpaceId,
880        va: VirtAddr,
881        page_order: PageOrder,
882        page: Arc<PageObject>,
883        frame_offset: usize,
884        resident_kind: Option<RssKind>,
885    ) -> Option<Self> {
886        let bytes = PAGE_SIZE_4K.checked_shl(page_order.get().into())?;
887        let end = frame_offset.checked_add(bytes)?;
888        if !frame_offset.is_multiple_of(PAGE_SIZE_4K) || end > page.frame().size() {
889            return None;
890        }
891        Some(Self {
892            mapping,
893            mm_id,
894            va,
895            page_order,
896            page,
897            frame_offset,
898            resident_kind: AtomicU8::new(RssKind::slot_value(resident_kind)),
899            huge_split_deposit: IrqMutex::new(None),
900            state: AtomicU8::new(SlotState::Reserved as u8),
901        })
902    }
903
904    pub(crate) const fn frame_offset(&self) -> usize {
905        self.frame_offset
906    }
907
908    pub(crate) fn mapped_paddr(&self) -> Option<PhysAddr> {
909        self.page.frame().paddr().checked_add(self.frame_offset)
910    }
911
912    /// Attaches the deposit before the slot is published into the address
913    /// space.  A second deposit would mean two page-table-frame owners for one
914    /// huge leaf and is rejected by returning ownership to the caller.
915    pub(crate) fn attach_huge_split_deposit(
916        self,
917        deposit: HugeSplitDeposit,
918    ) -> Result<Self, HugeSplitDeposit> {
919        {
920            let mut owner = self.huge_split_deposit.lock();
921            if owner.is_some() {
922                return Err(deposit);
923            }
924            *owner = Some(deposit);
925        }
926        Ok(self)
927    }
928
929    pub(crate) fn has_huge_split_deposit(&self) -> bool {
930        self.huge_split_deposit.lock().is_some()
931    }
932
933    pub(crate) fn take_huge_split_deposit(&self) -> Option<HugeSplitDeposit> {
934        self.huge_split_deposit.lock().take()
935    }
936
937    pub(crate) fn restore_huge_split_deposit(
938        &self,
939        deposit: HugeSplitDeposit,
940    ) -> Result<(), HugeSplitDeposit> {
941        let mut owner = self.huge_split_deposit.lock();
942        if owner.is_some() {
943            return Err(deposit);
944        }
945        *owner = Some(deposit);
946        Ok(())
947    }
948
949    pub(crate) fn overlaps(&self, range: VirtAddrRange) -> bool {
950        let Some(bytes) = PAGE_SIZE_4K.checked_shl(self.page_order.get().into()) else {
951            return false;
952        };
953        let Some(end) = self.va.checked_add(bytes) else {
954            return false;
955        };
956        self.va < range.end && range.start < end
957    }
958
959    pub fn state(&self) -> SlotState {
960        SlotState::from_u8(self.state.load(Ordering::Acquire))
961    }
962
963    pub fn resident_kind(&self) -> Option<RssKind> {
964        RssKind::from_slot_value(self.resident_kind.load(Ordering::Acquire))
965    }
966
967    /// Reclassifies one published resident fact (for example File -> Anon on
968    /// a private COW write).  The owning address-space mutation gate provides
969    /// serialization; release/acquire keeps lock-free statistics snapshots
970    /// from observing a torn category.
971    pub(crate) fn set_resident_kind(&self, kind: Option<RssKind>) {
972        self.resident_kind
973            .store(RssKind::slot_value(kind), Ordering::Release);
974    }
975
976    pub(crate) fn publish(&self) -> bool {
977        if self
978            .state
979            .compare_exchange(
980                SlotState::Reserved as u8,
981                SlotState::Present as u8,
982                Ordering::AcqRel,
983                Ordering::Acquire,
984            )
985            .is_err()
986        {
987            return false;
988        }
989        if self.page.publish_slot_graph(MappingSlotKey {
990            space_id: self.mm_id,
991            va: self.va,
992        }) {
993            true
994        } else {
995            let _ = self.state.compare_exchange(
996                SlotState::Present as u8,
997                SlotState::Reserved as u8,
998                Ordering::AcqRel,
999                Ordering::Acquire,
1000            );
1001            false
1002        }
1003    }
1004
1005    pub(crate) fn detach(&self) -> bool {
1006        self.detach_slot()
1007    }
1008
1009    /// Restores a detached slot and its mapping reference together. Mutation
1010    /// rollback uses the same Arc identity captured in the preimage, so rmap,
1011    /// refcount and slot state cannot be reconstructed as unrelated
1012    /// best-effort operations.
1013    pub(crate) fn restore(&self) -> bool {
1014        if self
1015            .state
1016            .compare_exchange(
1017                SlotState::Detached as u8,
1018                SlotState::Present as u8,
1019                Ordering::AcqRel,
1020                Ordering::Acquire,
1021            )
1022            .is_err()
1023        {
1024            return false;
1025        }
1026        if self.page.publish_slot_graph(MappingSlotKey {
1027            space_id: self.mm_id,
1028            va: self.va,
1029        }) {
1030            true
1031        } else {
1032            let _ = self.state.compare_exchange(
1033                SlotState::Present as u8,
1034                SlotState::Detached as u8,
1035                Ordering::AcqRel,
1036                Ordering::Acquire,
1037            );
1038            false
1039        }
1040    }
1041
1042    fn detach_slot(&self) -> bool {
1043        if self
1044            .state
1045            .compare_exchange(
1046                SlotState::Present as u8,
1047                SlotState::Detached as u8,
1048                Ordering::AcqRel,
1049                Ordering::Acquire,
1050            )
1051            .is_err()
1052        {
1053            return false;
1054        }
1055        if !self.page.detach_slot_graph(MappingSlotKey {
1056            space_id: self.mm_id,
1057            va: self.va,
1058        }) {
1059            let _ = self.state.compare_exchange(
1060                SlotState::Detached as u8,
1061                SlotState::Present as u8,
1062                Ordering::AcqRel,
1063                Ordering::Acquire,
1064            );
1065            return false;
1066        }
1067        true
1068    }
1069
1070    pub(crate) fn publish_after_graph_replace(&self) -> bool {
1071        self.state
1072            .compare_exchange(
1073                SlotState::Reserved as u8,
1074                SlotState::Present as u8,
1075                Ordering::AcqRel,
1076                Ordering::Acquire,
1077            )
1078            .is_ok()
1079    }
1080
1081    pub(crate) fn detach_after_graph_replace(&self) -> bool {
1082        self.state
1083            .compare_exchange(
1084                SlotState::Present as u8,
1085                SlotState::Detached as u8,
1086                Ordering::AcqRel,
1087                Ordering::Acquire,
1088            )
1089            .is_ok()
1090    }
1091
1092    /// Relocates one installed mapping record without changing the page's
1093    /// mapping-reference cardinality.
1094    ///
1095    /// Linux `move_ptes()` moves a PTE while retaining the same folio/rmap
1096    /// ownership.  This is the equivalent software-ownership transition: the
1097    /// reverse-mapping key is replaced under the PageObject graph lock, then
1098    /// the replacement slot becomes Present and the old slot becomes Detached.
1099    /// A failed state transition restores the old graph before returning.
1100    pub(crate) fn relocate_to(&self, replacement: &Self) -> Result<(), MappingGraphError> {
1101        if self.state() != SlotState::Present || replacement.state() != SlotState::Reserved {
1102            return Err(MappingGraphError::SlotStateConflict);
1103        }
1104        if self.mm_id != replacement.mm_id
1105            || self.page_order != replacement.page_order
1106            || !Arc::ptr_eq(&self.page, &replacement.page)
1107        {
1108            return Err(MappingGraphError::SlotIdentityMismatch);
1109        }
1110
1111        let old_key = MappingSlotKey {
1112            space_id: self.mm_id,
1113            va: self.va,
1114        };
1115        let new_key = MappingSlotKey {
1116            space_id: replacement.mm_id,
1117            va: replacement.va,
1118        };
1119        if old_key == new_key {
1120            return Err(MappingGraphError::DuplicateNewSlot);
1121        }
1122
1123        self.page.replace_mapping_graph(&[old_key], &[new_key])?;
1124        if !replacement.publish_after_graph_replace() {
1125            if self
1126                .page
1127                .replace_mapping_graph(&[new_key], &[old_key])
1128                .is_err()
1129            {
1130                return Err(MappingGraphError::RollbackFailed);
1131            }
1132            return Err(MappingGraphError::SlotStateConflict);
1133        }
1134        if self.detach_after_graph_replace() {
1135            return Ok(());
1136        }
1137
1138        let replacement_reserved = replacement.reserve_after_graph_replace();
1139        let graph_restored = replacement_reserved
1140            && self
1141                .page
1142                .replace_mapping_graph(&[new_key], &[old_key])
1143                .is_ok();
1144        if !replacement_reserved || !graph_restored {
1145            return Err(MappingGraphError::RollbackFailed);
1146        }
1147        Err(MappingGraphError::SlotStateConflict)
1148    }
1149
1150    pub(crate) fn restore_after_graph_replace(&self) -> bool {
1151        self.state
1152            .compare_exchange(
1153                SlotState::Detached as u8,
1154                SlotState::Present as u8,
1155                Ordering::AcqRel,
1156                Ordering::Acquire,
1157            )
1158            .is_ok()
1159    }
1160
1161    pub(crate) fn reserve_after_graph_replace(&self) -> bool {
1162        self.state
1163            .compare_exchange(
1164                SlotState::Present as u8,
1165                SlotState::Reserved as u8,
1166                Ordering::AcqRel,
1167                Ordering::Acquire,
1168            )
1169            .is_ok()
1170    }
1171}
1172
1173#[cfg(test)]
1174mod tests {
1175    #[cfg(not(axtest))]
1176    use core::sync::atomic::AtomicUsize;
1177
1178    use super::*;
1179
1180    #[cfg(not(axtest))]
1181    static RELEASES: AtomicUsize = AtomicUsize::new(0);
1182
1183    #[cfg(not(axtest))]
1184    fn record_release(_paddr: PhysAddr, bytes: usize) {
1185        assert_eq!(bytes, PAGE_SIZE_4K * 4);
1186        RELEASES.fetch_add(1, Ordering::Relaxed);
1187    }
1188
1189    #[test]
1190    fn shared_page_tracks_slots_until_detach() {
1191        let page = PageObject::new(
1192            PageId::new(1),
1193            FrameLease::new(PhysAddr::from_usize(0x1000)),
1194        );
1195        assert!(page.transition(PageState::Reserved, PageState::Present));
1196        let id = AddressSpaceId::allocate();
1197        let slot_a = MappingSlot::new(
1198            MappingId::new(1),
1199            id,
1200            VirtAddr::from_usize(0x4000),
1201            PageOrder::BASE,
1202            page.clone(),
1203            Some(RssKind::Anon),
1204        );
1205        let slot_b = MappingSlot::new(
1206            MappingId::new(1),
1207            id,
1208            VirtAddr::from_usize(0x5000),
1209            PageOrder::BASE,
1210            page.clone(),
1211            Some(RssKind::Anon),
1212        );
1213        assert!(slot_a.publish());
1214        assert!(slot_b.publish());
1215        assert_eq!(page.rmap.snapshot().len(), 2);
1216        assert!(slot_a.detach());
1217        assert_eq!(page.rmap.snapshot().len(), 1);
1218        assert!(slot_b.detach());
1219        assert!(page.rmap.is_empty());
1220    }
1221
1222    #[cfg_attr(axtest, axtest::axtest)]
1223    #[cfg_attr(not(axtest), test)]
1224    fn concurrent_mapping_publication_preserves_reserved_capacity() {
1225        let page = PageObject::new_present(
1226            PageId::new(7),
1227            FrameLease::new(PhysAddr::from_usize(0x1000)),
1228        );
1229        let first = MappingSlotKey {
1230            space_id: AddressSpaceId::allocate(),
1231            va: VirtAddr::from_usize(0x4000),
1232        };
1233        let second = MappingSlotKey {
1234            space_id: AddressSpaceId::allocate(),
1235            va: first.va,
1236        };
1237        // Both MMs prepare against the same page before either publishes.
1238        // Completing the second fork must not consume the first fork's space.
1239        let mut first_reservation = page.prepare_mapping_graph_replace(&[], &[first]).unwrap();
1240        page.replace_mapping_graph(&[], &[second]).unwrap();
1241        page.replace_mapping_graph_reserved(&[], &[first], &mut first_reservation)
1242            .unwrap();
1243        drop(first_reservation);
1244        assert_eq!(page.mapping_refs(), 2);
1245        let mappings = page.rmap.snapshot();
1246        assert_eq!(mappings.len(), 2);
1247        assert!(mappings.contains(&first));
1248        assert!(mappings.contains(&second));
1249        let cancelled = page.prepare_mapping_graph_replace(&[], &[first]).unwrap();
1250        drop(cancelled);
1251        assert_eq!(page.rmap.entries.lock().reserved, 0);
1252        assert_eq!(page.mapping_refs(), 2);
1253        page.replace_mapping_graph(&[first, second], &[]).unwrap();
1254        assert_eq!(page.mapping_refs(), 0);
1255        assert!(page.rmap.is_empty());
1256    }
1257
1258    #[cfg_attr(axtest, axtest::axtest)]
1259    #[cfg_attr(not(axtest), test)]
1260    fn lazy_free_page_requeues_when_a_shared_mapping_becomes_exclusive() {
1261        let page = PageObject::new_present(
1262            PageId::new(5),
1263            FrameLease::new(PhysAddr::from_usize(0xa000)),
1264        );
1265        let mapping = MappingId::new(5);
1266        let parent = MappingSlot::new(
1267            mapping,
1268            AddressSpaceId::allocate(),
1269            VirtAddr::from_usize(0xa000),
1270            PageOrder::BASE,
1271            page.clone(),
1272            Some(RssKind::Anon),
1273        );
1274        let child = MappingSlot::new(
1275            mapping,
1276            AddressSpaceId::allocate(),
1277            VirtAddr::from_usize(0xb000),
1278            PageOrder::BASE,
1279            page.clone(),
1280            Some(RssKind::Anon),
1281        );
1282        assert!(parent.publish());
1283        assert!(child.publish());
1284        assert!(page.mark_lazy_free());
1285        assert_eq!(page.mapping_refs(), 2);
1286
1287        let requests = super::super::lifecycle::lazy_free_reclaim_request_count_for_test();
1288        assert!(child.detach());
1289        assert_eq!(page.mapping_refs(), 1);
1290        assert!(
1291            super::super::lifecycle::lazy_free_reclaim_request_count_for_test() > requests,
1292            "the 2 -> 1 mapping transition must publish a new reclaim edge"
1293        );
1294        assert!(parent.detach());
1295    }
1296
1297    #[cfg_attr(axtest, axtest::axtest)]
1298    #[cfg_attr(not(axtest), test)]
1299    fn lazy_free_page_requeues_after_a_batch_graph_replacement() {
1300        let page = PageObject::new_present(
1301            PageId::new(6),
1302            FrameLease::new(PhysAddr::from_usize(0xc000)),
1303        );
1304        let first = MappingSlotKey {
1305            space_id: AddressSpaceId::allocate(),
1306            va: VirtAddr::from_usize(0xc000),
1307        };
1308        let second = MappingSlotKey {
1309            space_id: AddressSpaceId::allocate(),
1310            va: VirtAddr::from_usize(0xd000),
1311        };
1312        let replacement = MappingSlotKey {
1313            space_id: first.space_id,
1314            va: VirtAddr::from_usize(0xe000),
1315        };
1316        page.replace_mapping_graph(&[], &[first, second]).unwrap();
1317        assert!(page.mark_lazy_free());
1318        assert_eq!(page.mapping_refs(), 2);
1319
1320        let requests = super::super::lifecycle::lazy_free_reclaim_request_count_for_test();
1321        let mut reservation = page
1322            .prepare_mapping_graph_replace(&[first, second], &[replacement])
1323            .unwrap();
1324        page.replace_mapping_graph_reserved(&[first, second], &[replacement], &mut reservation)
1325            .unwrap();
1326        drop(reservation);
1327
1328        assert_eq!(page.mapping_refs(), 1);
1329        assert!(
1330            super::super::lifecycle::lazy_free_reclaim_request_count_for_test() > requests,
1331            "a batch graph replacement ending at one mapping must publish a reclaim edge"
1332        );
1333        page.replace_mapping_graph(&[replacement], &[]).unwrap();
1334    }
1335
1336    #[test]
1337    fn reserved_page_cannot_publish_a_mapping_slot() {
1338        let page = PageObject::new(
1339            PageId::new(2),
1340            FrameLease::new(PhysAddr::from_usize(0x2000)),
1341        );
1342        let slot = MappingSlot::new(
1343            MappingId::new(2),
1344            AddressSpaceId::allocate(),
1345            VirtAddr::from_usize(0x6000),
1346            PageOrder::BASE,
1347            page,
1348            None,
1349        );
1350        assert!(!slot.publish());
1351        assert_eq!(slot.state(), SlotState::Reserved);
1352    }
1353
1354    #[cfg_attr(axtest, axtest::axtest)]
1355    #[cfg_attr(not(axtest), test)]
1356    fn relocation_replaces_rmap_key_without_changing_mapping_refs() {
1357        let page = PageObject::new_present(
1358            PageId::new(4),
1359            FrameLease::new(PhysAddr::from_usize(0x4000)),
1360        );
1361        let mm_id = AddressSpaceId::allocate();
1362        let old_va = VirtAddr::from_usize(0x8000);
1363        let new_va = VirtAddr::from_usize(0x9000);
1364        let old = MappingSlot::new(
1365            MappingId::new(4),
1366            mm_id,
1367            old_va,
1368            PageOrder::BASE,
1369            page.clone(),
1370            Some(RssKind::Anon),
1371        );
1372        let replacement = MappingSlot::new(
1373            MappingId::new(4),
1374            mm_id,
1375            new_va,
1376            PageOrder::BASE,
1377            page.clone(),
1378            Some(RssKind::Anon),
1379        );
1380        assert!(old.publish());
1381        assert_eq!(page.mapping_refs(), 1);
1382
1383        old.relocate_to(&replacement).unwrap();
1384
1385        assert_eq!(old.state(), SlotState::Detached);
1386        assert_eq!(replacement.state(), SlotState::Present);
1387        assert_eq!(page.mapping_refs(), 1);
1388        assert_eq!(
1389            page.rmap.snapshot(),
1390            alloc::vec![MappingSlotKey {
1391                space_id: mm_id,
1392                va: new_va,
1393            }]
1394        );
1395        assert!(replacement.detach());
1396        assert_eq!(page.mapping_refs(), 0);
1397    }
1398
1399    #[test]
1400    fn eviction_cannot_resume_before_tlb_retirement() {
1401        let page = PageObject::new_present(
1402            PageId::new(3),
1403            FrameLease::new(PhysAddr::from_usize(0x3000)),
1404        );
1405        let lease = page.eviction_lease().unwrap();
1406
1407        // A published eviction deliberately drops its lease while the remote
1408        // receipt is outstanding.  Neither a second reclaimer nor the cache
1409        // may turn the page back into a reusable Present page at this point.
1410        drop(lease);
1411        assert_eq!(page.state(), PageState::Evicting);
1412        assert!(matches!(
1413            page.resume_eviction_lease(),
1414            Err(EvictionError::Busy)
1415        ));
1416
1417        page.complete_eviction_tlb();
1418        let resumed = page.resume_eviction_lease().unwrap();
1419        assert!(resumed.cancel());
1420        assert_eq!(page.state(), PageState::Present);
1421    }
1422
1423    #[test]
1424    fn split_frame_leases_release_the_allocation_once_after_the_last_subpage() {
1425        RELEASES.store(0, Ordering::Relaxed);
1426        let owner = FrameLease::owned_with_releaser(
1427            PhysAddr::from_usize(0x20_0000),
1428            PAGE_SIZE_4K * 4,
1429            record_release,
1430        );
1431        let first = owner.sublease(0, PAGE_SIZE_4K).unwrap();
1432        let last = owner.sublease(PAGE_SIZE_4K * 3, PAGE_SIZE_4K).unwrap();
1433        assert_eq!(first.paddr(), PhysAddr::from_usize(0x20_0000));
1434        assert_eq!(last.paddr(), PhysAddr::from_usize(0x20_3000));
1435        assert!(owner.sublease(PAGE_SIZE_4K * 4, PAGE_SIZE_4K).is_none());
1436
1437        drop(owner);
1438        drop(first);
1439        assert_eq!(RELEASES.load(Ordering::Relaxed), 0);
1440        drop(last);
1441        assert_eq!(RELEASES.load(Ordering::Relaxed), 1);
1442    }
1443}
starry_kernel/mm/aspace/objects.rs

starry_kernel/mm/aspace/
objects.rs
