Skip to main content

starry_kernel/mm/aspace/
mod.rs

1use alloc::{
2    collections::{BTreeMap, BTreeSet},
3    sync::Arc,
4    vec::Vec,
5};
6use core::{fmt, sync::atomic::AtomicUsize};
7
8use ax_fs_ng::file::CachedPagePin;
9use ax_memory_addr::{
10    MemoryAddr, PAGE_SIZE_4K, PageIter4K, PhysAddr, VirtAddr, VirtAddrRange, is_aligned_4k,
11};
12#[cfg(not(any(target_arch = "aarch64", target_arch = "loongarch64")))]
13use ax_mm::RootEntryShare;
14use ax_runtime::hal::{
15    mem::phys_to_virt,
16    paging::{
17        DeferredPageTableFrames, InstalledHugeSplit, MappingFlags, PageTable, PageTableEntry,
18        PageTableMapDeposit, PageTableMapPlan, PageTableMovePlan, PagingAllocator, PagingError,
19    },
20    trap::PageFaultFlags,
21};
22
23use crate::{
24    StarryError, StarryResult,
25    config::USER_HEAP_BASE,
26    mm::{ProcessVmStat, ProcessVmStatSnapshot, UserVirtualAddressLayout},
27    sync::{IrqMutex, LockdepMutexExt, Mutex, try_reserve_irq_vec},
28};
29
30#[cfg(test)]
31fn complete_page_fault_with(
32    handled: bool,
33    vaddr: VirtAddr,
34    update_mmu_cache: impl FnOnce(VirtAddr),
35) -> bool {
36    if handled {
37        update_mmu_cache(vaddr);
38    }
39    handled
40}
41
42mod accounting;
43mod backend;
44pub(crate) mod domain;
45pub mod lifecycle;
46pub(crate) mod mutation;
47pub(crate) mod objects;
48pub(crate) mod reclaim;
49pub(crate) mod vma;
50
51pub(crate) use self::domain::PageTableDomain;
52// These are intentionally exported as the new ownership/transaction surface;
53// they are consumed by the migration work as call sites leave the legacy
54// `MappingOperation` facade.
55pub use self::mutation::{
56    AppliedMutation, EvictionResult, MappingDelta, MutationError, MutationGate, MutationReceipt,
57    MutationState, PreparedMutation, PteDelta, PublishEvent, PublishedMutation,
58    PublishedPendingTlb, QuarantineError, QuarantineFailure, ResidentDelta, TlbQuarantine,
59    TlbRange, TlbRequest, VmaDelta,
60};
61use self::{
62    accounting::ResidentWatermark,
63    backend::{
64        FaultFallback, FaultMaterialization, FaultPteSnapshot, MappingMutationContext,
65        PopulateRequest, PreparedPteOwner, ProviderPublication, PteMaterialization,
66        PteOwnerTransition,
67    },
68};
69pub use self::{
70    backend::*,
71    lifecycle::*,
72    objects::{
73        EvictionError, EvictionLease, FrameLease, MappingGraphError, MappingSlot, MappingSlotKey,
74        PageId, PageObject, PageState, RmapSet, SlotState, WritebackError, WritebackLease,
75    },
76    reclaim::*,
77    vma::*,
78};
79
80#[cfg(all(test, axtest))]
81static MAPPING_GRAPH_SNAPSHOT_CALLS: AtomicUsize = AtomicUsize::new(0);
82
83#[derive(Clone, Copy, PartialEq, Eq)]
84enum MovedPageDestination {
85    /// The destination PTE was empty and now names the source PageObject.
86    SourceOwner,
87    /// An eager target backend had already materialized the destination PTE.
88    TargetOwner { slot_va: VirtAddr },
89}
90
91#[derive(Clone, Copy)]
92struct MovedPage {
93    src_va: VirtAddr,
94    dst_va: VirtAddr,
95    paddr: PhysAddr,
96    page_size: usize,
97    destination: MovedPageDestination,
98}
99
100enum PreparedMovedSlot {
101    Relocate {
102        source_key: MappingSlotKey,
103        target_key: MappingSlotKey,
104        source: Arc<MappingSlot>,
105        replacement: Arc<MappingSlot>,
106    },
107    DetachSource {
108        source_key: MappingSlotKey,
109        target_key: MappingSlotKey,
110        source: Arc<MappingSlot>,
111    },
112}
113const CLONED_ADDR_SPACE_LOCK_SUBCLASS: u32 = 1;
114
115#[derive(Clone, Copy)]
116struct ForkParentPteProtection {
117    va: VirtAddr,
118    paddr: PhysAddr,
119    page_size: usize,
120    original_flags: MappingFlags,
121    protected_flags: MappingFlags,
122}
123
124struct PreparedForkParentMutation {
125    mutation: PreparedMutation,
126    ptes: Vec<ForkParentPteProtection>,
127    ranges: Vec<VirtAddrRange>,
128}
129
130#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
131pub(crate) struct ResidentPageCounts {
132    pub anon: u64,
133    pub file: u64,
134    pub shmem: u64,
135}
136
137impl ResidentPageCounts {
138    pub const fn total(self) -> u64 {
139        self.anon
140            .saturating_add(self.file)
141            .saturating_add(self.shmem)
142    }
143
144    fn checked_delta_to(self, after: Self) -> StarryResult<ResidentDelta> {
145        fn delta(before: u64, after: u64) -> StarryResult<i64> {
146            let value = i128::from(after) - i128::from(before);
147            i64::try_from(value).map_err(|_| StarryError::BadState)
148        }
149
150        Ok(ResidentDelta {
151            anon: delta(self.anon, after.anon)?,
152            file: delta(self.file, after.file)?,
153            shmem: delta(self.shmem, after.shmem)?,
154        })
155    }
156
157    fn checked_apply(&mut self, delta: ResidentDelta) -> StarryResult {
158        fn apply(current: u64, delta: i64) -> StarryResult<u64> {
159            if delta >= 0 {
160                current
161                    .checked_add(u64::try_from(delta).map_err(|_| StarryError::BadState)?)
162                    .ok_or(StarryError::BadState)
163            } else {
164                current
165                    .checked_sub(delta.unsigned_abs())
166                    .ok_or(StarryError::BadState)
167            }
168        }
169
170        self.anon = apply(self.anon, delta.anon)?;
171        self.file = apply(self.file, delta.file)?;
172        self.shmem = apply(self.shmem, delta.shmem)?;
173        Ok(())
174    }
175
176    fn checked_add_pages(&mut self, kind: Option<RssKind>, pages: u64) -> StarryResult {
177        let bucket = match kind {
178            Some(RssKind::Anon) => &mut self.anon,
179            Some(RssKind::File) => &mut self.file,
180            Some(RssKind::Shmem) => &mut self.shmem,
181            None => return Ok(()),
182        };
183        *bucket = bucket.checked_add(pages).ok_or(StarryError::BadState)?;
184        Ok(())
185    }
186
187    fn checked_negated_delta(self) -> StarryResult<ResidentDelta> {
188        Ok(ResidentDelta {
189            anon: -i64::try_from(self.anon).map_err(|_| StarryError::BadState)?,
190            file: -i64::try_from(self.file).map_err(|_| StarryError::BadState)?,
191            shmem: -i64::try_from(self.shmem).map_err(|_| StarryError::BadState)?,
192        })
193    }
194
195    fn checked_positive_delta(self) -> StarryResult<ResidentDelta> {
196        Ok(ResidentDelta {
197            anon: i64::try_from(self.anon).map_err(|_| StarryError::BadState)?,
198            file: i64::try_from(self.file).map_err(|_| StarryError::BadState)?,
199            shmem: i64::try_from(self.shmem).map_err(|_| StarryError::BadState)?,
200        })
201    }
202}
203
204impl ResidentDelta {
205    fn for_pages(kind: Option<RssKind>, pages: i64) -> Self {
206        match kind {
207            Some(RssKind::Anon) => Self {
208                anon: pages,
209                ..Self::default()
210            },
211            Some(RssKind::File) => Self {
212                file: pages,
213                ..Self::default()
214            },
215            Some(RssKind::Shmem) => Self {
216                shmem: pages,
217                ..Self::default()
218            },
219            None => Self::default(),
220        }
221    }
222
223    fn checked_add_assign(&mut self, other: Self) -> StarryResult {
224        self.anon = self
225            .anon
226            .checked_add(other.anon)
227            .ok_or(StarryError::BadState)?;
228        self.file = self
229            .file
230            .checked_add(other.file)
231            .ok_or(StarryError::BadState)?;
232        self.shmem = self
233            .shmem
234            .checked_add(other.shmem)
235            .ok_or(StarryError::BadState)?;
236        Ok(())
237    }
238}
239
240#[derive(Debug, Clone, Copy, Default)]
241struct PteOwnerPublication {
242    satisfied_pages: usize,
243    mapping_delta: MappingDelta,
244    resident_delta: ResidentDelta,
245}
246
247struct PreparedSlotPublication {
248    key: MappingSlotKey,
249    previous: Option<Arc<MappingSlot>>,
250    replacement: Option<Arc<MappingSlot>>,
251    resident_kind: Option<RssKind>,
252    provider_publication: ProviderPublication,
253    mapping_delta: MappingDelta,
254    resident_delta: ResidentDelta,
255}
256
257#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
258struct MappingSlotFingerprint {
259    key: MappingSlotKey,
260    mapping: MappingId,
261    page: PageId,
262    page_order: PageOrder,
263}
264
265/// Pre-apply facts used to derive one receipt from the published mapping
266/// graph.  This snapshot owns no page or frame reference; transaction
267/// preimages retain the actual objects needed for rollback.
268struct MappingGraphSnapshot {
269    slots: Vec<MappingSlotFingerprint>,
270    resident: ResidentPageCounts,
271}
272
273impl MappingGraphSnapshot {
274    fn delta_to(&self, after: &Self) -> StarryResult<(MappingDelta, ResidentDelta)> {
275        let mut before_index = 0usize;
276        let mut after_index = 0usize;
277        let mut attached = 0usize;
278        let mut detached = 0usize;
279        while before_index < self.slots.len() && after_index < after.slots.len() {
280            match self.slots[before_index].cmp(&after.slots[after_index]) {
281                core::cmp::Ordering::Less => {
282                    detached = detached.checked_add(1).ok_or(StarryError::BadState)?;
283                    before_index += 1;
284                }
285                core::cmp::Ordering::Equal => {
286                    before_index += 1;
287                    after_index += 1;
288                }
289                core::cmp::Ordering::Greater => {
290                    attached = attached.checked_add(1).ok_or(StarryError::BadState)?;
291                    after_index += 1;
292                }
293            }
294        }
295        detached = detached
296            .checked_add(self.slots.len() - before_index)
297            .ok_or(StarryError::BadState)?;
298        attached = attached
299            .checked_add(after.slots.len() - after_index)
300            .ok_or(StarryError::BadState)?;
301        Ok((
302            MappingDelta {
303                attached: u32::try_from(attached).map_err(|_| StarryError::BadState)?,
304                detached: u32::try_from(detached).map_err(|_| StarryError::BadState)?,
305            },
306            self.resident.checked_delta_to(after.resident)?,
307        ))
308    }
309}
310
311/// Permission views carried by one mapping operation. Keeping current,
312/// user-visible and maximum rights together prevents a caller from
313/// accidentally treating a lowered permission as the immutable VM envelope.
314#[derive(Debug, Clone, Copy)]
315pub struct MappingPermissions {
316    pub current: MappingFlags,
317    pub reported: MappingFlags,
318    pub maximum: MappingFlags,
319}
320
321#[derive(Debug, Clone, Copy)]
322pub(crate) struct MappingPublication {
323    replace: bool,
324    huge_page_advice: HugePageAdvice,
325    lock_mode: VmaLockMode,
326    advice_policy: VmaAdvicePolicy,
327    memlock_limit: Option<MemlockLimit>,
328}
329
330/// Per-syscall view of Linux `RLIMIT_MEMLOCK` and `CAP_IPC_LOCK`.
331///
332/// The limit is only an authorization input. Charged pages are always derived
333/// from the immutable VMA root, so rollback, split, merge and unmap cannot
334/// leave a second counter out of sync.
335#[derive(Debug, Clone, Copy)]
336pub(crate) struct MemlockLimit {
337    page_limit: u64,
338    bypass_limit: bool,
339    may_lock: bool,
340    exceeded_error: MemlockLimitError,
341}
342
343#[derive(Debug, Clone, Copy)]
344enum MemlockLimitError {
345    NoMemory,
346    WouldBlock,
347}
348
349impl MemlockLimit {
350    pub(crate) const fn for_mlock(byte_limit: u64, bypass_limit: bool) -> Self {
351        Self {
352            page_limit: byte_limit / PAGE_SIZE_4K as u64,
353            bypass_limit,
354            may_lock: byte_limit != 0 || bypass_limit,
355            exceeded_error: MemlockLimitError::NoMemory,
356        }
357    }
358
359    pub(crate) const fn for_mapping(byte_limit: u64, bypass_limit: bool) -> Self {
360        Self {
361            page_limit: byte_limit / PAGE_SIZE_4K as u64,
362            bypass_limit,
363            may_lock: byte_limit != 0 || bypass_limit,
364            exceeded_error: MemlockLimitError::WouldBlock,
365        }
366    }
367
368    pub(crate) const fn can_lock(self) -> bool {
369        self.may_lock
370    }
371
372    fn validate(self, locked_pages: u64) -> StarryResult {
373        if self.bypass_limit || locked_pages <= self.page_limit {
374            return Ok(());
375        }
376        Err(match self.exceeded_error {
377            MemlockLimitError::NoMemory => StarryError::NoMemory,
378            MemlockLimitError::WouldBlock => StarryError::WouldBlock,
379        })
380    }
381}
382
383impl MappingPublication {
384    const fn new(replace: bool) -> Self {
385        Self {
386            replace,
387            huge_page_advice: HugePageAdvice::Default,
388            lock_mode: VmaLockMode::Unlocked,
389            advice_policy: VmaAdvicePolicy::DEFAULT,
390            memlock_limit: None,
391        }
392    }
393
394    pub(crate) const fn mmap(
395        replace: bool,
396        lock_mode: VmaLockMode,
397        memlock_limit: Option<MemlockLimit>,
398    ) -> Self {
399        Self {
400            replace,
401            huge_page_advice: HugePageAdvice::Default,
402            lock_mode,
403            advice_policy: VmaAdvicePolicy::DEFAULT,
404            memlock_limit,
405        }
406    }
407
408    const fn mremap(
409        replace: bool,
410        huge_page_advice: HugePageAdvice,
411        lock_mode: VmaLockMode,
412        advice_policy: VmaAdvicePolicy,
413        memlock_limit: Option<MemlockLimit>,
414    ) -> Self {
415        Self {
416            replace,
417            huge_page_advice,
418            lock_mode,
419            advice_policy,
420            memlock_limit,
421        }
422    }
423}
424
425/// Publication result of one address-space mutation.
426///
427/// A pending TLB acknowledgement is a published mutation: metadata and the
428/// matching scalar state must remain visible while the receipt owns the
429/// detached resources.  Keeping it distinct from an unpublished error avoids
430/// reconstructing publication state by comparing epochs at syscall call sites.
431pub(crate) enum AddressSpaceMutationOutcome {
432    Complete,
433    PublishedPendingTlb(StarryError),
434}
435
436impl AddressSpaceMutationOutcome {
437    fn into_result(self) -> StarryResult {
438        match self {
439            Self::Complete => Ok(()),
440            Self::PublishedPendingTlb(error) => Err(error),
441        }
442    }
443}
444
445/// Linux keeps `start_brk` and `brk` in `mm_struct`, under `mmap_lock`.
446/// Keeping the equivalent values inside `AddrSpace` gives `CLONE_VM` one
447/// shared fact and makes a forked MM receive a snapshot with its VMA tree.
448#[derive(Debug, Clone, Copy)]
449struct HeapState {
450    start: usize,
451    current: usize,
452}
453
454impl HeapState {
455    const fn new(start: usize) -> Self {
456        Self {
457            start,
458            current: start,
459        }
460    }
461}
462
463/// Linux `mm->start_data`/`mm->end_data` metadata for the main executable.
464///
465/// The ELF loader publishes this once while the MM is still unreachable.  It
466/// is then copied with fork and read by both `brk` and procfs, preventing the
467/// resource-limit calculation and its observable metadata from diverging.
468#[derive(Debug, Clone, Copy, Default)]
469struct ExecutableDataLayout {
470    start: usize,
471    end: usize,
472}
473
474impl ExecutableDataLayout {
475    fn try_new(start: usize, end: usize) -> StarryResult<Self> {
476        if start == 0 || end < start {
477            return Err(StarryError::MalformedExecutable);
478        }
479        Ok(Self { start, end })
480    }
481
482    fn size(self) -> Option<usize> {
483        self.end.checked_sub(self.start)
484    }
485}
486
487fn checked_page_align_up(value: usize) -> StarryResult<usize> {
488    value
489        .checked_add(PAGE_SIZE_4K - 1)
490        .map(|rounded| rounded & !(PAGE_SIZE_4K - 1))
491        .ok_or(StarryError::InvalidInput)
492}
493
494struct ResidentLeafPreimage {
495    va: VirtAddr,
496    paddr: PhysAddr,
497    page_size: usize,
498    flags: MappingFlags,
499    backend: MappingOperation,
500    page: Arc<PageObject>,
501    slot: Arc<MappingSlot>,
502}
503
504#[derive(Clone, Copy)]
505struct OccupiedPteLeaf {
506    range: VirtAddrRange,
507    paddr: PhysAddr,
508    flags: MappingFlags,
509}
510
511struct MappingPreimage {
512    vma_root: Arc<VmaMap>,
513    vm_stat: ProcessVmStatSnapshot,
514    leaves: Vec<ResidentLeafPreimage>,
515}
516
517struct AppliedHugeSplit {
518    installed: InstalledHugeSplit,
519    old_slot: Arc<MappingSlot>,
520    child_keys: Vec<MappingSlotKey>,
521    child_slots: Vec<Arc<MappingSlot>>,
522    previous_mapping_slots: BTreeMap<MappingSlotKey, Arc<MappingSlot>>,
523}
524
525#[derive(Clone, Copy)]
526struct ProtectionLeafPreimage {
527    va: VirtAddr,
528    paddr: PhysAddr,
529    page_size: usize,
530    flags: MappingFlags,
531}
532
533/// Ownership retained after a PTE is detached and before its TLB receipt is
534/// acknowledged.  Each vector has a distinct ownership role: backend clones
535/// retain shared/device anchors, page objects retain anonymous frames, and
536/// cache pins retain file frames whose PageObject lease is intentionally
537/// non-owning.
538#[derive(Default)]
539struct RetiredMappingOwners {
540    backends: Vec<MappingOperation>,
541    pages: Vec<Arc<PageObject>>,
542    cache_pins: Vec<CachedPagePin>,
543    page_tables: Vec<DeferredPageTableFrames>,
544    /// File pages whose eviction stopped after PTE publication but before
545    /// every target CPU acknowledged the receipt.  Releasing this owner marks
546    /// the existing EvictionLease resumable; it must never make the page
547    /// Present again.
548    deferred_evictions: Vec<Arc<PageObject>>,
549}
550
551impl RetiredMappingOwners {
552    fn is_empty(&self) -> bool {
553        self.backends.is_empty()
554            && self.pages.is_empty()
555            && self.cache_pins.is_empty()
556            && self.page_tables.is_empty()
557            && self.deferred_evictions.is_empty()
558    }
559}
560
561struct RetiredMappingBatch {
562    epoch: VmEpoch,
563    owners: RetiredMappingOwners,
564}
565
566#[derive(Debug)]
567enum CommitMutationError {
568    /// The epoch/VMA root was not published.  A caller that retained an exact
569    /// preimage may restore it and return the original error.
570    Unpublished(StarryError),
571    /// Publication already advanced the epoch and retained the receipt.  The
572    /// visible mapping must not be rolled back while a remote CPU may still
573    /// hold the preimage translation.
574    PublishedPendingTlb(StarryError),
575}
576
577#[derive(Debug, Clone, Copy, PartialEq, Eq)]
578enum MutationPublication {
579    Complete,
580    PendingTlb,
581}
582
583struct PageFaultPlan {
584    base_epoch: VmEpoch,
585    space_id: AddressSpaceId,
586    vaddr: VirtAddr,
587    range: VirtAddrRange,
588    vma_flags: MappingFlags,
589    access_flags: MappingFlags,
590    operation: MappingOperation,
591    request: PopulateRequest,
592    preimage: FaultPteSnapshot,
593    map_plans: Option<PageFaultMapPlans>,
594}
595
596struct PageFaultMapPlans {
597    preferred: PageTableMapPlan,
598    fallback: Option<PageTableMapPlan>,
599}
600
601fn prepare_mapping_publication_mutation(
602    gate: &MutationGate,
603    space_id: AddressSpaceId,
604    active_targets: &Arc<AtomicUsize>,
605    start: VirtAddr,
606    size: usize,
607    replaces_existing: bool,
608) -> PreparedMutation {
609    // A non-replacing mmap publishes into a range whose current VMA and PTE
610    // preimage are empty. It can use Linux's fresh-PTE fast path only if no
611    // older shootdown for that VA is still pending. MAP_FIXED-style
612    // replacement retains the live target source until commit so a CPU
613    // activated during apply is still included in the shootdown receipt.
614    let mut mutation = if replaces_existing {
615        gate.begin_with_active_targets(space_id, active_targets.clone())
616    } else {
617        gate.begin_fresh_mapping(space_id)
618    };
619    if let Some(range) = TlbRange::new(start, size) {
620        mutation.add_tlb_range(range);
621    }
622    mutation
623}
624
625struct PreparedPageFault {
626    plan: PageFaultPlan,
627    materialization: FaultMaterialization,
628    map_deposit: Option<PageTableMapDeposit>,
629}
630
631impl PreparedPageFault {
632    fn into_apply_attempt(self) -> PageFaultApplyAttempt {
633        PageFaultApplyAttempt {
634            prepared: Some(self),
635            orphaned_map_deposit: None,
636        }
637    }
638
639    fn cancel(self) -> StarryResult {
640        self.plan
641            .operation
642            .cancel_prepared_fault_publication(self.materialization)
643    }
644}
645
646/// Retains ownership of a prepared fault while the address-space lock is held.
647///
648/// Cancellation leaves the token populated so the caller can release backend
649/// reservations after dropping the address-space lock. Successful publication
650/// consumes it before returning, keeping the apply result small without adding
651/// a heap allocation to every page fault.
652struct PageFaultApplyAttempt {
653    prepared: Option<PreparedPageFault>,
654    /// An internally duplicated deposit is retained here so even a corrupted
655    /// state never releases page-table frames below the address-space mutex.
656    orphaned_map_deposit: Option<PageTableMapDeposit>,
657}
658
659impl PageFaultApplyAttempt {
660    fn prepared(&self) -> &PreparedPageFault {
661        self.prepared
662            .as_ref()
663            .expect("page-fault apply attempt must retain its prepared token")
664    }
665
666    fn take_prepared(&mut self) -> PreparedPageFault {
667        self.prepared
668            .take()
669            .expect("page-fault apply attempt must consume its prepared token once")
670    }
671
672    fn take_map_deposit(&mut self) -> Option<PageTableMapDeposit> {
673        self.prepared.as_mut()?.map_deposit.take()
674    }
675
676    fn restore_map_deposit(&mut self, deposit: PageTableMapDeposit) {
677        if let Some(prepared) = self.prepared.as_mut()
678            && prepared.map_deposit.is_none()
679        {
680            prepared.map_deposit = Some(deposit);
681            return;
682        }
683        // This slot is reachable only if the internal prepared-token invariant
684        // was already violated. Retain the extra owner for lock-free release
685        // instead of dropping either page-table path here.
686        debug_assert!(self.orphaned_map_deposit.is_none());
687        self.orphaned_map_deposit = Some(deposit);
688    }
689
690    fn cancel(self) -> StarryResult {
691        let Self {
692            prepared,
693            orphaned_map_deposit,
694        } = self;
695        drop(orphaned_map_deposit);
696        prepared
697            .expect("cancelled page-fault apply attempt must retain its prepared token")
698            .cancel()
699    }
700
701    /// Releases only the caller-side token after an indeterminate apply.
702    ///
703    /// A Pending backend publication remains the explicit quarantine owner:
704    /// CowPageIndex retains the allocated PageObject and FilePageDomain retains
705    /// both the PageObject and CachedPagePin. The map deposit, if any, is still
706    /// unreachable and is deliberately dropped here after the address-space
707    /// mutex and every PTE stripe have been released.
708    fn release_to_repair_state(self) {
709        debug_assert!(self.prepared.is_some());
710        drop(self);
711    }
712}
713
714enum PageFaultApplyOutcome {
715    Complete(FaultResult),
716    Cancel(FaultResult),
717    NeedsRepair(FaultResult),
718    /// No PTE or owner was published. Cancel the prepared candidate before
719    /// servicing this older obligation, then plan the fault again.
720    CancelPendingTlb {
721        request: TlbRequest,
722        targets: Arc<AtomicUsize>,
723    },
724    PendingTlb {
725        request: TlbRequest,
726        targets: Arc<AtomicUsize>,
727    },
728}
729
730/// Result of revoking one file-cache reverse mapping.  A pending TLB result is
731/// successful publication, not a rollback-safe error.  `NeedsRepair` likewise
732/// keeps the page in Evicting so it cannot be reused from an unproved state.
733#[derive(Debug, Clone, Copy, PartialEq, Eq)]
734pub(crate) enum EvictMappingOutcome {
735    Complete,
736    PublishedPendingTlb,
737    NeedsRepair,
738}
739
740/// The virtual memory address space.
741pub struct AddrSpace {
742    id: AddressSpaceId,
743    /// Immutable ABI and hardware address limits captured at MM creation.
744    layout: UserVirtualAddressLayout,
745    /// The sole VMA metadata and executable-operation publication owner.
746    /// Readers receive metadata-only snapshots; mutations path-copy a complete
747    /// successor before any PTE apply phase begins.
748    vma_root: Arc<VmaMap>,
749    /// Heap boundaries owned by this MM and protected by the same lock as its
750    /// VMA/PTE mutation.  This is the Rust equivalent of Linux `mm->start_brk`
751    /// and `mm->brk`, not a process-side mirror.
752    heap: HeapState,
753    executable_data: ExecutableDataLayout,
754    /// The main image remains write-protected until this MM is retired.
755    executable_file: Option<Arc<ax_fs_ng::file::ExecutableFile>>,
756    pt: PageTable,
757    /// Fixed-order PTE/structure lock domains.  The page-table root remains a
758    /// materialized view; ownership is carried by VMA/page records.
759    pte_domain: PageTableDomain,
760    /// Monotonic publication epoch for VMA/PTE mutations.
761    mutation_gate: MutationGate,
762    /// Lock-free scheduler view of the last published mutation epoch.  It is
763    /// shared with `MmInner`; every commit updates it before returning so an
764    /// activation cannot install a stale generation.
765    published_epoch: Arc<core::sync::atomic::AtomicU64>,
766    /// All VmX counters for this address space.  Maintained automatically by
767    /// `map`, `unmap`, `clear`, and `try_clone`; never touch from outside mm/.
768    pub vm_stat: ProcessVmStat,
769    /// Current Linux RSS buckets. This value changes only when one published
770    /// mutation receipt is committed; MappingSlot remains the corresponding
771    /// installed-page ownership fact and rollback never touches these counters.
772    resident_pages: ResidentPageCounts,
773    resident_watermark: ResidentWatermark,
774    /// Frames detached by a mutation remain here until every TLB obligation
775    /// acknowledges the corresponding `(space_id, epoch)` request.
776    tlb_quarantine: TlbQuarantine,
777    /// Shared with the typed MM lifecycle object.  It is a materialized
778    /// active-CPU mask used only to form shootdown targets; ownership remains
779    /// in `MmInner`, and no lock is taken on the scheduler hot path.
780    tlb_targets: Arc<AtomicUsize>,
781    /// Authoritative per-address-space resident mapping records.  Legacy
782    /// backends still perform the hardware write, then this index publishes
783    /// the corresponding `MappingSlot`/rmap record in the same mutation path.
784    mapping_slots: BTreeMap<MappingSlotKey, Arc<MappingSlot>>,
785    /// Mapping owners detached by published mutations.  A batch is removed
786    /// only after the matching epoch's active-CPU shootdown completes.  An
787    /// uncommitted mutation that entered NeedsRepair deliberately leaves its
788    /// batch here, preventing teardown from fabricating a successful retire.
789    retired_mapping_batches: IrqMutex<Vec<RetiredMappingBatch>>,
790}
791
792impl AddrSpace {
793    /// Returns the address space base.
794    pub const fn base(&self) -> VirtAddr {
795        self.layout.range().start
796    }
797
798    /// Returns the address space end.
799    pub const fn end(&self) -> VirtAddr {
800        self.layout.task_size()
801    }
802
803    /// Returns the address space size.
804    pub fn size(&self) -> usize {
805        self.layout.range().size()
806    }
807
808    /// Returns the initial-stack ceiling captured by this MM.
809    pub const fn stack_top(&self) -> VirtAddr {
810        self.layout.stack_top()
811    }
812
813    /// Returns the immutable heap start recorded for `/proc/[pid]/stat` and
814    /// resource-limit calculations.
815    pub(crate) const fn heap_start(&self) -> usize {
816        self.heap.start
817    }
818
819    /// Returns the current program break while the address-space lock is held.
820    pub(crate) const fn heap_break(&self) -> usize {
821        self.heap.current
822    }
823
824    /// Transfers the loader's exclusion lease to the address-space lifetime.
825    pub(crate) fn set_executable_file(&mut self, file: ax_fs_ng::file::ExecutableFile) {
826        self.executable_file = Some(Arc::new(file));
827    }
828
829    /// Ends file exclusion at last-user exit, separately from physical reclaim.
830    /// Returned leases must be destroyed after releasing the MM metadata lock.
831    fn take_file_accesses(
832        &mut self,
833    ) -> (
834        Option<Arc<ax_fs_ng::file::ExecutableFile>>,
835        Vec<Arc<ax_fs_ng::file::WriteAccess>>,
836    ) {
837        let mut writers = Vec::new();
838        for entry in self.vma_root.iter_entries() {
839            if let Some(access) = entry.operation().take_write_access() {
840                writers.push(access);
841            }
842        }
843        (self.executable_file.take(), writers)
844    }
845
846    /// Publishes the main executable's Linux `start_data`/`end_data` pair.
847    /// This is only called for an unpublished loader-owned address space.
848    pub(crate) fn set_executable_data_layout(&mut self, start: usize, end: usize) -> StarryResult {
849        let layout = ExecutableDataLayout::try_new(start, end)?;
850        if start < self.base().as_usize() || end > self.end().as_usize() {
851            return Err(StarryError::MalformedExecutable);
852        }
853        self.executable_data = layout;
854        Ok(())
855    }
856
857    pub(crate) const fn executable_data_bounds(&self) -> (usize, usize) {
858        (self.executable_data.start, self.executable_data.end)
859    }
860
861    pub(crate) fn executable_data_size(&self) -> Option<usize> {
862        self.executable_data.size()
863    }
864
865    /// Applies one Linux-style `brk` VMA change and publishes the scalar break
866    /// under the same address-space lock.
867    ///
868    /// Unpublished mapping failures leave both values unchanged.  A mutation
869    /// whose VMA/PTE epoch is already published also publishes `brk` before
870    /// returning [`AddressSpaceMutationOutcome::PublishedPendingTlb`]; its
871    /// retained receipt prevents reuse until shootdown acknowledgement.
872    pub(crate) fn resize_heap_break(
873        &mut self,
874        requested: usize,
875        initial_mapping_end: usize,
876    ) -> StarryResult<AddressSpaceMutationOutcome> {
877        let old_break = self.heap.current;
878        let old_aligned = checked_page_align_up(old_break)?;
879        let new_aligned = checked_page_align_up(requested)?;
880
881        let outcome = if new_aligned > old_aligned {
882            let map_start = initial_mapping_end.max(old_aligned);
883            let map_size = new_aligned.saturating_sub(map_start);
884            if map_size == 0 {
885                AddressSpaceMutationOutcome::Complete
886            } else {
887                let start = VirtAddr::from(map_start);
888                let flags = MappingFlags::READ | MappingFlags::WRITE | MappingFlags::USER;
889                self.map_with_permissions_mode_classified(
890                    start,
891                    map_size,
892                    MappingPermissions {
893                        current: flags,
894                        reported: flags,
895                        maximum: flags,
896                    },
897                    false,
898                    MappingOperation::new_alloc(start, PAGE_SIZE_4K, "[heap]"),
899                    MappingPublication::new(false),
900                )?
901            }
902        } else if new_aligned < old_aligned {
903            let unmap_start = initial_mapping_end.max(new_aligned);
904            let unmap_size = old_aligned.saturating_sub(unmap_start);
905            if unmap_size == 0 {
906                AddressSpaceMutationOutcome::Complete
907            } else {
908                self.unmap_classified(VirtAddr::from(unmap_start), unmap_size)?
909            }
910        } else {
911            AddressSpaceMutationOutcome::Complete
912        };
913
914        // Linux preserves the exact unaligned user request while VMA changes
915        // use page-aligned boundaries.
916        self.heap.current = requested;
917        Ok(outcome)
918    }
919
920    /// Translates one materialized user address without exposing the page
921    /// table implementation to callers.
922    pub(crate) fn translate(&self, vaddr: VirtAddr) -> StarryResult<PhysAddr> {
923        self.pt
924            .query(vaddr)
925            .map(|(paddr, ..)| paddr)
926            .map_err(Into::into)
927    }
928
929    /// Returns the size of the resident leaf containing `vaddr`.
930    ///
931    /// Absence means the VMA is lazy or has been reclaimed; callers must use
932    /// the immutable VMA snapshot to distinguish that from an invalid VA.
933    pub(crate) fn resident_span(&self, vaddr: VirtAddr) -> Option<usize> {
934        self.pt.query(vaddr).ok().map(|(_, _, size)| size)
935    }
936
937    /// Returns resident bytes from an address to its owning leaf's end,
938    /// including permissionless leaves. This is a residency snapshot, not an
939    /// access capability; callers must validate VMA coverage separately.
940    pub(crate) fn resident_bytes_from(&self, vaddr: VirtAddr) -> Option<usize> {
941        let key = MappingSlotKey {
942            space_id: self.id,
943            va: vaddr,
944        };
945        let (_, slot) = self.mapping_slots.range(..=key).next_back()?;
946        if slot.state() != SlotState::Present {
947            return None;
948        }
949        let bytes = PAGE_SIZE_4K.checked_shl(slot.page_order.get().into())?;
950        let end = slot.va.checked_add(bytes)?;
951        (vaddr >= slot.va && vaddr < end).then(|| end.as_usize() - vaddr.as_usize())
952    }
953
954    /// Iterates only MappingSlots that can overlap `range`.
955    ///
956    /// A huge leaf may begin before `range.start`, so the ordered walk includes
957    /// the immediate predecessor before visiting keys whose start lies inside
958    /// the range. No earlier slot can overlap because published slots within
959    /// one address space never overlap each other.
960    fn mapping_slots_overlapping(
961        &self,
962        range: VirtAddrRange,
963    ) -> impl Iterator<Item = (&MappingSlotKey, &Arc<MappingSlot>)> {
964        let start = MappingSlotKey {
965            space_id: self.id,
966            va: range.start,
967        };
968        let end = MappingSlotKey {
969            space_id: self.id,
970            va: range.end,
971        };
972        let predecessor = self
973            .mapping_slots
974            .range(..start)
975            .next_back()
976            .filter(move |(key, slot)| key.space_id == self.id && slot.overlaps(range));
977        let inside = self
978            .mapping_slots
979            .range(start..end)
980            .filter(move |(key, slot)| key.space_id == self.id && slot.overlaps(range));
981        predecessor.into_iter().chain(inside)
982    }
983
984    fn mapping_slot_summary(
985        &self,
986        range: VirtAddrRange,
987    ) -> StarryResult<(usize, usize, ResidentPageCounts)> {
988        let mut slots = 0usize;
989        let mut materialized_pages = 0usize;
990        let mut resident = ResidentPageCounts::default();
991        for (_, slot) in self.mapping_slots_overlapping(range) {
992            if slot.state() != SlotState::Present {
993                return Err(StarryError::BadState);
994            }
995            let pages = 1usize
996                .checked_shl(slot.page_order.get().into())
997                .ok_or(StarryError::BadState)?;
998            slots = slots.checked_add(1).ok_or(StarryError::BadState)?;
999            materialized_pages = materialized_pages
1000                .checked_add(pages)
1001                .ok_or(StarryError::BadState)?;
1002            resident.checked_add_pages(
1003                slot.resident_kind(),
1004                u64::try_from(pages).map_err(|_| StarryError::BadState)?,
1005            )?;
1006        }
1007        Ok((slots, materialized_pages, resident))
1008    }
1009
1010    fn resident_counts_from_all_slots(&self) -> StarryResult<ResidentPageCounts> {
1011        let mut resident = ResidentPageCounts::default();
1012        for slot in self.mapping_slots.values() {
1013            if slot.state() != SlotState::Present {
1014                return Err(StarryError::BadState);
1015            }
1016            let pages = 1u64
1017                .checked_shl(slot.page_order.get().into())
1018                .ok_or(StarryError::BadState)?;
1019            resident.checked_add_pages(slot.resident_kind(), pages)?;
1020        }
1021        Ok(resident)
1022    }
1023
1024    /// Returns occupied page-table leaves overlapping any requested range.
1025    ///
1026    /// This includes retained non-present leaves: protection may remove
1027    /// hardware access without releasing the PTE's frame ownership. Walking
1028    /// allocated page-table frames keeps sparse transactions proportional to
1029    /// materialized state instead of the virtual address span.
1030    fn occupied_pte_leaves_overlapping(
1031        &self,
1032        ranges: &[VirtAddrRange],
1033    ) -> StarryResult<Vec<OccupiedPteLeaf>> {
1034        let mut leaves = Vec::new();
1035        for range in ranges {
1036            for entry in self.pt.walk_occupied_range(range.start, range.end) {
1037                let leaf_start = entry.vaddr;
1038                let page_size = self
1039                    .pt
1040                    .mapping_size_for_level(entry.level)
1041                    .ok_or(StarryError::BadState)?;
1042                let leaf_end = leaf_start
1043                    .checked_add(page_size)
1044                    .ok_or(StarryError::BadState)?;
1045                if leaf_start >= range.end || leaf_end <= range.start {
1046                    continue;
1047                }
1048                if leaf_start < range.start || leaf_end > range.end {
1049                    return Err(StarryError::OperationNotSupported);
1050                }
1051                let is_directory_level = entry.level > 1;
1052                leaves.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
1053                leaves.push(OccupiedPteLeaf {
1054                    range: VirtAddrRange::new(leaf_start, leaf_end),
1055                    paddr: entry.pte.paddr(is_directory_level),
1056                    flags: entry.pte.config(is_directory_level),
1057                });
1058            }
1059        }
1060        Ok(leaves)
1061    }
1062
1063    /// Retains the published ownership records matching the occupied PTE
1064    /// leaves in the requested ranges. Both directions are verified so neither
1065    /// a PTE without a MappingSlot nor a Present slot without a PTE can enter a
1066    /// transaction preimage.
1067    fn materialized_slots_overlapping(
1068        &self,
1069        ranges: &[VirtAddrRange],
1070    ) -> StarryResult<Vec<(MappingSlotKey, Arc<MappingSlot>, OccupiedPteLeaf)>> {
1071        let leaves = self.occupied_pte_leaves_overlapping(ranges)?;
1072        let mut slots = Vec::new();
1073        slots
1074            .try_reserve(leaves.len())
1075            .map_err(|_| StarryError::NoMemory)?;
1076        for leaf in leaves {
1077            let key = MappingSlotKey {
1078                space_id: self.id,
1079                va: leaf.range.start,
1080            };
1081            let slot = self
1082                .mapping_slots
1083                .get(&key)
1084                .cloned()
1085                .ok_or(StarryError::BadState)?;
1086            let expected_size = PAGE_SIZE_4K
1087                .checked_shl(slot.page_order.get().into())
1088                .ok_or(StarryError::BadState)?;
1089            if slot.state() != SlotState::Present
1090                || slot.mm_id != self.id
1091                || slot.va != key.va
1092                || expected_size != leaf.range.size()
1093                || slot.mapped_paddr() != Some(leaf.paddr)
1094            {
1095                return Err(StarryError::BadState);
1096            }
1097            slots.push((key, slot, leaf));
1098        }
1099        let overlapping_slots = ranges
1100            .iter()
1101            .map(|range| self.mapping_slots_overlapping(*range).count())
1102            .sum::<usize>();
1103        if overlapping_slots != slots.len() {
1104            return Err(StarryError::BadState);
1105        }
1106        Ok(slots)
1107    }
1108
1109    /// Rejects an operation that would carve only part of a materialized
1110    /// huge-page leaf.  Until the typed THP split receipt is wired through all
1111    /// four architectures, moving or replacing such a leaf must fail before
1112    /// either source or destination metadata changes.
1113    pub(crate) fn validate_materialized_leaf_boundaries(
1114        &self,
1115        start: VirtAddr,
1116        size: usize,
1117    ) -> StarryResult {
1118        self.validate_region(start, size)?;
1119        let range =
1120            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
1121        for (key, slot, leaf) in self.materialized_slots_overlapping(&[range])? {
1122            let page_size = leaf.range.size();
1123            let leaf_end = slot
1124                .va
1125                .checked_add(page_size)
1126                .ok_or(StarryError::BadState)?;
1127            if slot.va < range.start || leaf_end > range.end {
1128                return Err(StarryError::OperationNotSupported);
1129            }
1130            if key.va != slot.va
1131                || slot.mm_id != self.id
1132                || slot.state() != SlotState::Present
1133                || slot.mapped_paddr() != Some(leaf.paddr)
1134            {
1135                return Err(StarryError::BadState);
1136            }
1137        }
1138        Ok(())
1139    }
1140
1141    /// Splits only huge leaves crossed by a mutation boundary.  Allocation of
1142    /// child MappingSlots and the replacement BTreeMap happens before the PTE
1143    /// stripe is acquired; apply itself consumes the leaf's deposited table and
1144    /// publishes one rmap/refcount cardinality change.
1145    fn apply_partial_huge_splits(
1146        &mut self,
1147        range: VirtAddrRange,
1148    ) -> StarryResult<Vec<AppliedHugeSplit>> {
1149        let mut candidates = Vec::new();
1150        candidates
1151            .try_reserve(2)
1152            .map_err(|_| StarryError::NoMemory)?;
1153        for (key, slot) in self.mapping_slots_overlapping(range) {
1154            if slot.page_order == PageOrder::BASE || !slot.overlaps(range) {
1155                continue;
1156            }
1157            let slot_size = PAGE_SIZE_4K
1158                .checked_shl(slot.page_order.get().into())
1159                .ok_or(StarryError::BadState)?;
1160            let slot_end = slot
1161                .va
1162                .checked_add(slot_size)
1163                .ok_or(StarryError::BadState)?;
1164            if range.start <= slot.va && slot_end <= range.end {
1165                continue;
1166            }
1167            candidates
1168                .try_reserve(1)
1169                .map_err(|_| StarryError::NoMemory)?;
1170            candidates.push((*key, slot.clone()));
1171        }
1172
1173        let mut applied = Vec::new();
1174        applied
1175            .try_reserve(candidates.len())
1176            .map_err(|_| StarryError::NoMemory)?;
1177        for (key, slot) in candidates {
1178            match self.apply_one_partial_huge_split(key, slot) {
1179                Ok(split) => applied.push(split),
1180                Err(error) => {
1181                    if !self.rollback_applied_huge_splits(applied) {
1182                        self.mutation_gate.mark_needs_repair();
1183                        return Err(StarryError::BadState);
1184                    }
1185                    return Err(error);
1186                }
1187            }
1188        }
1189        Ok(applied)
1190    }
1191
1192    /// Applies boundary splits for several already-validated, disjoint
1193    /// mutation ranges.  Earlier ranges are rolled back if a later range
1194    /// cannot consume its deposit, so callers either receive every split
1195    /// receipt or observe the original huge-leaf graph.
1196    fn apply_partial_huge_splits_for_ranges(
1197        &mut self,
1198        ranges: &[VirtAddrRange],
1199    ) -> StarryResult<Vec<AppliedHugeSplit>> {
1200        let capacity = ranges.len().checked_mul(2).ok_or(StarryError::NoMemory)?;
1201        let mut applied = Vec::new();
1202        applied
1203            .try_reserve_exact(capacity)
1204            .map_err(|_| StarryError::NoMemory)?;
1205        for range in ranges {
1206            match self.apply_partial_huge_splits(*range) {
1207                Ok(mut splits) => applied.append(&mut splits),
1208                Err(error) => {
1209                    if !self.rollback_applied_huge_splits(applied) {
1210                        self.mutation_gate.mark_needs_repair();
1211                        return Err(StarryError::BadState);
1212                    }
1213                    return Err(error);
1214                }
1215            }
1216        }
1217        Ok(applied)
1218    }
1219
1220    fn apply_one_partial_huge_split(
1221        &mut self,
1222        old_key: MappingSlotKey,
1223        old_slot: Arc<MappingSlot>,
1224    ) -> StarryResult<AppliedHugeSplit> {
1225        // Starry's first THP implementation is order-9.  A larger block needs
1226        // another deposited level and must not be represented as 4 KiB slots.
1227        if old_slot.page_order != PageOrder::new(9)
1228            || old_slot.state() != SlotState::Present
1229            || !self
1230                .mapping_slots
1231                .get(&old_key)
1232                .is_some_and(|slot| Arc::ptr_eq(slot, &old_slot))
1233        {
1234            return Err(StarryError::OperationNotSupported);
1235        }
1236        let block_size = PAGE_SIZE_4K
1237            .checked_shl(old_slot.page_order.get().into())
1238            .ok_or(StarryError::BadState)?;
1239        let block_range = VirtAddrRange::try_from_start_size(old_slot.va, block_size)
1240            .ok_or(StarryError::BadState)?;
1241        let (mapped_paddr, _, mapped_size) = self.pt.query(old_slot.va)?;
1242        if old_slot.mapped_paddr() != Some(mapped_paddr)
1243            || mapped_size != block_size
1244            || old_slot.page.frame().size() < block_size
1245        {
1246            return Err(StarryError::BadState);
1247        }
1248
1249        let child_count = block_size / PAGE_SIZE_4K;
1250        let mut child_keys = Vec::new();
1251        let mut child_slots = Vec::new();
1252        child_keys
1253            .try_reserve_exact(child_count)
1254            .map_err(|_| StarryError::NoMemory)?;
1255        child_slots
1256            .try_reserve_exact(child_count)
1257            .map_err(|_| StarryError::NoMemory)?;
1258        for index in 0..child_count {
1259            let offset = index
1260                .checked_mul(PAGE_SIZE_4K)
1261                .ok_or(StarryError::BadState)?;
1262            let va = old_slot
1263                .va
1264                .checked_add(offset)
1265                .ok_or(StarryError::BadState)?;
1266            let key = MappingSlotKey {
1267                space_id: self.id,
1268                va,
1269            };
1270            let child = MappingSlot::new_with_frame_offset(
1271                old_slot.mapping,
1272                self.id,
1273                va,
1274                PageOrder::BASE,
1275                old_slot.page.clone(),
1276                old_slot
1277                    .frame_offset()
1278                    .checked_add(offset)
1279                    .ok_or(StarryError::BadState)?,
1280                old_slot.resident_kind(),
1281            )
1282            .ok_or(StarryError::BadState)?;
1283            child_keys.push(key);
1284            child_slots.push(Arc::new(child));
1285        }
1286
1287        // Build the published slot root before touching the materialized page
1288        // table. BTreeMap has no fallible reserve API on this toolchain.
1289        let mut next_mapping_slots = self.mapping_slots.clone();
1290        let removed = next_mapping_slots
1291            .remove(&old_key)
1292            .ok_or(StarryError::BadState)?;
1293        if !Arc::ptr_eq(&removed, &old_slot) {
1294            return Err(StarryError::BadState);
1295        }
1296        for (key, slot) in child_keys.iter().copied().zip(child_slots.iter().cloned()) {
1297            if next_mapping_slots.insert(key, slot).is_some() {
1298                return Err(StarryError::BadState);
1299            }
1300        }
1301
1302        // THP split grows one rmap entry into 512. Reserve any replacement
1303        // backing store and claim capacity before taking the PTE stripe; apply
1304        // below only changes keys under the IRQ-saving graph lock.
1305        let old_keys = [old_key];
1306        let mut graph_reservation = old_slot
1307            .page
1308            .prepare_mapping_graph_replace(&old_keys, &child_keys)
1309            .map_err(|error| match error {
1310                MappingGraphError::ResourceExhausted | MappingGraphError::RefOverflow => {
1311                    StarryError::NoMemory
1312                }
1313                _ => StarryError::BadState,
1314            })?;
1315
1316        let deposit = old_slot
1317            .take_huge_split_deposit()
1318            .ok_or(StarryError::BadState)?;
1319        let mutation_gate = &self.mutation_gate;
1320        let pte_domain = &self.pte_domain;
1321        let pt = &mut self.pt;
1322        let stripe = pte_domain.lock_range(block_range);
1323        let installed = match pt.try_split_huge_page_with(deposit) {
1324            Ok(installed) => installed,
1325            Err(failure) => {
1326                let (error, deposit) = failure.into_parts();
1327                match old_slot.restore_huge_split_deposit(deposit) {
1328                    Ok(()) => {
1329                        drop(stripe);
1330                        drop(graph_reservation);
1331                        return Err(error.into());
1332                    }
1333                    Err(orphaned) => {
1334                        // The detached table is unreachable, but releasing its
1335                        // frame while IRQs are disabled would nest the global
1336                        // allocator below the PTE stripe.
1337                        drop(stripe);
1338                        drop(graph_reservation);
1339                        drop(orphaned);
1340                        mutation_gate.mark_needs_repair();
1341                        return Err(StarryError::BadState);
1342                    }
1343                }
1344            }
1345        };
1346
1347        if let Err(graph_error) = old_slot.page.replace_mapping_graph_reserved(
1348            &old_keys,
1349            &child_keys,
1350            &mut graph_reservation,
1351        ) {
1352            let restored = pt.restore_huge_split(installed);
1353            match restored {
1354                Ok(deposit) => match old_slot.restore_huge_split_deposit(deposit) {
1355                    Ok(()) => {
1356                        drop(stripe);
1357                        drop(graph_reservation);
1358                        return Err(match graph_error {
1359                            MappingGraphError::ResourceExhausted
1360                            | MappingGraphError::RefOverflow => StarryError::NoMemory,
1361                            _ => StarryError::BadState,
1362                        });
1363                    }
1364                    Err(orphaned) => {
1365                        drop(stripe);
1366                        drop(graph_reservation);
1367                        drop(orphaned);
1368                    }
1369                },
1370                Err(_) => {
1371                    drop(stripe);
1372                    drop(graph_reservation);
1373                }
1374            }
1375            mutation_gate.mark_needs_repair();
1376            return Err(StarryError::BadState);
1377        }
1378
1379        let mut published_children = 0usize;
1380        for child in &child_slots {
1381            if !child.publish_after_graph_replace() {
1382                break;
1383            }
1384            published_children += 1;
1385        }
1386        let old_detached =
1387            published_children == child_slots.len() && old_slot.detach_after_graph_replace();
1388        if !old_detached {
1389            for child in child_slots[..published_children].iter().rev() {
1390                let _ = child.reserve_after_graph_replace();
1391            }
1392            let graph_restored = old_slot
1393                .page
1394                .replace_mapping_graph_reserved(&child_keys, &old_keys, &mut graph_reservation)
1395                .is_ok();
1396            let restored = pt.restore_huge_split(installed);
1397            let (deposit_restored, orphaned) = match restored {
1398                Ok(deposit) => match old_slot.restore_huge_split_deposit(deposit) {
1399                    Ok(()) => (true, None),
1400                    Err(orphaned) => (false, Some(orphaned)),
1401                },
1402                Err(_) => (false, None),
1403            };
1404            drop(stripe);
1405            drop(graph_reservation);
1406            drop(orphaned);
1407            if graph_restored && deposit_restored {
1408                return Err(StarryError::BadState);
1409            }
1410            mutation_gate.mark_needs_repair();
1411            return Err(StarryError::BadState);
1412        }
1413
1414        drop(stripe);
1415        drop(graph_reservation);
1416        let previous_mapping_slots =
1417            core::mem::replace(&mut self.mapping_slots, next_mapping_slots);
1418        Ok(AppliedHugeSplit {
1419            installed,
1420            old_slot,
1421            child_keys,
1422            child_slots,
1423            previous_mapping_slots,
1424        })
1425    }
1426
1427    fn rollback_applied_huge_splits(&mut self, mut splits: Vec<AppliedHugeSplit>) -> bool {
1428        while let Some(split) = splits.pop() {
1429            let block_range = VirtAddrRange::try_from_start_size(
1430                split.installed.block_vaddr(),
1431                split.installed.block_size(),
1432            );
1433            let Some(block_range) = block_range else {
1434                return false;
1435            };
1436            let old_key = MappingSlotKey {
1437                space_id: self.id,
1438                va: split.old_slot.va,
1439            };
1440            let old_keys = [old_key];
1441            let mut graph_reservation = match split
1442                .old_slot
1443                .page
1444                .prepare_mapping_graph_replace(&split.child_keys, &old_keys)
1445            {
1446                Ok(reservation) => reservation,
1447                Err(_) => return false,
1448            };
1449            let pte_domain = &self.pte_domain;
1450            let pt = &mut self.pt;
1451            let stripe = pte_domain.lock_range(block_range);
1452            let Ok(deposit) = pt.restore_huge_split(split.installed) else {
1453                drop(stripe);
1454                drop(graph_reservation);
1455                return false;
1456            };
1457            if split
1458                .old_slot
1459                .page
1460                .replace_mapping_graph_reserved(
1461                    &split.child_keys,
1462                    &old_keys,
1463                    &mut graph_reservation,
1464                )
1465                .is_err()
1466            {
1467                let orphaned = split.old_slot.restore_huge_split_deposit(deposit).err();
1468                drop(stripe);
1469                drop(graph_reservation);
1470                drop(orphaned);
1471                return false;
1472            }
1473            let slots_restored = !split
1474                .child_slots
1475                .iter()
1476                .any(|slot| !slot.detach_after_graph_replace())
1477                && split.old_slot.restore_after_graph_replace();
1478            let orphaned = split.old_slot.restore_huge_split_deposit(deposit).err();
1479            let deposit_restored = orphaned.is_none();
1480            drop(stripe);
1481            drop(graph_reservation);
1482            drop(orphaned);
1483            if !slots_restored || !deposit_restored {
1484                return false;
1485            }
1486            self.mapping_slots = split.previous_mapping_slots;
1487        }
1488        true
1489    }
1490
1491    fn capture_protection_leaf_preimage(
1492        &self,
1493        range: VirtAddrRange,
1494    ) -> StarryResult<Vec<ProtectionLeafPreimage>> {
1495        let occupied = self.occupied_pte_leaves_overlapping(&[range])?;
1496        let mut leaves = Vec::new();
1497        leaves
1498            .try_reserve(occupied.len())
1499            .map_err(|_| StarryError::NoMemory)?;
1500        for leaf in occupied {
1501            leaves.push(ProtectionLeafPreimage {
1502                va: leaf.range.start,
1503                paddr: leaf.paddr,
1504                page_size: leaf.range.size(),
1505                flags: leaf.flags,
1506            });
1507        }
1508        Ok(leaves)
1509    }
1510
1511    fn restore_protection_leaf_preimage(&mut self, leaves: &[ProtectionLeafPreimage]) -> bool {
1512        for leaf in leaves.iter().rev() {
1513            let Ok((paddr, _, page_size)) = self.pt.query(leaf.va) else {
1514                return false;
1515            };
1516            if paddr != leaf.paddr || page_size != leaf.page_size {
1517                return false;
1518            }
1519            if self.pt.protect_page(leaf.va, leaf.flags) != Ok(leaf.page_size) {
1520                return false;
1521            }
1522        }
1523        true
1524    }
1525
1526    fn abort_unpublished_protection(
1527        &mut self,
1528        vma_root: Arc<VmaMap>,
1529        vm_stat: ProcessVmStatSnapshot,
1530        leaves: &[ProtectionLeafPreimage],
1531        splits: Vec<AppliedHugeSplit>,
1532        original_error: StarryError,
1533    ) -> StarryResult {
1534        let ptes_restored = self.restore_protection_leaf_preimage(leaves);
1535        self.vma_root = vma_root;
1536        self.vm_stat.restore(vm_stat);
1537        let splits_restored = self.rollback_applied_huge_splits(splits);
1538        if ptes_restored && splits_restored {
1539            self.mutation_gate.clear_repair();
1540            Err(original_error)
1541        } else {
1542            self.mutation_gate.mark_needs_repair();
1543            Err(StarryError::BadState)
1544        }
1545    }
1546
1547    /// Installs the kernel root entries required by architectures that share a
1548    /// single hardware root between kernel and userspace.
1549    ///
1550    /// This intent-specific operation deliberately does not expose a mutable
1551    /// page-table reference to callers.
1552    ///
1553    /// # Safety
1554    ///
1555    /// `source` and every shared intermediate node must outlive this address
1556    /// space, and the capability's range must not overlap its user-owned range.
1557    #[cfg(not(any(target_arch = "aarch64", target_arch = "loongarch64")))]
1558    pub(crate) unsafe fn share_kernel_root_entries_from(
1559        &mut self,
1560        source: RootEntryShare<'_>,
1561    ) -> Result<(), PagingError> {
1562        // SAFETY: the caller provides the lifetime and non-overlap proof stated
1563        // by this method's contract.
1564        unsafe { source.install_into(&mut self.pt) }
1565    }
1566
1567    /// Returns the materialized hardware root for lifecycle publication.
1568    const fn materialized_root(&self) -> PhysAddr {
1569        self.pt.root_paddr()
1570    }
1571
1572    /// Checks if the address space contains the given address range.
1573    pub fn contains_range(&self, start: VirtAddr, size: usize) -> bool {
1574        let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
1575            return false;
1576        };
1577        range.start >= self.base() && range.end <= self.end()
1578    }
1579
1580    /// Creates a new empty address space.
1581    pub fn new_empty(base: VirtAddr, size: usize) -> StarryResult<Self> {
1582        Self::new_with_layout(UserVirtualAddressLayout::from_range(base, size)?)
1583    }
1584
1585    /// Creates a user MM from one already validated immutable layout.
1586    pub(crate) fn new_user(layout: UserVirtualAddressLayout) -> StarryResult<Self> {
1587        Self::new_with_layout(layout)
1588    }
1589
1590    fn new_with_layout(layout: UserVirtualAddressLayout) -> StarryResult<Self> {
1591        Ok(Self {
1592            id: AddressSpaceId::allocate(),
1593            layout,
1594            vma_root: Arc::new(VmaMap::default()),
1595            heap: HeapState::new(USER_HEAP_BASE),
1596            executable_data: ExecutableDataLayout::default(),
1597            executable_file: None,
1598            pt: PageTable::new(PagingAllocator).map_err(|_| StarryError::NoMemory)?,
1599            pte_domain: PageTableDomain::new(),
1600            mutation_gate: MutationGate::new(),
1601            published_epoch: Arc::new(core::sync::atomic::AtomicU64::new(0)),
1602            vm_stat: ProcessVmStat::new(),
1603            resident_pages: ResidentPageCounts::default(),
1604            resident_watermark: ResidentWatermark::new(),
1605            tlb_quarantine: TlbQuarantine::default(),
1606            tlb_targets: Arc::new(AtomicUsize::new(0)),
1607            mapping_slots: BTreeMap::new(),
1608            retired_mapping_batches: IrqMutex::new(Vec::new()),
1609        })
1610    }
1611
1612    /// Returns the stable identity used by scheduler activation and TLB
1613    /// obligations. It is independent of the page-table root address.
1614    pub const fn address_space_id(&self) -> AddressSpaceId {
1615        self.id
1616    }
1617
1618    /// Returns the current VMA/PTE publication epoch.
1619    pub fn vm_epoch(&self) -> VmEpoch {
1620        self.mutation_gate.current_epoch()
1621    }
1622
1623    pub(crate) fn tlb_targets(&self) -> Arc<AtomicUsize> {
1624        self.tlb_targets.clone()
1625    }
1626
1627    pub(crate) fn published_epoch_source(&self) -> Arc<core::sync::atomic::AtomicU64> {
1628        self.published_epoch.clone()
1629    }
1630
1631    fn publish_mutation_classified(
1632        &mut self,
1633        mutation: PreparedMutation,
1634    ) -> Result<MutationPublication, CommitMutationError> {
1635        let mut next_resident = self.resident_pages;
1636        next_resident
1637            .checked_apply(mutation.receipt().resident_delta)
1638            .map_err(CommitMutationError::Unpublished)?;
1639        match self.mutation_gate.commit(mutation) {
1640            Ok(receipt) => {
1641                self.resident_pages = next_resident;
1642                self.published_epoch.store(
1643                    receipt.new_epoch.get(),
1644                    core::sync::atomic::Ordering::Release,
1645                );
1646                self.resident_watermark
1647                    .observe_resident_total(self.resident_pages.total());
1648                Ok(MutationPublication::Complete)
1649            }
1650            Err(MutationError::TlbPending) => {
1651                self.resident_pages = next_resident;
1652                self.published_epoch.store(
1653                    self.mutation_gate.current_epoch().get(),
1654                    core::sync::atomic::Ordering::Release,
1655                );
1656                // The epoch and mapping graph are already published even
1657                // though old translations and owners remain quarantined.
1658                // Linux likewise records RSS high-water before reclaiming the
1659                // old page-table view; a TLB timeout must not hide the new RSS.
1660                self.resident_watermark
1661                    .observe_resident_total(self.resident_pages.total());
1662                Ok(MutationPublication::PendingTlb)
1663            }
1664            Err(error) => Err(CommitMutationError::Unpublished(
1665                Self::map_unpublished_mutation_error(error),
1666            )),
1667        }
1668    }
1669
1670    fn map_unpublished_mutation_error(error: MutationError) -> StarryError {
1671        match error {
1672            MutationError::ResourceExhausted => StarryError::NoMemory,
1673            MutationError::PendingTlbOverlap => StarryError::ResourceBusy,
1674            MutationError::NeedsRepair
1675            | MutationError::EpochExhausted
1676            | MutationError::EpochConflict
1677            | MutationError::WrongState
1678            | MutationError::ApplyFailed
1679            | MutationError::TlbPending => StarryError::BadState,
1680        }
1681    }
1682
1683    fn commit_mutation_classified(
1684        &mut self,
1685        mutation: PreparedMutation,
1686    ) -> Result<(), CommitMutationError> {
1687        match self.publish_mutation_classified(mutation)? {
1688            MutationPublication::Complete => Ok(()),
1689            MutationPublication::PendingTlb => {
1690                // Compatibility callers still use the synchronous service.
1691                // Faults use `publish_mutation_classified` directly, drop the
1692                // address-space mutex, and complete this platform operation
1693                // outside every VMA/PTE/rmap lock.
1694                self.service_pending_tlb()
1695                    .map(|_| ())
1696                    .map_err(CommitMutationError::PublishedPendingTlb)
1697            }
1698        }
1699    }
1700
1701    fn commit_mutation(&mut self, mutation: PreparedMutation) -> StarryResult {
1702        match self.commit_mutation_classified(mutation) {
1703            Ok(()) => Ok(()),
1704            Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
1705            Err(CommitMutationError::Unpublished(error)) => {
1706                // Most compatibility callers have already applied a
1707                // backend/PTE delta and do not retain an inverse.  They must
1708                // remain quarantined.  Transactional callers such as fault
1709                // use `commit_mutation_classified` directly and restore their
1710                // preimage before returning.
1711                self.mutation_gate.mark_needs_repair();
1712                Err(error)
1713            }
1714        }
1715    }
1716
1717    /// Completes outstanding address-space-tagged TLB obligations.
1718    ///
1719    /// The request snapshot and active-CPU source can outlive the address-space
1720    /// mutex. Platform shootdown therefore runs without a VMA, PTE, rmap, or
1721    /// page-cache lock, after which the short acknowledgement phase retires the
1722    /// matching receipts and quarantined owners.
1723    pub fn service_pending_tlb(&self) -> StarryResult<usize> {
1724        let requests = self
1725            .mutation_gate
1726            .pending_requests()
1727            .map_err(|_| StarryError::NoMemory)?;
1728        Self::flush_tlb_requests(&requests, &self.tlb_targets)?;
1729        self.acknowledge_tlb_requests(&requests)
1730    }
1731
1732    fn flush_tlb_requests(requests: &[TlbRequest], tlb_targets: &AtomicUsize) -> StarryResult {
1733        for request in requests {
1734            let pending_targets = request.pending();
1735            // `tlb_targets` is the full-flush fallback's equivalent of
1736            // Linux's loaded-mm footprint. A bit can only be cleared after
1737            // another root is installed and the local TLB is flushed.
1738            let live_targets =
1739                pending_targets & tlb_targets.load(core::sync::atomic::Ordering::Acquire);
1740            if request.ranges.is_empty() && live_targets != 0 {
1741                ax_runtime::hal::cache::flush_tlb_all_on_cpus(live_targets)
1742                    .map_err(Self::map_tlb_shootdown_error)?;
1743            } else if live_targets != 0 {
1744                for range in &request.ranges {
1745                    ax_runtime::hal::cache::flush_tlb_range_on_cpus(
1746                        live_targets,
1747                        range.start,
1748                        range.size,
1749                    )
1750                    .map_err(Self::map_tlb_shootdown_error)?;
1751                }
1752            }
1753        }
1754        Ok(())
1755    }
1756
1757    fn map_tlb_shootdown_error(error: ax_runtime::hal::cache::TlbShootdownError) -> StarryError {
1758        match error {
1759            ax_runtime::hal::cache::TlbShootdownError::Timeout => StarryError::TimedOut,
1760            ax_runtime::hal::cache::TlbShootdownError::Unsupported
1761            | ax_runtime::hal::cache::TlbShootdownError::CpuOffline => StarryError::Unsupported,
1762            ax_runtime::hal::cache::TlbShootdownError::GenerationExhausted => {
1763                StarryError::Errno(syscalls::Errno::EOVERFLOW)
1764            }
1765            ax_runtime::hal::cache::TlbShootdownError::Platform => StarryError::Io,
1766        }
1767    }
1768
1769    fn acknowledge_tlb_requests(&self, requests: &[TlbRequest]) -> StarryResult<usize> {
1770        let mut completed = 0;
1771        for request in requests {
1772            let pending_targets = request.pending();
1773            // Every target now has either a shootdown proof or a root-switch
1774            // proof. A later activation installs the published epoch and
1775            // performs its required local tag/full flush before use.
1776            for cpu in 0..usize::BITS as usize {
1777                if pending_targets & (1usize << cpu) == 0 {
1778                    continue;
1779                }
1780                match self.mutation_gate.acknowledge(self.id, request.epoch, cpu) {
1781                    Ok(_) | Err(MutationError::WrongState) | Err(MutationError::TlbPending) => {}
1782                    Err(_) => return Err(StarryError::BadState),
1783                }
1784                self.tlb_quarantine
1785                    .acknowledge(self.id, request.epoch, cpu)
1786                    .map_err(|_| StarryError::NoMemory)?;
1787            }
1788            if self
1789                .mutation_gate
1790                .pending_request(self.id, request.epoch)
1791                .is_none()
1792            {
1793                self.release_retired_mapping_owners(request.epoch);
1794            }
1795            completed += 1;
1796        }
1797        Ok(completed)
1798    }
1799
1800    fn prepare_mutation(&self) -> PreparedMutation {
1801        self.mutation_gate
1802            .begin_with_active_targets(self.id, self.tlb_targets.clone())
1803    }
1804
1805    /// Begins a metadata-only publication that cannot leave stale hardware
1806    /// translations.  Unlike a PTE mutation, it deliberately captures no
1807    /// active CPU targets and therefore cannot manufacture a full-flush
1808    /// obligation for a VMA advice-bit change.
1809    fn prepare_metadata_mutation(&self) -> PreparedMutation {
1810        self.mutation_gate.begin(self.id, 0)
1811    }
1812
1813    fn prepare_mutation_range(&self, start: VirtAddr, size: usize) -> PreparedMutation {
1814        let mut mutation = self.prepare_mutation();
1815        if let Some(range) = TlbRange::new(start, size) {
1816            mutation.add_tlb_range(range);
1817        }
1818        mutation
1819    }
1820
1821    /// Begins a fresh-PTE publication that cannot invalidate an older present
1822    /// translation. Linux installs a previously-none fault PTE under the PTL
1823    /// and returns through `update_mmu_cache()` without a remote shootdown;
1824    /// only permission changes and page replacement require an active-CPU TLB
1825    /// obligation. The range remains in the receipt for auditability.
1826    fn prepare_fresh_pte_mutation_range(&self, start: VirtAddr, size: usize) -> PreparedMutation {
1827        prepare_mapping_publication_mutation(
1828            &self.mutation_gate,
1829            self.id,
1830            &self.tlb_targets,
1831            start,
1832            size,
1833            false,
1834        )
1835    }
1836
1837    /// Reserves and captures every owner that may become unreachable when a
1838    /// materialized mapping in `range` is detached.  This runs before the PTE
1839    /// mutation, so allocation or cache-identity failure leaves the published
1840    /// state untouched.
1841    /// Records one VMA's backend, and the file-cache pins its shared lease
1842    /// keeps alive, into the retired-mapping owners.
1843    fn collect_retired_mapping_owner(
1844        &self,
1845        entry: &Arc<VmaEntry>,
1846        range: VirtAddrRange,
1847        owners: &mut RetiredMappingOwners,
1848    ) -> StarryResult {
1849        owners
1850            .backends
1851            .try_reserve(1)
1852            .map_err(|_| StarryError::NoMemory)?;
1853        let backend = entry.operation_clone();
1854
1855        if backend.shared_file_lease().is_some() {
1856            let start = entry.start().max(range.start).align_down_4k();
1857            let end = entry.end().min(range.end);
1858            let file_range = VirtAddrRange::new(start, end);
1859            let count = self.mapping_slots_overlapping(file_range).count();
1860            owners
1861                .cache_pins
1862                .try_reserve(count)
1863                .map_err(|_| StarryError::NoMemory)?;
1864            for (_, slot) in self.mapping_slots_overlapping(file_range) {
1865                if slot.state() != SlotState::Present
1866                    || slot.mapping != backend.mapping_id()
1867                    || slot.page_order != PageOrder::BASE
1868                {
1869                    return Err(StarryError::BadState);
1870                }
1871                let paddr = slot.mapped_paddr().ok_or(StarryError::BadState)?;
1872                let (installed, _, page_size) = self.pt.query(slot.va)?;
1873                if installed != paddr || page_size != PAGE_SIZE_4K {
1874                    return Err(StarryError::BadState);
1875                }
1876                let pin = backend
1877                    .pin_file_cache_owner_for_mapping(slot.va, paddr)?
1878                    .ok_or(StarryError::BadState)?;
1879                owners.cache_pins.push(pin);
1880            }
1881        }
1882        owners.backends.push(backend);
1883        Ok(())
1884    }
1885
1886    fn prepare_retired_mapping_owners(
1887        &self,
1888        range: VirtAddrRange,
1889    ) -> StarryResult<RetiredMappingOwners> {
1890        try_reserve_irq_vec(&self.retired_mapping_batches, 1).map_err(|_| StarryError::NoMemory)?;
1891
1892        let mut owners = RetiredMappingOwners::default();
1893        // Only the VMAs the range covers matter here. Breaking out of a
1894        // full-map iteration saved nothing: `iter_entries` builds the entire
1895        // vector before the loop reads its first entry.
1896        let mut failure = None;
1897        self.vma_root.for_each_overlapping_entry(range, |entry| {
1898            match self.collect_retired_mapping_owner(entry, range, &mut owners) {
1899                Ok(()) => true,
1900                Err(err) => {
1901                    failure = Some(err);
1902                    false
1903                }
1904            }
1905        });
1906        if let Some(err) = failure {
1907            return Err(err);
1908        }
1909
1910        let matching_slots = self.mapping_slots_overlapping(range).count();
1911        owners
1912            .pages
1913            .try_reserve(matching_slots)
1914            .map_err(|_| StarryError::NoMemory)?;
1915        for (_, slot) in self.mapping_slots_overlapping(range) {
1916            if slot.state() != SlotState::Present {
1917                return Err(StarryError::BadState);
1918            }
1919            owners.pages.push(slot.page.clone());
1920        }
1921        Ok(owners)
1922    }
1923
1924    /// Reserves ownership for every intermediate page-table frame that an
1925    /// unmap may detach. The occupied-leaf count is a strict upper bound: one
1926    /// leaf removal returns at most one move-only reclaim token.
1927    fn prepare_page_table_reclaims(
1928        &self,
1929        ranges: &[VirtAddrRange],
1930    ) -> StarryResult<MappingMutationContext> {
1931        try_reserve_irq_vec(&self.retired_mapping_batches, 1).map_err(|_| StarryError::NoMemory)?;
1932        let capacity = self.occupied_pte_leaves_overlapping(ranges)?.len();
1933        MappingMutationContext::for_published_mutation(capacity)
1934    }
1935
1936    /// Reserves the post-publication owner used by rmap-driven file eviction.
1937    /// Both allocations happen before the PTE is detached, so an allocation
1938    /// failure is an ordinary retry with no visible state change.
1939    fn prepare_deferred_eviction_owner(
1940        &self,
1941        page: &Arc<PageObject>,
1942    ) -> StarryResult<RetiredMappingOwners> {
1943        try_reserve_irq_vec(&self.retired_mapping_batches, 1).map_err(|_| StarryError::NoMemory)?;
1944        let mut owners = RetiredMappingOwners::default();
1945        owners
1946            .deferred_evictions
1947            .try_reserve(1)
1948            .map_err(|_| StarryError::NoMemory)?;
1949        owners
1950            .page_tables
1951            .try_reserve(1)
1952            .map_err(|_| StarryError::NoMemory)?;
1953        owners.deferred_evictions.push(page.clone());
1954        Ok(owners)
1955    }
1956
1957    fn park_retired_mapping_owners(&self, epoch: VmEpoch, owners: RetiredMappingOwners) {
1958        if owners.is_empty() {
1959            return;
1960        }
1961        // Capacity was reserved by `prepare_retired_mapping_owners` before
1962        // any PTE changed, so this publication cannot allocate or fail.
1963        self.retired_mapping_batches
1964            .lock()
1965            .push(RetiredMappingBatch { epoch, owners });
1966    }
1967
1968    fn park_page_table_reclaims(&self, epoch: VmEpoch, context: MappingMutationContext) {
1969        if context.is_empty() {
1970            return;
1971        }
1972        let owners = RetiredMappingOwners {
1973            page_tables: context.into_page_tables(),
1974            ..RetiredMappingOwners::default()
1975        };
1976        self.park_retired_mapping_owners(epoch, owners);
1977    }
1978
1979    fn release_retired_mapping_owners(&self, epoch: VmEpoch) {
1980        loop {
1981            let batch = {
1982                let mut batches = self.retired_mapping_batches.lock();
1983                batches
1984                    .iter()
1985                    .position(|batch| batch.epoch == epoch)
1986                    .map(|index| batches.swap_remove(index))
1987            };
1988            let Some(mut batch) = batch else {
1989                break;
1990            };
1991            debug_assert!(!batch.owners.is_empty());
1992            for page in &batch.owners.deferred_evictions {
1993                page.complete_eviction_tlb();
1994            }
1995            for tables in batch.owners.page_tables.drain(..) {
1996                // SAFETY: this batch is removed only after the matching
1997                // address-space receipt has no pending CPU acknowledgement.
1998                unsafe { tables.reclaim() };
1999            }
2000            // Cache pins may take the page-cache lock in Drop.  Release them
2001            // only after the IRQ-safe batch lock is gone.
2002            drop(batch);
2003        }
2004    }
2005
2006    fn pending_retired_mapping_batches(&self) -> usize {
2007        self.retired_mapping_batches.lock().len()
2008    }
2009
2010    #[cfg(test)]
2011    fn pending_retired_page_table_reclaims(&self) -> usize {
2012        self.retired_mapping_batches
2013            .lock()
2014            .iter()
2015            .map(|batch| batch.owners.page_tables.len())
2016            .sum()
2017    }
2018
2019    fn confirm_restored_mapping_owners(
2020        &self,
2021        epoch: VmEpoch,
2022        ranges: &[VirtAddrRange],
2023    ) -> StarryResult {
2024        for range in ranges {
2025            crate::mm::flush_tlb_range_sync(range.start, range.size())?;
2026        }
2027        self.release_retired_mapping_owners(epoch);
2028        Ok(())
2029    }
2030
2031    /// Number of mutations whose remote TLB obligations are not complete.
2032    pub fn pending_tlb_obligations(&self) -> usize {
2033        self.mutation_gate.pending_count()
2034    }
2035
2036    /// Snapshot of outstanding shootdown requests.  The caller may submit
2037    /// these to the platform IPI layer without retaining an address-space
2038    /// metadata lock.
2039    pub fn pending_tlb_requests(&self) -> StarryResult<Vec<TlbRequest>> {
2040        self.mutation_gate
2041            .pending_requests()
2042            .map_err(|_| StarryError::NoMemory)
2043    }
2044
2045    /// Records one remote acknowledgement and returns frames that became safe
2046    /// to reclaim from the quarantine.
2047    pub fn acknowledge_tlb(
2048        &self,
2049        space_id: AddressSpaceId,
2050        epoch: VmEpoch,
2051        cpu: usize,
2052    ) -> StarryResult<Vec<FrameLease>> {
2053        match self.mutation_gate.acknowledge(space_id, epoch, cpu) {
2054            Ok(_) | Err(MutationError::TlbPending) => {}
2055            Err(MutationError::WrongState)
2056                if self.tlb_quarantine.contains_request(space_id, epoch) =>
2057            {
2058                // The last gate acknowledgement may have removed the receipt
2059                // just before this caller drained the frame quarantine.  The
2060                // frame-level obligation is still authoritative, so accept
2061                // this idempotent late acknowledgement.
2062            }
2063            Err(_) => return Err(StarryError::ResourceBusy),
2064        }
2065        let released = self
2066            .tlb_quarantine
2067            .acknowledge(space_id, epoch, cpu)
2068            .map_err(|_| StarryError::NoMemory)?;
2069        if self
2070            .mutation_gate
2071            .pending_request(space_id, epoch)
2072            .is_none()
2073        {
2074            self.release_retired_mapping_owners(epoch);
2075        }
2076        Ok(released)
2077    }
2078
2079    pub fn quarantine_frame(
2080        &self,
2081        frame: FrameLease,
2082        request: TlbRequest,
2083    ) -> Result<(), QuarantineFailure> {
2084        self.tlb_quarantine.try_defer(frame, request)
2085    }
2086
2087    fn validate_region(&self, start: VirtAddr, size: usize) -> StarryResult {
2088        if self.mutation_gate.needs_repair() {
2089            return Err(StarryError::BadState);
2090        }
2091        if size == 0 || !self.contains_range(start, size) {
2092            return Err(StarryError::NoMemory);
2093        }
2094        if !start.is_aligned_4k() || !is_aligned_4k(size) {
2095            return Err(StarryError::InvalidInput);
2096        }
2097        Ok(())
2098    }
2099
2100    /// Retains the exact materialized leaves and their frame owners before a
2101    /// mapping replacement starts.  The immutable VMA root is deliberately
2102    /// not changed here; it remains the publication preimage until commit.
2103    fn capture_mapping_preimage(&self, range: VirtAddrRange) -> StarryResult<MappingPreimage> {
2104        self.capture_mapping_preimage_ranges(&[range])
2105    }
2106
2107    /// Retains one coherent metadata and resident-leaf preimage for several
2108    /// disjoint ranges.  `mremap` uses this before publishing its destination
2109    /// so a later PTE or metadata failure can restore both the source and a
2110    /// replaced fixed target, rather than merely deleting the new target.
2111    fn capture_mapping_preimage_ranges(
2112        &self,
2113        ranges: &[VirtAddrRange],
2114    ) -> StarryResult<MappingPreimage> {
2115        for (index, range) in ranges.iter().enumerate() {
2116            self.validate_region(range.start, range.size())?;
2117            if ranges[..index]
2118                .iter()
2119                .any(|previous| previous.overlaps(*range))
2120            {
2121                return Err(StarryError::InvalidInput);
2122            }
2123        }
2124        let resident_slots = self.materialized_slots_overlapping(ranges)?;
2125        let mut leaves = Vec::new();
2126        leaves
2127            .try_reserve(resident_slots.len())
2128            .map_err(|_| StarryError::NoMemory)?;
2129        for (key, slot, occupied_leaf) in resident_slots {
2130            let page_size = occupied_leaf.range.size();
2131            let end = slot
2132                .va
2133                .checked_add(page_size)
2134                .ok_or(StarryError::BadState)?;
2135            let range = ranges
2136                .iter()
2137                .find(|range| slot.overlaps(**range))
2138                .ok_or(StarryError::BadState)?;
2139            // A partial huge-leaf replacement needs the THP split receipt.
2140            // Reject it before mutation until that typed split path is active;
2141            // restoring only part of a block descriptor would be unsound.
2142            if slot.va < range.start || end > range.end {
2143                return Err(StarryError::OperationNotSupported);
2144            }
2145            let paddr = occupied_leaf.paddr;
2146            let backend = self
2147                .vma_root
2148                .lookup_entry(slot.va)
2149                .map(|entry| entry.operation_clone())
2150                .ok_or(StarryError::BadState)?;
2151            let page = slot.page.clone();
2152            let frame_start = page.frame().paddr().as_usize();
2153            let frame_end = frame_start
2154                .checked_add(page.frame().size())
2155                .ok_or(StarryError::BadState)?;
2156            let leaf_start = paddr.as_usize();
2157            let leaf_end = leaf_start
2158                .checked_add(page_size)
2159                .ok_or(StarryError::BadState)?;
2160            if key.va != slot.va
2161                || slot.state() != SlotState::Present
2162                || slot.mm_id != self.id
2163                || slot.mapping != backend.mapping_id()
2164                || slot.mapped_paddr() != Some(paddr)
2165                || leaf_start < frame_start
2166                || leaf_end > frame_end
2167            {
2168                return Err(StarryError::BadState);
2169            }
2170            leaves.push(ResidentLeafPreimage {
2171                va: slot.va,
2172                paddr,
2173                page_size,
2174                flags: occupied_leaf.flags,
2175                backend,
2176                page,
2177                slot,
2178            });
2179        }
2180        Ok(MappingPreimage {
2181            vma_root: self.vma_root.clone(),
2182            vm_stat: self.vm_stat.snapshot(),
2183            leaves,
2184        })
2185    }
2186
2187    /// Reverts a not-yet-published mapping using the retained materialized
2188    /// preimage.  Returning an error means the caller must enter NeedsRepair;
2189    /// it must never report the original syscall error as if state were old.
2190    fn restore_mapping_preimage(
2191        &mut self,
2192        range: VirtAddrRange,
2193        preimage: MappingPreimage,
2194    ) -> StarryResult {
2195        self.restore_mapping_preimage_ranges(&[range], preimage)
2196    }
2197
2198    /// Aborts an unpublished mapping mutation by restoring its exact software
2199    /// and materialized-page-table preimage.  `MemorySet` deliberately reports
2200    /// `NeedsRepair` after a partial PTE apply because it cannot prove its own
2201    /// metadata-only rollback repaired the page table.  At this layer we own
2202    /// the retained frame/backend references and can make that proof; only a
2203    /// complete restore is allowed to clear the repair latch and return the
2204    /// original syscall error.
2205    fn abort_unpublished_mapping_mutation(
2206        &mut self,
2207        range: VirtAddrRange,
2208        preimage: MappingPreimage,
2209        original_error: StarryError,
2210    ) -> StarryResult {
2211        self.abort_unpublished_parked_mapping_mutation(range, preimage, None, original_error)
2212    }
2213
2214    /// Restores a file-eviction delta that failed before epoch publication.
2215    /// The caller may cancel its EvictionLease only when this returns `Err`:
2216    /// `NeedsRepair` means restoration could not be proved and deliberately
2217    /// leaves the page pinned in Evicting.
2218    fn abort_file_eviction_mutation(
2219        &mut self,
2220        range: VirtAddrRange,
2221        preimage: MappingPreimage,
2222        parked_epoch: Option<VmEpoch>,
2223        original_error: StarryError,
2224    ) -> Result<EvictMappingOutcome, StarryError> {
2225        match self.restore_mapping_preimage(range, preimage) {
2226            Ok(()) => {
2227                if let Some(epoch) = parked_epoch
2228                    && self
2229                        .confirm_restored_mapping_owners(epoch, &[range])
2230                        .is_err()
2231                {
2232                    self.mutation_gate.mark_needs_repair();
2233                    return Ok(EvictMappingOutcome::NeedsRepair);
2234                }
2235                self.mutation_gate.clear_repair();
2236                Err(original_error)
2237            }
2238            Err(_) => {
2239                self.mutation_gate.mark_needs_repair();
2240                Ok(EvictMappingOutcome::NeedsRepair)
2241            }
2242        }
2243    }
2244
2245    /// Variant of [`Self::abort_unpublished_mapping_mutation`] for an unmap
2246    /// or replacement whose detached owners were already parked for the next
2247    /// epoch.  The batch is released only after the old PTE/rmap graph has
2248    /// been proved restored; an indeterminate restore deliberately keeps the
2249    /// extra owners alive for a repair worker.
2250    fn abort_unpublished_parked_mapping_mutation(
2251        &mut self,
2252        range: VirtAddrRange,
2253        preimage: MappingPreimage,
2254        parked_epoch: Option<VmEpoch>,
2255        original_error: StarryError,
2256    ) -> StarryResult {
2257        self.abort_unpublished_parked_mapping_mutation_ranges(
2258            &[range],
2259            preimage,
2260            parked_epoch,
2261            original_error,
2262        )
2263    }
2264
2265    fn abort_unpublished_parked_mapping_mutation_ranges(
2266        &mut self,
2267        ranges: &[VirtAddrRange],
2268        preimage: MappingPreimage,
2269        parked_epoch: Option<VmEpoch>,
2270        original_error: StarryError,
2271    ) -> StarryResult {
2272        match self.restore_mapping_preimage_ranges(ranges, preimage) {
2273            Ok(()) => {
2274                if let Some(epoch) = parked_epoch
2275                    && self
2276                        .confirm_restored_mapping_owners(epoch, ranges)
2277                        .is_err()
2278                {
2279                    self.mutation_gate.mark_needs_repair();
2280                    return Err(StarryError::BadState);
2281                }
2282                self.mutation_gate.clear_repair();
2283                Err(original_error)
2284            }
2285            Err(_) => {
2286                self.mutation_gate.mark_needs_repair();
2287                Err(StarryError::BadState)
2288            }
2289        }
2290    }
2291
2292    /// Aborts an unpublished mapping mutation that first materialized one or
2293    /// two THP boundary splits.  The exact leaf/slot preimage is restored
2294    /// before collapsing the deposited child table back into the original
2295    /// huge descriptor; only then may parked owners be released.
2296    fn abort_unpublished_split_mapping_mutation(
2297        &mut self,
2298        range: VirtAddrRange,
2299        preimage: MappingPreimage,
2300        parked_epoch: Option<VmEpoch>,
2301        splits: Vec<AppliedHugeSplit>,
2302        original_error: StarryError,
2303    ) -> StarryResult {
2304        self.abort_unpublished_split_mapping_mutation_ranges(
2305            &[range],
2306            preimage,
2307            parked_epoch,
2308            splits,
2309            original_error,
2310        )
2311    }
2312
2313    /// Multi-range form used by mremap.  Resident source/target preimages are
2314    /// restored before deposited child tables are collapsed, matching the
2315    /// inverse of prepare/apply ordering.
2316    fn abort_unpublished_split_mapping_mutation_ranges(
2317        &mut self,
2318        ranges: &[VirtAddrRange],
2319        preimage: MappingPreimage,
2320        parked_epoch: Option<VmEpoch>,
2321        splits: Vec<AppliedHugeSplit>,
2322        original_error: StarryError,
2323    ) -> StarryResult {
2324        if self
2325            .restore_mapping_preimage_ranges(ranges, preimage)
2326            .is_ok()
2327            && self.rollback_applied_huge_splits(splits)
2328        {
2329            if let Some(epoch) = parked_epoch
2330                && self
2331                    .confirm_restored_mapping_owners(epoch, ranges)
2332                    .is_err()
2333            {
2334                self.mutation_gate.mark_needs_repair();
2335                return Err(StarryError::BadState);
2336            }
2337            self.mutation_gate.clear_repair();
2338            Err(original_error)
2339        } else {
2340            self.mutation_gate.mark_needs_repair();
2341            Err(StarryError::BadState)
2342        }
2343    }
2344
2345    /// Reverts a prepared boundary split when no mapping leaf has otherwise
2346    /// changed.  This is used for allocation/capture failures between split
2347    /// apply and the first backend unmap.
2348    fn abort_unpublished_huge_splits(
2349        &mut self,
2350        splits: Vec<AppliedHugeSplit>,
2351        original_error: StarryError,
2352    ) -> StarryResult {
2353        if self.rollback_applied_huge_splits(splits) {
2354            self.mutation_gate.clear_repair();
2355            Err(original_error)
2356        } else {
2357            self.mutation_gate.mark_needs_repair();
2358            Err(StarryError::BadState)
2359        }
2360    }
2361
2362    fn restore_mapping_preimage_ranges(
2363        &mut self,
2364        ranges: &[VirtAddrRange],
2365        preimage: MappingPreimage,
2366    ) -> StarryResult {
2367        let mut current_memfds = Vec::new();
2368        for range in ranges {
2369            current_memfds.extend(crate::syscall::memfd_collect_metas_touching_mprotect_range(
2370                self,
2371                range.start,
2372                range.size(),
2373            ));
2374        }
2375        let MappingPreimage {
2376            vma_root,
2377            vm_stat,
2378            leaves,
2379        } = preimage;
2380        if self.detach_current_materialized_ranges(ranges).is_err() {
2381            self.mutation_gate.mark_needs_repair();
2382            return Err(StarryError::BadState);
2383        }
2384        for range in ranges {
2385            self.detach_mapping_slots(*range)?;
2386        }
2387        self.vma_root = vma_root.clone();
2388        for leaf in leaves {
2389            if leaf
2390                .backend
2391                .restore_resident_preimage(
2392                    ResidentLeafRestore {
2393                        va: leaf.va,
2394                        paddr: leaf.paddr,
2395                        page_size: leaf.page_size,
2396                        flags: leaf.flags,
2397                        page: Some(&leaf.page),
2398                    },
2399                    &mut self.pt,
2400                )
2401                .is_err()
2402            {
2403                self.mutation_gate.mark_needs_repair();
2404                return Err(StarryError::BadState);
2405            }
2406            let key = MappingSlotKey {
2407                space_id: self.id,
2408                va: leaf.va,
2409            };
2410            if self.mapping_slots.contains_key(&key)
2411                || !leaf.slot.restore()
2412                || self.mapping_slots.insert(key, leaf.slot).is_some()
2413            {
2414                self.mutation_gate.mark_needs_repair();
2415                return Err(StarryError::BadState);
2416            }
2417            leaf.page.set_resident_kind(
2418                self.mapping_slots
2419                    .get(&key)
2420                    .and_then(|slot| slot.resident_kind()),
2421            );
2422        }
2423        self.vma_root = vma_root;
2424        self.vm_stat.restore(vm_stat);
2425        crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(self, &current_memfds);
2426        for range in ranges {
2427            let restored_memfds = crate::syscall::memfd_collect_metas_touching_mprotect_range(
2428                self,
2429                range.start,
2430                range.size(),
2431            );
2432            crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
2433                self,
2434                &restored_memfds,
2435            );
2436        }
2437        Ok(())
2438    }
2439
2440    /// Finds a free area that can accommodate the given size.
2441    ///
2442    /// The search starts from the given hint address, and the area should be
2443    /// within the given limit range.
2444    ///
2445    /// Returns the start address of the free area. Returns None if no such area
2446    /// is found.
2447    pub fn find_free_area(
2448        &self,
2449        hint: VirtAddr,
2450        size: usize,
2451        limit: VirtAddrRange,
2452        align: usize,
2453    ) -> Option<VirtAddr> {
2454        self.vma_root.find_free_area(hint, size, limit, align)
2455    }
2456
2457    /// Returns immutable VMA metadata that can cross a lock or I/O boundary.
2458    pub fn find_area_snapshot(&self, vaddr: VirtAddr) -> Option<Arc<VmaSnapshot>> {
2459        self.vma_root.lookup(vaddr)
2460    }
2461
2462    /// Publishes an immutable VMA index snapshot for procfs and fault readers.
2463    pub fn vma_map_snapshot(&self) -> Arc<VmaMap> {
2464        self.vma_root.clone()
2465    }
2466
2467    /// Returns owned VMA snapshots intersecting a checked range.
2468    pub fn vma_snapshots_in_range(
2469        &self,
2470        start: VirtAddr,
2471        size: usize,
2472    ) -> StarryResult<Vec<Arc<VmaSnapshot>>> {
2473        let range =
2474            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2475        if range.is_empty() {
2476            return Ok(Vec::new());
2477        }
2478        Ok(self.vma_root.lookup_range(range))
2479    }
2480
2481    pub(crate) fn vma_inspection_records(&self) -> StarryResult<Vec<VmaInspectionRecord>> {
2482        let mut records = Vec::new();
2483        records
2484            .try_reserve(self.vma_root.len())
2485            .map_err(|_| StarryError::NoMemory)?;
2486        for entry in self.vma_root.iter_entries() {
2487            records.push(entry.inspection_record()?);
2488        }
2489        Ok(records)
2490    }
2491
2492    pub(crate) fn max_mapped_end(&self) -> Option<VirtAddr> {
2493        self.vma_root.iter().last().map(|vma| vma.range.end)
2494    }
2495
2496    pub(crate) fn next_advice_fragment(
2497        &self,
2498        cursor: VirtAddr,
2499        end: VirtAddr,
2500    ) -> Option<VmaAdviceFragment> {
2501        self.vma_root
2502            .iter_entries()
2503            .find(|entry| entry.end() > cursor && entry.start() < end)
2504            .and_then(|entry| entry.advice_fragment(cursor, end))
2505    }
2506
2507    pub(crate) fn validate_mprotect_mapping_capabilities(
2508        &self,
2509        start: VirtAddr,
2510        size: usize,
2511        flags: MappingFlags,
2512    ) -> StarryResult<Vec<SharedFileMappingLease>> {
2513        let range =
2514            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2515        let operations = self.mapping_operation_fragments(range, true)?;
2516        let mut files = Vec::new();
2517        files
2518            .try_reserve(operations.len())
2519            .map_err(|_| StarryError::NoMemory)?;
2520        for (_, operation) in operations {
2521            operation.check_mprotect_flags(flags)?;
2522            if let Some(file) = operation.shared_file_lease() {
2523                files.push(file);
2524            }
2525        }
2526        Ok(files)
2527    }
2528
2529    pub(crate) fn shared_futex_identity(&self, address: VirtAddr) -> Option<SharedFutexIdentity> {
2530        self.vma_root
2531            .lookup_entry(address)
2532            .and_then(|entry| entry.operation().shared_futex_identity(address))
2533    }
2534
2535    pub(crate) fn mincore_probe(&self, address: VirtAddr) -> Option<VmaResidencyProbe> {
2536        self.vma_root
2537            .lookup_entry(address)
2538            .map(|entry| entry.residency_probe())
2539    }
2540
2541    pub(crate) fn mremap_source(&self, address: VirtAddr) -> Option<VmaMremapSource> {
2542        self.vma_root
2543            .lookup_entry(address)
2544            .map(|entry| entry.mremap_source())
2545    }
2546
2547    pub(crate) fn shared_file_vma_at(&self, address: VirtAddr) -> Option<SharedFileVmaRecord> {
2548        self.vma_root
2549            .lookup_entry(address)
2550            .and_then(|entry| entry.shared_file_record())
2551    }
2552
2553    pub(crate) fn shared_file_vmas(&self) -> Vec<SharedFileVmaRecord> {
2554        self.vma_root
2555            .iter_entries()
2556            .filter_map(|entry| entry.shared_file_record())
2557            .collect()
2558    }
2559
2560    #[allow(clippy::too_many_arguments)]
2561    pub(crate) fn mremap_move_from_source(
2562        &mut self,
2563        source: &VmaMremapSource,
2564        src: VirtAddr,
2565        src_size: usize,
2566        target: VirtAddr,
2567        target_size: usize,
2568        huge_page_advice: HugePageAdvice,
2569        dontunmap: bool,
2570        source_offset: usize,
2571        replace_target: bool,
2572        memlock_limit: Option<MemlockLimit>,
2573    ) -> StarryResult {
2574        let operation = source.relocated_operation(target, source_offset, target_size)?;
2575        self.mremap_move_transaction(
2576            src,
2577            src_size,
2578            target,
2579            target_size,
2580            MappingPermissions {
2581                current: source.rights(),
2582                reported: source.reported_rights(),
2583                maximum: source.max_rights(),
2584            },
2585            operation,
2586            huge_page_advice,
2587            source.lock_mode(),
2588            source.advice_policy(),
2589            dontunmap,
2590            replace_target,
2591            memlock_limit,
2592        )
2593    }
2594
2595    pub(crate) fn duplicate_shared_mremap_source(
2596        &mut self,
2597        source: &VmaMremapSource,
2598        target: VirtAddr,
2599        target_size: usize,
2600        source_offset: usize,
2601        replace_target: bool,
2602        memlock_limit: Option<MemlockLimit>,
2603    ) -> StarryResult {
2604        let object = source.shared_object().ok_or(StarryError::InvalidInput)?;
2605        let backend_start = target
2606            .as_usize()
2607            .checked_sub(source_offset)
2608            .map(VirtAddr::from_usize)
2609            .ok_or(StarryError::InvalidInput)?;
2610        self.map_mremap_duplicate(
2611            target,
2612            target_size,
2613            MappingPermissions {
2614                current: source.rights(),
2615                reported: source.reported_rights(),
2616                maximum: source.rights(),
2617            },
2618            MappingOperation::new_shared(backend_start, object),
2619            MappingPublication::mremap(
2620                replace_target,
2621                source.huge_page_advice(),
2622                source.lock_mode(),
2623                source.advice_policy(),
2624                memlock_limit,
2625            ),
2626        )
2627    }
2628
2629    fn validate_memlock_successor(
2630        &self,
2631        successor: &VmaMap,
2632        memlock_limit: Option<MemlockLimit>,
2633    ) -> StarryResult {
2634        let previous_locked = self.vma_root.locked_pages().ok_or(StarryError::BadState)?;
2635        let successor_locked = successor.locked_pages().ok_or(StarryError::BadState)?;
2636        if successor_locked <= previous_locked {
2637            return Ok(());
2638        }
2639        memlock_limit
2640            .ok_or(StarryError::BadState)?
2641            .validate(successor_locked)
2642    }
2643
2644    fn publish_vma_metadata_successor(
2645        &mut self,
2646        previous_root: Arc<VmaMap>,
2647        successor: VmaMap,
2648        operation: &'static str,
2649    ) -> StarryResult {
2650        let before_vmas = previous_root.len();
2651        let after_vmas = successor.len();
2652        let mut mutation = self.prepare_metadata_mutation();
2653        mutation.set_vma_delta(VmaDelta {
2654            split: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
2655            merged: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
2656            ..VmaDelta::default()
2657        });
2658        self.vma_root = Arc::new(successor);
2659        match self.commit_mutation_classified(mutation) {
2660            Ok(()) => Ok(()),
2661            Err(CommitMutationError::Unpublished(error)) => {
2662                self.vma_root = previous_root;
2663                Err(error)
2664            }
2665            Err(CommitMutationError::PublishedPendingTlb(error)) => {
2666                // Metadata-only receipts have an empty target mask. Reaching
2667                // this branch is an invariant failure, not a recoverable TLB
2668                // timeout that a syscall may report as partially committed.
2669                self.mutation_gate.mark_needs_repair();
2670                warn!(
2671                    "metadata-only {operation} unexpectedly required TLB acknowledgement: {error}"
2672                );
2673                Err(StarryError::BadState)
2674            }
2675        }
2676    }
2677
2678    /// Publishes Linux-compatible transparent-huge-page advice for a mapped
2679    /// VMA range without changing any materialized PTE.
2680    ///
2681    /// The immutable root is the sole owner of this per-VMA policy.  A failed
2682    /// metadata commit restores the previous root; because no translation is
2683    /// changed, the receipt carries no TLB target or retirement obligation.
2684    pub fn advise_huge_pages(
2685        &mut self,
2686        start: VirtAddr,
2687        size: usize,
2688        advice: HugePageAdvice,
2689    ) -> StarryResult {
2690        self.validate_region(start, size)?;
2691        let range =
2692            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2693        let previous_root = self.vma_root.clone();
2694        let affected = previous_root.lookup_range(range);
2695        if affected.is_empty() || !previous_root.contains_range(start, size) {
2696            return Err(StarryError::NoMemory);
2697        }
2698        if affected.iter().all(|vma| vma.huge_page_advice == advice) {
2699            return Ok(());
2700        }
2701
2702        let successor = previous_root
2703            .with_huge_page_advice(range, advice)
2704            .ok_or(StarryError::NoMemory)?;
2705        self.publish_vma_metadata_successor(previous_root, successor, "VMA THP advice")
2706    }
2707
2708    /// Publishes one Linux access, fork-inheritance or dump policy update.
2709    /// The immutable VMA root is both the current state and rollback preimage.
2710    pub(crate) fn advise_vma_policy(
2711        &mut self,
2712        start: VirtAddr,
2713        size: usize,
2714        update: VmaAdviceUpdate,
2715    ) -> StarryResult {
2716        self.validate_region(start, size)?;
2717        let range =
2718            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2719        let previous_root = self.vma_root.clone();
2720        let affected = previous_root.lookup_range(range);
2721        if affected.is_empty() || !previous_root.contains_range(start, size) {
2722            return Err(StarryError::NoMemory);
2723        }
2724        if affected
2725            .iter()
2726            .all(|vma| vma.advice_policy.apply(update) == vma.advice_policy)
2727        {
2728            return Ok(());
2729        }
2730        let successor = previous_root
2731            .with_advice_update(range, update)
2732            .ok_or(StarryError::NoMemory)?;
2733        self.publish_vma_metadata_successor(previous_root, successor, "VMA madvise policy")
2734    }
2735
2736    /// Publishes Linux `VM_LOCKED`/`VM_LOCKONFAULT` policy for a fully mapped
2737    /// range. The policy is part of the immutable VMA root, so split, merge,
2738    /// `mprotect`, `mremap`, `msync`, and proc readers observe one coherent
2739    /// fact. Page population is deliberately a separate operation: Linux
2740    /// publishes the lock policy before `__mm_populate`, and a later populate
2741    /// failure does not undo the VMA flags.
2742    fn set_vma_lock_mode(
2743        &mut self,
2744        start: VirtAddr,
2745        size: usize,
2746        lock_mode: VmaLockMode,
2747        memlock_limit: Option<MemlockLimit>,
2748    ) -> StarryResult {
2749        self.validate_region(start, size)?;
2750        let range =
2751            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2752        let previous_root = self.vma_root.clone();
2753        let affected = previous_root.lookup_range(range);
2754        if affected.is_empty() || !previous_root.contains_range(start, size) {
2755            return Err(StarryError::NoMemory);
2756        }
2757        if affected.iter().all(|vma| vma.lock_mode == lock_mode) {
2758            return Ok(());
2759        }
2760
2761        let successor = previous_root
2762            .with_lock_mode(range, lock_mode)
2763            .ok_or(StarryError::NoMemory)?;
2764        self.validate_memlock_successor(&successor, memlock_limit)?;
2765        self.publish_vma_metadata_successor(previous_root, successor, "VMA lock update")
2766    }
2767
2768    pub(crate) fn lock_vma_range(
2769        &mut self,
2770        start: VirtAddr,
2771        size: usize,
2772        lock_mode: VmaLockMode,
2773        memlock_limit: MemlockLimit,
2774    ) -> StarryResult {
2775        if !lock_mode.is_locked() {
2776            return Err(StarryError::InvalidInput);
2777        }
2778        self.set_vma_lock_mode(start, size, lock_mode, Some(memlock_limit))
2779    }
2780
2781    pub(crate) fn unlock_vma_range(&mut self, start: VirtAddr, size: usize) -> StarryResult {
2782        self.set_vma_lock_mode(start, size, VmaLockMode::Unlocked, None)
2783    }
2784
2785    /// Publishes the exact PageObject owners returned by one backend PTE
2786    /// operation.
2787    ///
2788    /// Linux fault paths carry a referenced folio into the PTL critical
2789    /// section, recheck the PTE, establish the new rmap and only then publish
2790    /// or replace the PTE. Starry's backend currently applies the PTE before
2791    /// returning to this outer mutation gate, so this method performs the
2792    /// corresponding identity recheck and publishes the MappingSlot/rmap
2793    /// before the mutation receipt can become visible. The PTE's raw physical
2794    /// address is validation data only; it is never used to discover or create
2795    /// a PageObject.
2796    fn publish_prepared_pte_owners(
2797        &mut self,
2798        operation: &MappingOperation,
2799        range: VirtAddrRange,
2800        materialization: &PteMaterialization,
2801    ) -> StarryResult<PteOwnerPublication> {
2802        let owners = materialization.owners();
2803        let mut publications = Vec::new();
2804        publications
2805            .try_reserve(owners.len())
2806            .map_err(|_| StarryError::NoMemory)?;
2807        let mut seen = Vec::new();
2808        seen.try_reserve(owners.len())
2809            .map_err(|_| StarryError::NoMemory)?;
2810        let mut mapping_delta = MappingDelta::default();
2811        let mut resident_delta = ResidentDelta::default();
2812
2813        // Bulk populate may carry multiple leaves. Prepare every fallible slot
2814        // and rmap owner before publishing the first one so rollback retains a
2815        // complete inverse operation.
2816        for owner in owners {
2817            let publication = self.prepare_slot_publication(operation, range, owner)?;
2818            if seen.contains(&publication.key) {
2819                return Err(StarryError::BadState);
2820            }
2821            seen.push(publication.key);
2822            mapping_delta.attached = mapping_delta
2823                .attached
2824                .checked_add(publication.mapping_delta.attached)
2825                .ok_or(StarryError::BadState)?;
2826            mapping_delta.detached = mapping_delta
2827                .detached
2828                .checked_add(publication.mapping_delta.detached)
2829                .ok_or(StarryError::BadState)?;
2830            resident_delta.checked_add_assign(publication.resident_delta)?;
2831            publications.push(publication);
2832        }
2833
2834        for publication in publications {
2835            self.apply_slot_publication(operation, publication)?;
2836        }
2837        Ok(PteOwnerPublication {
2838            satisfied_pages: materialization.satisfied_pages(),
2839            mapping_delta,
2840            resident_delta,
2841        })
2842    }
2843
2844    fn publish_prepared_fault_owner(
2845        &mut self,
2846        operation: &MappingOperation,
2847        range: VirtAddrRange,
2848        materialization: &FaultMaterialization,
2849    ) -> StarryResult<PteOwnerPublication> {
2850        let Some(owner) = materialization.owner() else {
2851            return Ok(PteOwnerPublication {
2852                satisfied_pages: materialization.satisfied_pages(),
2853                ..PteOwnerPublication::default()
2854            });
2855        };
2856        // A hardware fault carries one owner inline, so neither preparation
2857        // nor publication needs a temporary Vec.
2858        let publication = self.prepare_slot_publication(operation, range, owner)?;
2859        let mapping_delta = publication.mapping_delta;
2860        let resident_delta = publication.resident_delta;
2861        self.apply_slot_publication(operation, publication)?;
2862        Ok(PteOwnerPublication {
2863            satisfied_pages: materialization.satisfied_pages(),
2864            mapping_delta,
2865            resident_delta,
2866        })
2867    }
2868
2869    fn prepare_slot_publication(
2870        &mut self,
2871        operation: &MappingOperation,
2872        range: VirtAddrRange,
2873        owner: &PreparedPteOwner,
2874    ) -> StarryResult<PreparedSlotPublication> {
2875        let va = owner.va;
2876        let paddr = owner.paddr;
2877        let page_size = owner.page_size;
2878        let page = &owner.page;
2879        let resident_kind = owner.resident_kind;
2880        let transition = owner.transition;
2881        let provider_publication = owner.provider_publication;
2882        if page_size < PAGE_SIZE_4K || !page_size.is_power_of_two() || !va.is_aligned(page_size) {
2883            return Err(StarryError::BadState);
2884        }
2885        let leaf_range =
2886            VirtAddrRange::try_from_start_size(va, page_size).ok_or(StarryError::BadState)?;
2887        if !range.contains_range(leaf_range) {
2888            return Err(StarryError::BadState);
2889        }
2890        let frame_start = page.frame().paddr().as_usize();
2891        let frame_end = frame_start
2892            .checked_add(page.frame().size())
2893            .ok_or(StarryError::BadState)?;
2894        let leaf_start = paddr.as_usize();
2895        let leaf_end = leaf_start
2896            .checked_add(page_size)
2897            .ok_or(StarryError::BadState)?;
2898        if leaf_start < frame_start || leaf_end > frame_end {
2899            return Err(StarryError::BadState);
2900        }
2901        match self.pt.query(va) {
2902            Ok((installed, _, installed_size))
2903                if installed == paddr && installed_size == page_size => {}
2904            Ok(_) | Err(_) => return Err(StarryError::BadState),
2905        }
2906        if !matches!(page.state(), PageState::Present | PageState::LazyFree) {
2907            return Err(StarryError::BadState);
2908        }
2909
2910        let key = MappingSlotKey {
2911            space_id: self.id,
2912            va,
2913        };
2914        let previous = self.mapping_slots.get(&key).cloned();
2915        let order = page_size
2916            .trailing_zeros()
2917            .checked_sub(PAGE_SIZE_4K.trailing_zeros())
2918            .and_then(|order| u8::try_from(order).ok())
2919            .map(PageOrder::new)
2920            .ok_or(StarryError::BadState)?;
2921        let same_owner = previous.as_ref().is_some_and(|slot| {
2922            slot.state() == SlotState::Present
2923                && slot.mapping == operation.mapping_id()
2924                && slot.page_order == order
2925                && slot.mapped_paddr() == Some(paddr)
2926                && Arc::ptr_eq(&slot.page, page)
2927                && (order == PageOrder::BASE || slot.has_huge_split_deposit())
2928        });
2929        match transition {
2930            PteOwnerTransition::Updated if !same_owner => return Err(StarryError::BadState),
2931            PteOwnerTransition::Replaced if previous.is_none() || same_owner => {
2932                return Err(StarryError::BadState);
2933            }
2934            PteOwnerTransition::Installed
2935            | PteOwnerTransition::Replaced
2936            | PteOwnerTransition::Updated => {}
2937        }
2938
2939        let replacement = if same_owner {
2940            None
2941        } else {
2942            let split_deposit = if order == PageOrder::BASE {
2943                None
2944            } else {
2945                Some(self.pt.prepare_huge_split(va)?)
2946            };
2947            let frame_offset = paddr
2948                .as_usize()
2949                .checked_sub(page.frame().paddr().as_usize())
2950                .ok_or(StarryError::BadState)?;
2951            let slot = MappingSlot::new_with_frame_offset(
2952                operation.mapping_id(),
2953                self.id,
2954                va,
2955                order,
2956                page.clone(),
2957                frame_offset,
2958                resident_kind,
2959            )
2960            .ok_or(StarryError::BadState)?;
2961            let slot = match split_deposit {
2962                Some(deposit) => slot
2963                    .attach_huge_split_deposit(deposit)
2964                    .map_err(|_| StarryError::BadState)?,
2965                None => slot,
2966            };
2967            Some(Arc::new(slot))
2968        };
2969        let mut mapping_delta = MappingDelta::default();
2970        let mut resident_delta = ResidentDelta::default();
2971        if !same_owner {
2972            mapping_delta.attached = 1;
2973            mapping_delta.detached = u32::from(previous.is_some());
2974        }
2975        if let Some(previous) = &previous {
2976            let pages = 1i64
2977                .checked_shl(previous.page_order.get().into())
2978                .ok_or(StarryError::BadState)?;
2979            resident_delta
2980                .checked_add_assign(ResidentDelta::for_pages(previous.resident_kind(), -pages))?;
2981        }
2982        let pages = 1i64
2983            .checked_shl(order.get().into())
2984            .ok_or(StarryError::BadState)?;
2985        resident_delta.checked_add_assign(ResidentDelta::for_pages(resident_kind, pages))?;
2986        Ok(PreparedSlotPublication {
2987            key,
2988            previous,
2989            replacement,
2990            resident_kind,
2991            provider_publication,
2992            mapping_delta,
2993            resident_delta,
2994        })
2995    }
2996
2997    fn apply_slot_publication(
2998        &mut self,
2999        operation: &MappingOperation,
3000        publication: PreparedSlotPublication,
3001    ) -> StarryResult {
3002        let PreparedSlotPublication {
3003            key,
3004            previous,
3005            replacement,
3006            resident_kind,
3007            provider_publication,
3008            mapping_delta: _,
3009            resident_delta: _,
3010        } = publication;
3011        let Some(replacement) = replacement else {
3012            let current = self.mapping_slots.get(&key).ok_or(StarryError::BadState)?;
3013            if previous
3014                .as_ref()
3015                .is_none_or(|previous| !Arc::ptr_eq(previous, current))
3016            {
3017                return Err(StarryError::BadState);
3018            }
3019            current.set_resident_kind(resident_kind);
3020            current.page.set_resident_kind(resident_kind);
3021            if provider_publication == ProviderPublication::Pending {
3022                operation.finish_page_publication(key.va, &current.page)?;
3023            }
3024            return Ok(());
3025        };
3026
3027        if let Some(previous) = &previous {
3028            let Some(current) = self.mapping_slots.remove(&key) else {
3029                return Err(StarryError::BadState);
3030            };
3031            if !Arc::ptr_eq(previous, &current) || !current.detach() {
3032                self.mapping_slots.insert(key, current);
3033                return Err(StarryError::BadState);
3034            }
3035        } else if self.mapping_slots.contains_key(&key) {
3036            return Err(StarryError::BadState);
3037        }
3038
3039        if !replacement.publish() {
3040            if let Some(previous) = previous
3041                && (!previous.restore() || self.mapping_slots.insert(key, previous).is_some())
3042            {
3043                self.mutation_gate.mark_needs_repair();
3044            }
3045            return Err(StarryError::BadState);
3046        }
3047        if provider_publication == ProviderPublication::Pending
3048            && let Err(error) = operation.finish_page_publication(key.va, &replacement.page)
3049        {
3050            let replacement_detached = replacement.detach();
3051            let previous_restored = previous.is_none_or(|previous| {
3052                previous.restore() && self.mapping_slots.insert(key, previous).is_none()
3053            });
3054            if !replacement_detached || !previous_restored {
3055                self.mutation_gate.mark_needs_repair();
3056                return Err(StarryError::BadState);
3057            }
3058            return Err(error);
3059        }
3060        if self.mapping_slots.insert(key, replacement).is_some() {
3061            self.mutation_gate.mark_needs_repair();
3062            return Err(StarryError::BadState);
3063        }
3064        Ok(())
3065    }
3066
3067    fn detach_mapping_slots(&mut self, range: VirtAddrRange) -> StarryResult {
3068        let keys: Vec<_> = self
3069            .mapping_slots_overlapping(range)
3070            .map(|(key, _)| *key)
3071            .collect();
3072        for key in keys {
3073            if let Some(slot) = self.mapping_slots.remove(&key)
3074                && !slot.detach()
3075            {
3076                self.mapping_slots.insert(key, slot);
3077                return Err(StarryError::BadState);
3078            }
3079        }
3080        Ok(())
3081    }
3082
3083    /// Revokes one file-cache reverse mapping.  The PageObject and MappingSlot
3084    /// identities are checked before the PTE is touched, then the remote TLB is
3085    /// acknowledged before the backend's mapping reference is released.
3086    pub(crate) fn evict_file_mapping_slot(
3087        &mut self,
3088        key: MappingSlotKey,
3089        page: &Arc<PageObject>,
3090    ) -> Result<EvictMappingOutcome, StarryError> {
3091        if key.space_id != self.id {
3092            return Err(StarryError::BadState);
3093        }
3094        let slot = self
3095            .mapping_slots
3096            .get(&key)
3097            .cloned()
3098            .ok_or(StarryError::BadState)?;
3099        if !Arc::ptr_eq(&slot.page, page) {
3100            return Err(StarryError::BadState);
3101        }
3102
3103        let range = VirtAddrRange::from_start_size(key.va, PAGE_SIZE_4K);
3104        let preimage = self.capture_mapping_preimage(range)?;
3105        let mut retired_owner = self.prepare_deferred_eviction_owner(page)?;
3106        let mut mutation = self.prepare_mutation_range(key.va, PAGE_SIZE_4K);
3107        let retire_epoch = mutation
3108            .receipt()
3109            .base_epoch
3110            .checked_next()
3111            .ok_or(StarryError::BadState)?;
3112        mutation.set_pte_delta(PteDelta {
3113            unmapped: 1,
3114            ..PteDelta::default()
3115        });
3116        mutation.set_mapping_delta(MappingDelta {
3117            detached: 1,
3118            ..MappingDelta::default()
3119        });
3120        mutation.set_resident_delta(ResidentDelta::for_pages(slot.resident_kind(), -1));
3121
3122        let unmap_plan = self.pt.plan_unmap_page(key.va)?;
3123        if slot.mapped_paddr() != Some(unmap_plan.paddr()) || unmap_plan.page_size() != PAGE_SIZE_4K
3124        {
3125            return Err(StarryError::BadState);
3126        }
3127        let apply_result = (|| -> StarryResult {
3128            let unmapped = {
3129                let pte_domain = &self.pte_domain;
3130                let pt = &mut self.pt;
3131                let range = VirtAddrRange::from_start_size(key.va, PAGE_SIZE_4K);
3132                let _structure = pte_domain.lock_structure();
3133                let _stripe = pte_domain.lock_range(range);
3134                pt.unmap_page_deferred(key.va)?
3135            };
3136            if slot.mapped_paddr() != Some(unmapped.0) || unmapped.2 != PAGE_SIZE_4K {
3137                return Err(StarryError::BadState);
3138            }
3139            if !unmapped.3.is_empty() {
3140                retired_owner.page_tables.push(unmapped.3);
3141            }
3142            // `page` and the caller's EvictionLease keep the frame owned until
3143            // this receipt is acknowledged.  No independent all-CPU flush is
3144            // allowed here: the receipt is the sole retirement obligation.
3145            let removed = self
3146                .mapping_slots
3147                .remove(&key)
3148                .ok_or(StarryError::BadState)?;
3149            if !Arc::ptr_eq(&removed, &slot) || !removed.detach() {
3150                return Err(StarryError::BadState);
3151            }
3152            Ok(())
3153        })();
3154        if let Err(error) = apply_result {
3155            return self.abort_file_eviction_mutation(range, preimage, None, error);
3156        }
3157
3158        self.park_retired_mapping_owners(retire_epoch, retired_owner);
3159        match self.commit_mutation_classified(mutation) {
3160            Ok(()) => {
3161                self.release_retired_mapping_owners(retire_epoch);
3162                Ok(EvictMappingOutcome::Complete)
3163            }
3164            Err(CommitMutationError::PublishedPendingTlb(_)) => {
3165                Ok(EvictMappingOutcome::PublishedPendingTlb)
3166            }
3167            Err(CommitMutationError::Unpublished(error)) => {
3168                self.abort_file_eviction_mutation(range, preimage, Some(retire_epoch), error)
3169            }
3170        }
3171    }
3172
3173    /// Write-protects one dirty file-cache reverse mapping for a writeback
3174    /// generation.  No file/cache lock is held by this operation.
3175    pub(crate) fn protect_file_mapping_slot(
3176        &mut self,
3177        key: MappingSlotKey,
3178        page: &Arc<PageObject>,
3179    ) -> StarryResult {
3180        if key.space_id != self.id {
3181            return Err(StarryError::BadState);
3182        }
3183        let slot = self.mapping_slots.get(&key).ok_or(StarryError::BadState)?;
3184        if !Arc::ptr_eq(&slot.page, page) {
3185            return Err(StarryError::BadState);
3186        }
3187
3188        let (paddr, flags, page_size) = self.pt.query(key.va)?;
3189        if slot.mapped_paddr() != Some(paddr) || page_size != PAGE_SIZE_4K {
3190            return Err(StarryError::BadState);
3191        }
3192        if !flags.contains(MappingFlags::WRITE) {
3193            return Ok(());
3194        }
3195        let mut mutation = self.prepare_mutation_range(key.va, PAGE_SIZE_4K);
3196        mutation.set_pte_delta(PteDelta {
3197            protected: 1,
3198            ..PteDelta::default()
3199        });
3200        {
3201            let pt = &mut self.pt;
3202            let _stripe = self
3203                .pte_domain
3204                .lock_range(VirtAddrRange::from_start_size(key.va, PAGE_SIZE_4K));
3205            // Disjoint field borrows retain stripe exclusion without aliasing
3206            // the mutable page-table owner through a raw pointer.
3207            pt.remap_page(key.va, paddr, flags - MappingFlags::WRITE)?;
3208        }
3209        // `commit_mutation` publishes the PTE delta and synchronously services
3210        // its tagged TLB request.  The WritebackLease is completed only after
3211        // this method returns, so dirty-page snapshotting cannot race ahead of
3212        // the acknowledgement.
3213        self.commit_mutation(mutation)
3214    }
3215
3216    pub fn resident_mapping_slots(&self) -> Vec<Arc<MappingSlot>> {
3217        self.mapping_slots.values().cloned().collect()
3218    }
3219
3220    fn capture_mapping_graph_snapshot(
3221        &self,
3222        ranges: &[VirtAddrRange],
3223    ) -> StarryResult<MappingGraphSnapshot> {
3224        #[cfg(all(test, axtest))]
3225        MAPPING_GRAPH_SNAPSHOT_CALLS.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
3226        let slot_count = ranges
3227            .iter()
3228            .map(|range| self.mapping_slots_overlapping(*range).count())
3229            .sum();
3230        let mut slots = Vec::new();
3231        slots
3232            .try_reserve(slot_count)
3233            .map_err(|_| StarryError::NoMemory)?;
3234        let mut seen = BTreeSet::new();
3235        let mut resident = ResidentPageCounts::default();
3236        for range in ranges {
3237            for (key, slot) in self.mapping_slots_overlapping(*range) {
3238                if slot.state() != SlotState::Present || !seen.insert(*key) {
3239                    continue;
3240                }
3241                slots.push(MappingSlotFingerprint {
3242                    key: *key,
3243                    mapping: slot.mapping,
3244                    page: slot.page.id,
3245                    page_order: slot.page_order,
3246                });
3247                let pages = 1u64
3248                    .checked_shl(u32::from(slot.page_order.get()))
3249                    .ok_or(StarryError::BadState)?;
3250                match slot.resident_kind() {
3251                    Some(RssKind::Anon) => {
3252                        resident.anon = resident
3253                            .anon
3254                            .checked_add(pages)
3255                            .ok_or(StarryError::BadState)?;
3256                    }
3257                    Some(RssKind::File) => {
3258                        resident.file = resident
3259                            .file
3260                            .checked_add(pages)
3261                            .ok_or(StarryError::BadState)?;
3262                    }
3263                    Some(RssKind::Shmem) => {
3264                        resident.shmem = resident
3265                            .shmem
3266                            .checked_add(pages)
3267                            .ok_or(StarryError::BadState)?;
3268                    }
3269                    None => {}
3270                }
3271            }
3272        }
3273        slots.sort_unstable();
3274        Ok(MappingGraphSnapshot { slots, resident })
3275    }
3276
3277    fn set_mapping_graph_receipt_delta(
3278        &self,
3279        mutation: &mut PreparedMutation,
3280        before: &MappingGraphSnapshot,
3281        ranges: &[VirtAddrRange],
3282    ) -> StarryResult {
3283        let after = self.capture_mapping_graph_snapshot(ranges)?;
3284        let (mapping_delta, resident_delta) = before.delta_to(&after)?;
3285        mutation.set_mapping_delta(mapping_delta);
3286        mutation.set_resident_delta(resident_delta);
3287        Ok(())
3288    }
3289
3290    /// Returns the current resident set published by mutation receipts.
3291    pub(crate) fn resident_page_counts(&self) -> ResidentPageCounts {
3292        self.resident_pages
3293    }
3294
3295    #[cfg(all(test, axtest))]
3296    fn reset_mapping_graph_snapshot_calls_for_test(&self) {
3297        MAPPING_GRAPH_SNAPSHOT_CALLS.store(0, core::sync::atomic::Ordering::Relaxed);
3298    }
3299
3300    #[cfg(all(test, axtest))]
3301    fn mapping_graph_snapshot_calls_for_test(&self) -> usize {
3302        MAPPING_GRAPH_SNAPSHOT_CALLS.load(core::sync::atomic::Ordering::Relaxed)
3303    }
3304
3305    pub(crate) fn resident_hiwater_pages(&self) -> u64 {
3306        self.resident_watermark.hiwater_pages()
3307    }
3308
3309    /// Collects executable operations for every VMA intersection in virtual
3310    /// order.  The returned values are owned, so no persistent-tree node is
3311    /// borrowed while a backend touches page tables or enters file I/O.
3312    fn mapping_operation_fragments(
3313        &self,
3314        range: VirtAddrRange,
3315        require_full_coverage: bool,
3316    ) -> StarryResult<Vec<(VirtAddrRange, MappingOperation)>> {
3317        let mut fragments = Vec::new();
3318        fragments
3319            .try_reserve(self.vma_root.len())
3320            .map_err(|_| StarryError::NoMemory)?;
3321        let mut covered = range.start;
3322        for entry in self.vma_root.iter_entries() {
3323            if entry.start() >= range.end {
3324                break;
3325            }
3326            if entry.end() <= range.start {
3327                continue;
3328            }
3329            let fragment =
3330                VirtAddrRange::new(entry.start().max(range.start), entry.end().min(range.end));
3331            if require_full_coverage && fragment.start > covered {
3332                return Err(StarryError::NoMemory);
3333            }
3334            covered = covered.max(fragment.end);
3335            fragments.push((fragment, entry.operation_clone()));
3336        }
3337        if require_full_coverage && covered < range.end {
3338            return Err(StarryError::NoMemory);
3339        }
3340        Ok(fragments)
3341    }
3342
3343    /// Detaches only leaves that are actually materialized for `operation`.
3344    /// This is used to clean a backend that reported failure after installing
3345    /// a prefix; unlike a whole-range unmap it is valid when the remaining
3346    /// addresses never acquired PTEs.
3347    fn detach_materialized_operation(
3348        &mut self,
3349        range: VirtAddrRange,
3350        operation: &MappingOperation,
3351    ) -> bool {
3352        let Ok(occupied) = self.occupied_pte_leaves_overlapping(&[range]) else {
3353            return false;
3354        };
3355        let leaves: Vec<_> = occupied.into_iter().map(|leaf| leaf.range).collect();
3356        if leaves
3357            .iter()
3358            .any(|leaf| !operation.validate_unmap_range(*leaf, &self.pt))
3359        {
3360            return false;
3361        }
3362        let Ok(mut context) = MappingMutationContext::for_rollback(leaves.len()) else {
3363            return false;
3364        };
3365        let detached = leaves
3366            .into_iter()
3367            .all(|leaf| {
3368                operation
3369                    .unmap_range(leaf, &mut context, &mut self.pt)
3370                    .is_ok()
3371            });
3372        if !context.pte_changed() {
3373            return detached;
3374        }
3375        if crate::mm::flush_tlb_range_sync(range.start, range.size()).is_err() {
3376            return false;
3377        }
3378        // SAFETY: the synchronous all-CPU invalidation above covers every
3379        // detached leaf in this rollback range.
3380        unsafe { context.reclaim() };
3381        detached
3382    }
3383
3384    /// Detaches every occupied leaf covered by the currently unpublished VMA
3385    /// root. The page-table walk visits allocated tables only, so rollback of
3386    /// a sparse multi-gigabyte mapping does not scan every virtual base page.
3387    fn detach_current_materialized_ranges(&mut self, ranges: &[VirtAddrRange]) -> StarryResult {
3388        let leaves = self.occupied_pte_leaves_overlapping(ranges)?;
3389        let mut context = MappingMutationContext::for_rollback(leaves.len())?;
3390        let mut operations = Vec::new();
3391        operations
3392            .try_reserve(leaves.len())
3393            .map_err(|_| StarryError::NoMemory)?;
3394        for leaf in leaves {
3395            let operation = self
3396                .vma_root
3397                .lookup_entry(leaf.range.start)
3398                .map(|entry| entry.operation_clone())
3399                .ok_or(StarryError::BadState)?;
3400            operations.push((leaf.range, operation));
3401        }
3402        if operations
3403            .iter()
3404            .any(|(leaf, operation)| !operation.validate_unmap_range(*leaf, &self.pt))
3405        {
3406            return Err(StarryError::BadState);
3407        }
3408        for (leaf, operation) in operations {
3409            if let Err(error) = operation.unmap_range(leaf, &mut context, &mut self.pt) {
3410                for range in ranges {
3411                    crate::mm::flush_tlb_range_sync(range.start, range.size())?;
3412                }
3413                // SAFETY: the synchronous invalidations cover every token
3414                // collected before the backend reported failure.
3415                unsafe { context.reclaim() };
3416                return Err(error);
3417            }
3418        }
3419        for range in ranges {
3420            crate::mm::flush_tlb_range_sync(range.start, range.size())?;
3421        }
3422        // SAFETY: all detached ranges completed synchronous system-wide
3423        // invalidation before any intermediate table frame is returned.
3424        unsafe { context.reclaim() };
3425        Ok(())
3426    }
3427
3428    #[allow(clippy::too_many_arguments)]
3429    fn prepare_mapping_successor(
3430        &self,
3431        range: VirtAddrRange,
3432        permissions: MappingPermissions,
3433        operation: &MappingOperation,
3434        huge_page_advice: HugePageAdvice,
3435        lock_mode: VmaLockMode,
3436        advice_policy: VmaAdvicePolicy,
3437        replace: bool,
3438    ) -> StarryResult<VmaMap> {
3439        let entry = self
3440            .vma_root
3441            .prepare_mapping_entry(
3442                range,
3443                permissions.current,
3444                permissions.reported,
3445                permissions.maximum,
3446                huge_page_advice,
3447                lock_mode,
3448                advice_policy,
3449                operation.clone(),
3450            )
3451            .ok_or(StarryError::BadState)?;
3452        let successor = self.vma_root.with_mapping_entry(entry, replace).ok_or(
3453            if self.vma_root.overlaps(range) && !replace {
3454                StarryError::AlreadyExists
3455            } else {
3456                StarryError::BadState
3457            },
3458        )?;
3459        Ok(successor)
3460    }
3461
3462    /// Applies only the materialized PTE half of a prepared mapping. The
3463    /// caller owns the successor root and publishes it only after this phase
3464    /// has completed.
3465    fn apply_mapping_pages_unpublished(
3466        &mut self,
3467        range: VirtAddrRange,
3468        permissions: MappingPermissions,
3469        operation: &MappingOperation,
3470        replace: bool,
3471        context: &mut MappingMutationContext,
3472    ) -> StarryResult<PteMaterialization> {
3473        let replaced = if replace {
3474            self.mapping_operation_fragments(range, false)?
3475        } else {
3476            Vec::new()
3477        };
3478        if replaced
3479            .iter()
3480            .any(|(fragment, old)| !old.validate_unmap_range(*fragment, &self.pt))
3481            || (!replace && !operation.validate_map_range(range, &self.pt))
3482        {
3483            return Err(StarryError::BadState);
3484        }
3485        for (fragment, old) in &replaced {
3486            old.unmap_range(*fragment, context, &mut self.pt)?;
3487        }
3488        match operation.map_range(range, permissions.current, &mut self.pt) {
3489            Ok(materialization) => Ok(materialization),
3490            Err(error) => {
3491                if !self.detach_materialized_operation(range, operation) {
3492                    self.mutation_gate.mark_needs_repair();
3493                    return Err(StarryError::BadState);
3494                }
3495                Err(error)
3496            }
3497        }
3498    }
3499
3500    /// Applies a fresh mapping or a `MAP_FIXED` replacement while keeping the
3501    /// immutable successor unpublished until every backend step succeeds.
3502    #[allow(clippy::too_many_arguments)]
3503    fn apply_mapping_unpublished(
3504        &mut self,
3505        range: VirtAddrRange,
3506        permissions: MappingPermissions,
3507        operation: &MappingOperation,
3508        huge_page_advice: HugePageAdvice,
3509        lock_mode: VmaLockMode,
3510        advice_policy: VmaAdvicePolicy,
3511        memlock_limit: Option<MemlockLimit>,
3512        replace: bool,
3513        context: &mut MappingMutationContext,
3514    ) -> StarryResult<PteMaterialization> {
3515        let successor = self.prepare_mapping_successor(
3516            range,
3517            permissions,
3518            operation,
3519            huge_page_advice,
3520            lock_mode,
3521            advice_policy,
3522            replace,
3523        )?;
3524        self.validate_memlock_successor(&successor, memlock_limit)?;
3525        let materialization = self.apply_mapping_pages_unpublished(
3526            range,
3527            permissions,
3528            operation,
3529            replace,
3530            context,
3531        )?;
3532        self.vma_root = Arc::new(successor);
3533        Ok(materialization)
3534    }
3535
3536    fn apply_unmap_pages_unpublished(
3537        &mut self,
3538        range: VirtAddrRange,
3539        context: &mut MappingMutationContext,
3540    ) -> StarryResult {
3541        let operations = self.mapping_operation_fragments(range, false)?;
3542        if operations
3543            .iter()
3544            .any(|(fragment, operation)| !operation.validate_unmap_range(*fragment, &self.pt))
3545        {
3546            return Err(StarryError::BadState);
3547        }
3548        for (fragment, operation) in operations {
3549            operation.unmap_range(fragment, context, &mut self.pt)?;
3550        }
3551        Ok(())
3552    }
3553
3554    /// Applies VMA/PTE removal from one precomputed persistent-tree successor.
3555    /// Holes are accepted, matching Linux `munmap`; every intersecting backend
3556    /// is validated before the first PTE is detached.
3557    fn apply_unmap_unpublished(
3558        &mut self,
3559        range: VirtAddrRange,
3560        context: &mut MappingMutationContext,
3561    ) -> StarryResult {
3562        let successor = self
3563            .vma_root
3564            .without_range(range)
3565            .ok_or(StarryError::BadState)?;
3566        self.apply_unmap_pages_unpublished(range, context)?;
3567        self.vma_root = Arc::new(successor);
3568        Ok(())
3569    }
3570
3571    /// Applies a complete mprotect carve from one immutable successor.
3572    fn apply_protection_unpublished(
3573        &mut self,
3574        range: VirtAddrRange,
3575        flags: MappingFlags,
3576        reported_flags: MappingFlags,
3577    ) -> StarryResult {
3578        let successor = self
3579            .vma_root
3580            .with_permissions(range, flags, reported_flags)
3581            .ok_or(StarryError::NoMemory)?;
3582        let operations = self.mapping_operation_fragments(range, true)?;
3583        if operations
3584            .iter()
3585            .any(|(fragment, operation)| !operation.validate_protect_range(*fragment, &self.pt))
3586        {
3587            return Err(StarryError::BadState);
3588        }
3589        for (fragment, operation) in operations {
3590            operation.protect_range(fragment, flags, &mut self.pt)?;
3591        }
3592        self.vma_root = Arc::new(successor);
3593        Ok(())
3594    }
3595
3596    fn apply_extend_unpublished(
3597        &mut self,
3598        address: VirtAddr,
3599        additional_size: usize,
3600        memlock_limit: Option<MemlockLimit>,
3601    ) -> StarryResult<(VirtAddrRange, MappingOperation, PteMaterialization)> {
3602        let entry = self
3603            .vma_root
3604            .lookup_entry(address)
3605            .ok_or(StarryError::InvalidInput)?;
3606        let suffix = VirtAddrRange::try_from_start_size(entry.end(), additional_size)
3607            .ok_or(StarryError::InvalidInput)?;
3608        let operation = entry.operation_clone();
3609        let flags = entry.rights();
3610        let successor = self
3611            .vma_root
3612            .with_extended_right(address, additional_size)
3613            .ok_or(StarryError::AlreadyExists)?;
3614        self.validate_memlock_successor(&successor, memlock_limit)?;
3615        if !operation.validate_map_range(suffix, &self.pt) {
3616            return Err(StarryError::BadState);
3617        }
3618        let materialization = match operation.map_range(suffix, flags, &mut self.pt) {
3619            Ok(materialization) => materialization,
3620            Err(error) => {
3621                if !self.detach_materialized_operation(suffix, &operation) {
3622                    self.mutation_gate.mark_needs_repair();
3623                    return Err(StarryError::BadState);
3624                }
3625                return Err(error);
3626            }
3627        };
3628        self.vma_root = Arc::new(successor);
3629        Ok((suffix, operation, materialization))
3630    }
3631
3632    /// Add a new linear mapping.
3633    ///
3634    /// See [`MappingOperation`] for more details about the mapping backends.
3635    ///
3636    /// The `flags` parameter indicates the mapping permissions and attributes.
3637    ///
3638    /// Returns an error if the address range is out of the address space or not
3639    /// aligned.
3640    pub fn map_linear(
3641        &mut self,
3642        start_vaddr: VirtAddr,
3643        start_paddr: PhysAddr,
3644        size: usize,
3645        flags: MappingFlags,
3646    ) -> StarryResult {
3647        self.validate_region(start_vaddr, size)?;
3648        let range = VirtAddrRange::from_start_size(start_vaddr, size);
3649        let preimage = self.capture_mapping_preimage(range)?;
3650        let graph_preimage = self.capture_mapping_graph_snapshot(&[range])?;
3651        let mut mutation = self.prepare_mutation_range(start_vaddr, size);
3652
3653        if !start_paddr.is_aligned_4k() {
3654            return Err(StarryError::InvalidInput);
3655        }
3656        if start_paddr.checked_add(size).is_none() {
3657            return Err(StarryError::InvalidInput);
3658        }
3659
3660        let operation = MappingOperation::new_linear(start_vaddr, start_paddr, false);
3661        let mut page_table_reclaims = MappingMutationContext::for_published_mutation(0)?;
3662        let materialization = match self.apply_mapping_unpublished(
3663            range,
3664            MappingPermissions {
3665                current: flags,
3666                reported: flags,
3667                maximum: flags,
3668            },
3669            &operation,
3670            HugePageAdvice::Default,
3671            VmaLockMode::Unlocked,
3672            VmaAdvicePolicy::default(),
3673            None,
3674            false,
3675            &mut page_table_reclaims,
3676        ) {
3677            Ok(materialization) => materialization,
3678            Err(error) => {
3679                return self.abort_unpublished_mapping_mutation(range, preimage, error);
3680            }
3681        };
3682        mutation.set_vma_delta(VmaDelta {
3683            inserted: 1,
3684            ..VmaDelta::default()
3685        });
3686        mutation.set_pte_delta(PteDelta {
3687            mapped: u32::try_from(size / PAGE_SIZE_4K).unwrap_or(u32::MAX),
3688            ..PteDelta::default()
3689        });
3690        self.vm_stat.on_map((size / PAGE_SIZE_4K) as u64);
3691        if let Err(error) = self.publish_prepared_pte_owners(&operation, range, &materialization) {
3692            return self.abort_unpublished_mapping_mutation(range, preimage, error);
3693        }
3694        if let Err(error) =
3695            self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[range])
3696        {
3697            return self.abort_unpublished_mapping_mutation(range, preimage, error);
3698        }
3699        match self.commit_mutation_classified(mutation) {
3700            Ok(()) => Ok(()),
3701            Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
3702            Err(CommitMutationError::Unpublished(error)) => {
3703                self.abort_unpublished_mapping_mutation(range, preimage, error)
3704            }
3705        }
3706    }
3707
3708    pub fn map(
3709        &mut self,
3710        start: VirtAddr,
3711        size: usize,
3712        flags: MappingFlags,
3713        populate: bool,
3714        backend: MappingOperation,
3715    ) -> StarryResult {
3716        self.map_with_reported_flags(start, size, flags, flags, populate, backend)
3717    }
3718
3719    /// Applies a mapping and preserves the distinction between an unpublished
3720    /// failure and a published mutation whose TLB obligation is still
3721    /// pending.  Syscalls that also update an external ownership index (for
3722    /// example SysV SHM) must use this outcome-aware entry point so they can
3723    /// finish that bookkeeping even when the hardware shootdown cannot be
3724    /// acknowledged synchronously.
3725    pub(crate) fn map_outcome(
3726        &mut self,
3727        start: VirtAddr,
3728        size: usize,
3729        flags: MappingFlags,
3730        populate: bool,
3731        backend: MappingOperation,
3732    ) -> StarryResult<AddressSpaceMutationOutcome> {
3733        self.map_with_permissions_mode_classified(
3734            start,
3735            size,
3736            MappingPermissions {
3737                current: flags,
3738                reported: flags,
3739                maximum: flags,
3740            },
3741            populate,
3742            backend,
3743            MappingPublication::new(false),
3744        )
3745    }
3746
3747    pub fn map_with_reported_flags(
3748        &mut self,
3749        start: VirtAddr,
3750        size: usize,
3751        flags: MappingFlags,
3752        reported_flags: MappingFlags,
3753        populate: bool,
3754        backend: MappingOperation,
3755    ) -> StarryResult {
3756        self.map_with_permissions(
3757            start,
3758            size,
3759            MappingPermissions {
3760                current: flags,
3761                reported: reported_flags,
3762                maximum: flags,
3763            },
3764            populate,
3765            backend,
3766        )
3767    }
3768
3769    /// Maps a region and, when `replace` is true, atomically replaces any
3770    /// overlapping VMAs (the `MAP_FIXED` operation).  The replacement bit is
3771    /// deliberately explicit instead of making callers unmap first: an
3772    /// allocation, permission, or backend failure must leave the old mapping
3773    /// untouched.  [`MemorySet::map`] owns the preimage/rollback of the
3774    /// overlapping page-table fragments.
3775    pub fn map_with_permissions_replace(
3776        &mut self,
3777        start: VirtAddr,
3778        size: usize,
3779        permissions: MappingPermissions,
3780        populate: bool,
3781        backend: MappingOperation,
3782        replace: bool,
3783    ) -> StarryResult {
3784        self.map_with_permissions_mode(
3785            start,
3786            size,
3787            permissions,
3788            populate,
3789            backend,
3790            MappingPublication::new(replace),
3791        )
3792    }
3793
3794    /// Maps a region with one prepared metadata publication policy. Keeping
3795    /// replacement, huge-page advice, and VMA lock policy in one value avoids
3796    /// independently publishing fields that describe the same mapping.
3797    pub(crate) fn map_with_permissions_publication(
3798        &mut self,
3799        start: VirtAddr,
3800        size: usize,
3801        permissions: MappingPermissions,
3802        populate: bool,
3803        backend: MappingOperation,
3804        publication: MappingPublication,
3805    ) -> StarryResult {
3806        self.map_with_permissions_mode(start, size, permissions, populate, backend, publication)
3807    }
3808
3809    /// Installs the duplicate created by Linux's `mremap(old_size == 0)`
3810    /// special case while preserving the source VMA's THP advice in the same
3811    /// mapping receipt.
3812    pub(crate) fn map_mremap_duplicate(
3813        &mut self,
3814        start: VirtAddr,
3815        size: usize,
3816        permissions: MappingPermissions,
3817        backend: MappingOperation,
3818        publication: MappingPublication,
3819    ) -> StarryResult {
3820        self.map_with_permissions_mode(start, size, permissions, false, backend, publication)
3821    }
3822
3823    /// Maps a region while retaining the maximum permission envelope allowed
3824    /// by the original file/VM policy.  `mprotect` may lower permissions and
3825    /// later restore them only inside this envelope; the current PTE flags
3826    /// alone are not sufficient to express that Linux invariant.
3827    pub fn map_with_permissions(
3828        &mut self,
3829        start: VirtAddr,
3830        size: usize,
3831        permissions: MappingPermissions,
3832        populate: bool,
3833        backend: MappingOperation,
3834    ) -> StarryResult {
3835        self.map_with_permissions_mode(
3836            start,
3837            size,
3838            permissions,
3839            populate,
3840            backend,
3841            MappingPublication::new(false),
3842        )
3843    }
3844
3845    fn map_with_permissions_mode(
3846        &mut self,
3847        start: VirtAddr,
3848        size: usize,
3849        permissions: MappingPermissions,
3850        populate: bool,
3851        backend: MappingOperation,
3852        publication: MappingPublication,
3853    ) -> StarryResult {
3854        self.map_with_permissions_mode_classified(
3855            start,
3856            size,
3857            permissions,
3858            populate,
3859            backend,
3860            publication,
3861        )?
3862        .into_result()
3863    }
3864
3865    fn map_with_permissions_mode_classified(
3866        &mut self,
3867        start: VirtAddr,
3868        size: usize,
3869        permissions: MappingPermissions,
3870        populate: bool,
3871        backend: MappingOperation,
3872        publication: MappingPublication,
3873    ) -> StarryResult<AddressSpaceMutationOutcome> {
3874        let MappingPublication {
3875            replace,
3876            huge_page_advice,
3877            lock_mode,
3878            advice_policy,
3879            memlock_limit,
3880        } = publication;
3881        self.validate_region(start, size)?;
3882        if !permissions.maximum.contains(permissions.current) {
3883            return Err(StarryError::PermissionDenied);
3884        }
3885        let range =
3886            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
3887        let mut mutation = prepare_mapping_publication_mutation(
3888            &self.mutation_gate,
3889            self.id,
3890            &self.tlb_targets,
3891            start,
3892            size,
3893            replace,
3894        );
3895        self.mutation_gate
3896            .validate_publish_preconditions(&mutation)
3897            .map_err(Self::map_unpublished_mutation_error)?;
3898        // Count the old VSS before the persistent root performs replacement. This is
3899        // metadata only and therefore cannot make a failed map visible.
3900        let removed_pages = if replace {
3901            self.vma_root
3902                .iter_entries()
3903                .filter(|entry| entry.start() < range.end && entry.end() > range.start)
3904                .try_fold(0u64, |pages, entry| {
3905                    let lo = entry.start().max(range.start);
3906                    let hi = entry.end().min(range.end);
3907                    let fragment = hi.checked_sub_addr(lo).ok_or(StarryError::InvalidInput)?;
3908                    pages
3909                        .checked_add((fragment / PAGE_SIZE_4K) as u64)
3910                        .ok_or(StarryError::InvalidInput)
3911                })?
3912        } else {
3913            0
3914        };
3915        let mapping_preimage = self.capture_mapping_preimage(range)?;
3916        let graph_preimage = self.capture_mapping_graph_snapshot(&[range])?;
3917        let retire_epoch = mutation
3918            .receipt()
3919            .base_epoch
3920            .checked_next()
3921            .ok_or(StarryError::BadState)?;
3922        let mut retired_owners = replace
3923            .then(|| self.prepare_retired_mapping_owners(range))
3924            .transpose()?;
3925        let mut page_table_reclaims = if replace {
3926            self.prepare_page_table_reclaims(&[range])?
3927        } else {
3928            MappingMutationContext::for_published_mutation(0)?
3929        };
3930
3931        // Keep the identities of shared memfd VMAs so their writable-count
3932        // side band can be recomputed only after the replacement commits.  A
3933        // event before the VMA/PTE apply would make an allocation failure
3934        // externally visible despite the old mapping being restored.
3935        let touched_memfds = if replace {
3936            crate::syscall::memfd_collect_metas_touching_mprotect_range(self, start, size)
3937        } else {
3938            Vec::new()
3939        };
3940
3941        let map_materialization = match self.apply_mapping_unpublished(
3942            range,
3943            permissions,
3944            &backend,
3945            huge_page_advice,
3946            lock_mode,
3947            advice_policy,
3948            memlock_limit,
3949            replace,
3950            &mut page_table_reclaims,
3951        ) {
3952            Ok(materialization) => materialization,
3953            Err(error) => {
3954                if let Some(owners) = retired_owners.take() {
3955                    self.park_retired_mapping_owners(retire_epoch, owners);
3956                }
3957                self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
3958                return self
3959                    .abort_unpublished_parked_mapping_mutation(
3960                        range,
3961                        mapping_preimage,
3962                        replace.then_some(retire_epoch),
3963                        error,
3964                    )
3965                    .map(|()| AddressSpaceMutationOutcome::Complete);
3966            }
3967        };
3968        if let Some(owners) = retired_owners.take() {
3969            self.park_retired_mapping_owners(retire_epoch, owners);
3970        }
3971        self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
3972        if removed_pages != 0
3973            && let Err(error) = self.detach_mapping_slots(range)
3974        {
3975            return self
3976                .abort_unpublished_parked_mapping_mutation(
3977                    range,
3978                    mapping_preimage,
3979                    replace.then_some(retire_epoch),
3980                    error,
3981                )
3982                .map(|()| AddressSpaceMutationOutcome::Complete);
3983        }
3984        if let Err(error) = self.publish_prepared_pte_owners(&backend, range, &map_materialization)
3985        {
3986            return self
3987                .abort_unpublished_parked_mapping_mutation(
3988                    range,
3989                    mapping_preimage,
3990                    replace.then_some(retire_epoch),
3991                    error,
3992                )
3993                .map(|()| AddressSpaceMutationOutcome::Complete);
3994        }
3995        if populate
3996            && let Err(populate_error) = self.apply_populate_area(start, size, permissions.current)
3997        {
3998            return self
3999                .abort_unpublished_parked_mapping_mutation(
4000                    range,
4001                    mapping_preimage,
4002                    replace.then_some(retire_epoch),
4003                    populate_error,
4004                )
4005                .map(|()| AddressSpaceMutationOutcome::Complete);
4006        }
4007        mutation.set_vma_delta(VmaDelta {
4008            inserted: 1,
4009            removed: u32::try_from(removed_pages).unwrap_or(u32::MAX),
4010            ..VmaDelta::default()
4011        });
4012        self.vm_stat.on_map((size / PAGE_SIZE_4K) as u64);
4013        if removed_pages != 0 {
4014            self.vm_stat.on_unmap(removed_pages);
4015        }
4016        if let Err(error) =
4017            self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[range])
4018        {
4019            return self
4020                .abort_unpublished_parked_mapping_mutation(
4021                    range,
4022                    mapping_preimage,
4023                    replace.then_some(retire_epoch),
4024                    error,
4025                )
4026                .map(|()| AddressSpaceMutationOutcome::Complete);
4027        }
4028        match self.commit_mutation_classified(mutation) {
4029            Ok(()) => {
4030                if replace {
4031                    self.release_retired_mapping_owners(retire_epoch);
4032                    crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
4033                        self,
4034                        &touched_memfds,
4035                    );
4036                } else {
4037                    crate::syscall::memfd_on_after_map(self, start);
4038                }
4039                Ok(AddressSpaceMutationOutcome::Complete)
4040            }
4041            Err(CommitMutationError::PublishedPendingTlb(error)) => {
4042                if replace {
4043                    crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
4044                        self,
4045                        &touched_memfds,
4046                    );
4047                } else {
4048                    crate::syscall::memfd_on_after_map(self, start);
4049                }
4050                Ok(AddressSpaceMutationOutcome::PublishedPendingTlb(error))
4051            }
4052            Err(CommitMutationError::Unpublished(error)) => self
4053                .abort_unpublished_parked_mapping_mutation(
4054                    range,
4055                    mapping_preimage,
4056                    replace.then_some(retire_epoch),
4057                    error,
4058                )
4059                .map(|()| AddressSpaceMutationOutcome::Complete),
4060        }
4061    }
4062
4063    /// Applies backend population and its typed MappingSlot/rmap owners without
4064    /// publishing an epoch or external event. The caller owns the retained
4065    /// mapping preimage until the surrounding mutation commits.
4066    fn apply_populate_area(
4067        &mut self,
4068        mut start: VirtAddr,
4069        size: usize,
4070        access_flags: MappingFlags,
4071    ) -> StarryResult<usize> {
4072        self.validate_region(start, size)?;
4073        let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4074        let mut populated = 0usize;
4075
4076        loop {
4077            let area_end = {
4078                let Some(entry) = self.vma_root.lookup_entry(start) else {
4079                    break;
4080                };
4081                let entry_end = entry.end();
4082                let range = VirtAddrRange::new(start, entry_end.min(end));
4083                let flags = entry.rights();
4084                let backend = entry.operation_clone();
4085                let request = PopulateRequest::area(range, backend.page_size())?;
4086                let materialization =
4087                    backend.populate(self.id, request, flags, access_flags, &mut self.pt)?;
4088                let publication =
4089                    self.publish_prepared_pte_owners(&backend, range, &materialization)?;
4090                populated = populated
4091                    .checked_add(publication.satisfied_pages)
4092                    .ok_or(StarryError::NoMemory)?;
4093                entry_end
4094            };
4095            start = area_end;
4096            assert!(start.is_aligned_4k());
4097            if start >= end {
4098                break;
4099            }
4100        }
4101
4102        if start < end {
4103            // If the area is not fully mapped, we return ENOMEM.
4104            return Err(StarryError::NoMemory);
4105        }
4106
4107        Ok(populated)
4108    }
4109
4110    /// Returns whether every materialized leaf in `range` already permits the
4111    /// requested user access.
4112    ///
4113    /// This is a software page-table check under the address-space mutex, so it
4114    /// is the architecture-independent fallback for CPUs without a cheap user
4115    /// translation probe.  It deliberately requires `USER` even though
4116    /// `UserAccessIntent` only carries read/write intent.  A present supervisor
4117    /// mapping must never make a user-copy preparation succeed.
4118    fn materialized_range_satisfies_access(
4119        &self,
4120        range: VirtAddrRange,
4121        access_flags: MappingFlags,
4122    ) -> bool {
4123        let required = access_flags | MappingFlags::USER;
4124        let mut cursor = range.start;
4125        while cursor < range.end {
4126            let Ok((_, flags, leaf_size)) = self.pt.query(cursor) else {
4127                return false;
4128            };
4129            if leaf_size < PAGE_SIZE_4K || !leaf_size.is_power_of_two() || !flags.contains(required)
4130            {
4131                return false;
4132            }
4133            let Some(leaf_end) = cursor.align_down(leaf_size).checked_add(leaf_size) else {
4134                return false;
4135            };
4136            if leaf_end <= cursor {
4137                return false;
4138            }
4139            cursor = leaf_end.min(range.end);
4140        }
4141        true
4142    }
4143
4144    /// Populates an already-published area with physical frames.
4145    pub fn populate_area(
4146        &mut self,
4147        start: VirtAddr,
4148        size: usize,
4149        access_flags: MappingFlags,
4150    ) -> StarryResult {
4151        let range =
4152            VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
4153        self.validate_region(start, size)?;
4154        if self.can_access_range(start, size, access_flags)
4155            && self.materialized_range_satisfies_access(range, access_flags)
4156        {
4157            return Ok(());
4158        }
4159        let preimage = self.capture_mapping_preimage(range)?;
4160        let graph_preimage = self.capture_mapping_graph_snapshot(&[range])?;
4161        let mut mutation = self.prepare_mutation_range(start, size);
4162        let retire_epoch = mutation
4163            .receipt()
4164            .base_epoch
4165            .checked_next()
4166            .ok_or(StarryError::BadState)?;
4167        let retired_owners = (!graph_preimage.slots.is_empty())
4168            .then(|| self.prepare_retired_mapping_owners(range))
4169            .transpose()?;
4170        let populated = match self.apply_populate_area(start, size, access_flags) {
4171            Ok(populated) => populated,
4172            Err(populate_error) => {
4173                if self.restore_mapping_preimage(range, preimage).is_err() {
4174                    return Err(StarryError::BadState);
4175                }
4176                return Err(populate_error);
4177            }
4178        };
4179
4180        mutation.set_pte_delta(PteDelta {
4181            mapped: u32::try_from(populated).unwrap_or(u32::MAX),
4182            ..PteDelta::default()
4183        });
4184        if let Err(error) =
4185            self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[range])
4186        {
4187            return self.abort_unpublished_mapping_mutation(range, preimage, error);
4188        }
4189        if let Some(owners) = retired_owners {
4190            self.park_retired_mapping_owners(retire_epoch, owners);
4191        }
4192        match self.commit_mutation_classified(mutation) {
4193            Ok(()) => {
4194                self.release_retired_mapping_owners(retire_epoch);
4195                Ok(())
4196            }
4197            Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
4198            Err(CommitMutationError::Unpublished(error)) => self
4199                .abort_unpublished_parked_mapping_mutation(
4200                    range,
4201                    preimage,
4202                    Some(retire_epoch),
4203                    error,
4204                ),
4205        }
4206    }
4207
4208    /// Discards the physical pages backing `[start, start+size)` while keeping
4209    /// the VMA metadata intact (Linux `MADV_DONTNEED` semantics).
4210    pub fn discard_range(&mut self, start: VirtAddr, size: usize) -> StarryResult {
4211        self.validate_region(start, size)?;
4212        let retired_range = VirtAddrRange::from_start_size(start, size);
4213        let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4214
4215        let mut frags: alloc::vec::Vec<(VirtAddrRange, MappingOperation)> = alloc::vec::Vec::new();
4216        frags
4217            .try_reserve(self.vma_root.len())
4218            .map_err(|_| StarryError::NoMemory)?;
4219        let mut covered = start;
4220        for entry in self.vma_root.iter_entries() {
4221            if entry.start() >= end {
4222                break;
4223            }
4224            if entry.end() <= start {
4225                continue;
4226            }
4227            let frag_start = entry.start().max(start);
4228            let frag_end = entry.end().min(end);
4229            if frag_start > covered {
4230                return Err(StarryError::NoMemory);
4231            }
4232            let backend = entry.operation_clone();
4233            // Device/linear mappings cannot reconstruct a discarded PTE.
4234            // External huge-page providers reject a partial-page carve before
4235            // any split or PTE mutation is published.
4236            backend.validate_discard_fragment(VirtAddrRange::new(frag_start, frag_end))?;
4237            frags.push((VirtAddrRange::new(frag_start, frag_end), backend));
4238            covered = frag_end;
4239        }
4240        if covered < end {
4241            return Err(StarryError::NoMemory);
4242        }
4243
4244        let mut mutation = self.prepare_mutation_range(start, size);
4245        mutation
4246            .try_reserve_tlb_ranges(2)
4247            .map_err(|_| StarryError::NoMemory)?;
4248        let retire_epoch = mutation
4249            .receipt()
4250            .base_epoch
4251            .checked_next()
4252            .ok_or(StarryError::BadState)?;
4253        let splits = self.apply_partial_huge_splits(retired_range)?;
4254        for index in 0..splits.len() {
4255            let split = &splits[index];
4256            let Some(tlb_range) =
4257                TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
4258            else {
4259                return self.abort_unpublished_huge_splits(splits, StarryError::BadState);
4260            };
4261            mutation.add_tlb_range(tlb_range);
4262        }
4263        if frags
4264            .iter()
4265            .any(|(range, backend)| !backend.validate_unmap_range(*range, &self.pt))
4266        {
4267            return self.abort_unpublished_huge_splits(splits, StarryError::OperationNotSupported);
4268        }
4269
4270        let preimage = match self.capture_mapping_preimage(retired_range) {
4271            Ok(preimage) => preimage,
4272            Err(error) => return self.abort_unpublished_huge_splits(splits, error),
4273        };
4274        let retired_owners = match self.prepare_retired_mapping_owners(retired_range) {
4275            Ok(owners) => owners,
4276            Err(error) => {
4277                return self.abort_unpublished_split_mapping_mutation(
4278                    retired_range,
4279                    preimage,
4280                    None,
4281                    splits,
4282                    error,
4283                );
4284            }
4285        };
4286        let mut page_table_reclaims = match self.prepare_page_table_reclaims(&[retired_range]) {
4287            Ok(reclaims) => reclaims,
4288            Err(error) => {
4289                return self.abort_unpublished_split_mapping_mutation(
4290                    retired_range,
4291                    preimage,
4292                    None,
4293                    splits,
4294                    error,
4295                );
4296            }
4297        };
4298        let Ok((detached_slots, retired_pages, retired_resident)) =
4299            self.mapping_slot_summary(retired_range)
4300        else {
4301            return self.abort_unpublished_split_mapping_mutation(
4302                retired_range,
4303                preimage,
4304                None,
4305                splits,
4306                StarryError::BadState,
4307            );
4308        };
4309        let split_slots = splits.iter().try_fold(0usize, |slots, split| {
4310            slots.checked_add(split.child_slots.len().saturating_sub(1))
4311        });
4312        let Some(split_slots) = split_slots else {
4313            return self.abort_unpublished_split_mapping_mutation(
4314                retired_range,
4315                preimage,
4316                None,
4317                splits,
4318                StarryError::BadState,
4319            );
4320        };
4321
4322        for (range, backend) in frags {
4323            if let Err(error) =
4324                backend.unmap_range(range, &mut page_table_reclaims, &mut self.pt)
4325            {
4326                self.park_retired_mapping_owners(retire_epoch, retired_owners);
4327                self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4328                return self.abort_unpublished_split_mapping_mutation(
4329                    retired_range,
4330                    preimage,
4331                    Some(retire_epoch),
4332                    splits,
4333                    error,
4334                );
4335            }
4336        }
4337        self.park_retired_mapping_owners(retire_epoch, retired_owners);
4338        self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4339        if let Err(error) = self.detach_mapping_slots(retired_range) {
4340            return self.abort_unpublished_split_mapping_mutation(
4341                retired_range,
4342                preimage,
4343                Some(retire_epoch),
4344                splits,
4345                error,
4346            );
4347        }
4348        mutation.set_pte_delta(PteDelta {
4349            unmapped: u32::try_from(retired_pages).unwrap_or(u32::MAX),
4350            ..PteDelta::default()
4351        });
4352        mutation.set_mapping_delta(MappingDelta {
4353            attached: u32::try_from(split_slots).unwrap_or(u32::MAX),
4354            detached: u32::try_from(detached_slots).unwrap_or(u32::MAX),
4355        });
4356        mutation.set_resident_delta(retired_resident.checked_negated_delta()?);
4357
4358        match self.commit_mutation_classified(mutation) {
4359            Ok(()) => {
4360                self.release_retired_mapping_owners(retire_epoch);
4361                Ok(())
4362            }
4363            Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
4364            Err(CommitMutationError::Unpublished(error)) => self
4365                .abort_unpublished_split_mapping_mutation(
4366                    retired_range,
4367                    preimage,
4368                    Some(retire_epoch),
4369                    splits,
4370                    error,
4371                ),
4372        }
4373    }
4374
4375    /// Marks exclusive private-anonymous PageObjects as lazily free.
4376    ///
4377    /// The PageObject is the sole mark owner.  Writable leaves are protected
4378    /// in the same receipt so a later store faults and changes `LazyFree` back
4379    /// to `Present` before write permission is republished.  COW-shared pages
4380    /// are intentionally skipped: a mapping-local hint must not mark another
4381    /// process's shared PageObject reclaimable.
4382    pub fn mark_lazy_free(&mut self, start: VirtAddr, size: usize) -> StarryResult {
4383        self.validate_region(start, size)?;
4384        let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4385        let mut covered = start;
4386        for entry in self.vma_root.iter_entries() {
4387            if entry.start() >= end {
4388                break;
4389            }
4390            if entry.end() <= start {
4391                continue;
4392            }
4393            let fragment_start = entry.start().max(start);
4394            if fragment_start > covered {
4395                return Err(StarryError::NoMemory);
4396            }
4397            if !entry.operation().is_private_anonymous() {
4398                return Err(StarryError::InvalidInput);
4399            }
4400            covered = entry.end().min(end);
4401        }
4402        if covered < end {
4403            return Err(StarryError::NoMemory);
4404        }
4405
4406        let range = VirtAddrRange::from_start_size(start, size);
4407        let mut candidates = Vec::new();
4408        candidates
4409            .try_reserve(self.mapping_slots_overlapping(range).count())
4410            .map_err(|_| StarryError::NoMemory)?;
4411        for (_, slot) in self.mapping_slots_overlapping(range) {
4412            if slot.page_order != PageOrder::BASE || slot.page.mapping_refs() != 1 {
4413                continue;
4414            }
4415            match slot.page.state() {
4416                PageState::LazyFree => continue,
4417                PageState::Present => {}
4418                PageState::Reserved
4419                | PageState::Evicting
4420                | PageState::Writeback
4421                | PageState::Retired => return Err(StarryError::ResourceBusy),
4422            }
4423            let (paddr, flags, page_size) = self.pt.query(slot.va)?;
4424            if slot.mapped_paddr() != Some(paddr) || page_size != PAGE_SIZE_4K {
4425                return Err(StarryError::BadState);
4426            }
4427            candidates.push((slot.va, paddr, flags, slot.page.clone()));
4428        }
4429        if candidates.is_empty() {
4430            return Ok(());
4431        }
4432
4433        let mut mutation = self.prepare_mutation_range(start, size);
4434        let pt = &mut self.pt;
4435        let stripes = self.pte_domain.lock_range(range);
4436        let mut protected = 0usize;
4437        for &(va, paddr, flags, _) in &candidates {
4438            if !flags.contains(MappingFlags::WRITE) {
4439                continue;
4440            }
4441            // The ordered stripe cursor borrows only the lock domain; the
4442            // page table remains exclusively borrowed throughout apply.
4443            if pt
4444                .remap_page(va, paddr, flags - MappingFlags::WRITE)
4445                .is_err()
4446            {
4447                for &(old_va, old_paddr, old_flags, _) in candidates.iter().rev() {
4448                    if old_flags.contains(MappingFlags::WRITE) {
4449                        // Covered by the same stripe cursor.
4450                        let _ = pt.remap_page(old_va, old_paddr, old_flags);
4451                    }
4452                }
4453                return Err(StarryError::BadState);
4454            }
4455            protected += 1;
4456        }
4457        drop(stripes);
4458
4459        for (marked, (_, _, _, page)) in candidates.iter().enumerate() {
4460            if !page.mark_lazy_free() {
4461                for (_, _, _, marked_page) in candidates[..marked].iter().rev() {
4462                    let _ = marked_page.clear_lazy_free();
4463                }
4464                let pt = &mut self.pt;
4465                let _stripes = self.pte_domain.lock_range(range);
4466                for &(va, paddr, flags, _) in &candidates {
4467                    if flags.contains(MappingFlags::WRITE) {
4468                        // Covered by the ordered stripe cursor.
4469                        let _ = pt.remap_page(va, paddr, flags);
4470                    }
4471                }
4472                return Err(StarryError::ResourceBusy);
4473            }
4474        }
4475
4476        mutation.set_pte_delta(PteDelta {
4477            protected: u32::try_from(protected).unwrap_or(u32::MAX),
4478            ..PteDelta::default()
4479        });
4480        match self.commit_mutation_classified(mutation) {
4481            Ok(()) => {
4482                lifecycle::request_lazy_free_reclaim();
4483                Ok(())
4484            }
4485            Err(CommitMutationError::PublishedPendingTlb(error)) => {
4486                // The LazyFree state is already visible even though detached
4487                // owners remain quarantined for the outstanding shootdown.
4488                lifecycle::request_lazy_free_reclaim();
4489                Err(error)
4490            }
4491            Err(CommitMutationError::Unpublished(error)) => {
4492                for (_, _, _, page) in candidates.iter().rev() {
4493                    if !page.clear_lazy_free() {
4494                        self.mutation_gate.mark_needs_repair();
4495                        return Err(StarryError::BadState);
4496                    }
4497                }
4498                let pt = &mut self.pt;
4499                let _stripes = self.pte_domain.lock_range(range);
4500                for &(va, paddr, flags, _) in &candidates {
4501                    if flags.contains(MappingFlags::WRITE)
4502                        && pt.remap_page(va, paddr, flags).is_err()
4503                    {
4504                        self.mutation_gate.mark_needs_repair();
4505                        return Err(StarryError::BadState);
4506                    }
4507                }
4508                Err(error)
4509            }
4510        }
4511    }
4512
4513    /// Reclaims up to `limit` exclusive anonymous pages previously marked by
4514    /// `MADV_FREE`.  Each detached PTE uses the ordinary discard receipt, so
4515    /// the PageObject cannot reach `Retired` until the active-CPU TLB request
4516    /// has completed and its reverse mapping is gone.
4517    pub(crate) fn reclaim_lazy_free_pages(&mut self, limit: usize) -> StarryResult<usize> {
4518        if limit == 0 {
4519            return Ok(0);
4520        }
4521        let mut reclaimed = 0;
4522        let mut cursor = None;
4523        while reclaimed < limit {
4524            let eligible = |(key, slot): (&MappingSlotKey, &Arc<MappingSlot>)| {
4525                (slot.page_order == PageOrder::BASE
4526                    && slot.page.state() == PageState::LazyFree
4527                    && slot.page.mapping_refs() == 1)
4528                    .then(|| (*key, slot.page.clone()))
4529            };
4530            let candidate = if let Some(after) = cursor {
4531                self.mapping_slots
4532                    .range((
4533                        core::ops::Bound::Excluded(after),
4534                        core::ops::Bound::Unbounded,
4535                    ))
4536                    .find_map(eligible)
4537            } else {
4538                self.mapping_slots.iter().find_map(eligible)
4539            };
4540            let Some((key, page)) = candidate else {
4541                break;
4542            };
4543            cursor = Some(key);
4544            let Some(entry) = self.vma_root.lookup_entry(key.va) else {
4545                return Err(StarryError::BadState);
4546            };
4547            if !entry.operation().is_private_anonymous() {
4548                return Err(StarryError::BadState);
4549            }
4550            let still_reclaimable = self.mapping_slots.get(&key).is_some_and(|slot| {
4551                Arc::ptr_eq(&slot.page, &page)
4552                    && page.state() == PageState::LazyFree
4553                    && page.mapping_refs() == 1
4554            });
4555            if !still_reclaimable {
4556                continue;
4557            }
4558            self.discard_range(key.va, PAGE_SIZE_4K)?;
4559            if !page.rmap.is_empty()
4560                || page.mapping_refs() != 0
4561                || !page.transition(PageState::LazyFree, PageState::Retired)
4562            {
4563                self.mutation_gate.mark_needs_repair();
4564                return Err(StarryError::BadState);
4565            }
4566            reclaimed += 1;
4567        }
4568        Ok(reclaimed)
4569    }
4570
4571    /// Removes mappings within the specified virtual address range.
4572    ///
4573    /// Returns an error if the address range is out of the address space or not
4574    /// aligned.
4575    pub fn unmap(&mut self, start: VirtAddr, size: usize) -> StarryResult {
4576        self.unmap_classified(start, size)?.into_result()
4577    }
4578
4579    /// Outcome-aware counterpart to [`Self::unmap`].  The returned pending
4580    /// error means that the VMA/PTE/rmap removal is already published; callers
4581    /// must not restore the old mapping or skip their corresponding ownership
4582    /// index update.
4583    pub(crate) fn unmap_outcome(
4584        &mut self,
4585        start: VirtAddr,
4586        size: usize,
4587    ) -> StarryResult<AddressSpaceMutationOutcome> {
4588        self.unmap_classified(start, size)
4589    }
4590
4591    fn unmap_classified(
4592        &mut self,
4593        start: VirtAddr,
4594        size: usize,
4595    ) -> StarryResult<AddressSpaceMutationOutcome> {
4596        self.validate_region(start, size)?;
4597        let range = VirtAddrRange::from_start_size(start, size);
4598        let before_vmas = self.vma_root.len();
4599        // Memfd's writable-map counter is a side-band fact derived from the
4600        // VMA tree.  Prepare its delta while the old metadata is still
4601        // visible, but publish it only after the VMA/PTE transaction below.
4602        let memfd_deltas = crate::syscall::memfd_prepare_aspace_unmap_deltas(self, start, size);
4603        let mut mutation = self.prepare_mutation_range(start, size);
4604        let retire_epoch = mutation
4605            .receipt()
4606            .base_epoch
4607            .checked_next()
4608            .ok_or(StarryError::BadState)?;
4609        mutation
4610            .try_reserve_tlb_ranges(2)
4611            .map_err(|_| StarryError::NoMemory)?;
4612        let splits = self.apply_partial_huge_splits(range)?;
4613        for index in 0..splits.len() {
4614            let split = &splits[index];
4615            let Some(tlb_range) =
4616                TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
4617            else {
4618                return self
4619                    .abort_unpublished_huge_splits(splits, StarryError::BadState)
4620                    .map(|()| AddressSpaceMutationOutcome::Complete);
4621            };
4622            mutation.add_tlb_range(tlb_range);
4623        }
4624        let preimage = match self.capture_mapping_preimage(range) {
4625            Ok(preimage) => preimage,
4626            Err(error) => {
4627                return self
4628                    .abort_unpublished_huge_splits(splits, error)
4629                    .map(|()| AddressSpaceMutationOutcome::Complete);
4630            }
4631        };
4632        let retired_owners = match self.prepare_retired_mapping_owners(range) {
4633            Ok(owners) => owners,
4634            Err(error) => {
4635                return self
4636                    .abort_unpublished_huge_splits(splits, error)
4637                .map(|()| AddressSpaceMutationOutcome::Complete);
4638            }
4639        };
4640        let mut page_table_reclaims = match self.prepare_page_table_reclaims(&[range]) {
4641            Ok(reclaims) => reclaims,
4642            Err(error) => {
4643                return self
4644                    .abort_unpublished_split_mapping_mutation(
4645                        range,
4646                        preimage,
4647                        None,
4648                        splits,
4649                        error,
4650                    )
4651                    .map(|()| AddressSpaceMutationOutcome::Complete);
4652            }
4653        };
4654        let Ok((detached_slots, detached_resident_pages, detached_resident)) =
4655            self.mapping_slot_summary(range)
4656        else {
4657            return self
4658                .abort_unpublished_huge_splits(splits, StarryError::BadState)
4659                .map(|()| AddressSpaceMutationOutcome::Complete);
4660        };
4661        let split_slots = splits.iter().try_fold(0usize, |slots, split| {
4662            slots.checked_add(split.child_slots.len().saturating_sub(1))
4663        });
4664        let Some(split_slots) = split_slots else {
4665            return self
4666                .abort_unpublished_huge_splits(splits, StarryError::BadState)
4667                .map(|()| AddressSpaceMutationOutcome::Complete);
4668        };
4669
4670        // Compute the actual mapped bytes being removed (unmap is already O(n)).
4671        let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4672        let removed_pages: u64 = self
4673            .vma_root
4674            .iter_entries()
4675            .filter(|entry| entry.start() < end && entry.end() > start)
4676            .map(|entry| {
4677                let lo = entry.start().max(start);
4678                let hi = entry.end().min(end);
4679                ((hi - lo) / PAGE_SIZE_4K) as u64
4680            })
4681            .sum();
4682
4683        if let Err(error) = self.apply_unmap_unpublished(range, &mut page_table_reclaims) {
4684            self.park_retired_mapping_owners(retire_epoch, retired_owners);
4685            self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4686            return self
4687                .abort_unpublished_split_mapping_mutation(
4688                    range,
4689                    preimage,
4690                    Some(retire_epoch),
4691                    splits,
4692                    error,
4693                )
4694                .map(|()| AddressSpaceMutationOutcome::Complete);
4695        }
4696        self.park_retired_mapping_owners(retire_epoch, retired_owners);
4697        self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4698        if let Err(error) = self.detach_mapping_slots(range) {
4699            return self
4700                .abort_unpublished_split_mapping_mutation(
4701                    range,
4702                    preimage,
4703                    Some(retire_epoch),
4704                    splits,
4705                    error,
4706                )
4707                .map(|()| AddressSpaceMutationOutcome::Complete);
4708        }
4709        mutation.set_pte_delta(PteDelta {
4710            unmapped: u32::try_from(detached_resident_pages).unwrap_or(u32::MAX),
4711            ..PteDelta::default()
4712        });
4713        mutation.set_mapping_delta(MappingDelta {
4714            attached: u32::try_from(split_slots).unwrap_or(u32::MAX),
4715            detached: u32::try_from(detached_slots).unwrap_or(u32::MAX),
4716        });
4717        mutation.set_resident_delta(detached_resident.checked_negated_delta()?);
4718        self.vm_stat.on_unmap(removed_pages);
4719        let after_vmas = self.vma_root.len();
4720        mutation.set_vma_delta(VmaDelta {
4721            removed: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
4722            split: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
4723            ..VmaDelta::default()
4724        });
4725        match self.commit_mutation_classified(mutation) {
4726            Ok(()) => {
4727                self.release_retired_mapping_owners(retire_epoch);
4728                crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
4729                Ok(AddressSpaceMutationOutcome::Complete)
4730            }
4731            Err(CommitMutationError::PublishedPendingTlb(error)) => {
4732                crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
4733                Ok(AddressSpaceMutationOutcome::PublishedPendingTlb(error))
4734            }
4735            Err(CommitMutationError::Unpublished(error)) => self
4736                .abort_unpublished_split_mapping_mutation(
4737                    range,
4738                    preimage,
4739                    Some(retire_epoch),
4740                    splits,
4741                    error,
4742                )
4743                .map(|()| AddressSpaceMutationOutcome::Complete),
4744        }
4745    }
4746
4747    fn prepare_moved_slots(
4748        &mut self,
4749        moved_pages: &[MovedPage],
4750        target_mapping: MappingId,
4751    ) -> StarryResult<Vec<PreparedMovedSlot>> {
4752        let mut prepared = Vec::new();
4753        prepared
4754            .try_reserve(moved_pages.len())
4755            .map_err(|_| StarryError::NoMemory)?;
4756
4757        for moved in moved_pages {
4758            let source_key = MappingSlotKey {
4759                space_id: self.id,
4760                va: moved.src_va,
4761            };
4762            let target_slot_va = match moved.destination {
4763                MovedPageDestination::SourceOwner => moved.dst_va,
4764                MovedPageDestination::TargetOwner { slot_va } => slot_va,
4765            };
4766            let target_key = MappingSlotKey {
4767                space_id: self.id,
4768                va: target_slot_va,
4769            };
4770            if matches!(moved.destination, MovedPageDestination::SourceOwner)
4771                && prepared.iter().any(|entry| {
4772                    matches!(
4773                        entry,
4774                        PreparedMovedSlot::Relocate {
4775                            target_key: existing,
4776                            ..
4777                        } if *existing == target_key
4778                    )
4779                })
4780            {
4781                return Err(StarryError::BadState);
4782            }
4783            let source = self
4784                .mapping_slots
4785                .get(&source_key)
4786                .cloned()
4787                .ok_or(StarryError::BadState)?;
4788            let page_order = moved
4789                .page_size
4790                .trailing_zeros()
4791                .checked_sub(PAGE_SIZE_4K.trailing_zeros())
4792                .and_then(|order| u8::try_from(order).ok())
4793                .map(PageOrder::new)
4794                .ok_or(StarryError::BadState)?;
4795            let frame_start = source.page.frame().paddr().as_usize();
4796            let frame_end = frame_start
4797                .checked_add(source.page.frame().size())
4798                .ok_or(StarryError::BadState)?;
4799            let leaf_start = moved.paddr.as_usize();
4800            let leaf_end = leaf_start
4801                .checked_add(moved.page_size)
4802                .ok_or(StarryError::BadState)?;
4803            if source.state() != SlotState::Present
4804                || source.mm_id != self.id
4805                || source.va != moved.src_va
4806                || source.page_order != page_order
4807                || source.mapped_paddr() != Some(moved.paddr)
4808                || leaf_start < frame_start
4809                || leaf_end > frame_end
4810                || (page_order != PageOrder::BASE && !source.has_huge_split_deposit())
4811            {
4812                return Err(StarryError::BadState);
4813            }
4814
4815            match moved.destination {
4816                MovedPageDestination::SourceOwner => {
4817                    let replacement = MappingSlot::new_with_frame_offset(
4818                        target_mapping,
4819                        self.id,
4820                        moved.dst_va,
4821                        page_order,
4822                        source.page.clone(),
4823                        source.frame_offset(),
4824                        source.resident_kind(),
4825                    )
4826                    .ok_or(StarryError::BadState)?;
4827                    let replacement = if page_order == PageOrder::BASE {
4828                        replacement
4829                    } else {
4830                        replacement
4831                            .attach_huge_split_deposit(self.pt.prepare_huge_split(moved.dst_va)?)
4832                            .map_err(|_| StarryError::BadState)?
4833                    };
4834                    prepared.push(PreparedMovedSlot::Relocate {
4835                        source_key,
4836                        target_key,
4837                        source,
4838                        replacement: Arc::new(replacement),
4839                    });
4840                }
4841                MovedPageDestination::TargetOwner { .. } => {
4842                    prepared.push(PreparedMovedSlot::DetachSource {
4843                        source_key,
4844                        target_key,
4845                        source,
4846                    });
4847                }
4848            }
4849        }
4850        Ok(prepared)
4851    }
4852
4853    fn publish_moved_slots(&mut self, prepared: Vec<PreparedMovedSlot>) -> StarryResult {
4854        for entry in prepared {
4855            match entry {
4856                PreparedMovedSlot::Relocate {
4857                    source_key,
4858                    target_key,
4859                    source,
4860                    replacement,
4861                } => {
4862                    if self.mapping_slots.contains_key(&target_key) {
4863                        return Err(StarryError::BadState);
4864                    }
4865                    let removed = self
4866                        .mapping_slots
4867                        .remove(&source_key)
4868                        .ok_or(StarryError::BadState)?;
4869                    if !Arc::ptr_eq(&removed, &source) {
4870                        self.mapping_slots.insert(source_key, removed);
4871                        return Err(StarryError::BadState);
4872                    }
4873                    if let Err(error) = source.relocate_to(&replacement) {
4874                        let restored = source.state() == SlotState::Present
4875                            && self
4876                                .mapping_slots
4877                                .insert(source_key, source.clone())
4878                                .is_none();
4879                        if !restored || error == MappingGraphError::RollbackFailed {
4880                            self.mutation_gate.mark_needs_repair();
4881                        }
4882                        return Err(match error {
4883                            MappingGraphError::ResourceExhausted => StarryError::NoMemory,
4884                            _ => StarryError::BadState,
4885                        });
4886                    }
4887                    if self.mapping_slots.insert(target_key, replacement).is_some() {
4888                        // `&mut self` and the pre-insertion check make this
4889                        // unreachable unless the map itself was already corrupt.
4890                        self.mutation_gate.mark_needs_repair();
4891                        return Err(StarryError::BadState);
4892                    }
4893                }
4894                PreparedMovedSlot::DetachSource {
4895                    source_key,
4896                    target_key,
4897                    source,
4898                } => {
4899                    let target = self
4900                        .mapping_slots
4901                        .get(&target_key)
4902                        .ok_or(StarryError::BadState)?;
4903                    if target.state() != SlotState::Present {
4904                        return Err(StarryError::BadState);
4905                    }
4906                    let removed = self
4907                        .mapping_slots
4908                        .remove(&source_key)
4909                        .ok_or(StarryError::BadState)?;
4910                    if !Arc::ptr_eq(&removed, &source) || !removed.detach() {
4911                        self.mapping_slots.insert(source_key, removed);
4912                        return Err(StarryError::BadState);
4913                    }
4914                }
4915            }
4916        }
4917        Ok(())
4918    }
4919
4920    /// Applies only the materialized PTE portion of a relocation. Metadata,
4921    /// MappingSlot publication and the epoch receipt belong to the outer
4922    /// transaction. Pages already materialized at `dst` (shared backends) are
4923    /// kept, while an empty destination receives the exact source PTE owner.
4924    fn apply_move_pages(
4925        &mut self,
4926        src: VirtAddr,
4927        dst: VirtAddr,
4928        size: usize,
4929    ) -> StarryResult<Vec<MovedPage>> {
4930        let move_range =
4931            VirtAddrRange::try_from_start_size(src, size).ok_or(StarryError::InvalidInput)?;
4932        let dst_range =
4933            VirtAddrRange::try_from_start_size(dst, size).ok_or(StarryError::InvalidInput)?;
4934        if move_range.overlaps(dst_range) {
4935            // The low-level mover walks source leaves while writing the
4936            // destination.  Overlapping intervals would make a later source
4937            // lookup observe an already moved leaf (and can duplicate or
4938            // destroy data), so callers must first perform the explicit
4939            // overlap-aware mremap preparation.
4940            return Err(StarryError::InvalidInput);
4941        }
4942        self.validate_materialized_leaf_boundaries(src, size)?;
4943        let source_slots = self.materialized_slots_overlapping(&[move_range])?;
4944        let mut mapped_pages = alloc::vec::Vec::new();
4945        mapped_pages
4946            .try_reserve(source_slots.len())
4947            .map_err(|_| StarryError::NoMemory)?;
4948        for (key, slot, occupied_leaf) in source_slots {
4949            let offset = key.va.checked_sub_addr(src).ok_or(StarryError::BadState)?;
4950            let dst_va = dst.checked_add(offset).ok_or(StarryError::InvalidInput)?;
4951            let paddr = occupied_leaf.paddr;
4952            let flags = occupied_leaf.flags;
4953            let page_size = occupied_leaf.range.size();
4954            let expected_size = PAGE_SIZE_4K
4955                .checked_shl(slot.page_order.get().into())
4956                .ok_or(StarryError::BadState)?;
4957            if slot.state() != SlotState::Present
4958                || slot.va != key.va
4959                || slot.mm_id != self.id
4960                || slot.mapped_paddr() != Some(paddr)
4961                || page_size != expected_size
4962            {
4963                return Err(StarryError::BadState);
4964            }
4965            if !key.va.is_aligned(page_size) || !dst_va.is_aligned(page_size) {
4966                return Err(StarryError::OperationNotSupported);
4967            }
4968            mapped_pages.push((key.va, dst_va, paddr, flags, page_size));
4969        }
4970
4971        let mut moved_pages = alloc::vec::Vec::new();
4972        moved_pages
4973            .try_reserve(mapped_pages.len())
4974            .map_err(|_| StarryError::NoMemory)?;
4975        let mut move_plans = Vec::<PageTableMovePlan>::new();
4976        move_plans
4977            .try_reserve(mapped_pages.len())
4978            .map_err(|_| StarryError::NoMemory)?;
4979
4980        // Linux allocates destination PTE/PMD directories before taking the
4981        // leaf PTL.  Publish the same kind of empty structural deposit here:
4982        // allocation and loser destruction happen outside every IRQ-saving
4983        // page-table lock, and no materialized mapping is visible yet.
4984        for &(_, dst_va, _, _, page_size) in &mapped_pages {
4985            match self.pt.query_occupied(dst_va) {
4986                Ok(_) => {}
4987                Err(PagingError::NotMapped) => {
4988                    let plan = self.pt.plan_map_page(dst_va, page_size)?;
4989                    if let Some(deposit) = plan.prepare_path()? {
4990                        let apply_result = {
4991                            let _structure = self.pte_domain.lock_structure();
4992                            self.pt.try_install_map_path(deposit)
4993                        };
4994                        if let Err(failure) = apply_result {
4995                            let (error, deposit) = failure.into_parts();
4996                            // The deposit owns page-table frames.  It is
4997                            // intentionally destroyed after the IRQ-saving
4998                            // structure guard above has gone out of scope.
4999                            drop(deposit);
5000                            return Err(error.into());
5001                        }
5002                    }
5003                }
5004                Err(error) => return Err(error.into()),
5005            }
5006        }
5007
5008        for &(src_va, dst_va, paddr, flags, page_size) in &mapped_pages {
5009            let plan = self.pt.plan_move_page(src_va, dst_va)?;
5010            if plan.source_vaddr() != src_va
5011                || plan.destination_vaddr() != dst_va
5012                || plan.paddr() != paddr
5013                || plan.config() != flags
5014                || plan.page_size() != page_size
5015            {
5016                return Err(StarryError::BadState);
5017            }
5018            let destination = if plan.destination_is_occupied() {
5019                let target_size = plan.destination_page_size();
5020                if target_size < PAGE_SIZE_4K || !target_size.is_power_of_two() {
5021                    return Err(StarryError::BadState);
5022                }
5023                MovedPageDestination::TargetOwner {
5024                    slot_va: dst_va.align_down(target_size),
5025                }
5026            } else {
5027                MovedPageDestination::SourceOwner
5028            };
5029            moved_pages.push(MovedPage {
5030                src_va,
5031                dst_va,
5032                paddr,
5033                page_size,
5034                destination,
5035            });
5036            move_plans.push(plan);
5037        }
5038
5039        // Every fallible allocation and all path publication completed before
5040        // these IRQ-saving guards.  The generic batch first revalidates every
5041        // source/destination preimage and only then performs infallible leaf
5042        // stores, so rollback never allocates or waits for an IPI under PTL.
5043        let pte_domain = &self.pte_domain;
5044        let cursor = &mut self.pt;
5045        let apply_result = {
5046            let _structure = pte_domain.lock_structure();
5047            let pte_stripes = pte_domain.lock_ranges(&[move_range, dst_range]);
5048            debug_assert!(!pte_stripes.stripe_indices().is_empty());
5049            cursor.try_move_pages_with(&move_plans)
5050        };
5051        let applied = apply_result?;
5052        if applied != moved_pages.len() {
5053            self.mutation_gate.mark_needs_repair();
5054            return Err(StarryError::BadState);
5055        }
5056        Ok(moved_pages)
5057    }
5058
5059    /// Relocates one complete logical mapping under a single epoch receipt.
5060    /// The target VMA, source metadata, PTEs, rmap slots, RSS and memfd side
5061    /// bands are prepared from one preimage and become visible together.
5062    #[allow(clippy::too_many_arguments)]
5063    pub(crate) fn mremap_move_transaction(
5064        &mut self,
5065        src: VirtAddr,
5066        src_size: usize,
5067        target: VirtAddr,
5068        target_size: usize,
5069        permissions: MappingPermissions,
5070        target_backend: MappingOperation,
5071        huge_page_advice: HugePageAdvice,
5072        lock_mode: VmaLockMode,
5073        advice_policy: VmaAdvicePolicy,
5074        dontunmap: bool,
5075        replace_target: bool,
5076        memlock_limit: Option<MemlockLimit>,
5077    ) -> StarryResult {
5078        self.validate_region(src, src_size)?;
5079        self.validate_region(target, target_size)?;
5080        if !permissions.maximum.contains(permissions.current) {
5081            return Err(StarryError::PermissionDenied);
5082        }
5083        let source_range =
5084            VirtAddrRange::try_from_start_size(src, src_size).ok_or(StarryError::InvalidInput)?;
5085        let target_range = VirtAddrRange::try_from_start_size(target, target_size)
5086            .ok_or(StarryError::InvalidInput)?;
5087        if source_range.overlaps(target_range) {
5088            return Err(StarryError::InvalidInput);
5089        }
5090        let move_size = src_size.min(target_size);
5091        let moved_source_range =
5092            VirtAddrRange::try_from_start_size(src, move_size).ok_or(StarryError::InvalidInput)?;
5093
5094        let source_page_size = self
5095            .vma_root
5096            .lookup_entry(src)
5097            .map(|entry| entry.operation().page_size())
5098            .ok_or(StarryError::BadAddress)?;
5099        let source_replacement =
5100            dontunmap.then(|| MappingOperation::new_alloc(src, source_page_size, ""));
5101        let rollback_ranges = [source_range, target_range];
5102        let tail_range = if src_size > move_size {
5103            Some(
5104                VirtAddrRange::try_from_start_size(
5105                    src.checked_add(move_size)
5106                        .ok_or(StarryError::InvalidInput)?,
5107                    src_size - move_size,
5108                )
5109                .ok_or(StarryError::InvalidInput)?,
5110            )
5111        } else {
5112            None
5113        };
5114
5115        let target_removed_pages = if replace_target {
5116            self.vma_root
5117                .iter_entries()
5118                .filter(|entry| {
5119                    entry.start() < target_range.end && entry.end() > target_range.start
5120                })
5121                .try_fold(0u64, |pages, entry| {
5122                    let lo = entry.start().max(target_range.start);
5123                    let hi = entry.end().min(target_range.end);
5124                    let bytes = hi.checked_sub_addr(lo).ok_or(StarryError::BadState)?;
5125                    pages
5126                        .checked_add((bytes / PAGE_SIZE_4K) as u64)
5127                        .ok_or(StarryError::InvalidInput)
5128                })?
5129        } else {
5130            0
5131        };
5132        // Allocate and validate the complete target/source VMA successor
5133        // before any huge split or PTE mutation. This is the metadata prepare
5134        // phase of the mremap receipt.
5135        let target_successor = self.prepare_mapping_successor(
5136            target_range,
5137            permissions,
5138            &target_backend,
5139            huge_page_advice,
5140            lock_mode,
5141            advice_policy,
5142            replace_target,
5143        )?;
5144        let mut final_successor = target_successor.clone();
5145        if dontunmap {
5146            final_successor = final_successor
5147                .without_range(moved_source_range)
5148                .ok_or(StarryError::BadState)?;
5149            let replacement = source_replacement.as_ref().ok_or(StarryError::BadState)?;
5150            let replacement_entry = final_successor
5151                .prepare_mapping_entry(
5152                    moved_source_range,
5153                    permissions.current,
5154                    permissions.reported,
5155                    permissions.maximum,
5156                    huge_page_advice,
5157                    // Linux keeps the copied target locked but always clears
5158                    // VM_LOCKED and VM_LOCKONFAULT on the source VMA after a
5159                    // successful MREMAP_DONTUNMAP move.
5160                    VmaLockMode::Unlocked,
5161                    advice_policy,
5162                    replacement.clone(),
5163                )
5164                .ok_or(StarryError::BadState)?;
5165            final_successor = final_successor
5166                .with_mapping_entry(replacement_entry, false)
5167                .ok_or(StarryError::BadState)?;
5168        } else {
5169            if let Some(tail) = tail_range {
5170                final_successor = final_successor
5171                    .without_range(tail)
5172                    .ok_or(StarryError::BadState)?;
5173            }
5174            final_successor = final_successor
5175                .without_range(moved_source_range)
5176                .ok_or(StarryError::BadState)?;
5177        }
5178        self.validate_memlock_successor(&final_successor, memlock_limit)?;
5179        let before_vmas = self.vma_root.len();
5180        let graph_preimage = self.capture_mapping_graph_snapshot(&rollback_ranges)?;
5181
5182        let mut mutation = self.prepare_mutation_range(src, src_size);
5183        mutation
5184            .try_reserve_tlb_ranges(5)
5185            .map_err(|error| match error {
5186                MutationError::ResourceExhausted => StarryError::NoMemory,
5187                _ => StarryError::BadState,
5188            })?;
5189        mutation
5190            .add_tlb_range(TlbRange::new(target, target_size).ok_or(StarryError::InvalidInput)?);
5191        let retire_epoch = mutation
5192            .receipt()
5193            .base_epoch
5194            .checked_next()
5195            .ok_or(StarryError::BadState)?;
5196
5197        // Linux moves a complete PMD leaf directly, but splits a PMD when
5198        // either the moved source extent or the replacement target cuts only
5199        // part of it.  Consume every deposited table before capturing the
5200        // rollback preimage, and bind the full PMD invalidations to this same
5201        // mremap receipt.
5202        let split_ranges = [moved_source_range, target_range];
5203        let splits = self.apply_partial_huge_splits_for_ranges(&split_ranges)?;
5204        for index in 0..splits.len() {
5205            let split = &splits[index];
5206            let Some(tlb_range) =
5207                TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
5208            else {
5209                return self.abort_unpublished_huge_splits(splits, StarryError::BadState);
5210            };
5211            mutation.add_tlb_range(tlb_range);
5212        }
5213
5214        let preimage = match self.capture_mapping_preimage_ranges(&rollback_ranges) {
5215            Ok(preimage) => preimage,
5216            Err(error) => return self.abort_unpublished_huge_splits(splits, error),
5217        };
5218        let mut target_owners = match replace_target
5219            .then(|| self.prepare_retired_mapping_owners(target_range))
5220            .transpose()
5221        {
5222            Ok(owners) => owners,
5223            Err(error) => {
5224                return self.abort_unpublished_split_mapping_mutation_ranges(
5225                    &rollback_ranges,
5226                    preimage,
5227                    None,
5228                    splits,
5229                    error,
5230                );
5231            }
5232        };
5233        let mut tail_owners = match tail_range
5234            .map(|range| self.prepare_retired_mapping_owners(range))
5235            .transpose()
5236        {
5237            Ok(owners) => owners,
5238            Err(error) => {
5239                return self.abort_unpublished_split_mapping_mutation_ranges(
5240                    &rollback_ranges,
5241                    preimage,
5242                    None,
5243                    splits,
5244                    error,
5245                );
5246            }
5247        };
5248        let mut page_table_reclaims = if replace_target || tail_range.is_some() {
5249            match self.prepare_page_table_reclaims(&rollback_ranges) {
5250                Ok(reclaims) => reclaims,
5251                Err(error) => {
5252                    return self.abort_unpublished_split_mapping_mutation_ranges(
5253                        &rollback_ranges,
5254                        preimage,
5255                        None,
5256                        splits,
5257                        error,
5258                    );
5259                }
5260            }
5261        } else {
5262            MappingMutationContext::for_published_mutation(0)?
5263        };
5264
5265        let mut memfd_deltas = crate::syscall::memfd_prepare_aspace_replace_deltas(
5266            self,
5267            target,
5268            target_size,
5269            permissions.current,
5270            &target_backend,
5271        );
5272        if dontunmap {
5273            if let Some(replacement) = source_replacement.as_ref() {
5274                memfd_deltas.extend(crate::syscall::memfd_prepare_aspace_replace_deltas(
5275                    self,
5276                    src,
5277                    move_size,
5278                    permissions.current,
5279                    replacement,
5280                ));
5281            }
5282        } else {
5283            memfd_deltas.extend(crate::syscall::memfd_prepare_aspace_unmap_deltas(
5284                self, src, src_size,
5285            ));
5286        }
5287
5288        let apply_result = (|| -> StarryResult<usize> {
5289            let target_materialization = self.apply_mapping_pages_unpublished(
5290                target_range,
5291                permissions,
5292                &target_backend,
5293                replace_target,
5294                &mut page_table_reclaims,
5295            )?;
5296            self.vma_root = Arc::new(target_successor.clone());
5297
5298            let moved_pages = self.apply_move_pages(src, target, move_size)?;
5299            let prepared_moved_slots =
5300                self.prepare_moved_slots(&moved_pages, target_backend.mapping_id())?;
5301            if !dontunmap && let Some(tail) = tail_range {
5302                self.apply_unmap_pages_unpublished(tail, &mut page_table_reclaims)?;
5303            }
5304
5305            if replace_target {
5306                self.detach_mapping_slots(target_range)?;
5307            }
5308            self.publish_prepared_pte_owners(
5309                &target_backend,
5310                target_range,
5311                &target_materialization,
5312            )?;
5313            self.publish_moved_slots(prepared_moved_slots)?;
5314            if !dontunmap && let Some(tail) = tail_range {
5315                self.detach_mapping_slots(tail)?;
5316            }
5317            if self
5318                .mapping_slots_overlapping(moved_source_range)
5319                .next()
5320                .is_some()
5321            {
5322                return Err(StarryError::BadState);
5323            }
5324            self.vma_root = Arc::new(final_successor.clone());
5325
5326            self.vm_stat.on_map((target_size / PAGE_SIZE_4K) as u64);
5327            if target_removed_pages != 0 {
5328                self.vm_stat.on_unmap(target_removed_pages);
5329            }
5330            if !dontunmap {
5331                self.vm_stat.on_unmap((src_size / PAGE_SIZE_4K) as u64);
5332            }
5333            Ok(moved_pages.len())
5334        })();
5335
5336        let moved_leaves = match apply_result {
5337            Ok(moved) => moved,
5338            Err(error) => {
5339                let parked = target_owners.is_some()
5340                    || tail_owners.is_some()
5341                    || page_table_reclaims.pte_changed();
5342                if let Some(owners) = target_owners.take() {
5343                    self.park_retired_mapping_owners(retire_epoch, owners);
5344                }
5345                if let Some(owners) = tail_owners.take() {
5346                    self.park_retired_mapping_owners(retire_epoch, owners);
5347                }
5348                self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
5349                return self.abort_unpublished_split_mapping_mutation_ranges(
5350                    &rollback_ranges,
5351                    preimage,
5352                    parked.then_some(retire_epoch),
5353                    splits,
5354                    error,
5355                );
5356            }
5357        };
5358        if let Some(owners) = target_owners.take() {
5359            self.park_retired_mapping_owners(retire_epoch, owners);
5360        }
5361        if let Some(owners) = tail_owners.take() {
5362            self.park_retired_mapping_owners(retire_epoch, owners);
5363        }
5364        self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
5365
5366        let after_vmas = self.vma_root.len();
5367        mutation.set_vma_delta(VmaDelta {
5368            inserted: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
5369            removed: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
5370            ..VmaDelta::default()
5371        });
5372        mutation.set_pte_delta(PteDelta {
5373            mapped: u32::try_from(moved_leaves).unwrap_or(u32::MAX),
5374            unmapped: u32::try_from(moved_leaves).unwrap_or(u32::MAX),
5375            ..PteDelta::default()
5376        });
5377        if let Err(error) =
5378            self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &rollback_ranges)
5379        {
5380            return self.abort_unpublished_split_mapping_mutation_ranges(
5381                &rollback_ranges,
5382                preimage,
5383                Some(retire_epoch),
5384                splits,
5385                error,
5386            );
5387        }
5388
5389        match self.commit_mutation_classified(mutation) {
5390            Ok(()) => {
5391                self.release_retired_mapping_owners(retire_epoch);
5392                crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5393                Ok(())
5394            }
5395            Err(CommitMutationError::PublishedPendingTlb(error)) => {
5396                crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5397                Err(error)
5398            }
5399            Err(CommitMutationError::Unpublished(error)) => self
5400                .abort_unpublished_split_mapping_mutation_ranges(
5401                    &rollback_ranges,
5402                    preimage,
5403                    Some(retire_epoch),
5404                    splits,
5405                    error,
5406                ),
5407        }
5408    }
5409
5410    /// Grows the mapping containing `addr` by `additional_size` at its end.
5411    pub fn extend_area(&mut self, addr: VirtAddr, additional_size: usize) -> StarryResult {
5412        self.extend_area_with_memlock(addr, additional_size, None)
5413    }
5414
5415    pub(crate) fn extend_area_with_memlock(
5416        &mut self,
5417        addr: VirtAddr,
5418        additional_size: usize,
5419        memlock_limit: Option<MemlockLimit>,
5420    ) -> StarryResult {
5421        if additional_size == 0 {
5422            return Ok(());
5423        }
5424        let entry = self
5425            .vma_root
5426            .lookup_entry(addr)
5427            .ok_or(StarryError::InvalidInput)?;
5428        if !additional_size.is_multiple_of(PAGE_SIZE_4K) {
5429            return Err(StarryError::InvalidInput);
5430        }
5431        let old_end = entry.end();
5432        let grown = VirtAddrRange::try_from_start_size(old_end, additional_size)
5433            .ok_or(StarryError::InvalidInput)?;
5434        let preimage = self.capture_mapping_preimage(grown)?;
5435        let graph_preimage = self.capture_mapping_graph_snapshot(&[grown])?;
5436        let mut mutation = self.prepare_mutation_range(old_end, additional_size);
5437        if entry
5438            .end()
5439            .checked_add(additional_size)
5440            .is_none_or(|new_end| new_end > self.end())
5441        {
5442            return Err(StarryError::NoMemory);
5443        }
5444        let (materialized_range, operation, materialization) =
5445            match self.apply_extend_unpublished(addr, additional_size, memlock_limit) {
5446                Ok(applied) => applied,
5447                Err(error) => {
5448                    return self.abort_unpublished_mapping_mutation(grown, preimage, error);
5449                }
5450            };
5451        if materialized_range != grown {
5452            return self.abort_unpublished_mapping_mutation(grown, preimage, StarryError::BadState);
5453        }
5454        self.vm_stat.on_map((additional_size / PAGE_SIZE_4K) as u64);
5455        if let Err(error) = self.publish_prepared_pte_owners(&operation, grown, &materialization) {
5456            return self.abort_unpublished_mapping_mutation(grown, preimage, error);
5457        }
5458        if let Err(error) =
5459            self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[grown])
5460        {
5461            return self.abort_unpublished_mapping_mutation(grown, preimage, error);
5462        }
5463        match self.commit_mutation_classified(mutation) {
5464            Ok(()) => Ok(()),
5465            Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
5466            Err(CommitMutationError::Unpublished(error)) => {
5467                self.abort_unpublished_mapping_mutation(grown, preimage, error)
5468            }
5469        }
5470    }
5471
5472    /// To process data in this area with the given function.
5473    ///
5474    /// Now it supports reading and writing data in the given interval.
5475    fn process_area_data<F>(&self, start: VirtAddr, size: usize, mut f: F) -> StarryResult
5476    where
5477        F: FnMut(VirtAddr, usize, usize),
5478    {
5479        if size == 0 {
5480            return Ok(());
5481        }
5482        if !self.contains_range(start, size) {
5483            return Err(StarryError::InvalidInput);
5484        }
5485        let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
5486        // Aligning with the low-level helper can wrap at `usize::MAX`; use a
5487        // checked addition so user-copy never turns an invalid end into a
5488        // low address.
5489        let end_align_up = end
5490            .as_usize()
5491            .checked_add(PAGE_SIZE_4K - 1)
5492            .map(|value| VirtAddr::from_usize(value & !(PAGE_SIZE_4K - 1)))
5493            .ok_or(StarryError::InvalidInput)?;
5494        let page_start = start.align_down_4k();
5495        let pages = PageIter4K::new(page_start, end_align_up).ok_or(StarryError::InvalidInput)?;
5496        let mut copied = 0usize;
5497        for vaddr in pages {
5498            let (paddr, ..) = self.pt.query(vaddr).map_err(|_| StarryError::BadAddress)?;
5499            let page_offset = if vaddr == page_start {
5500                start.align_offset_4k()
5501            } else {
5502                0
5503            };
5504            let copy_size = (PAGE_SIZE_4K - page_offset).min(size - copied);
5505            if copy_size == 0 {
5506                break;
5507            }
5508            let paddr = paddr
5509                .checked_add(page_offset)
5510                .ok_or(StarryError::BadAddress)?;
5511            f(phys_to_virt(paddr), copied, copy_size);
5512            copied = copied
5513                .checked_add(copy_size)
5514                .ok_or(StarryError::InvalidInput)?;
5515        }
5516        (copied == size)
5517            .then_some(())
5518            .ok_or(StarryError::BadAddress)
5519    }
5520
5521    pub fn read(&self, start: VirtAddr, buf: &mut [u8]) -> StarryResult {
5522        self.process_area_data(start, buf.len(), |src, offset, read_size| unsafe {
5523            core::ptr::copy_nonoverlapping(src.as_ptr(), buf.as_mut_ptr().add(offset), read_size);
5524        })
5525    }
5526
5527    /// To write data to the address space.
5528    ///
5529    /// # Arguments
5530    ///
5531    /// * `start_vaddr` - The start virtual address to write.
5532    /// * `buf` - The buffer to write to the address space.
5533    pub fn write(&self, start: VirtAddr, buf: &[u8]) -> StarryResult {
5534        self.process_area_data(start, buf.len(), |dst, offset, write_size| unsafe {
5535            core::ptr::copy_nonoverlapping(buf.as_ptr().add(offset), dst.as_mut_ptr(), write_size);
5536        })
5537    }
5538
5539    /// Synchronizes instruction fetch after modifying executable memory through this address space.
5540    pub fn sync_modified_text(&self, start: VirtAddr, size: usize) -> StarryResult {
5541        if size == 0 {
5542            return Ok(());
5543        }
5544
5545        self.process_area_data(start, size, |dst, _offset, sync_size| {
5546            let range = ax_cpu::cache::CacheRange::new(dst, sync_size)
5547                .expect("mapped text chunk must not wrap");
5548            // SAFETY: process_area_data retains the translated backing mapping
5549            // for this chunk until the callback completes its cache maintenance.
5550            unsafe { ax_cpu::cache::clean_dcache_range_to_pou(range) };
5551        })?;
5552        ax_cpu::cache::flush_icache_all();
5553        Ok(())
5554    }
5555
5556    /// Updates mapping within the specified virtual address range.
5557    ///
5558    /// Returns an error if the address range is out of the address space or not
5559    /// aligned.
5560    pub fn protect(&mut self, start: VirtAddr, size: usize, flags: MappingFlags) -> StarryResult {
5561        self.protect_with_reported_flags(start, size, flags, flags)
5562    }
5563
5564    pub fn protect_with_reported_flags(
5565        &mut self,
5566        start: VirtAddr,
5567        size: usize,
5568        flags: MappingFlags,
5569        reported_flags: MappingFlags,
5570    ) -> StarryResult {
5571        self.validate_region(start, size)?;
5572        let range = VirtAddrRange::from_start_size(start, size);
5573
5574        // Validate against the immutable permission envelope before touching
5575        // any PTE or VMA fragment.  This is intentionally done here (rather
5576        // than in one backend) because a range can span several fragments and
5577        // the envelope belongs to the VMA record itself.
5578        start.checked_add(size).ok_or(StarryError::InvalidInput)?;
5579        // The bounds check reads only the VMAs the range covers. Walking the
5580        // whole map and breaking out of it saved nothing: `iter_entries` builds
5581        // the entire vector before the loop gets to look at the first entry.
5582        let mut denied = false;
5583        self.vma_root.for_each_overlapping_entry(range, |entry| {
5584            if entry.max_rights().contains(flags) {
5585                return true;
5586            }
5587            denied = true;
5588            false
5589        });
5590        if denied {
5591            return Err(StarryError::PermissionDenied);
5592        }
5593
5594        let vma_preimage = self.vma_root.clone();
5595        let vm_stat_preimage = self.vm_stat.snapshot();
5596        let before_vmas = self.vma_root.len();
5597        let mut mutation = self.prepare_mutation_range(start, size);
5598        mutation
5599            .try_reserve_tlb_ranges(2)
5600            .map_err(|_| StarryError::NoMemory)?;
5601        let splits = self.apply_partial_huge_splits(range)?;
5602        for split in &splits {
5603            let Some(tlb_range) =
5604                TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
5605            else {
5606                return self.abort_unpublished_protection(
5607                    vma_preimage,
5608                    vm_stat_preimage,
5609                    &[],
5610                    splits,
5611                    StarryError::BadState,
5612                );
5613            };
5614            mutation.add_tlb_range(tlb_range);
5615        }
5616        let protection_preimage = match self.capture_protection_leaf_preimage(range) {
5617            Ok(preimage) => preimage,
5618            Err(error) => {
5619                return self.abort_unpublished_protection(
5620                    vma_preimage,
5621                    vm_stat_preimage,
5622                    &[],
5623                    splits,
5624                    error,
5625                );
5626            }
5627        };
5628        let protected_leaves = protection_preimage.len();
5629        mutation.set_pte_delta(PteDelta {
5630            protected: u32::try_from(protected_leaves).unwrap_or(u32::MAX),
5631            ..PteDelta::default()
5632        });
5633        let split_slots = splits
5634            .iter()
5635            .map(|split| split.child_slots.len().saturating_sub(1))
5636            .sum::<usize>();
5637        mutation.set_mapping_delta(MappingDelta {
5638            attached: u32::try_from(split_slots).unwrap_or(u32::MAX),
5639            ..MappingDelta::default()
5640        });
5641        let touched_memfds =
5642            crate::syscall::memfd_collect_metas_touching_mprotect_range(self, start, size);
5643        if flags.contains(MappingFlags::EXECUTE) {
5644            for leaf in &protection_preimage {
5645                let key = MappingSlotKey {
5646                    space_id: self.id,
5647                    va: leaf.va,
5648                };
5649                let slot = self
5650                    .mapping_slots
5651                    .get(&key)
5652                    .expect("published executable leaf must retain its mapping owner");
5653                slot.page.prepare_executable_mapping(
5654                    leaf.paddr,
5655                    leaf.page_size,
5656                    flags | (leaf.flags & (MappingFlags::DEVICE | MappingFlags::UNCACHED)),
5657                );
5658            }
5659        }
5660        if let Err(error) = self.apply_protection_unpublished(range, flags, reported_flags) {
5661            return self.abort_unpublished_protection(
5662                vma_preimage,
5663                vm_stat_preimage,
5664                &protection_preimage,
5665                splits,
5666                error,
5667            );
5668        }
5669        let after_vmas = self.vma_root.len();
5670        mutation.set_vma_delta(VmaDelta {
5671            split: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
5672            merged: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
5673            ..VmaDelta::default()
5674        });
5675
5676        match self.commit_mutation_classified(mutation) {
5677            Ok(()) => {
5678                crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
5679                    self,
5680                    &touched_memfds,
5681                );
5682                Ok(())
5683            }
5684            Err(CommitMutationError::PublishedPendingTlb(error)) => {
5685                // Publication is already externally visible; side-band
5686                // accounting follows it even while old frame ownership stays
5687                // quarantined behind the outstanding TLB receipt.
5688                crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
5689                    self,
5690                    &touched_memfds,
5691                );
5692                Err(error)
5693            }
5694            Err(CommitMutationError::Unpublished(error)) => self.abort_unpublished_protection(
5695                vma_preimage,
5696                vm_stat_preimage,
5697                &protection_preimage,
5698                splits,
5699                error,
5700            ),
5701        }
5702    }
5703
5704    fn ensure_quiescent_for_content_clear(&self) -> StarryResult {
5705        if self.tlb_targets.load(core::sync::atomic::Ordering::Acquire) != 0
5706            || self.mutation_gate.pending_count() != 0
5707            || self.pending_retired_mapping_batches() != 0
5708        {
5709            return Err(StarryError::ResourceBusy);
5710        }
5711        Ok(())
5712    }
5713
5714    /// Removes every user mapping after the caller has proved that this page
5715    /// table cannot be installed on a CPU.  This is the shared apply step for
5716    /// unpublished-image abort and retired-MM reclaim; it deliberately does
5717    /// not publish an epoch or side-band event by itself.
5718    fn clear_quiescent_contents(&mut self) -> StarryResult {
5719        self.ensure_quiescent_for_content_clear()?;
5720        let range = self.layout.range();
5721        let operations = self.mapping_operation_fragments(range, false)?;
5722        if operations
5723            .iter()
5724            .any(|(fragment, operation)| !operation.validate_unmap_range(*fragment, &self.pt))
5725        {
5726            return Err(StarryError::BadState);
5727        }
5728
5729        let leaf_count = self.occupied_pte_leaves_overlapping(&[range])?.len();
5730        let mut page_table_reclaims =
5731            MappingMutationContext::for_published_mutation(leaf_count)?;
5732        // A retired MM has no users, pins, activations, pending receipts or
5733        // page-table walkers.  An unpublished loader image is likewise held by
5734        // one `&mut AddrSpace`.  Linux uses the same isolation proof to run
5735        // `free_pgtables()` without a PTL after VMAs have been detached.  Do
5736        // not acquire the IRQ-saving structure lock here: backend validation,
5737        // occupied-leaf vectors, page-table frame release and Arc destruction
5738        // are all allowed to allocate or enter the allocator's reclaim path.
5739        let clear_result = operations
5740            .into_iter()
5741            .try_for_each(|(fragment, operation)| {
5742                operation.unmap_range(fragment, &mut page_table_reclaims, &mut self.pt)
5743            });
5744        // SAFETY: `ensure_quiescent_for_content_clear` proved that this root
5745        // has no active CPU, pending receipt, or earlier retire batch.
5746        unsafe { page_table_reclaims.reclaim() };
5747        if let Err(error) = clear_result {
5748            self.mutation_gate.mark_needs_repair();
5749            return Err(error);
5750        }
5751        let slots = core::mem::take(&mut self.mapping_slots);
5752        for slot in slots.into_values() {
5753            slot.detach();
5754        }
5755        self.resident_watermark.reset();
5756        self.vm_stat.on_clear();
5757        self.vma_root = Arc::new(VmaMap::default());
5758        self.executable_file = None;
5759        // Once every materialized and software owner is empty, a prior repair
5760        // bit belonging solely to this unpublished/retired image is resolved.
5761        self.mutation_gate.clear_repair();
5762        Ok(())
5763    }
5764
5765    /// Aborts an address-space image that has never been registered in
5766    /// [`MmHandle`] or installed by the scheduler.
5767    ///
5768    /// Linux drops `bprm->mm` through `mmput()` before `begin_new_exec`; it does
5769    /// not publish an externally visible VMA mutation for a failed ELF or
5770    /// interpreter attempt.  Starry keeps the allocated root so the loader may
5771    /// reuse its borrowed kernel entries, but the discard has the same
5772    /// unpublished semantics: no [`MutationReceipt`] and no epoch advance.
5773    pub(crate) fn reset_uninstalled_for_loader(&mut self) -> StarryResult {
5774        self.ensure_quiescent_for_content_clear()?;
5775        let range = self.layout.range();
5776        let memfd_deltas =
5777            crate::syscall::memfd_prepare_aspace_unmap_deltas(self, range.start, range.size());
5778        self.clear_quiescent_contents()?;
5779        self.resident_pages = ResidentPageCounts::default();
5780        self.heap = HeapState::new(USER_HEAP_BASE);
5781        self.executable_data = ExecutableDataLayout::default();
5782        crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5783        Ok(())
5784    }
5785
5786    /// Clears a retired, formerly published MM and records that teardown in
5787    /// the ordinary mutation protocol before page-table frames are detached.
5788    fn clear_retired_contents(&mut self) -> StarryResult {
5789        self.ensure_quiescent_for_content_clear()?;
5790        let base_epoch = self.vm_epoch();
5791        base_epoch.checked_next().ok_or(StarryError::BadState)?;
5792        let range = self.layout.range();
5793        let removed_vmas = self.vma_root.len();
5794        let detached_slots = self.mapping_slots.len();
5795        let materialized_pages = self
5796            .mapping_slots
5797            .values()
5798            .try_fold(0usize, |pages, slot| {
5799                pages.checked_add(1usize.checked_shl(slot.page_order.get().into())?)
5800            })
5801            .ok_or(StarryError::BadState)?;
5802        let memfd_deltas =
5803            crate::syscall::memfd_prepare_aspace_unmap_deltas(self, range.start, range.size());
5804        let mut mutation = self.prepare_mutation_range(range.start, range.size());
5805        mutation.set_vma_delta(VmaDelta {
5806            removed: u32::try_from(removed_vmas).unwrap_or(u32::MAX),
5807            ..VmaDelta::default()
5808        });
5809        mutation.set_pte_delta(PteDelta {
5810            unmapped: u32::try_from(materialized_pages).unwrap_or(u32::MAX),
5811            ..PteDelta::default()
5812        });
5813        mutation.set_mapping_delta(MappingDelta {
5814            detached: u32::try_from(detached_slots).unwrap_or(u32::MAX),
5815            ..MappingDelta::default()
5816        });
5817        mutation.set_resident_delta(self.resident_pages.checked_negated_delta()?);
5818        self.clear_quiescent_contents()?;
5819        let result = self.commit_mutation(mutation);
5820        if self.vm_epoch() != base_epoch {
5821            crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5822        }
5823        result
5824    }
5825
5826    /// Reclaims all mappings after lifecycle quiescence.
5827    pub(crate) fn try_reclaim_contents(&mut self) -> StarryResult {
5828        // A retired permit is only valid after every CPU has switched away
5829        // and every earlier shootdown receipt has been acknowledged.  Keep
5830        // this check in the owning address-space object as a second line of
5831        // defence: callers must not be able to clear a root merely because an
5832        // `Arc` happened to be the last strong reference.
5833        if self.tlb_targets.load(core::sync::atomic::Ordering::Acquire) != 0
5834            || self.mutation_gate.pending_count() != 0
5835            || self.pending_retired_mapping_batches() != 0
5836        {
5837            return Err(StarryError::ResourceBusy);
5838        }
5839        self.clear_retired_contents()?;
5840        let epoch = self.vm_epoch();
5841
5842        // Detach page-table frames from the materialized tree before allocator
5843        // release. Lifecycle quiescence is the zero-target form of Linux's
5844        // mmu-gather contract: no CPU can still walk this root, so the typed
5845        // allocator capability may be consumed immediately. Published
5846        // mutations with remote observers take the ordinary TLB quarantine
5847        // path before an MM can reach Retired.
5848        let targets = self.tlb_targets.load(core::sync::atomic::Ordering::Acquire);
5849        let request = TlbRequest::new(self.id, epoch, targets);
5850        debug_assert!(request.is_complete());
5851        // SAFETY: lifecycle only calls this after all user/kernel references
5852        // and scheduler activations are quiescent.  `PageTable::detach` leaves
5853        // the owning table inert and transfers each frame to a token. The
5854        // completed zero-target request proves that consuming each token in
5855        // the callback cannot race an architectural page-table walk.
5856        unsafe {
5857            self.pt.detach(|token| token.reclaim());
5858        }
5859        Ok(())
5860    }
5861
5862    /// Checks whether an access to the specified memory region is valid.
5863    ///
5864    /// Returns `true` if the memory region given by `range` is all mapped and
5865    /// has proper permission flags (i.e. containing `access_flags`).
5866    pub fn can_access_range(
5867        &self,
5868        start: VirtAddr,
5869        size: usize,
5870        access_flags: MappingFlags,
5871    ) -> bool {
5872        let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
5873            return false;
5874        };
5875        if range.is_empty() {
5876            return false;
5877        }
5878        // This sits under every user-pointer check, so it must not allocate or
5879        // touch a reference count per VMA: an ordinary `clock_gettime` writing
5880        // one `timespec` was flattening the whole VMA tree.
5881        let mut cursor = range.start;
5882        let mut permitted = false;
5883        self.vma_root.for_each_overlapping(range, |vma| {
5884            if vma.range.end <= cursor {
5885                return true;
5886            }
5887            if vma.range.start > cursor || !vma.rights.contains(access_flags) {
5888                return false;
5889            }
5890            cursor = vma.range.end.min(range.end);
5891            if cursor >= range.end {
5892                permitted = true;
5893                return false;
5894            }
5895            true
5896        });
5897        permitted
5898    }
5899
5900    /// Chooses the materialized leaf size for one fault without changing the
5901    /// MappingGroup's long-term THP policy.
5902    ///
5903    /// A missing PTE inside a split PMD must be faulted as one base page.  It
5904    /// is safe to retry the policy-sized mapping only when the complete policy
5905    /// unit belongs to this VMA and no sibling PTE is still installed.  This
5906    /// is the same distinction Linux makes between a none PMD eligible for a
5907    /// new THP and a deposited PTE table containing a split folio.
5908    fn fault_transaction_page_size(
5909        &self,
5910        vaddr: VirtAddr,
5911        vma_range: VirtAddrRange,
5912        policy_size: usize,
5913    ) -> StarryResult<usize> {
5914        match self.pt.query(vaddr) {
5915            Ok((_, _, leaf_size)) => return Ok(leaf_size),
5916            Err(PagingError::NotMapped) => {}
5917            Err(error) => return Err(error.into()),
5918        }
5919        if policy_size == PAGE_SIZE_4K {
5920            return Ok(PAGE_SIZE_4K);
5921        }
5922        if policy_size < PAGE_SIZE_4K
5923            || !policy_size.is_power_of_two()
5924            || !policy_size.is_multiple_of(PAGE_SIZE_4K)
5925        {
5926            return Err(StarryError::BadState);
5927        }
5928
5929        let policy_start = vaddr.align_down(policy_size);
5930        let policy_range = VirtAddrRange::try_from_start_size(policy_start, policy_size)
5931            .ok_or(StarryError::BadState)?;
5932        if !vma_range.contains_range(policy_range) {
5933            return Ok(PAGE_SIZE_4K);
5934        }
5935
5936        // Any occupied sibling proves that this policy unit already owns a
5937        // base-page table. Replacing it with a huge leaf would overwrite live
5938        // or retained mappings and their rmap ownership. Follow allocated
5939        // page-table paths instead of issuing 512 base-page queries.
5940        if self
5941            .pt
5942            .walk_occupied_range(policy_range.start, policy_range.end)
5943            .next()
5944            .is_some()
5945        {
5946            return Ok(PAGE_SIZE_4K);
5947        }
5948        Ok(policy_size)
5949    }
5950
5951    fn plan_page_fault(
5952        &self,
5953        vaddr: VirtAddr,
5954        access_flags: PageFaultFlags,
5955        thp_mode: TransparentHugePageMode,
5956    ) -> Result<PageFaultPlan, FaultResult> {
5957        if self.mutation_gate.needs_repair() {
5958            // An indeterminate publication retains its owners in quarantine.
5959            // Neither a present PTE nor another allocation proves it usable.
5960            return Err(FaultResult::Sigbus(BusCode::ObjErr));
5961        }
5962        if !self.layout.range().contains(vaddr) {
5963            return Err(FaultResult::Unmapped);
5964        }
5965        let access_flags = MappingFlags::from(access_flags);
5966        let Some(entry) = self.vma_root.lookup_entry(vaddr) else {
5967            return Err(FaultResult::Unmapped);
5968        };
5969        let vma = entry.snapshot().clone();
5970        let flags = vma.rights;
5971        if !flags.contains(access_flags) {
5972            return Err(FaultResult::PermissionDenied);
5973        }
5974        let backend = entry.operation_clone();
5975        let Some(policy_size) = vma
5976            .group
5977            .page_policy
5978            .fault_leaf_size(vma.huge_page_advice, thp_mode)
5979        else {
5980            return Err(FaultResult::Unmapped);
5981        };
5982        let page_size = match self.fault_transaction_page_size(vaddr, vma.range, policy_size) {
5983            Ok(page_size) => page_size,
5984            Err(error) => {
5985                warn!("could not classify page-fault leaf for {vaddr:?}: {error}");
5986                return Err(FaultResult::Retry);
5987            }
5988        };
5989        let page_start = vaddr.align_down(page_size);
5990        let Some(range) = VirtAddrRange::try_from_start_size(page_start, page_size) else {
5991            return Err(FaultResult::Unmapped);
5992        };
5993        let fault_fallback = if vma.group.page_policy.permits_fault_fallback() {
5994            FaultFallback::BasePage
5995        } else {
5996            FaultFallback::Forbidden
5997        };
5998        let request = match PopulateRequest::fault(range, page_size, vaddr, fault_fallback) {
5999            Ok(request) => request,
6000            Err(_) => return Err(FaultResult::Unmapped),
6001        };
6002        let preimage = match FaultPteSnapshot::capture(&self.pt, page_start) {
6003            Ok(preimage) => preimage,
6004            Err(error) => {
6005                warn!("could not capture page-fault PTE at {page_start:?}: {error}");
6006                return Err(FaultResult::Retry);
6007            }
6008        };
6009        let map_plans = if preimage == FaultPteSnapshot::NotMapped {
6010            let preferred = match self.pt.plan_map_page(page_start, page_size) {
6011                Ok(plan) => plan,
6012                Err(error) => {
6013                    warn!("could not plan page-table path for {page_start:?}: {error}");
6014                    return Err(FaultResult::Retry);
6015                }
6016            };
6017            let fallback = if page_size > PAGE_SIZE_4K && fault_fallback == FaultFallback::BasePage
6018            {
6019                let fallback_start = vaddr.align_down_4k();
6020                match self.pt.plan_map_page(fallback_start, PAGE_SIZE_4K) {
6021                    Ok(plan) => Some(plan),
6022                    Err(error) => {
6023                        warn!(
6024                            "could not plan fallback page-table path for {fallback_start:?}: \
6025                             {error}"
6026                        );
6027                        return Err(FaultResult::Retry);
6028                    }
6029                }
6030            } else {
6031                None
6032            };
6033            Some(PageFaultMapPlans {
6034                preferred,
6035                fallback,
6036            })
6037        } else {
6038            None
6039        };
6040        Ok(PageFaultPlan {
6041            base_epoch: self.vm_epoch(),
6042            space_id: self.id,
6043            vaddr,
6044            range,
6045            vma_flags: flags,
6046            access_flags,
6047            operation: backend,
6048            request,
6049            preimage,
6050            map_plans,
6051        })
6052    }
6053
6054    fn classify_fault_error(file_backed: bool, error: StarryError) -> FaultResult {
6055        if matches!(
6056            error,
6057            StarryError::NoMemory
6058                | StarryError::Paging(PagingError::NoMemory)
6059                | StarryError::Vfs(axfs_ng_vfs::VfsError::NoMemory)
6060        ) {
6061            return FaultResult::NoMemory;
6062        }
6063        if matches!(error, StarryError::ResourceBusy) {
6064            return FaultResult::Retry;
6065        }
6066        if !file_backed {
6067            return FaultResult::Unmapped;
6068        }
6069        match error {
6070            StarryError::ResourceBusy | StarryError::Vfs(axfs_ng_vfs::VfsError::ResourceBusy) => {
6071                FaultResult::Retry
6072            }
6073            StarryError::BadAddress => FaultResult::Sigbus(BusCode::AdrErr),
6074            StarryError::Io | StarryError::Vfs(_) => FaultResult::Sigbus(BusCode::ObjErr),
6075            _ => FaultResult::Unmapped,
6076        }
6077    }
6078
6079    fn prepare_fault_materialization(
6080        plan: &PageFaultPlan,
6081        request: PopulateRequest,
6082    ) -> Result<FaultMaterialization, FaultResult> {
6083        match plan.operation.prepare_fault(
6084            plan.space_id,
6085            request,
6086            plan.vma_flags,
6087            plan.access_flags,
6088            plan.preimage,
6089        ) {
6090            Ok(materialization) => Ok(materialization),
6091            Err(error) => {
6092                warn!(
6093                    "failed to prepare page fault for {:?} ({:?}): {error}",
6094                    plan.vaddr, plan.vma_flags
6095                );
6096                Err(Self::classify_fault_error(
6097                    plan.operation.is_file_backed(),
6098                    error,
6099                ))
6100            }
6101        }
6102    }
6103
6104    fn cancel_fault_materialization(
6105        plan: &PageFaultPlan,
6106        materialization: FaultMaterialization,
6107    ) -> Result<(), FaultResult> {
6108        plan.operation
6109            .cancel_prepared_fault_publication(materialization)
6110            .map_err(|error| {
6111                warn!(
6112                    "failed to cancel prepared page fault for {:?}: {error}",
6113                    plan.vaddr
6114                );
6115                FaultResult::Retry
6116            })
6117    }
6118
6119    fn prepare_page_fault(mut plan: PageFaultPlan) -> Result<PreparedPageFault, FaultResult> {
6120        let mut materialization = Self::prepare_fault_materialization(&plan, plan.request)?;
6121        let installed_owner = materialization.owner().and_then(|owner| {
6122            (owner.transition == PteOwnerTransition::Installed).then_some((
6123                owner.va,
6124                owner.paddr,
6125                owner.page_size,
6126            ))
6127        });
6128        let map_deposit = if let Some((owner_va, owner_paddr, owner_page_size)) = installed_owner {
6129            let Some(plans) = plan.map_plans.take() else {
6130                Self::cancel_fault_materialization(&plan, materialization)?;
6131                return Err(FaultResult::Retry);
6132            };
6133            let PageFaultMapPlans {
6134                preferred,
6135                mut fallback,
6136            } = plans;
6137            let preferred_selected =
6138                preferred.vaddr() == owner_va && preferred.page_size() == owner_page_size;
6139            let fallback_selected = fallback.as_ref().is_some_and(|fallback| {
6140                fallback.vaddr() == owner_va && fallback.page_size() == owner_page_size
6141            });
6142            if !preferred_selected && !fallback_selected {
6143                Self::cancel_fault_materialization(&plan, materialization)?;
6144                return Err(FaultResult::Retry);
6145            }
6146            let Some(flags) = materialization.pte_flags() else {
6147                Self::cancel_fault_materialization(&plan, materialization)?;
6148                return Err(FaultResult::Retry);
6149            };
6150
6151            if fallback_selected {
6152                let Some(fallback_request) = plan.request.into_base_page_fallback() else {
6153                    Self::cancel_fault_materialization(&plan, materialization)?;
6154                    return Err(FaultResult::Retry);
6155                };
6156                plan.request = fallback_request;
6157                plan.range = fallback_request.range();
6158                let Some(fallback) = fallback.take() else {
6159                    Self::cancel_fault_materialization(&plan, materialization)?;
6160                    return Err(FaultResult::Retry);
6161                };
6162                match fallback.prepare(owner_paddr, flags) {
6163                    Ok(deposit) => Some(deposit),
6164                    Err(error) => {
6165                        warn!(
6166                            "could not prepare fallback page-table path for {owner_va:?}: {error}"
6167                        );
6168                        Self::cancel_fault_materialization(&plan, materialization)?;
6169                        return Err(Self::classify_fault_error(false, error.into()));
6170                    }
6171                }
6172            } else {
6173                match preferred.prepare(owner_paddr, flags) {
6174                    Ok(deposit) => Some(deposit),
6175                    Err(PagingError::NoMemory) if fallback.is_some() => {
6176                        // Releasing the huge PageObject first can make enough
6177                        // memory available for the base page plus its deeper
6178                        // table path. This matches Linux's preallocate, recheck,
6179                        // and retry boundary without allocating under the PTL.
6180                        Self::cancel_fault_materialization(&plan, materialization)?;
6181                        let Some(fallback_request) = plan.request.into_base_page_fallback() else {
6182                            return Err(FaultResult::Retry);
6183                        };
6184                        plan.request = fallback_request;
6185                        plan.range = fallback_request.range();
6186                        materialization =
6187                            Self::prepare_fault_materialization(&plan, fallback_request)?;
6188                        let Some(owner) = materialization.owner() else {
6189                            Self::cancel_fault_materialization(&plan, materialization)?;
6190                            return Err(FaultResult::Retry);
6191                        };
6192                        let Some(fallback) = fallback.take() else {
6193                            Self::cancel_fault_materialization(&plan, materialization)?;
6194                            return Err(FaultResult::Retry);
6195                        };
6196                        if owner.transition != PteOwnerTransition::Installed
6197                            || owner.va != fallback.vaddr()
6198                            || owner.page_size != fallback.page_size()
6199                        {
6200                            Self::cancel_fault_materialization(&plan, materialization)?;
6201                            return Err(FaultResult::Retry);
6202                        }
6203                        let Some(flags) = materialization.pte_flags() else {
6204                            Self::cancel_fault_materialization(&plan, materialization)?;
6205                            return Err(FaultResult::Retry);
6206                        };
6207                        match fallback.prepare(owner.paddr, flags) {
6208                            Ok(deposit) => Some(deposit),
6209                            Err(error) => {
6210                                warn!(
6211                                    "could not prepare base-page table path for {:?}: {error}",
6212                                    owner.va
6213                                );
6214                                Self::cancel_fault_materialization(&plan, materialization)?;
6215                                return Err(Self::classify_fault_error(false, error.into()));
6216                            }
6217                        }
6218                    }
6219                    Err(error) => {
6220                        warn!("could not prepare page-table path for {owner_va:?}: {error}");
6221                        Self::cancel_fault_materialization(&plan, materialization)?;
6222                        return Err(Self::classify_fault_error(false, error.into()));
6223                    }
6224                }
6225            }
6226        } else {
6227            None
6228        };
6229        Ok(PreparedPageFault {
6230            plan,
6231            materialization,
6232            map_deposit,
6233        })
6234    }
6235
6236    fn page_fault_plan_is_current(&self, plan: &PageFaultPlan) -> bool {
6237        if self.vm_epoch() != plan.base_epoch || !plan.preimage.matches(plan.range.start, &self.pt)
6238        {
6239            return false;
6240        }
6241        self.vma_root.lookup_entry(plan.vaddr).is_some_and(|entry| {
6242            entry.snapshot().rights == plan.vma_flags
6243                && entry.snapshot().range.contains_range(plan.range)
6244                && entry.operation().mapping_id() == plan.operation.mapping_id()
6245        })
6246    }
6247
6248    fn apply_prepared_page_fault(
6249        &mut self,
6250        attempt: &mut PageFaultApplyAttempt,
6251    ) -> PageFaultApplyOutcome {
6252        if !self.page_fault_plan_is_current(&attempt.prepared().plan) {
6253            return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6254        }
6255
6256        let (pages, file_backed, vaddr, vma_flags, range, access_flags, fault_preimage) = {
6257            let prepared = attempt.prepared();
6258            (
6259                prepared.materialization.satisfied_pages(),
6260                prepared.plan.operation.is_file_backed(),
6261                prepared.plan.vaddr,
6262                prepared.plan.vma_flags,
6263                prepared.plan.range,
6264                prepared.plan.access_flags,
6265                prepared.plan.preimage,
6266            )
6267        };
6268        if pages == 0 {
6269            let result = if file_backed {
6270                FaultResult::Sigbus(BusCode::AdrErr)
6271            } else {
6272                warn!("no pages prepared for {vaddr:?} ({vma_flags:?})");
6273                FaultResult::Unmapped
6274            };
6275            return PageFaultApplyOutcome::Cancel(result);
6276        }
6277        if attempt.prepared().materialization.owner().is_none() {
6278            return PageFaultApplyOutcome::Cancel(FaultResult::Handled);
6279        }
6280        let (owner_va, owner_paddr, owner_page_size, owner_transition, desired_flags) = {
6281            let prepared = attempt.prepared();
6282            let owner = prepared
6283                .materialization
6284                .owner()
6285                .expect("checked fault owner must remain present");
6286            let Some(desired_flags) = prepared.materialization.pte_flags() else {
6287                return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6288            };
6289            (
6290                owner.va,
6291                owner.paddr,
6292                owner.page_size,
6293                owner.transition,
6294                desired_flags,
6295            )
6296        };
6297        let mapping_preimage = match self.capture_mapping_preimage(range) {
6298            Ok(preimage) => preimage,
6299            Err(error) => {
6300                warn!("could not retain page-fault preimage for {vaddr:?}: {error}");
6301                return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6302            }
6303        };
6304        let replaces_owner = attempt
6305            .prepared()
6306            .materialization
6307            .owner()
6308            .is_some_and(|owner| owner.transition == PteOwnerTransition::Replaced);
6309        let retired_owners = if replaces_owner {
6310            match self.prepare_retired_mapping_owners(range) {
6311                Ok(owners) => Some(owners),
6312                Err(error) => {
6313                    warn!("could not reserve page-fault retire owners for {vaddr:?}: {error}");
6314                    return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6315                }
6316            }
6317        } else {
6318            None
6319        };
6320        let lazy_free_page = access_flags
6321            .contains(MappingFlags::WRITE)
6322            .then(|| {
6323                self.mapping_slots
6324                    .get(&MappingSlotKey {
6325                        space_id: self.id,
6326                        va: owner_va,
6327                    })
6328                    .filter(|slot| slot.page.state() == PageState::LazyFree)
6329                    .map(|slot| slot.page.clone())
6330            })
6331            .flatten();
6332        let fresh_install = matches!(fault_preimage, FaultPteSnapshot::NotMapped)
6333            && attempt
6334                .prepared()
6335                .materialization
6336                .owner()
6337                .is_some_and(|owner| owner.transition == PteOwnerTransition::Installed);
6338        let mut mutation = if fresh_install {
6339            self.prepare_fresh_pte_mutation_range(range.start, range.size())
6340        } else {
6341            self.prepare_mutation_range(range.start, range.size())
6342        };
6343        // A software-empty PTE can still have an older cached translation
6344        // after a failed discard shootdown. Check before touching the PTE or
6345        // publishing its new owner; commit repeats the non-cancellable check.
6346        // The outer MM mutex excludes new publishers until this apply ends.
6347        if let Some(request) = self.mutation_gate.pending_overlap_request(&mutation) {
6348            return PageFaultApplyOutcome::CancelPendingTlb {
6349                request,
6350                targets: self.tlb_targets(),
6351            };
6352        }
6353        let Some(retire_epoch) = mutation.receipt().base_epoch.checked_next() else {
6354            return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6355        };
6356
6357        let mut map_deposit = if owner_transition == PteOwnerTransition::Installed {
6358            match attempt.take_map_deposit() {
6359                Some(deposit) => Some(deposit),
6360                None => return PageFaultApplyOutcome::Cancel(FaultResult::Retry),
6361            }
6362        } else {
6363            None
6364        };
6365        let apply_result = {
6366            let _structure = (owner_transition == PteOwnerTransition::Installed)
6367                .then(|| self.pte_domain.lock_structure());
6368            let _stripe = self.pte_domain.lock_range(range);
6369            let pt = &mut self.pt;
6370            let preimage_matches = fault_preimage.matches(range.start, pt);
6371            let result = if !preimage_matches {
6372                Err(PagingError::stale_map_deposit(owner_va))
6373            } else {
6374                // Linux set_ptes synchronizes executable folios before making
6375                // the PTE visible, including read faults on executable VMAs.
6376                let owner = attempt
6377                    .prepared()
6378                    .materialization
6379                    .owner()
6380                    .expect("prepared fault retains its page until PTE publication");
6381                owner
6382                    .page
6383                    .prepare_executable_mapping(owner_paddr, owner_page_size, desired_flags);
6384                match owner_transition {
6385                    PteOwnerTransition::Installed => {
6386                        let deposit = map_deposit
6387                            .take()
6388                            .expect("fresh page fault must retain its map deposit");
6389                        match pt.try_map_page_with(deposit) {
6390                            Ok(()) => Ok(owner_page_size),
6391                            Err(failure) => {
6392                                let (error, deposit) = failure.into_parts();
6393                                map_deposit = Some(deposit);
6394                                Err(error)
6395                            }
6396                        }
6397                    }
6398                    PteOwnerTransition::Replaced | PteOwnerTransition::Updated => {
6399                        pt.remap_page(owner_va, owner_paddr, desired_flags)
6400                    }
6401                }
6402            };
6403            result.and_then(|installed_size| {
6404                (installed_size == owner_page_size)
6405                    .then_some(installed_size)
6406                    .ok_or(PagingError::NotMapped)
6407            })
6408        };
6409        if let Some(deposit) = map_deposit.take() {
6410            attempt.restore_map_deposit(deposit);
6411        }
6412        if let Err(error) = apply_result {
6413            warn!("could not apply prepared page fault for {vaddr:?}: {error}");
6414            if fault_preimage.matches(range.start, &self.pt) {
6415                return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6416            }
6417            if self
6418                .restore_mapping_preimage(range, mapping_preimage)
6419                .is_ok()
6420            {
6421                return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6422            } else {
6423                self.mutation_gate.mark_needs_repair();
6424            }
6425            return PageFaultApplyOutcome::NeedsRepair(FaultResult::Retry);
6426        }
6427
6428        mutation.set_pte_delta(PteDelta {
6429            mapped: u32::try_from(pages).unwrap_or(u32::MAX),
6430            ..PteDelta::default()
6431        });
6432        let publication = match self.publish_prepared_fault_owner(
6433            &attempt.prepared().plan.operation,
6434            range,
6435            &attempt.prepared().materialization,
6436        ) {
6437            Ok(publication) => publication,
6438            Err(error) => {
6439                warn!("could not publish prepared page owner for {vaddr:?}: {error}");
6440                if self
6441                    .restore_mapping_preimage(range, mapping_preimage)
6442                    .is_err()
6443                {
6444                    self.mutation_gate.mark_needs_repair();
6445                    return PageFaultApplyOutcome::NeedsRepair(FaultResult::Retry);
6446                }
6447                return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6448            }
6449        };
6450        let PreparedPageFault {
6451            plan: _,
6452            materialization: _,
6453            map_deposit,
6454        } = attempt.take_prepared();
6455        debug_assert!(map_deposit.is_none());
6456        mutation.set_mapping_delta(publication.mapping_delta);
6457        mutation.set_resident_delta(publication.resident_delta);
6458        if let Some(page) = &lazy_free_page
6459            && !page.clear_lazy_free()
6460        {
6461            if self
6462                .restore_mapping_preimage(range, mapping_preimage)
6463                .is_err()
6464            {
6465                self.mutation_gate.mark_needs_repair();
6466            }
6467            return PageFaultApplyOutcome::Complete(FaultResult::Retry);
6468        }
6469        if let Some(owners) = retired_owners {
6470            self.park_retired_mapping_owners(retire_epoch, owners);
6471        }
6472        match self.publish_mutation_classified(mutation) {
6473            Ok(MutationPublication::Complete) => {
6474                self.release_retired_mapping_owners(retire_epoch);
6475                PageFaultApplyOutcome::Complete(FaultResult::Handled)
6476            }
6477            Ok(MutationPublication::PendingTlb) => {
6478                let Some(request) = self.mutation_gate.pending_request(self.id, retire_epoch)
6479                else {
6480                    self.mutation_gate.mark_needs_repair();
6481                    return PageFaultApplyOutcome::Complete(FaultResult::Retry);
6482                };
6483                PageFaultApplyOutcome::PendingTlb {
6484                    request,
6485                    targets: self.tlb_targets(),
6486                }
6487            }
6488            Err(CommitMutationError::Unpublished(error)) => {
6489                warn!("page-fault publication for {vaddr:?} failed before publish: {error}");
6490                if self
6491                    .restore_mapping_preimage(range, mapping_preimage)
6492                    .is_err()
6493                {
6494                    self.mutation_gate.mark_needs_repair();
6495                } else {
6496                    self.release_retired_mapping_owners(retire_epoch);
6497                    if let Some(page) = &lazy_free_page
6498                        && !page.mark_lazy_free()
6499                    {
6500                        self.mutation_gate.mark_needs_repair();
6501                    }
6502                }
6503                PageFaultApplyOutcome::Complete(FaultResult::Retry)
6504            }
6505            Err(CommitMutationError::PublishedPendingTlb(error)) => {
6506                warn!("unexpected synchronous TLB result for page fault {vaddr:?}: {error}");
6507                let Some(request) = self.mutation_gate.pending_request(self.id, retire_epoch)
6508                else {
6509                    self.mutation_gate.mark_needs_repair();
6510                    return PageFaultApplyOutcome::Complete(FaultResult::Retry);
6511                };
6512                PageFaultApplyOutcome::PendingTlb {
6513                    request,
6514                    targets: self.tlb_targets(),
6515                }
6516            }
6517        }
6518    }
6519
6520    /// Test-only synchronous wrapper. Production faults are orchestrated by
6521    /// `MmPin`, which drops the address-space mutex around prepare and TLB IPI.
6522    #[cfg(all(test, axtest))]
6523    fn handle_page_fault_result(
6524        &mut self,
6525        vaddr: VirtAddr,
6526        access_flags: PageFaultFlags,
6527    ) -> FaultResult {
6528        let plan =
6529            match self.plan_page_fault(vaddr, access_flags, TransparentHugePageMode::default()) {
6530                Ok(plan) => plan,
6531                Err(result) => return result,
6532            };
6533        let prepared = match Self::prepare_page_fault(plan) {
6534            Ok(prepared) => prepared,
6535            Err(result) => return result,
6536        };
6537        let mut attempt = prepared.into_apply_attempt();
6538        let result = match self.apply_prepared_page_fault(&mut attempt) {
6539            PageFaultApplyOutcome::Complete(result) => result,
6540            PageFaultApplyOutcome::Cancel(result) => {
6541                if attempt.cancel().is_ok() {
6542                    result
6543                } else {
6544                    FaultResult::Retry
6545                }
6546            }
6547            PageFaultApplyOutcome::NeedsRepair(result) => {
6548                attempt.release_to_repair_state();
6549                result
6550            }
6551            PageFaultApplyOutcome::CancelPendingTlb { request, targets } => {
6552                if attempt.cancel().is_ok()
6553                    && Self::flush_tlb_requests(core::slice::from_ref(&request), &targets).is_ok()
6554                {
6555                    let _ = self.acknowledge_tlb_requests(core::slice::from_ref(&request));
6556                }
6557                FaultResult::Retry
6558            }
6559            PageFaultApplyOutcome::PendingTlb { request, targets } => {
6560                if Self::flush_tlb_requests(core::slice::from_ref(&request), &targets).is_ok()
6561                    && self
6562                        .acknowledge_tlb_requests(core::slice::from_ref(&request))
6563                        .is_ok()
6564                {
6565                    FaultResult::Handled
6566                } else {
6567                    FaultResult::Retry
6568                }
6569            }
6570        };
6571        complete_page_fault_with(
6572            matches!(result, FaultResult::Handled),
6573            vaddr,
6574            ax_cpu::mmu::update_mmu_cache,
6575        );
6576        result
6577    }
6578
6579    /// Captures every resident parent PTE that fork must make read-only.
6580    ///
6581    /// This is a pure prepare phase: all vectors and TLB ranges are reserved
6582    /// before either the parent or child page table is changed. The outer
6583    /// address-space lock keeps the captured leaf identity stable until apply.
6584    fn prepare_fork_parent_mutation(&self) -> StarryResult<Option<PreparedForkParentMutation>> {
6585        let mut mutation = self.prepare_mutation();
6586        let mut ptes = Vec::new();
6587        let mut ranges = Vec::new();
6588
6589        for entry in self.vma_root.iter_entries() {
6590            if entry.snapshot().advice_policy.dont_fork() {
6591                continue;
6592            }
6593            if !entry.operation().requires_fork_write_protect() {
6594                continue;
6595            }
6596            let mut range_changed = false;
6597            for leaf in self.occupied_pte_leaves_overlapping(&[entry.range()])? {
6598                let page_size = leaf.range.size();
6599                if page_size < PAGE_SIZE_4K || !page_size.is_power_of_two() {
6600                    return Err(StarryError::BadState);
6601                }
6602                let protected_flags = leaf.flags - MappingFlags::WRITE;
6603                if protected_flags != leaf.flags {
6604                    ptes.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
6605                    ptes.push(ForkParentPteProtection {
6606                        va: leaf.range.start,
6607                        paddr: leaf.paddr,
6608                        page_size,
6609                        original_flags: leaf.flags,
6610                        protected_flags,
6611                    });
6612                    range_changed = true;
6613                }
6614            }
6615            if range_changed {
6616                ranges.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
6617                ranges.push(entry.range());
6618                mutation
6619                    .try_add_tlb_range(
6620                        TlbRange::new(entry.start(), entry.size())
6621                            .ok_or(StarryError::InvalidInput)?,
6622                    )
6623                    .map_err(|error| match error {
6624                        MutationError::ResourceExhausted => StarryError::NoMemory,
6625                        _ => StarryError::BadState,
6626                    })?;
6627            }
6628        }
6629
6630        if ptes.is_empty() {
6631            return Ok(None);
6632        }
6633        mutation.set_pte_delta(PteDelta {
6634            protected: u32::try_from(ptes.len()).unwrap_or(u32::MAX),
6635            ..PteDelta::default()
6636        });
6637        Ok(Some(PreparedForkParentMutation {
6638            mutation,
6639            ptes,
6640            ranges,
6641        }))
6642    }
6643
6644    fn rollback_fork_parent_ptes(
6645        cursor: &mut PageTable,
6646        applied: &[ForkParentPteProtection],
6647    ) -> bool {
6648        let mut complete = true;
6649        for protection in applied.iter().rev() {
6650            let current_matches =
6651                cursor
6652                    .query(protection.va)
6653                    .is_ok_and(|(paddr, flags, page_size)| {
6654                        paddr == protection.paddr
6655                            && flags == protection.protected_flags
6656                            && page_size == protection.page_size
6657                    });
6658            if !current_matches
6659                || cursor
6660                    .protect_page(protection.va, protection.original_flags)
6661                    .is_err()
6662            {
6663                complete = false;
6664            }
6665        }
6666        complete
6667    }
6668
6669    /// Applies and publishes the parent half of fork after the child is fully
6670    /// prepared but still unreachable by the scheduler.
6671    fn apply_fork_parent_mutation(&mut self, prepared: PreparedForkParentMutation) -> StarryResult {
6672        let PreparedForkParentMutation {
6673            mutation,
6674            ptes,
6675            ranges,
6676        } = prepared;
6677        let pt = &mut self.pt;
6678        let pte_stripes = self.pte_domain.lock_ranges(&ranges);
6679        // The outer `&mut self` excludes other address-space mutations,
6680        // and `pte_stripes` covers every captured parent leaf in ascending
6681        // stripe order.
6682        let cursor = pt;
6683        for (applied, protection) in ptes.iter().enumerate() {
6684            let preimage_matches =
6685                cursor
6686                    .query(protection.va)
6687                    .is_ok_and(|(paddr, flags, page_size)| {
6688                        paddr == protection.paddr
6689                            && flags == protection.original_flags
6690                            && page_size == protection.page_size
6691                    });
6692            if !preimage_matches
6693                || cursor
6694                    .protect_page(protection.va, protection.protected_flags)
6695                    .is_err()
6696            {
6697                if !Self::rollback_fork_parent_ptes(cursor, &ptes[..applied]) {
6698                    self.mutation_gate.mark_needs_repair();
6699                    return Err(StarryError::BadState);
6700                }
6701                return Err(StarryError::BadState);
6702            }
6703        }
6704        drop(pte_stripes);
6705
6706        // Publication freezes the live active-CPU mask and does not return
6707        // success until every CPU that could have cached a writable parent
6708        // translation has acknowledged the shootdown.  A failure before
6709        // publication consumes the retained PTE preimage; a post-publication
6710        // shootdown failure must keep the read-only state visible.
6711        match self.commit_mutation_classified(mutation) {
6712            Ok(()) => Ok(()),
6713            Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
6714            Err(CommitMutationError::Unpublished(error)) => {
6715                let pt = &mut self.pt;
6716                let _pte_stripes = self.pte_domain.lock_ranges(&ranges);
6717                // Disjoint field borrows exclude competing address-space
6718                // mutations and the ordered stripe cursor covers every leaf
6719                // whose preimage is restored below.
6720                let restored = Self::rollback_fork_parent_ptes(pt, &ptes);
6721                if restored {
6722                    self.mutation_gate.clear_repair();
6723                    Err(error)
6724                } else {
6725                    self.mutation_gate.mark_needs_repair();
6726                    Err(StarryError::BadState)
6727                }
6728            }
6729        }
6730    }
6731
6732    fn abort_unpublished_clone(child: &mut Self) -> StarryResult {
6733        match child.reset_uninstalled_for_loader() {
6734            Ok(()) => Ok(()),
6735            Err(error)
6736                if child.vma_root.is_empty()
6737                    && child.mapping_slots.is_empty()
6738                    && child.pending_retired_mapping_batches() == 0 =>
6739            {
6740                // A prior child-only bookkeeping failure can leave its gate in
6741                // NeedsRepair, causing the final epoch bump to fail after clear
6742                // already removed every owned mapping. The object is still
6743                // safe to drop because it was never installed or published.
6744                warn!(
6745                    "unpublished fork child cleared all mappings but could not publish cleanup \
6746                     epoch: {error}"
6747                );
6748                Ok(())
6749            }
6750            Err(error) => Err(error),
6751        }
6752    }
6753
6754    /// Attempts to clone the current address space into a new one.
6755    ///
6756    /// This method creates a new empty address space with the same base and
6757    /// size, then iterates over all memory areas in the original address
6758    /// space to copy or share their mappings into the new one.
6759    ///
6760    /// Memfd shared-writable deltas are prepared while the unpublished child
6761    /// is built and applied only after the child's receipt is published.
6762    /// (`CLONE_VM` shares one address space and does not duplicate VMAs here.)
6763    pub fn try_clone(&mut self) -> StarryResult<Arc<Mutex<Self>>> {
6764        // Capture every fallible parent-side allocation and PTE preimage before
6765        // constructing the child. No published parent state changes in this
6766        // phase, so a child preparation failure is a true abort.
6767        let parent_mutation = self.prepare_fork_parent_mutation()?;
6768        let new_aspace = Arc::new(Mutex::new(Self::new_with_layout(self.layout)?));
6769
6770        // The caller holds the source AddrSpace lock while this fresh AddrSpace
6771        // is being populated. The new lock is not published yet, so this is a
6772        // structured source -> cloned-address-space nesting.
6773        let mut guard = new_aspace.lock_nested(CLONED_ADDR_SPACE_LOCK_SUBCLASS);
6774        guard.heap = self.heap;
6775        guard.executable_data = self.executable_data;
6776        guard.executable_file = self.executable_file.clone();
6777        let mut child_memfd_deltas = Vec::new();
6778        let mut child_vss_pages = 0u64;
6779
6780        let child_preparation = (|| -> StarryResult {
6781            let self_modify = &mut self.pt;
6782            for entry in self.vma_root.iter_entries() {
6783                if entry.snapshot().advice_policy.dont_fork() {
6784                    continue;
6785                }
6786                let (new_backend, materialization) = entry.operation().clone_map(
6787                    entry.range(),
6788                    entry.rights(),
6789                    self_modify,
6790                    &mut guard.pt,
6791                )?;
6792                let start = entry.start();
6793                child_memfd_deltas.extend(crate::syscall::memfd_prepare_aspace_replace_deltas(
6794                    &guard,
6795                    start,
6796                    entry.size(),
6797                    entry.rights(),
6798                    &new_backend,
6799                ));
6800
6801                let child_entry = guard
6802                    .vma_root
6803                    .prepare_mapping_entry(
6804                        entry.range(),
6805                        entry.rights(),
6806                        entry.reported_rights(),
6807                        entry.max_rights(),
6808                        entry.snapshot().huge_page_advice,
6809                        VmaLockMode::Unlocked,
6810                        entry.snapshot().advice_policy,
6811                        new_backend.clone(),
6812                    )
6813                    .ok_or(StarryError::BadState)?;
6814                let child_root = guard
6815                    .vma_root
6816                    .with_mapping_entry(child_entry, false)
6817                    .ok_or(StarryError::BadState)?;
6818                guard.vma_root = Arc::new(child_root);
6819                guard.publish_prepared_pte_owners(&new_backend, entry.range(), &materialization)?;
6820                child_vss_pages = child_vss_pages
6821                    .checked_add((entry.size() / PAGE_SIZE_4K) as u64)
6822                    .ok_or(StarryError::BadState)?;
6823            }
6824
6825            // VM_DONTCOPY areas are absent from the child, so derive both
6826            // total_vm and hiwater_vm from the root that was actually built.
6827            guard.vm_stat.seed_clone(child_vss_pages);
6828            Ok(())
6829        })();
6830
6831        if let Err(error) = child_preparation {
6832            if let Err(cleanup_error) = Self::abort_unpublished_clone(&mut guard) {
6833                warn!(
6834                    "fork child preparation failed ({error}); unpublished cleanup also failed \
6835                     ({cleanup_error})"
6836                );
6837                return Err(StarryError::BadState);
6838            }
6839            return Err(error);
6840        }
6841
6842        // Only now may the published parent lose write permission. A failed
6843        // shootdown leaves the parent mutation receipt pending and the child
6844        // unreachable; it is never returned with a stale writable parent TLB.
6845        if let Some(parent_mutation) = parent_mutation
6846            && let Err(error) = self.apply_fork_parent_mutation(parent_mutation)
6847        {
6848            if let Err(cleanup_error) = Self::abort_unpublished_clone(&mut guard) {
6849                warn!(
6850                    "fork parent publication failed ({error}); unpublished child cleanup also \
6851                     failed ({cleanup_error})"
6852                );
6853                return Err(StarryError::BadState);
6854            }
6855            return Err(error);
6856        }
6857
6858        // The child has no CPU activations, but its complete VMA/PTE/RSS/rmap
6859        // view still receives one auditable mutation receipt before the Arc is
6860        // handed to the caller.
6861        if !guard.vma_root.is_empty() {
6862            let mut child_mutation = guard.prepare_mutation();
6863            child_mutation.set_vma_delta(VmaDelta {
6864                inserted: u32::try_from(guard.vma_root.len()).unwrap_or(u32::MAX),
6865                ..VmaDelta::default()
6866            });
6867            child_mutation.set_pte_delta(PteDelta {
6868                mapped: u32::try_from(guard.mapping_slots.len()).unwrap_or(u32::MAX),
6869                ..PteDelta::default()
6870            });
6871            child_mutation.set_mapping_delta(MappingDelta {
6872                attached: u32::try_from(guard.mapping_slots.len()).unwrap_or(u32::MAX),
6873                ..MappingDelta::default()
6874            });
6875            child_mutation.set_resident_delta(
6876                guard
6877                    .resident_counts_from_all_slots()?
6878                    .checked_positive_delta()?,
6879            );
6880            if let Err(error) = guard.commit_mutation(child_mutation) {
6881                if let Err(cleanup_error) = Self::abort_unpublished_clone(&mut guard) {
6882                    warn!(
6883                        "fork child publication failed ({error}); unpublished cleanup also failed \
6884                         ({cleanup_error})"
6885                    );
6886                    return Err(StarryError::BadState);
6887                }
6888                return Err(error);
6889            }
6890        }
6891        crate::syscall::memfd_apply_shared_writable_deltas(&child_memfd_deltas);
6892        drop(guard);
6893
6894        Ok(new_aspace)
6895    }
6896}
6897
6898#[cfg(all(test, not(axtest)))]
6899fn page_fault_completion_updates_only_success_for_test() -> bool {
6900    use core::cell::Cell;
6901
6902    let calls = Cell::new(0);
6903    let observed = Cell::new(VirtAddr::from(0));
6904    let success = complete_page_fault_with(true, VirtAddr::from(0x4567), |vaddr| {
6905        calls.set(calls.get() + 1);
6906        observed.set(vaddr);
6907    });
6908    let rejected = complete_page_fault_with(false, VirtAddr::from(0x89ab), |_| {
6909        calls.set(calls.get() + 1);
6910    });
6911
6912    success && !rejected && calls.get() == 1 && observed.get() == VirtAddr::from(0x4567)
6913}
6914
6915impl fmt::Debug for AddrSpace {
6916    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6917        f.debug_struct("AddrSpace")
6918            .field("id", &self.id)
6919            .field("layout", &self.layout)
6920            .field("page_table_root", &self.pt.root_paddr())
6921            .field("vma_root", &self.vma_root)
6922            .field("vm_epoch", &self.vm_epoch())
6923            .finish()
6924    }
6925}
6926
6927impl Drop for AddrSpace {
6928    fn drop(&mut self) {
6929        // Destruction is not a recovery context: it cannot report a partial
6930        // page-table/backend failure and may run while an allocator lock is
6931        // held.  Normal owners must pass through `RetirePermit::reclaim`,
6932        // which performs the fallible clear/detach protocol.  A direct drop
6933        // therefore only records a diagnostic; `PageTable`'s own Drop handles
6934        // its frame bookkeeping, while any remaining mapping ownership stays
6935        // visible to the repair/leak detector instead of being half-freed.
6936        let has_retired_batches = self.pending_retired_mapping_batches() != 0;
6937        if !self.vma_root.is_empty() || !self.mapping_slots.is_empty() || has_retired_batches {
6938            warn!(
6939                "address space {} dropped before retire/reclaim; mappings intentionally retained",
6940                self.id.get()
6941            );
6942            // Do not let `PageTable`'s destructor recursively free
6943            // intermediate frames while a stale PTE/VMA or resident slot is
6944            // still observable.  The lifecycle repair path owns any later
6945            // reclamation decision; this destructor has no fallible return
6946            // channel and therefore leaks conservatively.
6947            self.pt.leak();
6948            if has_retired_batches {
6949                let batches = core::mem::take(&mut *self.retired_mapping_batches.lock());
6950                for batch in batches {
6951                    core::mem::forget(batch);
6952                }
6953            }
6954        }
6955    }
6956}
6957
6958#[cfg(test)]
6959mod tests {
6960    use alloc::sync::Arc;
6961    use core::sync::atomic::AtomicUsize;
6962
6963    use ax_memory_addr::{PAGE_SIZE_4K, VirtAddr};
6964
6965    use super::{
6966        AddressSpaceId, MutationError, MutationGate, TlbRange, VmEpoch,
6967        prepare_mapping_publication_mutation,
6968    };
6969
6970    #[cfg(all(test, not(axtest)))]
6971    #[test]
6972    fn page_fault_completion_updates_only_success() {
6973        assert!(super::page_fault_completion_updates_only_success_for_test());
6974    }
6975
6976    #[cfg_attr(axtest, axtest::axtest)]
6977    #[cfg_attr(not(axtest), test)]
6978    fn fresh_mapping_publication_has_no_tlb_targets() {
6979        let gate = MutationGate::new();
6980        let id = AddressSpaceId::allocate();
6981        let targets = Arc::new(AtomicUsize::new(0b1110));
6982        let start = VirtAddr::from(0x20_0000);
6983
6984        let fresh =
6985            prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, false);
6986        assert_eq!(fresh.receipt().tlb_obligation.targets(), 0);
6987
6988        let replacement =
6989            prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, true);
6990        assert_eq!(replacement.receipt().tlb_obligation.targets(), 0b1110);
6991    }
6992
6993    #[cfg_attr(axtest, axtest::axtest)]
6994    #[cfg_attr(not(axtest), test)]
6995    fn fresh_mapping_cannot_reuse_range_with_pending_shootdown() {
6996        let gate = MutationGate::new();
6997        let id = AddressSpaceId::allocate();
6998        let targets = Arc::new(AtomicUsize::new(0b1));
6999        let start = VirtAddr::from(0x20_0000);
7000        let mut unmap = gate.begin(id, 0b1);
7001        unmap.add_tlb_range(TlbRange::new(start, PAGE_SIZE_4K).unwrap());
7002        assert_eq!(gate.commit(unmap).unwrap_err(), MutationError::TlbPending);
7003
7004        let nonoverlapping = prepare_mapping_publication_mutation(
7005            &gate,
7006            id,
7007            &targets,
7008            start + PAGE_SIZE_4K * 2,
7009            PAGE_SIZE_4K,
7010            false,
7011        );
7012        gate.validate_publish_preconditions(&nonoverlapping)
7013            .unwrap();
7014        gate.commit(nonoverlapping).unwrap();
7015
7016        let fresh =
7017            prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, false);
7018        assert_eq!(
7019            gate.validate_publish_preconditions(&fresh),
7020            Err(MutationError::PendingTlbOverlap)
7021        );
7022        assert_eq!(
7023            gate.commit(fresh).unwrap_err(),
7024            MutationError::PendingTlbOverlap
7025        );
7026        assert_eq!(gate.current_epoch(), VmEpoch::new(2));
7027
7028        gate.acknowledge(id, VmEpoch::new(1), 0).unwrap().unwrap();
7029        let retry =
7030            prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, false);
7031        gate.validate_publish_preconditions(&retry).unwrap();
7032        gate.commit(retry).unwrap();
7033    }
7034
7035    #[cfg_attr(axtest, axtest::axtest)]
7036    #[cfg_attr(not(axtest), test)]
7037    fn pending_full_flush_blocks_every_fresh_mapping_range() {
7038        let gate = MutationGate::new();
7039        let id = AddressSpaceId::allocate();
7040        let targets = Arc::new(AtomicUsize::new(0b1));
7041        assert_eq!(
7042            gate.commit(gate.begin(id, 0b1)).unwrap_err(),
7043            MutationError::TlbPending
7044        );
7045
7046        let fresh = prepare_mapping_publication_mutation(
7047            &gate,
7048            id,
7049            &targets,
7050            VirtAddr::from(0x40_0000),
7051            PAGE_SIZE_4K,
7052            false,
7053        );
7054        assert_eq!(
7055            gate.commit(fresh).unwrap_err(),
7056            MutationError::PendingTlbOverlap
7057        );
7058        assert_eq!(gate.current_epoch(), VmEpoch::new(1));
7059    }
7060
7061    #[cfg(axtest)]
7062    fn refault_waits_for_discard_shootdown(full_flush: bool) {
7063        use ax_runtime::hal::trap::PageFaultFlags;
7064
7065        use super::{AddrSpace, FaultResult, MappingFlags, MappingOperation, PagingError};
7066
7067        let start = VirtAddr::from(0x7200_0000);
7068        let mut aspace = AddrSpace::new_empty(start, PAGE_SIZE_4K).unwrap();
7069        let flags = MappingFlags::READ | MappingFlags::WRITE | MappingFlags::USER;
7070        aspace
7071            .map(
7072                start,
7073                PAGE_SIZE_4K,
7074                flags,
7075                true,
7076                MappingOperation::new_alloc(start, PAGE_SIZE_4K, "[discard-refault]"),
7077            )
7078            .unwrap();
7079        aspace.discard_range(start, PAGE_SIZE_4K).unwrap();
7080        // Deterministically retain an unacknowledged discard obligation. The
7081        // page table is real, but no hardware CPU uses this test-only MM.
7082        let mut discard = aspace.mutation_gate.begin(aspace.id, 1);
7083        if !full_flush {
7084            discard.add_tlb_range(TlbRange::new(start, PAGE_SIZE_4K).unwrap());
7085        }
7086        assert_eq!(
7087            aspace.mutation_gate.commit(discard).unwrap_err(),
7088            MutationError::TlbPending
7089        );
7090        let epoch = aspace.vm_epoch();
7091        let plan = aspace
7092            .plan_page_fault(
7093                start,
7094                PageFaultFlags::READ | PageFaultFlags::USER,
7095                Default::default(),
7096            )
7097            .ok()
7098            .unwrap();
7099        let prepared = AddrSpace::prepare_page_fault(plan).ok().unwrap();
7100        let mut attempt = prepared.into_apply_attempt();
7101        let outcome = aspace.apply_prepared_page_fault(&mut attempt);
7102        let unpublished = matches!(aspace.pt.query(start), Err(PagingError::NotMapped))
7103            && aspace.vm_epoch() == epoch
7104            && aspace.mapping_slots.is_empty()
7105            && attempt.prepared.is_some()
7106            && !aspace.mutation_gate.needs_repair();
7107        drop(outcome);
7108        if attempt.prepared.is_some() {
7109            attempt.cancel().unwrap();
7110        }
7111        aspace
7112            .mutation_gate
7113            .acknowledge(aspace.id, epoch, 0)
7114            .unwrap()
7115            .unwrap();
7116        let retry =
7117            aspace.handle_page_fault_result(start, PageFaultFlags::READ | PageFaultFlags::USER);
7118        let recovered = matches!(retry, FaultResult::Handled) && aspace.pt.query(start).is_ok();
7119        aspace.reset_uninstalled_for_loader().unwrap();
7120        assert!(
7121            unpublished,
7122            "refault must leave the PTE, epoch and owner graph untouched until discard is \
7123             acknowledged"
7124        );
7125        assert!(
7126            recovered,
7127            "acknowledged discard must allow refault to make progress"
7128        );
7129    }
7130
7131    #[cfg(axtest)]
7132    #[axtest::axtest]
7133    fn refault_waits_for_pending_discard_range() {
7134        refault_waits_for_discard_shootdown(false);
7135    }
7136
7137    #[cfg(axtest)]
7138    #[axtest::axtest]
7139    fn refault_waits_for_pending_discard_full_flush() {
7140        refault_waits_for_discard_shootdown(true);
7141    }
7142
7143    #[cfg(axtest)]
7144    #[axtest::axtest]
7145    fn page_table_reclaim_waits_for_last_tlb_acknowledgement() {
7146        use ax_memory_addr::PhysAddr;
7147
7148        use super::{AddrSpace, MappingFlags, RetiredMappingOwners};
7149
7150        let start = VirtAddr::from(0x7300_0000);
7151        let mut aspace = AddrSpace::new_empty(start, PAGE_SIZE_4K).unwrap();
7152        aspace
7153            .pt
7154            .map_page(
7155                start,
7156                PhysAddr::from(0x1000_0000),
7157                PAGE_SIZE_4K,
7158                MappingFlags::READ | MappingFlags::USER,
7159            )
7160            .unwrap();
7161        let (_, _, _, deferred) = aspace.pt.unmap_page_deferred(start).unwrap();
7162        assert!(!deferred.is_empty());
7163
7164        let epoch = VmEpoch::new(1);
7165        let mut owners = RetiredMappingOwners::default();
7166        owners.page_tables.try_reserve(1).unwrap();
7167        owners.page_tables.push(deferred);
7168        aspace.park_retired_mapping_owners(epoch, owners);
7169
7170        let mut mutation = aspace.mutation_gate.begin(aspace.id, 0b11);
7171        mutation.add_tlb_range(TlbRange::new(start, PAGE_SIZE_4K).unwrap());
7172        assert!(matches!(
7173            aspace.mutation_gate.commit(mutation),
7174            Err(MutationError::TlbPending)
7175        ));
7176        assert_eq!(aspace.pending_retired_page_table_reclaims(), 1);
7177
7178        assert!(aspace.acknowledge_tlb(aspace.id, epoch, 0).unwrap().is_empty());
7179        assert_eq!(aspace.pending_retired_page_table_reclaims(), 1);
7180
7181        assert!(aspace.acknowledge_tlb(aspace.id, epoch, 1).unwrap().is_empty());
7182        assert_eq!(aspace.pending_retired_page_table_reclaims(), 0);
7183    }
7184}