1use alloc::{
2 collections::{BTreeMap, BTreeSet},
3 sync::Arc,
4 vec::Vec,
5};
6use core::{fmt, sync::atomic::AtomicUsize};
7
8use ax_fs_ng::file::CachedPagePin;
9use ax_memory_addr::{
10 MemoryAddr, PAGE_SIZE_4K, PageIter4K, PhysAddr, VirtAddr, VirtAddrRange, is_aligned_4k,
11};
12#[cfg(not(any(target_arch = "aarch64", target_arch = "loongarch64")))]
13use ax_mm::RootEntryShare;
14use ax_runtime::hal::{
15 mem::phys_to_virt,
16 paging::{
17 DeferredPageTableFrames, InstalledHugeSplit, MappingFlags, PageTable, PageTableEntry,
18 PageTableMapDeposit, PageTableMapPlan, PageTableMovePlan, PagingAllocator, PagingError,
19 },
20 trap::PageFaultFlags,
21};
22
23use crate::{
24 StarryError, StarryResult,
25 config::USER_HEAP_BASE,
26 mm::{ProcessVmStat, ProcessVmStatSnapshot, UserVirtualAddressLayout},
27 sync::{IrqMutex, LockdepMutexExt, Mutex, try_reserve_irq_vec},
28};
29
30#[cfg(test)]
31fn complete_page_fault_with(
32 handled: bool,
33 vaddr: VirtAddr,
34 update_mmu_cache: impl FnOnce(VirtAddr),
35) -> bool {
36 if handled {
37 update_mmu_cache(vaddr);
38 }
39 handled
40}
41
42mod accounting;
43mod backend;
44pub(crate) mod domain;
45pub mod lifecycle;
46pub(crate) mod mutation;
47pub(crate) mod objects;
48pub(crate) mod reclaim;
49pub(crate) mod vma;
50
51pub(crate) use self::domain::PageTableDomain;
52pub use self::mutation::{
56 AppliedMutation, EvictionResult, MappingDelta, MutationError, MutationGate, MutationReceipt,
57 MutationState, PreparedMutation, PteDelta, PublishEvent, PublishedMutation,
58 PublishedPendingTlb, QuarantineError, QuarantineFailure, ResidentDelta, TlbQuarantine,
59 TlbRange, TlbRequest, VmaDelta,
60};
61use self::{
62 accounting::ResidentWatermark,
63 backend::{
64 FaultFallback, FaultMaterialization, FaultPteSnapshot, MappingMutationContext,
65 PopulateRequest, PreparedPteOwner, ProviderPublication, PteMaterialization,
66 PteOwnerTransition,
67 },
68};
69pub use self::{
70 backend::*,
71 lifecycle::*,
72 objects::{
73 EvictionError, EvictionLease, FrameLease, MappingGraphError, MappingSlot, MappingSlotKey,
74 PageId, PageObject, PageState, RmapSet, SlotState, WritebackError, WritebackLease,
75 },
76 reclaim::*,
77 vma::*,
78};
79
80#[cfg(all(test, axtest))]
81static MAPPING_GRAPH_SNAPSHOT_CALLS: AtomicUsize = AtomicUsize::new(0);
82
83#[derive(Clone, Copy, PartialEq, Eq)]
84enum MovedPageDestination {
85 SourceOwner,
87 TargetOwner { slot_va: VirtAddr },
89}
90
91#[derive(Clone, Copy)]
92struct MovedPage {
93 src_va: VirtAddr,
94 dst_va: VirtAddr,
95 paddr: PhysAddr,
96 page_size: usize,
97 destination: MovedPageDestination,
98}
99
100enum PreparedMovedSlot {
101 Relocate {
102 source_key: MappingSlotKey,
103 target_key: MappingSlotKey,
104 source: Arc<MappingSlot>,
105 replacement: Arc<MappingSlot>,
106 },
107 DetachSource {
108 source_key: MappingSlotKey,
109 target_key: MappingSlotKey,
110 source: Arc<MappingSlot>,
111 },
112}
113const CLONED_ADDR_SPACE_LOCK_SUBCLASS: u32 = 1;
114
115#[derive(Clone, Copy)]
116struct ForkParentPteProtection {
117 va: VirtAddr,
118 paddr: PhysAddr,
119 page_size: usize,
120 original_flags: MappingFlags,
121 protected_flags: MappingFlags,
122}
123
124struct PreparedForkParentMutation {
125 mutation: PreparedMutation,
126 ptes: Vec<ForkParentPteProtection>,
127 ranges: Vec<VirtAddrRange>,
128}
129
130#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
131pub(crate) struct ResidentPageCounts {
132 pub anon: u64,
133 pub file: u64,
134 pub shmem: u64,
135}
136
137impl ResidentPageCounts {
138 pub const fn total(self) -> u64 {
139 self.anon
140 .saturating_add(self.file)
141 .saturating_add(self.shmem)
142 }
143
144 fn checked_delta_to(self, after: Self) -> StarryResult<ResidentDelta> {
145 fn delta(before: u64, after: u64) -> StarryResult<i64> {
146 let value = i128::from(after) - i128::from(before);
147 i64::try_from(value).map_err(|_| StarryError::BadState)
148 }
149
150 Ok(ResidentDelta {
151 anon: delta(self.anon, after.anon)?,
152 file: delta(self.file, after.file)?,
153 shmem: delta(self.shmem, after.shmem)?,
154 })
155 }
156
157 fn checked_apply(&mut self, delta: ResidentDelta) -> StarryResult {
158 fn apply(current: u64, delta: i64) -> StarryResult<u64> {
159 if delta >= 0 {
160 current
161 .checked_add(u64::try_from(delta).map_err(|_| StarryError::BadState)?)
162 .ok_or(StarryError::BadState)
163 } else {
164 current
165 .checked_sub(delta.unsigned_abs())
166 .ok_or(StarryError::BadState)
167 }
168 }
169
170 self.anon = apply(self.anon, delta.anon)?;
171 self.file = apply(self.file, delta.file)?;
172 self.shmem = apply(self.shmem, delta.shmem)?;
173 Ok(())
174 }
175
176 fn checked_add_pages(&mut self, kind: Option<RssKind>, pages: u64) -> StarryResult {
177 let bucket = match kind {
178 Some(RssKind::Anon) => &mut self.anon,
179 Some(RssKind::File) => &mut self.file,
180 Some(RssKind::Shmem) => &mut self.shmem,
181 None => return Ok(()),
182 };
183 *bucket = bucket.checked_add(pages).ok_or(StarryError::BadState)?;
184 Ok(())
185 }
186
187 fn checked_negated_delta(self) -> StarryResult<ResidentDelta> {
188 Ok(ResidentDelta {
189 anon: -i64::try_from(self.anon).map_err(|_| StarryError::BadState)?,
190 file: -i64::try_from(self.file).map_err(|_| StarryError::BadState)?,
191 shmem: -i64::try_from(self.shmem).map_err(|_| StarryError::BadState)?,
192 })
193 }
194
195 fn checked_positive_delta(self) -> StarryResult<ResidentDelta> {
196 Ok(ResidentDelta {
197 anon: i64::try_from(self.anon).map_err(|_| StarryError::BadState)?,
198 file: i64::try_from(self.file).map_err(|_| StarryError::BadState)?,
199 shmem: i64::try_from(self.shmem).map_err(|_| StarryError::BadState)?,
200 })
201 }
202}
203
204impl ResidentDelta {
205 fn for_pages(kind: Option<RssKind>, pages: i64) -> Self {
206 match kind {
207 Some(RssKind::Anon) => Self {
208 anon: pages,
209 ..Self::default()
210 },
211 Some(RssKind::File) => Self {
212 file: pages,
213 ..Self::default()
214 },
215 Some(RssKind::Shmem) => Self {
216 shmem: pages,
217 ..Self::default()
218 },
219 None => Self::default(),
220 }
221 }
222
223 fn checked_add_assign(&mut self, other: Self) -> StarryResult {
224 self.anon = self
225 .anon
226 .checked_add(other.anon)
227 .ok_or(StarryError::BadState)?;
228 self.file = self
229 .file
230 .checked_add(other.file)
231 .ok_or(StarryError::BadState)?;
232 self.shmem = self
233 .shmem
234 .checked_add(other.shmem)
235 .ok_or(StarryError::BadState)?;
236 Ok(())
237 }
238}
239
240#[derive(Debug, Clone, Copy, Default)]
241struct PteOwnerPublication {
242 satisfied_pages: usize,
243 mapping_delta: MappingDelta,
244 resident_delta: ResidentDelta,
245}
246
247struct PreparedSlotPublication {
248 key: MappingSlotKey,
249 previous: Option<Arc<MappingSlot>>,
250 replacement: Option<Arc<MappingSlot>>,
251 resident_kind: Option<RssKind>,
252 provider_publication: ProviderPublication,
253 mapping_delta: MappingDelta,
254 resident_delta: ResidentDelta,
255}
256
257#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
258struct MappingSlotFingerprint {
259 key: MappingSlotKey,
260 mapping: MappingId,
261 page: PageId,
262 page_order: PageOrder,
263}
264
265struct MappingGraphSnapshot {
269 slots: Vec<MappingSlotFingerprint>,
270 resident: ResidentPageCounts,
271}
272
273impl MappingGraphSnapshot {
274 fn delta_to(&self, after: &Self) -> StarryResult<(MappingDelta, ResidentDelta)> {
275 let mut before_index = 0usize;
276 let mut after_index = 0usize;
277 let mut attached = 0usize;
278 let mut detached = 0usize;
279 while before_index < self.slots.len() && after_index < after.slots.len() {
280 match self.slots[before_index].cmp(&after.slots[after_index]) {
281 core::cmp::Ordering::Less => {
282 detached = detached.checked_add(1).ok_or(StarryError::BadState)?;
283 before_index += 1;
284 }
285 core::cmp::Ordering::Equal => {
286 before_index += 1;
287 after_index += 1;
288 }
289 core::cmp::Ordering::Greater => {
290 attached = attached.checked_add(1).ok_or(StarryError::BadState)?;
291 after_index += 1;
292 }
293 }
294 }
295 detached = detached
296 .checked_add(self.slots.len() - before_index)
297 .ok_or(StarryError::BadState)?;
298 attached = attached
299 .checked_add(after.slots.len() - after_index)
300 .ok_or(StarryError::BadState)?;
301 Ok((
302 MappingDelta {
303 attached: u32::try_from(attached).map_err(|_| StarryError::BadState)?,
304 detached: u32::try_from(detached).map_err(|_| StarryError::BadState)?,
305 },
306 self.resident.checked_delta_to(after.resident)?,
307 ))
308 }
309}
310
311#[derive(Debug, Clone, Copy)]
315pub struct MappingPermissions {
316 pub current: MappingFlags,
317 pub reported: MappingFlags,
318 pub maximum: MappingFlags,
319}
320
321#[derive(Debug, Clone, Copy)]
322pub(crate) struct MappingPublication {
323 replace: bool,
324 huge_page_advice: HugePageAdvice,
325 lock_mode: VmaLockMode,
326 advice_policy: VmaAdvicePolicy,
327 memlock_limit: Option<MemlockLimit>,
328}
329
330#[derive(Debug, Clone, Copy)]
336pub(crate) struct MemlockLimit {
337 page_limit: u64,
338 bypass_limit: bool,
339 may_lock: bool,
340 exceeded_error: MemlockLimitError,
341}
342
343#[derive(Debug, Clone, Copy)]
344enum MemlockLimitError {
345 NoMemory,
346 WouldBlock,
347}
348
349impl MemlockLimit {
350 pub(crate) const fn for_mlock(byte_limit: u64, bypass_limit: bool) -> Self {
351 Self {
352 page_limit: byte_limit / PAGE_SIZE_4K as u64,
353 bypass_limit,
354 may_lock: byte_limit != 0 || bypass_limit,
355 exceeded_error: MemlockLimitError::NoMemory,
356 }
357 }
358
359 pub(crate) const fn for_mapping(byte_limit: u64, bypass_limit: bool) -> Self {
360 Self {
361 page_limit: byte_limit / PAGE_SIZE_4K as u64,
362 bypass_limit,
363 may_lock: byte_limit != 0 || bypass_limit,
364 exceeded_error: MemlockLimitError::WouldBlock,
365 }
366 }
367
368 pub(crate) const fn can_lock(self) -> bool {
369 self.may_lock
370 }
371
372 fn validate(self, locked_pages: u64) -> StarryResult {
373 if self.bypass_limit || locked_pages <= self.page_limit {
374 return Ok(());
375 }
376 Err(match self.exceeded_error {
377 MemlockLimitError::NoMemory => StarryError::NoMemory,
378 MemlockLimitError::WouldBlock => StarryError::WouldBlock,
379 })
380 }
381}
382
383impl MappingPublication {
384 const fn new(replace: bool) -> Self {
385 Self {
386 replace,
387 huge_page_advice: HugePageAdvice::Default,
388 lock_mode: VmaLockMode::Unlocked,
389 advice_policy: VmaAdvicePolicy::DEFAULT,
390 memlock_limit: None,
391 }
392 }
393
394 pub(crate) const fn mmap(
395 replace: bool,
396 lock_mode: VmaLockMode,
397 memlock_limit: Option<MemlockLimit>,
398 ) -> Self {
399 Self {
400 replace,
401 huge_page_advice: HugePageAdvice::Default,
402 lock_mode,
403 advice_policy: VmaAdvicePolicy::DEFAULT,
404 memlock_limit,
405 }
406 }
407
408 const fn mremap(
409 replace: bool,
410 huge_page_advice: HugePageAdvice,
411 lock_mode: VmaLockMode,
412 advice_policy: VmaAdvicePolicy,
413 memlock_limit: Option<MemlockLimit>,
414 ) -> Self {
415 Self {
416 replace,
417 huge_page_advice,
418 lock_mode,
419 advice_policy,
420 memlock_limit,
421 }
422 }
423}
424
425pub(crate) enum AddressSpaceMutationOutcome {
432 Complete,
433 PublishedPendingTlb(StarryError),
434}
435
436impl AddressSpaceMutationOutcome {
437 fn into_result(self) -> StarryResult {
438 match self {
439 Self::Complete => Ok(()),
440 Self::PublishedPendingTlb(error) => Err(error),
441 }
442 }
443}
444
445#[derive(Debug, Clone, Copy)]
449struct HeapState {
450 start: usize,
451 current: usize,
452}
453
454impl HeapState {
455 const fn new(start: usize) -> Self {
456 Self {
457 start,
458 current: start,
459 }
460 }
461}
462
463#[derive(Debug, Clone, Copy, Default)]
469struct ExecutableDataLayout {
470 start: usize,
471 end: usize,
472}
473
474impl ExecutableDataLayout {
475 fn try_new(start: usize, end: usize) -> StarryResult<Self> {
476 if start == 0 || end < start {
477 return Err(StarryError::MalformedExecutable);
478 }
479 Ok(Self { start, end })
480 }
481
482 fn size(self) -> Option<usize> {
483 self.end.checked_sub(self.start)
484 }
485}
486
487fn checked_page_align_up(value: usize) -> StarryResult<usize> {
488 value
489 .checked_add(PAGE_SIZE_4K - 1)
490 .map(|rounded| rounded & !(PAGE_SIZE_4K - 1))
491 .ok_or(StarryError::InvalidInput)
492}
493
494struct ResidentLeafPreimage {
495 va: VirtAddr,
496 paddr: PhysAddr,
497 page_size: usize,
498 flags: MappingFlags,
499 backend: MappingOperation,
500 page: Arc<PageObject>,
501 slot: Arc<MappingSlot>,
502}
503
504#[derive(Clone, Copy)]
505struct OccupiedPteLeaf {
506 range: VirtAddrRange,
507 paddr: PhysAddr,
508 flags: MappingFlags,
509}
510
511struct MappingPreimage {
512 vma_root: Arc<VmaMap>,
513 vm_stat: ProcessVmStatSnapshot,
514 leaves: Vec<ResidentLeafPreimage>,
515}
516
517struct AppliedHugeSplit {
518 installed: InstalledHugeSplit,
519 old_slot: Arc<MappingSlot>,
520 child_keys: Vec<MappingSlotKey>,
521 child_slots: Vec<Arc<MappingSlot>>,
522 previous_mapping_slots: BTreeMap<MappingSlotKey, Arc<MappingSlot>>,
523}
524
525#[derive(Clone, Copy)]
526struct ProtectionLeafPreimage {
527 va: VirtAddr,
528 paddr: PhysAddr,
529 page_size: usize,
530 flags: MappingFlags,
531}
532
533#[derive(Default)]
539struct RetiredMappingOwners {
540 backends: Vec<MappingOperation>,
541 pages: Vec<Arc<PageObject>>,
542 cache_pins: Vec<CachedPagePin>,
543 page_tables: Vec<DeferredPageTableFrames>,
544 deferred_evictions: Vec<Arc<PageObject>>,
549}
550
551impl RetiredMappingOwners {
552 fn is_empty(&self) -> bool {
553 self.backends.is_empty()
554 && self.pages.is_empty()
555 && self.cache_pins.is_empty()
556 && self.page_tables.is_empty()
557 && self.deferred_evictions.is_empty()
558 }
559}
560
561struct RetiredMappingBatch {
562 epoch: VmEpoch,
563 owners: RetiredMappingOwners,
564}
565
566#[derive(Debug)]
567enum CommitMutationError {
568 Unpublished(StarryError),
571 PublishedPendingTlb(StarryError),
575}
576
577#[derive(Debug, Clone, Copy, PartialEq, Eq)]
578enum MutationPublication {
579 Complete,
580 PendingTlb,
581}
582
583struct PageFaultPlan {
584 base_epoch: VmEpoch,
585 space_id: AddressSpaceId,
586 vaddr: VirtAddr,
587 range: VirtAddrRange,
588 vma_flags: MappingFlags,
589 access_flags: MappingFlags,
590 operation: MappingOperation,
591 request: PopulateRequest,
592 preimage: FaultPteSnapshot,
593 map_plans: Option<PageFaultMapPlans>,
594}
595
596struct PageFaultMapPlans {
597 preferred: PageTableMapPlan,
598 fallback: Option<PageTableMapPlan>,
599}
600
601fn prepare_mapping_publication_mutation(
602 gate: &MutationGate,
603 space_id: AddressSpaceId,
604 active_targets: &Arc<AtomicUsize>,
605 start: VirtAddr,
606 size: usize,
607 replaces_existing: bool,
608) -> PreparedMutation {
609 let mut mutation = if replaces_existing {
615 gate.begin_with_active_targets(space_id, active_targets.clone())
616 } else {
617 gate.begin_fresh_mapping(space_id)
618 };
619 if let Some(range) = TlbRange::new(start, size) {
620 mutation.add_tlb_range(range);
621 }
622 mutation
623}
624
625struct PreparedPageFault {
626 plan: PageFaultPlan,
627 materialization: FaultMaterialization,
628 map_deposit: Option<PageTableMapDeposit>,
629}
630
631impl PreparedPageFault {
632 fn into_apply_attempt(self) -> PageFaultApplyAttempt {
633 PageFaultApplyAttempt {
634 prepared: Some(self),
635 orphaned_map_deposit: None,
636 }
637 }
638
639 fn cancel(self) -> StarryResult {
640 self.plan
641 .operation
642 .cancel_prepared_fault_publication(self.materialization)
643 }
644}
645
646struct PageFaultApplyAttempt {
653 prepared: Option<PreparedPageFault>,
654 orphaned_map_deposit: Option<PageTableMapDeposit>,
657}
658
659impl PageFaultApplyAttempt {
660 fn prepared(&self) -> &PreparedPageFault {
661 self.prepared
662 .as_ref()
663 .expect("page-fault apply attempt must retain its prepared token")
664 }
665
666 fn take_prepared(&mut self) -> PreparedPageFault {
667 self.prepared
668 .take()
669 .expect("page-fault apply attempt must consume its prepared token once")
670 }
671
672 fn take_map_deposit(&mut self) -> Option<PageTableMapDeposit> {
673 self.prepared.as_mut()?.map_deposit.take()
674 }
675
676 fn restore_map_deposit(&mut self, deposit: PageTableMapDeposit) {
677 if let Some(prepared) = self.prepared.as_mut()
678 && prepared.map_deposit.is_none()
679 {
680 prepared.map_deposit = Some(deposit);
681 return;
682 }
683 debug_assert!(self.orphaned_map_deposit.is_none());
687 self.orphaned_map_deposit = Some(deposit);
688 }
689
690 fn cancel(self) -> StarryResult {
691 let Self {
692 prepared,
693 orphaned_map_deposit,
694 } = self;
695 drop(orphaned_map_deposit);
696 prepared
697 .expect("cancelled page-fault apply attempt must retain its prepared token")
698 .cancel()
699 }
700
701 fn release_to_repair_state(self) {
709 debug_assert!(self.prepared.is_some());
710 drop(self);
711 }
712}
713
714enum PageFaultApplyOutcome {
715 Complete(FaultResult),
716 Cancel(FaultResult),
717 NeedsRepair(FaultResult),
718 CancelPendingTlb {
721 request: TlbRequest,
722 targets: Arc<AtomicUsize>,
723 },
724 PendingTlb {
725 request: TlbRequest,
726 targets: Arc<AtomicUsize>,
727 },
728}
729
730#[derive(Debug, Clone, Copy, PartialEq, Eq)]
734pub(crate) enum EvictMappingOutcome {
735 Complete,
736 PublishedPendingTlb,
737 NeedsRepair,
738}
739
740pub struct AddrSpace {
742 id: AddressSpaceId,
743 layout: UserVirtualAddressLayout,
745 vma_root: Arc<VmaMap>,
749 heap: HeapState,
753 executable_data: ExecutableDataLayout,
754 executable_file: Option<Arc<ax_fs_ng::file::ExecutableFile>>,
756 pt: PageTable,
757 pte_domain: PageTableDomain,
760 mutation_gate: MutationGate,
762 published_epoch: Arc<core::sync::atomic::AtomicU64>,
766 pub vm_stat: ProcessVmStat,
769 resident_pages: ResidentPageCounts,
773 resident_watermark: ResidentWatermark,
774 tlb_quarantine: TlbQuarantine,
777 tlb_targets: Arc<AtomicUsize>,
781 mapping_slots: BTreeMap<MappingSlotKey, Arc<MappingSlot>>,
785 retired_mapping_batches: IrqMutex<Vec<RetiredMappingBatch>>,
790}
791
792impl AddrSpace {
793 pub const fn base(&self) -> VirtAddr {
795 self.layout.range().start
796 }
797
798 pub const fn end(&self) -> VirtAddr {
800 self.layout.task_size()
801 }
802
803 pub fn size(&self) -> usize {
805 self.layout.range().size()
806 }
807
808 pub const fn stack_top(&self) -> VirtAddr {
810 self.layout.stack_top()
811 }
812
813 pub(crate) const fn heap_start(&self) -> usize {
816 self.heap.start
817 }
818
819 pub(crate) const fn heap_break(&self) -> usize {
821 self.heap.current
822 }
823
824 pub(crate) fn set_executable_file(&mut self, file: ax_fs_ng::file::ExecutableFile) {
826 self.executable_file = Some(Arc::new(file));
827 }
828
829 fn take_file_accesses(
832 &mut self,
833 ) -> (
834 Option<Arc<ax_fs_ng::file::ExecutableFile>>,
835 Vec<Arc<ax_fs_ng::file::WriteAccess>>,
836 ) {
837 let mut writers = Vec::new();
838 for entry in self.vma_root.iter_entries() {
839 if let Some(access) = entry.operation().take_write_access() {
840 writers.push(access);
841 }
842 }
843 (self.executable_file.take(), writers)
844 }
845
846 pub(crate) fn set_executable_data_layout(&mut self, start: usize, end: usize) -> StarryResult {
849 let layout = ExecutableDataLayout::try_new(start, end)?;
850 if start < self.base().as_usize() || end > self.end().as_usize() {
851 return Err(StarryError::MalformedExecutable);
852 }
853 self.executable_data = layout;
854 Ok(())
855 }
856
857 pub(crate) const fn executable_data_bounds(&self) -> (usize, usize) {
858 (self.executable_data.start, self.executable_data.end)
859 }
860
861 pub(crate) fn executable_data_size(&self) -> Option<usize> {
862 self.executable_data.size()
863 }
864
865 pub(crate) fn resize_heap_break(
873 &mut self,
874 requested: usize,
875 initial_mapping_end: usize,
876 ) -> StarryResult<AddressSpaceMutationOutcome> {
877 let old_break = self.heap.current;
878 let old_aligned = checked_page_align_up(old_break)?;
879 let new_aligned = checked_page_align_up(requested)?;
880
881 let outcome = if new_aligned > old_aligned {
882 let map_start = initial_mapping_end.max(old_aligned);
883 let map_size = new_aligned.saturating_sub(map_start);
884 if map_size == 0 {
885 AddressSpaceMutationOutcome::Complete
886 } else {
887 let start = VirtAddr::from(map_start);
888 let flags = MappingFlags::READ | MappingFlags::WRITE | MappingFlags::USER;
889 self.map_with_permissions_mode_classified(
890 start,
891 map_size,
892 MappingPermissions {
893 current: flags,
894 reported: flags,
895 maximum: flags,
896 },
897 false,
898 MappingOperation::new_alloc(start, PAGE_SIZE_4K, "[heap]"),
899 MappingPublication::new(false),
900 )?
901 }
902 } else if new_aligned < old_aligned {
903 let unmap_start = initial_mapping_end.max(new_aligned);
904 let unmap_size = old_aligned.saturating_sub(unmap_start);
905 if unmap_size == 0 {
906 AddressSpaceMutationOutcome::Complete
907 } else {
908 self.unmap_classified(VirtAddr::from(unmap_start), unmap_size)?
909 }
910 } else {
911 AddressSpaceMutationOutcome::Complete
912 };
913
914 self.heap.current = requested;
917 Ok(outcome)
918 }
919
920 pub(crate) fn translate(&self, vaddr: VirtAddr) -> StarryResult<PhysAddr> {
923 self.pt
924 .query(vaddr)
925 .map(|(paddr, ..)| paddr)
926 .map_err(Into::into)
927 }
928
929 pub(crate) fn resident_span(&self, vaddr: VirtAddr) -> Option<usize> {
934 self.pt.query(vaddr).ok().map(|(_, _, size)| size)
935 }
936
937 pub(crate) fn resident_bytes_from(&self, vaddr: VirtAddr) -> Option<usize> {
941 let key = MappingSlotKey {
942 space_id: self.id,
943 va: vaddr,
944 };
945 let (_, slot) = self.mapping_slots.range(..=key).next_back()?;
946 if slot.state() != SlotState::Present {
947 return None;
948 }
949 let bytes = PAGE_SIZE_4K.checked_shl(slot.page_order.get().into())?;
950 let end = slot.va.checked_add(bytes)?;
951 (vaddr >= slot.va && vaddr < end).then(|| end.as_usize() - vaddr.as_usize())
952 }
953
954 fn mapping_slots_overlapping(
961 &self,
962 range: VirtAddrRange,
963 ) -> impl Iterator<Item = (&MappingSlotKey, &Arc<MappingSlot>)> {
964 let start = MappingSlotKey {
965 space_id: self.id,
966 va: range.start,
967 };
968 let end = MappingSlotKey {
969 space_id: self.id,
970 va: range.end,
971 };
972 let predecessor = self
973 .mapping_slots
974 .range(..start)
975 .next_back()
976 .filter(move |(key, slot)| key.space_id == self.id && slot.overlaps(range));
977 let inside = self
978 .mapping_slots
979 .range(start..end)
980 .filter(move |(key, slot)| key.space_id == self.id && slot.overlaps(range));
981 predecessor.into_iter().chain(inside)
982 }
983
984 fn mapping_slot_summary(
985 &self,
986 range: VirtAddrRange,
987 ) -> StarryResult<(usize, usize, ResidentPageCounts)> {
988 let mut slots = 0usize;
989 let mut materialized_pages = 0usize;
990 let mut resident = ResidentPageCounts::default();
991 for (_, slot) in self.mapping_slots_overlapping(range) {
992 if slot.state() != SlotState::Present {
993 return Err(StarryError::BadState);
994 }
995 let pages = 1usize
996 .checked_shl(slot.page_order.get().into())
997 .ok_or(StarryError::BadState)?;
998 slots = slots.checked_add(1).ok_or(StarryError::BadState)?;
999 materialized_pages = materialized_pages
1000 .checked_add(pages)
1001 .ok_or(StarryError::BadState)?;
1002 resident.checked_add_pages(
1003 slot.resident_kind(),
1004 u64::try_from(pages).map_err(|_| StarryError::BadState)?,
1005 )?;
1006 }
1007 Ok((slots, materialized_pages, resident))
1008 }
1009
1010 fn resident_counts_from_all_slots(&self) -> StarryResult<ResidentPageCounts> {
1011 let mut resident = ResidentPageCounts::default();
1012 for slot in self.mapping_slots.values() {
1013 if slot.state() != SlotState::Present {
1014 return Err(StarryError::BadState);
1015 }
1016 let pages = 1u64
1017 .checked_shl(slot.page_order.get().into())
1018 .ok_or(StarryError::BadState)?;
1019 resident.checked_add_pages(slot.resident_kind(), pages)?;
1020 }
1021 Ok(resident)
1022 }
1023
1024 fn occupied_pte_leaves_overlapping(
1031 &self,
1032 ranges: &[VirtAddrRange],
1033 ) -> StarryResult<Vec<OccupiedPteLeaf>> {
1034 let mut leaves = Vec::new();
1035 for range in ranges {
1036 for entry in self.pt.walk_occupied_range(range.start, range.end) {
1037 let leaf_start = entry.vaddr;
1038 let page_size = self
1039 .pt
1040 .mapping_size_for_level(entry.level)
1041 .ok_or(StarryError::BadState)?;
1042 let leaf_end = leaf_start
1043 .checked_add(page_size)
1044 .ok_or(StarryError::BadState)?;
1045 if leaf_start >= range.end || leaf_end <= range.start {
1046 continue;
1047 }
1048 if leaf_start < range.start || leaf_end > range.end {
1049 return Err(StarryError::OperationNotSupported);
1050 }
1051 let is_directory_level = entry.level > 1;
1052 leaves.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
1053 leaves.push(OccupiedPteLeaf {
1054 range: VirtAddrRange::new(leaf_start, leaf_end),
1055 paddr: entry.pte.paddr(is_directory_level),
1056 flags: entry.pte.config(is_directory_level),
1057 });
1058 }
1059 }
1060 Ok(leaves)
1061 }
1062
1063 fn materialized_slots_overlapping(
1068 &self,
1069 ranges: &[VirtAddrRange],
1070 ) -> StarryResult<Vec<(MappingSlotKey, Arc<MappingSlot>, OccupiedPteLeaf)>> {
1071 let leaves = self.occupied_pte_leaves_overlapping(ranges)?;
1072 let mut slots = Vec::new();
1073 slots
1074 .try_reserve(leaves.len())
1075 .map_err(|_| StarryError::NoMemory)?;
1076 for leaf in leaves {
1077 let key = MappingSlotKey {
1078 space_id: self.id,
1079 va: leaf.range.start,
1080 };
1081 let slot = self
1082 .mapping_slots
1083 .get(&key)
1084 .cloned()
1085 .ok_or(StarryError::BadState)?;
1086 let expected_size = PAGE_SIZE_4K
1087 .checked_shl(slot.page_order.get().into())
1088 .ok_or(StarryError::BadState)?;
1089 if slot.state() != SlotState::Present
1090 || slot.mm_id != self.id
1091 || slot.va != key.va
1092 || expected_size != leaf.range.size()
1093 || slot.mapped_paddr() != Some(leaf.paddr)
1094 {
1095 return Err(StarryError::BadState);
1096 }
1097 slots.push((key, slot, leaf));
1098 }
1099 let overlapping_slots = ranges
1100 .iter()
1101 .map(|range| self.mapping_slots_overlapping(*range).count())
1102 .sum::<usize>();
1103 if overlapping_slots != slots.len() {
1104 return Err(StarryError::BadState);
1105 }
1106 Ok(slots)
1107 }
1108
1109 pub(crate) fn validate_materialized_leaf_boundaries(
1114 &self,
1115 start: VirtAddr,
1116 size: usize,
1117 ) -> StarryResult {
1118 self.validate_region(start, size)?;
1119 let range =
1120 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
1121 for (key, slot, leaf) in self.materialized_slots_overlapping(&[range])? {
1122 let page_size = leaf.range.size();
1123 let leaf_end = slot
1124 .va
1125 .checked_add(page_size)
1126 .ok_or(StarryError::BadState)?;
1127 if slot.va < range.start || leaf_end > range.end {
1128 return Err(StarryError::OperationNotSupported);
1129 }
1130 if key.va != slot.va
1131 || slot.mm_id != self.id
1132 || slot.state() != SlotState::Present
1133 || slot.mapped_paddr() != Some(leaf.paddr)
1134 {
1135 return Err(StarryError::BadState);
1136 }
1137 }
1138 Ok(())
1139 }
1140
1141 fn apply_partial_huge_splits(
1146 &mut self,
1147 range: VirtAddrRange,
1148 ) -> StarryResult<Vec<AppliedHugeSplit>> {
1149 let mut candidates = Vec::new();
1150 candidates
1151 .try_reserve(2)
1152 .map_err(|_| StarryError::NoMemory)?;
1153 for (key, slot) in self.mapping_slots_overlapping(range) {
1154 if slot.page_order == PageOrder::BASE || !slot.overlaps(range) {
1155 continue;
1156 }
1157 let slot_size = PAGE_SIZE_4K
1158 .checked_shl(slot.page_order.get().into())
1159 .ok_or(StarryError::BadState)?;
1160 let slot_end = slot
1161 .va
1162 .checked_add(slot_size)
1163 .ok_or(StarryError::BadState)?;
1164 if range.start <= slot.va && slot_end <= range.end {
1165 continue;
1166 }
1167 candidates
1168 .try_reserve(1)
1169 .map_err(|_| StarryError::NoMemory)?;
1170 candidates.push((*key, slot.clone()));
1171 }
1172
1173 let mut applied = Vec::new();
1174 applied
1175 .try_reserve(candidates.len())
1176 .map_err(|_| StarryError::NoMemory)?;
1177 for (key, slot) in candidates {
1178 match self.apply_one_partial_huge_split(key, slot) {
1179 Ok(split) => applied.push(split),
1180 Err(error) => {
1181 if !self.rollback_applied_huge_splits(applied) {
1182 self.mutation_gate.mark_needs_repair();
1183 return Err(StarryError::BadState);
1184 }
1185 return Err(error);
1186 }
1187 }
1188 }
1189 Ok(applied)
1190 }
1191
1192 fn apply_partial_huge_splits_for_ranges(
1197 &mut self,
1198 ranges: &[VirtAddrRange],
1199 ) -> StarryResult<Vec<AppliedHugeSplit>> {
1200 let capacity = ranges.len().checked_mul(2).ok_or(StarryError::NoMemory)?;
1201 let mut applied = Vec::new();
1202 applied
1203 .try_reserve_exact(capacity)
1204 .map_err(|_| StarryError::NoMemory)?;
1205 for range in ranges {
1206 match self.apply_partial_huge_splits(*range) {
1207 Ok(mut splits) => applied.append(&mut splits),
1208 Err(error) => {
1209 if !self.rollback_applied_huge_splits(applied) {
1210 self.mutation_gate.mark_needs_repair();
1211 return Err(StarryError::BadState);
1212 }
1213 return Err(error);
1214 }
1215 }
1216 }
1217 Ok(applied)
1218 }
1219
1220 fn apply_one_partial_huge_split(
1221 &mut self,
1222 old_key: MappingSlotKey,
1223 old_slot: Arc<MappingSlot>,
1224 ) -> StarryResult<AppliedHugeSplit> {
1225 if old_slot.page_order != PageOrder::new(9)
1228 || old_slot.state() != SlotState::Present
1229 || !self
1230 .mapping_slots
1231 .get(&old_key)
1232 .is_some_and(|slot| Arc::ptr_eq(slot, &old_slot))
1233 {
1234 return Err(StarryError::OperationNotSupported);
1235 }
1236 let block_size = PAGE_SIZE_4K
1237 .checked_shl(old_slot.page_order.get().into())
1238 .ok_or(StarryError::BadState)?;
1239 let block_range = VirtAddrRange::try_from_start_size(old_slot.va, block_size)
1240 .ok_or(StarryError::BadState)?;
1241 let (mapped_paddr, _, mapped_size) = self.pt.query(old_slot.va)?;
1242 if old_slot.mapped_paddr() != Some(mapped_paddr)
1243 || mapped_size != block_size
1244 || old_slot.page.frame().size() < block_size
1245 {
1246 return Err(StarryError::BadState);
1247 }
1248
1249 let child_count = block_size / PAGE_SIZE_4K;
1250 let mut child_keys = Vec::new();
1251 let mut child_slots = Vec::new();
1252 child_keys
1253 .try_reserve_exact(child_count)
1254 .map_err(|_| StarryError::NoMemory)?;
1255 child_slots
1256 .try_reserve_exact(child_count)
1257 .map_err(|_| StarryError::NoMemory)?;
1258 for index in 0..child_count {
1259 let offset = index
1260 .checked_mul(PAGE_SIZE_4K)
1261 .ok_or(StarryError::BadState)?;
1262 let va = old_slot
1263 .va
1264 .checked_add(offset)
1265 .ok_or(StarryError::BadState)?;
1266 let key = MappingSlotKey {
1267 space_id: self.id,
1268 va,
1269 };
1270 let child = MappingSlot::new_with_frame_offset(
1271 old_slot.mapping,
1272 self.id,
1273 va,
1274 PageOrder::BASE,
1275 old_slot.page.clone(),
1276 old_slot
1277 .frame_offset()
1278 .checked_add(offset)
1279 .ok_or(StarryError::BadState)?,
1280 old_slot.resident_kind(),
1281 )
1282 .ok_or(StarryError::BadState)?;
1283 child_keys.push(key);
1284 child_slots.push(Arc::new(child));
1285 }
1286
1287 let mut next_mapping_slots = self.mapping_slots.clone();
1290 let removed = next_mapping_slots
1291 .remove(&old_key)
1292 .ok_or(StarryError::BadState)?;
1293 if !Arc::ptr_eq(&removed, &old_slot) {
1294 return Err(StarryError::BadState);
1295 }
1296 for (key, slot) in child_keys.iter().copied().zip(child_slots.iter().cloned()) {
1297 if next_mapping_slots.insert(key, slot).is_some() {
1298 return Err(StarryError::BadState);
1299 }
1300 }
1301
1302 let old_keys = [old_key];
1306 let mut graph_reservation = old_slot
1307 .page
1308 .prepare_mapping_graph_replace(&old_keys, &child_keys)
1309 .map_err(|error| match error {
1310 MappingGraphError::ResourceExhausted | MappingGraphError::RefOverflow => {
1311 StarryError::NoMemory
1312 }
1313 _ => StarryError::BadState,
1314 })?;
1315
1316 let deposit = old_slot
1317 .take_huge_split_deposit()
1318 .ok_or(StarryError::BadState)?;
1319 let mutation_gate = &self.mutation_gate;
1320 let pte_domain = &self.pte_domain;
1321 let pt = &mut self.pt;
1322 let stripe = pte_domain.lock_range(block_range);
1323 let installed = match pt.try_split_huge_page_with(deposit) {
1324 Ok(installed) => installed,
1325 Err(failure) => {
1326 let (error, deposit) = failure.into_parts();
1327 match old_slot.restore_huge_split_deposit(deposit) {
1328 Ok(()) => {
1329 drop(stripe);
1330 drop(graph_reservation);
1331 return Err(error.into());
1332 }
1333 Err(orphaned) => {
1334 drop(stripe);
1338 drop(graph_reservation);
1339 drop(orphaned);
1340 mutation_gate.mark_needs_repair();
1341 return Err(StarryError::BadState);
1342 }
1343 }
1344 }
1345 };
1346
1347 if let Err(graph_error) = old_slot.page.replace_mapping_graph_reserved(
1348 &old_keys,
1349 &child_keys,
1350 &mut graph_reservation,
1351 ) {
1352 let restored = pt.restore_huge_split(installed);
1353 match restored {
1354 Ok(deposit) => match old_slot.restore_huge_split_deposit(deposit) {
1355 Ok(()) => {
1356 drop(stripe);
1357 drop(graph_reservation);
1358 return Err(match graph_error {
1359 MappingGraphError::ResourceExhausted
1360 | MappingGraphError::RefOverflow => StarryError::NoMemory,
1361 _ => StarryError::BadState,
1362 });
1363 }
1364 Err(orphaned) => {
1365 drop(stripe);
1366 drop(graph_reservation);
1367 drop(orphaned);
1368 }
1369 },
1370 Err(_) => {
1371 drop(stripe);
1372 drop(graph_reservation);
1373 }
1374 }
1375 mutation_gate.mark_needs_repair();
1376 return Err(StarryError::BadState);
1377 }
1378
1379 let mut published_children = 0usize;
1380 for child in &child_slots {
1381 if !child.publish_after_graph_replace() {
1382 break;
1383 }
1384 published_children += 1;
1385 }
1386 let old_detached =
1387 published_children == child_slots.len() && old_slot.detach_after_graph_replace();
1388 if !old_detached {
1389 for child in child_slots[..published_children].iter().rev() {
1390 let _ = child.reserve_after_graph_replace();
1391 }
1392 let graph_restored = old_slot
1393 .page
1394 .replace_mapping_graph_reserved(&child_keys, &old_keys, &mut graph_reservation)
1395 .is_ok();
1396 let restored = pt.restore_huge_split(installed);
1397 let (deposit_restored, orphaned) = match restored {
1398 Ok(deposit) => match old_slot.restore_huge_split_deposit(deposit) {
1399 Ok(()) => (true, None),
1400 Err(orphaned) => (false, Some(orphaned)),
1401 },
1402 Err(_) => (false, None),
1403 };
1404 drop(stripe);
1405 drop(graph_reservation);
1406 drop(orphaned);
1407 if graph_restored && deposit_restored {
1408 return Err(StarryError::BadState);
1409 }
1410 mutation_gate.mark_needs_repair();
1411 return Err(StarryError::BadState);
1412 }
1413
1414 drop(stripe);
1415 drop(graph_reservation);
1416 let previous_mapping_slots =
1417 core::mem::replace(&mut self.mapping_slots, next_mapping_slots);
1418 Ok(AppliedHugeSplit {
1419 installed,
1420 old_slot,
1421 child_keys,
1422 child_slots,
1423 previous_mapping_slots,
1424 })
1425 }
1426
1427 fn rollback_applied_huge_splits(&mut self, mut splits: Vec<AppliedHugeSplit>) -> bool {
1428 while let Some(split) = splits.pop() {
1429 let block_range = VirtAddrRange::try_from_start_size(
1430 split.installed.block_vaddr(),
1431 split.installed.block_size(),
1432 );
1433 let Some(block_range) = block_range else {
1434 return false;
1435 };
1436 let old_key = MappingSlotKey {
1437 space_id: self.id,
1438 va: split.old_slot.va,
1439 };
1440 let old_keys = [old_key];
1441 let mut graph_reservation = match split
1442 .old_slot
1443 .page
1444 .prepare_mapping_graph_replace(&split.child_keys, &old_keys)
1445 {
1446 Ok(reservation) => reservation,
1447 Err(_) => return false,
1448 };
1449 let pte_domain = &self.pte_domain;
1450 let pt = &mut self.pt;
1451 let stripe = pte_domain.lock_range(block_range);
1452 let Ok(deposit) = pt.restore_huge_split(split.installed) else {
1453 drop(stripe);
1454 drop(graph_reservation);
1455 return false;
1456 };
1457 if split
1458 .old_slot
1459 .page
1460 .replace_mapping_graph_reserved(
1461 &split.child_keys,
1462 &old_keys,
1463 &mut graph_reservation,
1464 )
1465 .is_err()
1466 {
1467 let orphaned = split.old_slot.restore_huge_split_deposit(deposit).err();
1468 drop(stripe);
1469 drop(graph_reservation);
1470 drop(orphaned);
1471 return false;
1472 }
1473 let slots_restored = !split
1474 .child_slots
1475 .iter()
1476 .any(|slot| !slot.detach_after_graph_replace())
1477 && split.old_slot.restore_after_graph_replace();
1478 let orphaned = split.old_slot.restore_huge_split_deposit(deposit).err();
1479 let deposit_restored = orphaned.is_none();
1480 drop(stripe);
1481 drop(graph_reservation);
1482 drop(orphaned);
1483 if !slots_restored || !deposit_restored {
1484 return false;
1485 }
1486 self.mapping_slots = split.previous_mapping_slots;
1487 }
1488 true
1489 }
1490
1491 fn capture_protection_leaf_preimage(
1492 &self,
1493 range: VirtAddrRange,
1494 ) -> StarryResult<Vec<ProtectionLeafPreimage>> {
1495 let occupied = self.occupied_pte_leaves_overlapping(&[range])?;
1496 let mut leaves = Vec::new();
1497 leaves
1498 .try_reserve(occupied.len())
1499 .map_err(|_| StarryError::NoMemory)?;
1500 for leaf in occupied {
1501 leaves.push(ProtectionLeafPreimage {
1502 va: leaf.range.start,
1503 paddr: leaf.paddr,
1504 page_size: leaf.range.size(),
1505 flags: leaf.flags,
1506 });
1507 }
1508 Ok(leaves)
1509 }
1510
1511 fn restore_protection_leaf_preimage(&mut self, leaves: &[ProtectionLeafPreimage]) -> bool {
1512 for leaf in leaves.iter().rev() {
1513 let Ok((paddr, _, page_size)) = self.pt.query(leaf.va) else {
1514 return false;
1515 };
1516 if paddr != leaf.paddr || page_size != leaf.page_size {
1517 return false;
1518 }
1519 if self.pt.protect_page(leaf.va, leaf.flags) != Ok(leaf.page_size) {
1520 return false;
1521 }
1522 }
1523 true
1524 }
1525
1526 fn abort_unpublished_protection(
1527 &mut self,
1528 vma_root: Arc<VmaMap>,
1529 vm_stat: ProcessVmStatSnapshot,
1530 leaves: &[ProtectionLeafPreimage],
1531 splits: Vec<AppliedHugeSplit>,
1532 original_error: StarryError,
1533 ) -> StarryResult {
1534 let ptes_restored = self.restore_protection_leaf_preimage(leaves);
1535 self.vma_root = vma_root;
1536 self.vm_stat.restore(vm_stat);
1537 let splits_restored = self.rollback_applied_huge_splits(splits);
1538 if ptes_restored && splits_restored {
1539 self.mutation_gate.clear_repair();
1540 Err(original_error)
1541 } else {
1542 self.mutation_gate.mark_needs_repair();
1543 Err(StarryError::BadState)
1544 }
1545 }
1546
1547 #[cfg(not(any(target_arch = "aarch64", target_arch = "loongarch64")))]
1558 pub(crate) unsafe fn share_kernel_root_entries_from(
1559 &mut self,
1560 source: RootEntryShare<'_>,
1561 ) -> Result<(), PagingError> {
1562 unsafe { source.install_into(&mut self.pt) }
1565 }
1566
1567 const fn materialized_root(&self) -> PhysAddr {
1569 self.pt.root_paddr()
1570 }
1571
1572 pub fn contains_range(&self, start: VirtAddr, size: usize) -> bool {
1574 let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
1575 return false;
1576 };
1577 range.start >= self.base() && range.end <= self.end()
1578 }
1579
1580 pub fn new_empty(base: VirtAddr, size: usize) -> StarryResult<Self> {
1582 Self::new_with_layout(UserVirtualAddressLayout::from_range(base, size)?)
1583 }
1584
1585 pub(crate) fn new_user(layout: UserVirtualAddressLayout) -> StarryResult<Self> {
1587 Self::new_with_layout(layout)
1588 }
1589
1590 fn new_with_layout(layout: UserVirtualAddressLayout) -> StarryResult<Self> {
1591 Ok(Self {
1592 id: AddressSpaceId::allocate(),
1593 layout,
1594 vma_root: Arc::new(VmaMap::default()),
1595 heap: HeapState::new(USER_HEAP_BASE),
1596 executable_data: ExecutableDataLayout::default(),
1597 executable_file: None,
1598 pt: PageTable::new(PagingAllocator).map_err(|_| StarryError::NoMemory)?,
1599 pte_domain: PageTableDomain::new(),
1600 mutation_gate: MutationGate::new(),
1601 published_epoch: Arc::new(core::sync::atomic::AtomicU64::new(0)),
1602 vm_stat: ProcessVmStat::new(),
1603 resident_pages: ResidentPageCounts::default(),
1604 resident_watermark: ResidentWatermark::new(),
1605 tlb_quarantine: TlbQuarantine::default(),
1606 tlb_targets: Arc::new(AtomicUsize::new(0)),
1607 mapping_slots: BTreeMap::new(),
1608 retired_mapping_batches: IrqMutex::new(Vec::new()),
1609 })
1610 }
1611
1612 pub const fn address_space_id(&self) -> AddressSpaceId {
1615 self.id
1616 }
1617
1618 pub fn vm_epoch(&self) -> VmEpoch {
1620 self.mutation_gate.current_epoch()
1621 }
1622
1623 pub(crate) fn tlb_targets(&self) -> Arc<AtomicUsize> {
1624 self.tlb_targets.clone()
1625 }
1626
1627 pub(crate) fn published_epoch_source(&self) -> Arc<core::sync::atomic::AtomicU64> {
1628 self.published_epoch.clone()
1629 }
1630
1631 fn publish_mutation_classified(
1632 &mut self,
1633 mutation: PreparedMutation,
1634 ) -> Result<MutationPublication, CommitMutationError> {
1635 let mut next_resident = self.resident_pages;
1636 next_resident
1637 .checked_apply(mutation.receipt().resident_delta)
1638 .map_err(CommitMutationError::Unpublished)?;
1639 match self.mutation_gate.commit(mutation) {
1640 Ok(receipt) => {
1641 self.resident_pages = next_resident;
1642 self.published_epoch.store(
1643 receipt.new_epoch.get(),
1644 core::sync::atomic::Ordering::Release,
1645 );
1646 self.resident_watermark
1647 .observe_resident_total(self.resident_pages.total());
1648 Ok(MutationPublication::Complete)
1649 }
1650 Err(MutationError::TlbPending) => {
1651 self.resident_pages = next_resident;
1652 self.published_epoch.store(
1653 self.mutation_gate.current_epoch().get(),
1654 core::sync::atomic::Ordering::Release,
1655 );
1656 self.resident_watermark
1661 .observe_resident_total(self.resident_pages.total());
1662 Ok(MutationPublication::PendingTlb)
1663 }
1664 Err(error) => Err(CommitMutationError::Unpublished(
1665 Self::map_unpublished_mutation_error(error),
1666 )),
1667 }
1668 }
1669
1670 fn map_unpublished_mutation_error(error: MutationError) -> StarryError {
1671 match error {
1672 MutationError::ResourceExhausted => StarryError::NoMemory,
1673 MutationError::PendingTlbOverlap => StarryError::ResourceBusy,
1674 MutationError::NeedsRepair
1675 | MutationError::EpochExhausted
1676 | MutationError::EpochConflict
1677 | MutationError::WrongState
1678 | MutationError::ApplyFailed
1679 | MutationError::TlbPending => StarryError::BadState,
1680 }
1681 }
1682
1683 fn commit_mutation_classified(
1684 &mut self,
1685 mutation: PreparedMutation,
1686 ) -> Result<(), CommitMutationError> {
1687 match self.publish_mutation_classified(mutation)? {
1688 MutationPublication::Complete => Ok(()),
1689 MutationPublication::PendingTlb => {
1690 self.service_pending_tlb()
1695 .map(|_| ())
1696 .map_err(CommitMutationError::PublishedPendingTlb)
1697 }
1698 }
1699 }
1700
1701 fn commit_mutation(&mut self, mutation: PreparedMutation) -> StarryResult {
1702 match self.commit_mutation_classified(mutation) {
1703 Ok(()) => Ok(()),
1704 Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
1705 Err(CommitMutationError::Unpublished(error)) => {
1706 self.mutation_gate.mark_needs_repair();
1712 Err(error)
1713 }
1714 }
1715 }
1716
1717 pub fn service_pending_tlb(&self) -> StarryResult<usize> {
1724 let requests = self
1725 .mutation_gate
1726 .pending_requests()
1727 .map_err(|_| StarryError::NoMemory)?;
1728 Self::flush_tlb_requests(&requests, &self.tlb_targets)?;
1729 self.acknowledge_tlb_requests(&requests)
1730 }
1731
1732 fn flush_tlb_requests(requests: &[TlbRequest], tlb_targets: &AtomicUsize) -> StarryResult {
1733 for request in requests {
1734 let pending_targets = request.pending();
1735 let live_targets =
1739 pending_targets & tlb_targets.load(core::sync::atomic::Ordering::Acquire);
1740 if request.ranges.is_empty() && live_targets != 0 {
1741 ax_runtime::hal::cache::flush_tlb_all_on_cpus(live_targets)
1742 .map_err(Self::map_tlb_shootdown_error)?;
1743 } else if live_targets != 0 {
1744 for range in &request.ranges {
1745 ax_runtime::hal::cache::flush_tlb_range_on_cpus(
1746 live_targets,
1747 range.start,
1748 range.size,
1749 )
1750 .map_err(Self::map_tlb_shootdown_error)?;
1751 }
1752 }
1753 }
1754 Ok(())
1755 }
1756
1757 fn map_tlb_shootdown_error(error: ax_runtime::hal::cache::TlbShootdownError) -> StarryError {
1758 match error {
1759 ax_runtime::hal::cache::TlbShootdownError::Timeout => StarryError::TimedOut,
1760 ax_runtime::hal::cache::TlbShootdownError::Unsupported
1761 | ax_runtime::hal::cache::TlbShootdownError::CpuOffline => StarryError::Unsupported,
1762 ax_runtime::hal::cache::TlbShootdownError::GenerationExhausted => {
1763 StarryError::Errno(syscalls::Errno::EOVERFLOW)
1764 }
1765 ax_runtime::hal::cache::TlbShootdownError::Platform => StarryError::Io,
1766 }
1767 }
1768
1769 fn acknowledge_tlb_requests(&self, requests: &[TlbRequest]) -> StarryResult<usize> {
1770 let mut completed = 0;
1771 for request in requests {
1772 let pending_targets = request.pending();
1773 for cpu in 0..usize::BITS as usize {
1777 if pending_targets & (1usize << cpu) == 0 {
1778 continue;
1779 }
1780 match self.mutation_gate.acknowledge(self.id, request.epoch, cpu) {
1781 Ok(_) | Err(MutationError::WrongState) | Err(MutationError::TlbPending) => {}
1782 Err(_) => return Err(StarryError::BadState),
1783 }
1784 self.tlb_quarantine
1785 .acknowledge(self.id, request.epoch, cpu)
1786 .map_err(|_| StarryError::NoMemory)?;
1787 }
1788 if self
1789 .mutation_gate
1790 .pending_request(self.id, request.epoch)
1791 .is_none()
1792 {
1793 self.release_retired_mapping_owners(request.epoch);
1794 }
1795 completed += 1;
1796 }
1797 Ok(completed)
1798 }
1799
1800 fn prepare_mutation(&self) -> PreparedMutation {
1801 self.mutation_gate
1802 .begin_with_active_targets(self.id, self.tlb_targets.clone())
1803 }
1804
1805 fn prepare_metadata_mutation(&self) -> PreparedMutation {
1810 self.mutation_gate.begin(self.id, 0)
1811 }
1812
1813 fn prepare_mutation_range(&self, start: VirtAddr, size: usize) -> PreparedMutation {
1814 let mut mutation = self.prepare_mutation();
1815 if let Some(range) = TlbRange::new(start, size) {
1816 mutation.add_tlb_range(range);
1817 }
1818 mutation
1819 }
1820
1821 fn prepare_fresh_pte_mutation_range(&self, start: VirtAddr, size: usize) -> PreparedMutation {
1827 prepare_mapping_publication_mutation(
1828 &self.mutation_gate,
1829 self.id,
1830 &self.tlb_targets,
1831 start,
1832 size,
1833 false,
1834 )
1835 }
1836
1837 fn collect_retired_mapping_owner(
1844 &self,
1845 entry: &Arc<VmaEntry>,
1846 range: VirtAddrRange,
1847 owners: &mut RetiredMappingOwners,
1848 ) -> StarryResult {
1849 owners
1850 .backends
1851 .try_reserve(1)
1852 .map_err(|_| StarryError::NoMemory)?;
1853 let backend = entry.operation_clone();
1854
1855 if backend.shared_file_lease().is_some() {
1856 let start = entry.start().max(range.start).align_down_4k();
1857 let end = entry.end().min(range.end);
1858 let file_range = VirtAddrRange::new(start, end);
1859 let count = self.mapping_slots_overlapping(file_range).count();
1860 owners
1861 .cache_pins
1862 .try_reserve(count)
1863 .map_err(|_| StarryError::NoMemory)?;
1864 for (_, slot) in self.mapping_slots_overlapping(file_range) {
1865 if slot.state() != SlotState::Present
1866 || slot.mapping != backend.mapping_id()
1867 || slot.page_order != PageOrder::BASE
1868 {
1869 return Err(StarryError::BadState);
1870 }
1871 let paddr = slot.mapped_paddr().ok_or(StarryError::BadState)?;
1872 let (installed, _, page_size) = self.pt.query(slot.va)?;
1873 if installed != paddr || page_size != PAGE_SIZE_4K {
1874 return Err(StarryError::BadState);
1875 }
1876 let pin = backend
1877 .pin_file_cache_owner_for_mapping(slot.va, paddr)?
1878 .ok_or(StarryError::BadState)?;
1879 owners.cache_pins.push(pin);
1880 }
1881 }
1882 owners.backends.push(backend);
1883 Ok(())
1884 }
1885
1886 fn prepare_retired_mapping_owners(
1887 &self,
1888 range: VirtAddrRange,
1889 ) -> StarryResult<RetiredMappingOwners> {
1890 try_reserve_irq_vec(&self.retired_mapping_batches, 1).map_err(|_| StarryError::NoMemory)?;
1891
1892 let mut owners = RetiredMappingOwners::default();
1893 let mut failure = None;
1897 self.vma_root.for_each_overlapping_entry(range, |entry| {
1898 match self.collect_retired_mapping_owner(entry, range, &mut owners) {
1899 Ok(()) => true,
1900 Err(err) => {
1901 failure = Some(err);
1902 false
1903 }
1904 }
1905 });
1906 if let Some(err) = failure {
1907 return Err(err);
1908 }
1909
1910 let matching_slots = self.mapping_slots_overlapping(range).count();
1911 owners
1912 .pages
1913 .try_reserve(matching_slots)
1914 .map_err(|_| StarryError::NoMemory)?;
1915 for (_, slot) in self.mapping_slots_overlapping(range) {
1916 if slot.state() != SlotState::Present {
1917 return Err(StarryError::BadState);
1918 }
1919 owners.pages.push(slot.page.clone());
1920 }
1921 Ok(owners)
1922 }
1923
1924 fn prepare_page_table_reclaims(
1928 &self,
1929 ranges: &[VirtAddrRange],
1930 ) -> StarryResult<MappingMutationContext> {
1931 try_reserve_irq_vec(&self.retired_mapping_batches, 1).map_err(|_| StarryError::NoMemory)?;
1932 let capacity = self.occupied_pte_leaves_overlapping(ranges)?.len();
1933 MappingMutationContext::for_published_mutation(capacity)
1934 }
1935
1936 fn prepare_deferred_eviction_owner(
1940 &self,
1941 page: &Arc<PageObject>,
1942 ) -> StarryResult<RetiredMappingOwners> {
1943 try_reserve_irq_vec(&self.retired_mapping_batches, 1).map_err(|_| StarryError::NoMemory)?;
1944 let mut owners = RetiredMappingOwners::default();
1945 owners
1946 .deferred_evictions
1947 .try_reserve(1)
1948 .map_err(|_| StarryError::NoMemory)?;
1949 owners
1950 .page_tables
1951 .try_reserve(1)
1952 .map_err(|_| StarryError::NoMemory)?;
1953 owners.deferred_evictions.push(page.clone());
1954 Ok(owners)
1955 }
1956
1957 fn park_retired_mapping_owners(&self, epoch: VmEpoch, owners: RetiredMappingOwners) {
1958 if owners.is_empty() {
1959 return;
1960 }
1961 self.retired_mapping_batches
1964 .lock()
1965 .push(RetiredMappingBatch { epoch, owners });
1966 }
1967
1968 fn park_page_table_reclaims(&self, epoch: VmEpoch, context: MappingMutationContext) {
1969 if context.is_empty() {
1970 return;
1971 }
1972 let owners = RetiredMappingOwners {
1973 page_tables: context.into_page_tables(),
1974 ..RetiredMappingOwners::default()
1975 };
1976 self.park_retired_mapping_owners(epoch, owners);
1977 }
1978
1979 fn release_retired_mapping_owners(&self, epoch: VmEpoch) {
1980 loop {
1981 let batch = {
1982 let mut batches = self.retired_mapping_batches.lock();
1983 batches
1984 .iter()
1985 .position(|batch| batch.epoch == epoch)
1986 .map(|index| batches.swap_remove(index))
1987 };
1988 let Some(mut batch) = batch else {
1989 break;
1990 };
1991 debug_assert!(!batch.owners.is_empty());
1992 for page in &batch.owners.deferred_evictions {
1993 page.complete_eviction_tlb();
1994 }
1995 for tables in batch.owners.page_tables.drain(..) {
1996 unsafe { tables.reclaim() };
1999 }
2000 drop(batch);
2003 }
2004 }
2005
2006 fn pending_retired_mapping_batches(&self) -> usize {
2007 self.retired_mapping_batches.lock().len()
2008 }
2009
2010 #[cfg(test)]
2011 fn pending_retired_page_table_reclaims(&self) -> usize {
2012 self.retired_mapping_batches
2013 .lock()
2014 .iter()
2015 .map(|batch| batch.owners.page_tables.len())
2016 .sum()
2017 }
2018
2019 fn confirm_restored_mapping_owners(
2020 &self,
2021 epoch: VmEpoch,
2022 ranges: &[VirtAddrRange],
2023 ) -> StarryResult {
2024 for range in ranges {
2025 crate::mm::flush_tlb_range_sync(range.start, range.size())?;
2026 }
2027 self.release_retired_mapping_owners(epoch);
2028 Ok(())
2029 }
2030
2031 pub fn pending_tlb_obligations(&self) -> usize {
2033 self.mutation_gate.pending_count()
2034 }
2035
2036 pub fn pending_tlb_requests(&self) -> StarryResult<Vec<TlbRequest>> {
2040 self.mutation_gate
2041 .pending_requests()
2042 .map_err(|_| StarryError::NoMemory)
2043 }
2044
2045 pub fn acknowledge_tlb(
2048 &self,
2049 space_id: AddressSpaceId,
2050 epoch: VmEpoch,
2051 cpu: usize,
2052 ) -> StarryResult<Vec<FrameLease>> {
2053 match self.mutation_gate.acknowledge(space_id, epoch, cpu) {
2054 Ok(_) | Err(MutationError::TlbPending) => {}
2055 Err(MutationError::WrongState)
2056 if self.tlb_quarantine.contains_request(space_id, epoch) =>
2057 {
2058 }
2063 Err(_) => return Err(StarryError::ResourceBusy),
2064 }
2065 let released = self
2066 .tlb_quarantine
2067 .acknowledge(space_id, epoch, cpu)
2068 .map_err(|_| StarryError::NoMemory)?;
2069 if self
2070 .mutation_gate
2071 .pending_request(space_id, epoch)
2072 .is_none()
2073 {
2074 self.release_retired_mapping_owners(epoch);
2075 }
2076 Ok(released)
2077 }
2078
2079 pub fn quarantine_frame(
2080 &self,
2081 frame: FrameLease,
2082 request: TlbRequest,
2083 ) -> Result<(), QuarantineFailure> {
2084 self.tlb_quarantine.try_defer(frame, request)
2085 }
2086
2087 fn validate_region(&self, start: VirtAddr, size: usize) -> StarryResult {
2088 if self.mutation_gate.needs_repair() {
2089 return Err(StarryError::BadState);
2090 }
2091 if size == 0 || !self.contains_range(start, size) {
2092 return Err(StarryError::NoMemory);
2093 }
2094 if !start.is_aligned_4k() || !is_aligned_4k(size) {
2095 return Err(StarryError::InvalidInput);
2096 }
2097 Ok(())
2098 }
2099
2100 fn capture_mapping_preimage(&self, range: VirtAddrRange) -> StarryResult<MappingPreimage> {
2104 self.capture_mapping_preimage_ranges(&[range])
2105 }
2106
2107 fn capture_mapping_preimage_ranges(
2112 &self,
2113 ranges: &[VirtAddrRange],
2114 ) -> StarryResult<MappingPreimage> {
2115 for (index, range) in ranges.iter().enumerate() {
2116 self.validate_region(range.start, range.size())?;
2117 if ranges[..index]
2118 .iter()
2119 .any(|previous| previous.overlaps(*range))
2120 {
2121 return Err(StarryError::InvalidInput);
2122 }
2123 }
2124 let resident_slots = self.materialized_slots_overlapping(ranges)?;
2125 let mut leaves = Vec::new();
2126 leaves
2127 .try_reserve(resident_slots.len())
2128 .map_err(|_| StarryError::NoMemory)?;
2129 for (key, slot, occupied_leaf) in resident_slots {
2130 let page_size = occupied_leaf.range.size();
2131 let end = slot
2132 .va
2133 .checked_add(page_size)
2134 .ok_or(StarryError::BadState)?;
2135 let range = ranges
2136 .iter()
2137 .find(|range| slot.overlaps(**range))
2138 .ok_or(StarryError::BadState)?;
2139 if slot.va < range.start || end > range.end {
2143 return Err(StarryError::OperationNotSupported);
2144 }
2145 let paddr = occupied_leaf.paddr;
2146 let backend = self
2147 .vma_root
2148 .lookup_entry(slot.va)
2149 .map(|entry| entry.operation_clone())
2150 .ok_or(StarryError::BadState)?;
2151 let page = slot.page.clone();
2152 let frame_start = page.frame().paddr().as_usize();
2153 let frame_end = frame_start
2154 .checked_add(page.frame().size())
2155 .ok_or(StarryError::BadState)?;
2156 let leaf_start = paddr.as_usize();
2157 let leaf_end = leaf_start
2158 .checked_add(page_size)
2159 .ok_or(StarryError::BadState)?;
2160 if key.va != slot.va
2161 || slot.state() != SlotState::Present
2162 || slot.mm_id != self.id
2163 || slot.mapping != backend.mapping_id()
2164 || slot.mapped_paddr() != Some(paddr)
2165 || leaf_start < frame_start
2166 || leaf_end > frame_end
2167 {
2168 return Err(StarryError::BadState);
2169 }
2170 leaves.push(ResidentLeafPreimage {
2171 va: slot.va,
2172 paddr,
2173 page_size,
2174 flags: occupied_leaf.flags,
2175 backend,
2176 page,
2177 slot,
2178 });
2179 }
2180 Ok(MappingPreimage {
2181 vma_root: self.vma_root.clone(),
2182 vm_stat: self.vm_stat.snapshot(),
2183 leaves,
2184 })
2185 }
2186
2187 fn restore_mapping_preimage(
2191 &mut self,
2192 range: VirtAddrRange,
2193 preimage: MappingPreimage,
2194 ) -> StarryResult {
2195 self.restore_mapping_preimage_ranges(&[range], preimage)
2196 }
2197
2198 fn abort_unpublished_mapping_mutation(
2206 &mut self,
2207 range: VirtAddrRange,
2208 preimage: MappingPreimage,
2209 original_error: StarryError,
2210 ) -> StarryResult {
2211 self.abort_unpublished_parked_mapping_mutation(range, preimage, None, original_error)
2212 }
2213
2214 fn abort_file_eviction_mutation(
2219 &mut self,
2220 range: VirtAddrRange,
2221 preimage: MappingPreimage,
2222 parked_epoch: Option<VmEpoch>,
2223 original_error: StarryError,
2224 ) -> Result<EvictMappingOutcome, StarryError> {
2225 match self.restore_mapping_preimage(range, preimage) {
2226 Ok(()) => {
2227 if let Some(epoch) = parked_epoch
2228 && self
2229 .confirm_restored_mapping_owners(epoch, &[range])
2230 .is_err()
2231 {
2232 self.mutation_gate.mark_needs_repair();
2233 return Ok(EvictMappingOutcome::NeedsRepair);
2234 }
2235 self.mutation_gate.clear_repair();
2236 Err(original_error)
2237 }
2238 Err(_) => {
2239 self.mutation_gate.mark_needs_repair();
2240 Ok(EvictMappingOutcome::NeedsRepair)
2241 }
2242 }
2243 }
2244
2245 fn abort_unpublished_parked_mapping_mutation(
2251 &mut self,
2252 range: VirtAddrRange,
2253 preimage: MappingPreimage,
2254 parked_epoch: Option<VmEpoch>,
2255 original_error: StarryError,
2256 ) -> StarryResult {
2257 self.abort_unpublished_parked_mapping_mutation_ranges(
2258 &[range],
2259 preimage,
2260 parked_epoch,
2261 original_error,
2262 )
2263 }
2264
2265 fn abort_unpublished_parked_mapping_mutation_ranges(
2266 &mut self,
2267 ranges: &[VirtAddrRange],
2268 preimage: MappingPreimage,
2269 parked_epoch: Option<VmEpoch>,
2270 original_error: StarryError,
2271 ) -> StarryResult {
2272 match self.restore_mapping_preimage_ranges(ranges, preimage) {
2273 Ok(()) => {
2274 if let Some(epoch) = parked_epoch
2275 && self
2276 .confirm_restored_mapping_owners(epoch, ranges)
2277 .is_err()
2278 {
2279 self.mutation_gate.mark_needs_repair();
2280 return Err(StarryError::BadState);
2281 }
2282 self.mutation_gate.clear_repair();
2283 Err(original_error)
2284 }
2285 Err(_) => {
2286 self.mutation_gate.mark_needs_repair();
2287 Err(StarryError::BadState)
2288 }
2289 }
2290 }
2291
2292 fn abort_unpublished_split_mapping_mutation(
2297 &mut self,
2298 range: VirtAddrRange,
2299 preimage: MappingPreimage,
2300 parked_epoch: Option<VmEpoch>,
2301 splits: Vec<AppliedHugeSplit>,
2302 original_error: StarryError,
2303 ) -> StarryResult {
2304 self.abort_unpublished_split_mapping_mutation_ranges(
2305 &[range],
2306 preimage,
2307 parked_epoch,
2308 splits,
2309 original_error,
2310 )
2311 }
2312
2313 fn abort_unpublished_split_mapping_mutation_ranges(
2317 &mut self,
2318 ranges: &[VirtAddrRange],
2319 preimage: MappingPreimage,
2320 parked_epoch: Option<VmEpoch>,
2321 splits: Vec<AppliedHugeSplit>,
2322 original_error: StarryError,
2323 ) -> StarryResult {
2324 if self
2325 .restore_mapping_preimage_ranges(ranges, preimage)
2326 .is_ok()
2327 && self.rollback_applied_huge_splits(splits)
2328 {
2329 if let Some(epoch) = parked_epoch
2330 && self
2331 .confirm_restored_mapping_owners(epoch, ranges)
2332 .is_err()
2333 {
2334 self.mutation_gate.mark_needs_repair();
2335 return Err(StarryError::BadState);
2336 }
2337 self.mutation_gate.clear_repair();
2338 Err(original_error)
2339 } else {
2340 self.mutation_gate.mark_needs_repair();
2341 Err(StarryError::BadState)
2342 }
2343 }
2344
2345 fn abort_unpublished_huge_splits(
2349 &mut self,
2350 splits: Vec<AppliedHugeSplit>,
2351 original_error: StarryError,
2352 ) -> StarryResult {
2353 if self.rollback_applied_huge_splits(splits) {
2354 self.mutation_gate.clear_repair();
2355 Err(original_error)
2356 } else {
2357 self.mutation_gate.mark_needs_repair();
2358 Err(StarryError::BadState)
2359 }
2360 }
2361
2362 fn restore_mapping_preimage_ranges(
2363 &mut self,
2364 ranges: &[VirtAddrRange],
2365 preimage: MappingPreimage,
2366 ) -> StarryResult {
2367 let mut current_memfds = Vec::new();
2368 for range in ranges {
2369 current_memfds.extend(crate::syscall::memfd_collect_metas_touching_mprotect_range(
2370 self,
2371 range.start,
2372 range.size(),
2373 ));
2374 }
2375 let MappingPreimage {
2376 vma_root,
2377 vm_stat,
2378 leaves,
2379 } = preimage;
2380 if self.detach_current_materialized_ranges(ranges).is_err() {
2381 self.mutation_gate.mark_needs_repair();
2382 return Err(StarryError::BadState);
2383 }
2384 for range in ranges {
2385 self.detach_mapping_slots(*range)?;
2386 }
2387 self.vma_root = vma_root.clone();
2388 for leaf in leaves {
2389 if leaf
2390 .backend
2391 .restore_resident_preimage(
2392 ResidentLeafRestore {
2393 va: leaf.va,
2394 paddr: leaf.paddr,
2395 page_size: leaf.page_size,
2396 flags: leaf.flags,
2397 page: Some(&leaf.page),
2398 },
2399 &mut self.pt,
2400 )
2401 .is_err()
2402 {
2403 self.mutation_gate.mark_needs_repair();
2404 return Err(StarryError::BadState);
2405 }
2406 let key = MappingSlotKey {
2407 space_id: self.id,
2408 va: leaf.va,
2409 };
2410 if self.mapping_slots.contains_key(&key)
2411 || !leaf.slot.restore()
2412 || self.mapping_slots.insert(key, leaf.slot).is_some()
2413 {
2414 self.mutation_gate.mark_needs_repair();
2415 return Err(StarryError::BadState);
2416 }
2417 leaf.page.set_resident_kind(
2418 self.mapping_slots
2419 .get(&key)
2420 .and_then(|slot| slot.resident_kind()),
2421 );
2422 }
2423 self.vma_root = vma_root;
2424 self.vm_stat.restore(vm_stat);
2425 crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(self, ¤t_memfds);
2426 for range in ranges {
2427 let restored_memfds = crate::syscall::memfd_collect_metas_touching_mprotect_range(
2428 self,
2429 range.start,
2430 range.size(),
2431 );
2432 crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
2433 self,
2434 &restored_memfds,
2435 );
2436 }
2437 Ok(())
2438 }
2439
2440 pub fn find_free_area(
2448 &self,
2449 hint: VirtAddr,
2450 size: usize,
2451 limit: VirtAddrRange,
2452 align: usize,
2453 ) -> Option<VirtAddr> {
2454 self.vma_root.find_free_area(hint, size, limit, align)
2455 }
2456
2457 pub fn find_area_snapshot(&self, vaddr: VirtAddr) -> Option<Arc<VmaSnapshot>> {
2459 self.vma_root.lookup(vaddr)
2460 }
2461
2462 pub fn vma_map_snapshot(&self) -> Arc<VmaMap> {
2464 self.vma_root.clone()
2465 }
2466
2467 pub fn vma_snapshots_in_range(
2469 &self,
2470 start: VirtAddr,
2471 size: usize,
2472 ) -> StarryResult<Vec<Arc<VmaSnapshot>>> {
2473 let range =
2474 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2475 if range.is_empty() {
2476 return Ok(Vec::new());
2477 }
2478 Ok(self.vma_root.lookup_range(range))
2479 }
2480
2481 pub(crate) fn vma_inspection_records(&self) -> StarryResult<Vec<VmaInspectionRecord>> {
2482 let mut records = Vec::new();
2483 records
2484 .try_reserve(self.vma_root.len())
2485 .map_err(|_| StarryError::NoMemory)?;
2486 for entry in self.vma_root.iter_entries() {
2487 records.push(entry.inspection_record()?);
2488 }
2489 Ok(records)
2490 }
2491
2492 pub(crate) fn max_mapped_end(&self) -> Option<VirtAddr> {
2493 self.vma_root.iter().last().map(|vma| vma.range.end)
2494 }
2495
2496 pub(crate) fn next_advice_fragment(
2497 &self,
2498 cursor: VirtAddr,
2499 end: VirtAddr,
2500 ) -> Option<VmaAdviceFragment> {
2501 self.vma_root
2502 .iter_entries()
2503 .find(|entry| entry.end() > cursor && entry.start() < end)
2504 .and_then(|entry| entry.advice_fragment(cursor, end))
2505 }
2506
2507 pub(crate) fn validate_mprotect_mapping_capabilities(
2508 &self,
2509 start: VirtAddr,
2510 size: usize,
2511 flags: MappingFlags,
2512 ) -> StarryResult<Vec<SharedFileMappingLease>> {
2513 let range =
2514 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2515 let operations = self.mapping_operation_fragments(range, true)?;
2516 let mut files = Vec::new();
2517 files
2518 .try_reserve(operations.len())
2519 .map_err(|_| StarryError::NoMemory)?;
2520 for (_, operation) in operations {
2521 operation.check_mprotect_flags(flags)?;
2522 if let Some(file) = operation.shared_file_lease() {
2523 files.push(file);
2524 }
2525 }
2526 Ok(files)
2527 }
2528
2529 pub(crate) fn shared_futex_identity(&self, address: VirtAddr) -> Option<SharedFutexIdentity> {
2530 self.vma_root
2531 .lookup_entry(address)
2532 .and_then(|entry| entry.operation().shared_futex_identity(address))
2533 }
2534
2535 pub(crate) fn mincore_probe(&self, address: VirtAddr) -> Option<VmaResidencyProbe> {
2536 self.vma_root
2537 .lookup_entry(address)
2538 .map(|entry| entry.residency_probe())
2539 }
2540
2541 pub(crate) fn mremap_source(&self, address: VirtAddr) -> Option<VmaMremapSource> {
2542 self.vma_root
2543 .lookup_entry(address)
2544 .map(|entry| entry.mremap_source())
2545 }
2546
2547 pub(crate) fn shared_file_vma_at(&self, address: VirtAddr) -> Option<SharedFileVmaRecord> {
2548 self.vma_root
2549 .lookup_entry(address)
2550 .and_then(|entry| entry.shared_file_record())
2551 }
2552
2553 pub(crate) fn shared_file_vmas(&self) -> Vec<SharedFileVmaRecord> {
2554 self.vma_root
2555 .iter_entries()
2556 .filter_map(|entry| entry.shared_file_record())
2557 .collect()
2558 }
2559
2560 #[allow(clippy::too_many_arguments)]
2561 pub(crate) fn mremap_move_from_source(
2562 &mut self,
2563 source: &VmaMremapSource,
2564 src: VirtAddr,
2565 src_size: usize,
2566 target: VirtAddr,
2567 target_size: usize,
2568 huge_page_advice: HugePageAdvice,
2569 dontunmap: bool,
2570 source_offset: usize,
2571 replace_target: bool,
2572 memlock_limit: Option<MemlockLimit>,
2573 ) -> StarryResult {
2574 let operation = source.relocated_operation(target, source_offset, target_size)?;
2575 self.mremap_move_transaction(
2576 src,
2577 src_size,
2578 target,
2579 target_size,
2580 MappingPermissions {
2581 current: source.rights(),
2582 reported: source.reported_rights(),
2583 maximum: source.max_rights(),
2584 },
2585 operation,
2586 huge_page_advice,
2587 source.lock_mode(),
2588 source.advice_policy(),
2589 dontunmap,
2590 replace_target,
2591 memlock_limit,
2592 )
2593 }
2594
2595 pub(crate) fn duplicate_shared_mremap_source(
2596 &mut self,
2597 source: &VmaMremapSource,
2598 target: VirtAddr,
2599 target_size: usize,
2600 source_offset: usize,
2601 replace_target: bool,
2602 memlock_limit: Option<MemlockLimit>,
2603 ) -> StarryResult {
2604 let object = source.shared_object().ok_or(StarryError::InvalidInput)?;
2605 let backend_start = target
2606 .as_usize()
2607 .checked_sub(source_offset)
2608 .map(VirtAddr::from_usize)
2609 .ok_or(StarryError::InvalidInput)?;
2610 self.map_mremap_duplicate(
2611 target,
2612 target_size,
2613 MappingPermissions {
2614 current: source.rights(),
2615 reported: source.reported_rights(),
2616 maximum: source.rights(),
2617 },
2618 MappingOperation::new_shared(backend_start, object),
2619 MappingPublication::mremap(
2620 replace_target,
2621 source.huge_page_advice(),
2622 source.lock_mode(),
2623 source.advice_policy(),
2624 memlock_limit,
2625 ),
2626 )
2627 }
2628
2629 fn validate_memlock_successor(
2630 &self,
2631 successor: &VmaMap,
2632 memlock_limit: Option<MemlockLimit>,
2633 ) -> StarryResult {
2634 let previous_locked = self.vma_root.locked_pages().ok_or(StarryError::BadState)?;
2635 let successor_locked = successor.locked_pages().ok_or(StarryError::BadState)?;
2636 if successor_locked <= previous_locked {
2637 return Ok(());
2638 }
2639 memlock_limit
2640 .ok_or(StarryError::BadState)?
2641 .validate(successor_locked)
2642 }
2643
2644 fn publish_vma_metadata_successor(
2645 &mut self,
2646 previous_root: Arc<VmaMap>,
2647 successor: VmaMap,
2648 operation: &'static str,
2649 ) -> StarryResult {
2650 let before_vmas = previous_root.len();
2651 let after_vmas = successor.len();
2652 let mut mutation = self.prepare_metadata_mutation();
2653 mutation.set_vma_delta(VmaDelta {
2654 split: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
2655 merged: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
2656 ..VmaDelta::default()
2657 });
2658 self.vma_root = Arc::new(successor);
2659 match self.commit_mutation_classified(mutation) {
2660 Ok(()) => Ok(()),
2661 Err(CommitMutationError::Unpublished(error)) => {
2662 self.vma_root = previous_root;
2663 Err(error)
2664 }
2665 Err(CommitMutationError::PublishedPendingTlb(error)) => {
2666 self.mutation_gate.mark_needs_repair();
2670 warn!(
2671 "metadata-only {operation} unexpectedly required TLB acknowledgement: {error}"
2672 );
2673 Err(StarryError::BadState)
2674 }
2675 }
2676 }
2677
2678 pub fn advise_huge_pages(
2685 &mut self,
2686 start: VirtAddr,
2687 size: usize,
2688 advice: HugePageAdvice,
2689 ) -> StarryResult {
2690 self.validate_region(start, size)?;
2691 let range =
2692 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2693 let previous_root = self.vma_root.clone();
2694 let affected = previous_root.lookup_range(range);
2695 if affected.is_empty() || !previous_root.contains_range(start, size) {
2696 return Err(StarryError::NoMemory);
2697 }
2698 if affected.iter().all(|vma| vma.huge_page_advice == advice) {
2699 return Ok(());
2700 }
2701
2702 let successor = previous_root
2703 .with_huge_page_advice(range, advice)
2704 .ok_or(StarryError::NoMemory)?;
2705 self.publish_vma_metadata_successor(previous_root, successor, "VMA THP advice")
2706 }
2707
2708 pub(crate) fn advise_vma_policy(
2711 &mut self,
2712 start: VirtAddr,
2713 size: usize,
2714 update: VmaAdviceUpdate,
2715 ) -> StarryResult {
2716 self.validate_region(start, size)?;
2717 let range =
2718 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2719 let previous_root = self.vma_root.clone();
2720 let affected = previous_root.lookup_range(range);
2721 if affected.is_empty() || !previous_root.contains_range(start, size) {
2722 return Err(StarryError::NoMemory);
2723 }
2724 if affected
2725 .iter()
2726 .all(|vma| vma.advice_policy.apply(update) == vma.advice_policy)
2727 {
2728 return Ok(());
2729 }
2730 let successor = previous_root
2731 .with_advice_update(range, update)
2732 .ok_or(StarryError::NoMemory)?;
2733 self.publish_vma_metadata_successor(previous_root, successor, "VMA madvise policy")
2734 }
2735
2736 fn set_vma_lock_mode(
2743 &mut self,
2744 start: VirtAddr,
2745 size: usize,
2746 lock_mode: VmaLockMode,
2747 memlock_limit: Option<MemlockLimit>,
2748 ) -> StarryResult {
2749 self.validate_region(start, size)?;
2750 let range =
2751 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
2752 let previous_root = self.vma_root.clone();
2753 let affected = previous_root.lookup_range(range);
2754 if affected.is_empty() || !previous_root.contains_range(start, size) {
2755 return Err(StarryError::NoMemory);
2756 }
2757 if affected.iter().all(|vma| vma.lock_mode == lock_mode) {
2758 return Ok(());
2759 }
2760
2761 let successor = previous_root
2762 .with_lock_mode(range, lock_mode)
2763 .ok_or(StarryError::NoMemory)?;
2764 self.validate_memlock_successor(&successor, memlock_limit)?;
2765 self.publish_vma_metadata_successor(previous_root, successor, "VMA lock update")
2766 }
2767
2768 pub(crate) fn lock_vma_range(
2769 &mut self,
2770 start: VirtAddr,
2771 size: usize,
2772 lock_mode: VmaLockMode,
2773 memlock_limit: MemlockLimit,
2774 ) -> StarryResult {
2775 if !lock_mode.is_locked() {
2776 return Err(StarryError::InvalidInput);
2777 }
2778 self.set_vma_lock_mode(start, size, lock_mode, Some(memlock_limit))
2779 }
2780
2781 pub(crate) fn unlock_vma_range(&mut self, start: VirtAddr, size: usize) -> StarryResult {
2782 self.set_vma_lock_mode(start, size, VmaLockMode::Unlocked, None)
2783 }
2784
2785 fn publish_prepared_pte_owners(
2797 &mut self,
2798 operation: &MappingOperation,
2799 range: VirtAddrRange,
2800 materialization: &PteMaterialization,
2801 ) -> StarryResult<PteOwnerPublication> {
2802 let owners = materialization.owners();
2803 let mut publications = Vec::new();
2804 publications
2805 .try_reserve(owners.len())
2806 .map_err(|_| StarryError::NoMemory)?;
2807 let mut seen = Vec::new();
2808 seen.try_reserve(owners.len())
2809 .map_err(|_| StarryError::NoMemory)?;
2810 let mut mapping_delta = MappingDelta::default();
2811 let mut resident_delta = ResidentDelta::default();
2812
2813 for owner in owners {
2817 let publication = self.prepare_slot_publication(operation, range, owner)?;
2818 if seen.contains(&publication.key) {
2819 return Err(StarryError::BadState);
2820 }
2821 seen.push(publication.key);
2822 mapping_delta.attached = mapping_delta
2823 .attached
2824 .checked_add(publication.mapping_delta.attached)
2825 .ok_or(StarryError::BadState)?;
2826 mapping_delta.detached = mapping_delta
2827 .detached
2828 .checked_add(publication.mapping_delta.detached)
2829 .ok_or(StarryError::BadState)?;
2830 resident_delta.checked_add_assign(publication.resident_delta)?;
2831 publications.push(publication);
2832 }
2833
2834 for publication in publications {
2835 self.apply_slot_publication(operation, publication)?;
2836 }
2837 Ok(PteOwnerPublication {
2838 satisfied_pages: materialization.satisfied_pages(),
2839 mapping_delta,
2840 resident_delta,
2841 })
2842 }
2843
2844 fn publish_prepared_fault_owner(
2845 &mut self,
2846 operation: &MappingOperation,
2847 range: VirtAddrRange,
2848 materialization: &FaultMaterialization,
2849 ) -> StarryResult<PteOwnerPublication> {
2850 let Some(owner) = materialization.owner() else {
2851 return Ok(PteOwnerPublication {
2852 satisfied_pages: materialization.satisfied_pages(),
2853 ..PteOwnerPublication::default()
2854 });
2855 };
2856 let publication = self.prepare_slot_publication(operation, range, owner)?;
2859 let mapping_delta = publication.mapping_delta;
2860 let resident_delta = publication.resident_delta;
2861 self.apply_slot_publication(operation, publication)?;
2862 Ok(PteOwnerPublication {
2863 satisfied_pages: materialization.satisfied_pages(),
2864 mapping_delta,
2865 resident_delta,
2866 })
2867 }
2868
2869 fn prepare_slot_publication(
2870 &mut self,
2871 operation: &MappingOperation,
2872 range: VirtAddrRange,
2873 owner: &PreparedPteOwner,
2874 ) -> StarryResult<PreparedSlotPublication> {
2875 let va = owner.va;
2876 let paddr = owner.paddr;
2877 let page_size = owner.page_size;
2878 let page = &owner.page;
2879 let resident_kind = owner.resident_kind;
2880 let transition = owner.transition;
2881 let provider_publication = owner.provider_publication;
2882 if page_size < PAGE_SIZE_4K || !page_size.is_power_of_two() || !va.is_aligned(page_size) {
2883 return Err(StarryError::BadState);
2884 }
2885 let leaf_range =
2886 VirtAddrRange::try_from_start_size(va, page_size).ok_or(StarryError::BadState)?;
2887 if !range.contains_range(leaf_range) {
2888 return Err(StarryError::BadState);
2889 }
2890 let frame_start = page.frame().paddr().as_usize();
2891 let frame_end = frame_start
2892 .checked_add(page.frame().size())
2893 .ok_or(StarryError::BadState)?;
2894 let leaf_start = paddr.as_usize();
2895 let leaf_end = leaf_start
2896 .checked_add(page_size)
2897 .ok_or(StarryError::BadState)?;
2898 if leaf_start < frame_start || leaf_end > frame_end {
2899 return Err(StarryError::BadState);
2900 }
2901 match self.pt.query(va) {
2902 Ok((installed, _, installed_size))
2903 if installed == paddr && installed_size == page_size => {}
2904 Ok(_) | Err(_) => return Err(StarryError::BadState),
2905 }
2906 if !matches!(page.state(), PageState::Present | PageState::LazyFree) {
2907 return Err(StarryError::BadState);
2908 }
2909
2910 let key = MappingSlotKey {
2911 space_id: self.id,
2912 va,
2913 };
2914 let previous = self.mapping_slots.get(&key).cloned();
2915 let order = page_size
2916 .trailing_zeros()
2917 .checked_sub(PAGE_SIZE_4K.trailing_zeros())
2918 .and_then(|order| u8::try_from(order).ok())
2919 .map(PageOrder::new)
2920 .ok_or(StarryError::BadState)?;
2921 let same_owner = previous.as_ref().is_some_and(|slot| {
2922 slot.state() == SlotState::Present
2923 && slot.mapping == operation.mapping_id()
2924 && slot.page_order == order
2925 && slot.mapped_paddr() == Some(paddr)
2926 && Arc::ptr_eq(&slot.page, page)
2927 && (order == PageOrder::BASE || slot.has_huge_split_deposit())
2928 });
2929 match transition {
2930 PteOwnerTransition::Updated if !same_owner => return Err(StarryError::BadState),
2931 PteOwnerTransition::Replaced if previous.is_none() || same_owner => {
2932 return Err(StarryError::BadState);
2933 }
2934 PteOwnerTransition::Installed
2935 | PteOwnerTransition::Replaced
2936 | PteOwnerTransition::Updated => {}
2937 }
2938
2939 let replacement = if same_owner {
2940 None
2941 } else {
2942 let split_deposit = if order == PageOrder::BASE {
2943 None
2944 } else {
2945 Some(self.pt.prepare_huge_split(va)?)
2946 };
2947 let frame_offset = paddr
2948 .as_usize()
2949 .checked_sub(page.frame().paddr().as_usize())
2950 .ok_or(StarryError::BadState)?;
2951 let slot = MappingSlot::new_with_frame_offset(
2952 operation.mapping_id(),
2953 self.id,
2954 va,
2955 order,
2956 page.clone(),
2957 frame_offset,
2958 resident_kind,
2959 )
2960 .ok_or(StarryError::BadState)?;
2961 let slot = match split_deposit {
2962 Some(deposit) => slot
2963 .attach_huge_split_deposit(deposit)
2964 .map_err(|_| StarryError::BadState)?,
2965 None => slot,
2966 };
2967 Some(Arc::new(slot))
2968 };
2969 let mut mapping_delta = MappingDelta::default();
2970 let mut resident_delta = ResidentDelta::default();
2971 if !same_owner {
2972 mapping_delta.attached = 1;
2973 mapping_delta.detached = u32::from(previous.is_some());
2974 }
2975 if let Some(previous) = &previous {
2976 let pages = 1i64
2977 .checked_shl(previous.page_order.get().into())
2978 .ok_or(StarryError::BadState)?;
2979 resident_delta
2980 .checked_add_assign(ResidentDelta::for_pages(previous.resident_kind(), -pages))?;
2981 }
2982 let pages = 1i64
2983 .checked_shl(order.get().into())
2984 .ok_or(StarryError::BadState)?;
2985 resident_delta.checked_add_assign(ResidentDelta::for_pages(resident_kind, pages))?;
2986 Ok(PreparedSlotPublication {
2987 key,
2988 previous,
2989 replacement,
2990 resident_kind,
2991 provider_publication,
2992 mapping_delta,
2993 resident_delta,
2994 })
2995 }
2996
2997 fn apply_slot_publication(
2998 &mut self,
2999 operation: &MappingOperation,
3000 publication: PreparedSlotPublication,
3001 ) -> StarryResult {
3002 let PreparedSlotPublication {
3003 key,
3004 previous,
3005 replacement,
3006 resident_kind,
3007 provider_publication,
3008 mapping_delta: _,
3009 resident_delta: _,
3010 } = publication;
3011 let Some(replacement) = replacement else {
3012 let current = self.mapping_slots.get(&key).ok_or(StarryError::BadState)?;
3013 if previous
3014 .as_ref()
3015 .is_none_or(|previous| !Arc::ptr_eq(previous, current))
3016 {
3017 return Err(StarryError::BadState);
3018 }
3019 current.set_resident_kind(resident_kind);
3020 current.page.set_resident_kind(resident_kind);
3021 if provider_publication == ProviderPublication::Pending {
3022 operation.finish_page_publication(key.va, ¤t.page)?;
3023 }
3024 return Ok(());
3025 };
3026
3027 if let Some(previous) = &previous {
3028 let Some(current) = self.mapping_slots.remove(&key) else {
3029 return Err(StarryError::BadState);
3030 };
3031 if !Arc::ptr_eq(previous, ¤t) || !current.detach() {
3032 self.mapping_slots.insert(key, current);
3033 return Err(StarryError::BadState);
3034 }
3035 } else if self.mapping_slots.contains_key(&key) {
3036 return Err(StarryError::BadState);
3037 }
3038
3039 if !replacement.publish() {
3040 if let Some(previous) = previous
3041 && (!previous.restore() || self.mapping_slots.insert(key, previous).is_some())
3042 {
3043 self.mutation_gate.mark_needs_repair();
3044 }
3045 return Err(StarryError::BadState);
3046 }
3047 if provider_publication == ProviderPublication::Pending
3048 && let Err(error) = operation.finish_page_publication(key.va, &replacement.page)
3049 {
3050 let replacement_detached = replacement.detach();
3051 let previous_restored = previous.is_none_or(|previous| {
3052 previous.restore() && self.mapping_slots.insert(key, previous).is_none()
3053 });
3054 if !replacement_detached || !previous_restored {
3055 self.mutation_gate.mark_needs_repair();
3056 return Err(StarryError::BadState);
3057 }
3058 return Err(error);
3059 }
3060 if self.mapping_slots.insert(key, replacement).is_some() {
3061 self.mutation_gate.mark_needs_repair();
3062 return Err(StarryError::BadState);
3063 }
3064 Ok(())
3065 }
3066
3067 fn detach_mapping_slots(&mut self, range: VirtAddrRange) -> StarryResult {
3068 let keys: Vec<_> = self
3069 .mapping_slots_overlapping(range)
3070 .map(|(key, _)| *key)
3071 .collect();
3072 for key in keys {
3073 if let Some(slot) = self.mapping_slots.remove(&key)
3074 && !slot.detach()
3075 {
3076 self.mapping_slots.insert(key, slot);
3077 return Err(StarryError::BadState);
3078 }
3079 }
3080 Ok(())
3081 }
3082
3083 pub(crate) fn evict_file_mapping_slot(
3087 &mut self,
3088 key: MappingSlotKey,
3089 page: &Arc<PageObject>,
3090 ) -> Result<EvictMappingOutcome, StarryError> {
3091 if key.space_id != self.id {
3092 return Err(StarryError::BadState);
3093 }
3094 let slot = self
3095 .mapping_slots
3096 .get(&key)
3097 .cloned()
3098 .ok_or(StarryError::BadState)?;
3099 if !Arc::ptr_eq(&slot.page, page) {
3100 return Err(StarryError::BadState);
3101 }
3102
3103 let range = VirtAddrRange::from_start_size(key.va, PAGE_SIZE_4K);
3104 let preimage = self.capture_mapping_preimage(range)?;
3105 let mut retired_owner = self.prepare_deferred_eviction_owner(page)?;
3106 let mut mutation = self.prepare_mutation_range(key.va, PAGE_SIZE_4K);
3107 let retire_epoch = mutation
3108 .receipt()
3109 .base_epoch
3110 .checked_next()
3111 .ok_or(StarryError::BadState)?;
3112 mutation.set_pte_delta(PteDelta {
3113 unmapped: 1,
3114 ..PteDelta::default()
3115 });
3116 mutation.set_mapping_delta(MappingDelta {
3117 detached: 1,
3118 ..MappingDelta::default()
3119 });
3120 mutation.set_resident_delta(ResidentDelta::for_pages(slot.resident_kind(), -1));
3121
3122 let unmap_plan = self.pt.plan_unmap_page(key.va)?;
3123 if slot.mapped_paddr() != Some(unmap_plan.paddr()) || unmap_plan.page_size() != PAGE_SIZE_4K
3124 {
3125 return Err(StarryError::BadState);
3126 }
3127 let apply_result = (|| -> StarryResult {
3128 let unmapped = {
3129 let pte_domain = &self.pte_domain;
3130 let pt = &mut self.pt;
3131 let range = VirtAddrRange::from_start_size(key.va, PAGE_SIZE_4K);
3132 let _structure = pte_domain.lock_structure();
3133 let _stripe = pte_domain.lock_range(range);
3134 pt.unmap_page_deferred(key.va)?
3135 };
3136 if slot.mapped_paddr() != Some(unmapped.0) || unmapped.2 != PAGE_SIZE_4K {
3137 return Err(StarryError::BadState);
3138 }
3139 if !unmapped.3.is_empty() {
3140 retired_owner.page_tables.push(unmapped.3);
3141 }
3142 let removed = self
3146 .mapping_slots
3147 .remove(&key)
3148 .ok_or(StarryError::BadState)?;
3149 if !Arc::ptr_eq(&removed, &slot) || !removed.detach() {
3150 return Err(StarryError::BadState);
3151 }
3152 Ok(())
3153 })();
3154 if let Err(error) = apply_result {
3155 return self.abort_file_eviction_mutation(range, preimage, None, error);
3156 }
3157
3158 self.park_retired_mapping_owners(retire_epoch, retired_owner);
3159 match self.commit_mutation_classified(mutation) {
3160 Ok(()) => {
3161 self.release_retired_mapping_owners(retire_epoch);
3162 Ok(EvictMappingOutcome::Complete)
3163 }
3164 Err(CommitMutationError::PublishedPendingTlb(_)) => {
3165 Ok(EvictMappingOutcome::PublishedPendingTlb)
3166 }
3167 Err(CommitMutationError::Unpublished(error)) => {
3168 self.abort_file_eviction_mutation(range, preimage, Some(retire_epoch), error)
3169 }
3170 }
3171 }
3172
3173 pub(crate) fn protect_file_mapping_slot(
3176 &mut self,
3177 key: MappingSlotKey,
3178 page: &Arc<PageObject>,
3179 ) -> StarryResult {
3180 if key.space_id != self.id {
3181 return Err(StarryError::BadState);
3182 }
3183 let slot = self.mapping_slots.get(&key).ok_or(StarryError::BadState)?;
3184 if !Arc::ptr_eq(&slot.page, page) {
3185 return Err(StarryError::BadState);
3186 }
3187
3188 let (paddr, flags, page_size) = self.pt.query(key.va)?;
3189 if slot.mapped_paddr() != Some(paddr) || page_size != PAGE_SIZE_4K {
3190 return Err(StarryError::BadState);
3191 }
3192 if !flags.contains(MappingFlags::WRITE) {
3193 return Ok(());
3194 }
3195 let mut mutation = self.prepare_mutation_range(key.va, PAGE_SIZE_4K);
3196 mutation.set_pte_delta(PteDelta {
3197 protected: 1,
3198 ..PteDelta::default()
3199 });
3200 {
3201 let pt = &mut self.pt;
3202 let _stripe = self
3203 .pte_domain
3204 .lock_range(VirtAddrRange::from_start_size(key.va, PAGE_SIZE_4K));
3205 pt.remap_page(key.va, paddr, flags - MappingFlags::WRITE)?;
3208 }
3209 self.commit_mutation(mutation)
3214 }
3215
3216 pub fn resident_mapping_slots(&self) -> Vec<Arc<MappingSlot>> {
3217 self.mapping_slots.values().cloned().collect()
3218 }
3219
3220 fn capture_mapping_graph_snapshot(
3221 &self,
3222 ranges: &[VirtAddrRange],
3223 ) -> StarryResult<MappingGraphSnapshot> {
3224 #[cfg(all(test, axtest))]
3225 MAPPING_GRAPH_SNAPSHOT_CALLS.fetch_add(1, core::sync::atomic::Ordering::Relaxed);
3226 let slot_count = ranges
3227 .iter()
3228 .map(|range| self.mapping_slots_overlapping(*range).count())
3229 .sum();
3230 let mut slots = Vec::new();
3231 slots
3232 .try_reserve(slot_count)
3233 .map_err(|_| StarryError::NoMemory)?;
3234 let mut seen = BTreeSet::new();
3235 let mut resident = ResidentPageCounts::default();
3236 for range in ranges {
3237 for (key, slot) in self.mapping_slots_overlapping(*range) {
3238 if slot.state() != SlotState::Present || !seen.insert(*key) {
3239 continue;
3240 }
3241 slots.push(MappingSlotFingerprint {
3242 key: *key,
3243 mapping: slot.mapping,
3244 page: slot.page.id,
3245 page_order: slot.page_order,
3246 });
3247 let pages = 1u64
3248 .checked_shl(u32::from(slot.page_order.get()))
3249 .ok_or(StarryError::BadState)?;
3250 match slot.resident_kind() {
3251 Some(RssKind::Anon) => {
3252 resident.anon = resident
3253 .anon
3254 .checked_add(pages)
3255 .ok_or(StarryError::BadState)?;
3256 }
3257 Some(RssKind::File) => {
3258 resident.file = resident
3259 .file
3260 .checked_add(pages)
3261 .ok_or(StarryError::BadState)?;
3262 }
3263 Some(RssKind::Shmem) => {
3264 resident.shmem = resident
3265 .shmem
3266 .checked_add(pages)
3267 .ok_or(StarryError::BadState)?;
3268 }
3269 None => {}
3270 }
3271 }
3272 }
3273 slots.sort_unstable();
3274 Ok(MappingGraphSnapshot { slots, resident })
3275 }
3276
3277 fn set_mapping_graph_receipt_delta(
3278 &self,
3279 mutation: &mut PreparedMutation,
3280 before: &MappingGraphSnapshot,
3281 ranges: &[VirtAddrRange],
3282 ) -> StarryResult {
3283 let after = self.capture_mapping_graph_snapshot(ranges)?;
3284 let (mapping_delta, resident_delta) = before.delta_to(&after)?;
3285 mutation.set_mapping_delta(mapping_delta);
3286 mutation.set_resident_delta(resident_delta);
3287 Ok(())
3288 }
3289
3290 pub(crate) fn resident_page_counts(&self) -> ResidentPageCounts {
3292 self.resident_pages
3293 }
3294
3295 #[cfg(all(test, axtest))]
3296 fn reset_mapping_graph_snapshot_calls_for_test(&self) {
3297 MAPPING_GRAPH_SNAPSHOT_CALLS.store(0, core::sync::atomic::Ordering::Relaxed);
3298 }
3299
3300 #[cfg(all(test, axtest))]
3301 fn mapping_graph_snapshot_calls_for_test(&self) -> usize {
3302 MAPPING_GRAPH_SNAPSHOT_CALLS.load(core::sync::atomic::Ordering::Relaxed)
3303 }
3304
3305 pub(crate) fn resident_hiwater_pages(&self) -> u64 {
3306 self.resident_watermark.hiwater_pages()
3307 }
3308
3309 fn mapping_operation_fragments(
3313 &self,
3314 range: VirtAddrRange,
3315 require_full_coverage: bool,
3316 ) -> StarryResult<Vec<(VirtAddrRange, MappingOperation)>> {
3317 let mut fragments = Vec::new();
3318 fragments
3319 .try_reserve(self.vma_root.len())
3320 .map_err(|_| StarryError::NoMemory)?;
3321 let mut covered = range.start;
3322 for entry in self.vma_root.iter_entries() {
3323 if entry.start() >= range.end {
3324 break;
3325 }
3326 if entry.end() <= range.start {
3327 continue;
3328 }
3329 let fragment =
3330 VirtAddrRange::new(entry.start().max(range.start), entry.end().min(range.end));
3331 if require_full_coverage && fragment.start > covered {
3332 return Err(StarryError::NoMemory);
3333 }
3334 covered = covered.max(fragment.end);
3335 fragments.push((fragment, entry.operation_clone()));
3336 }
3337 if require_full_coverage && covered < range.end {
3338 return Err(StarryError::NoMemory);
3339 }
3340 Ok(fragments)
3341 }
3342
3343 fn detach_materialized_operation(
3348 &mut self,
3349 range: VirtAddrRange,
3350 operation: &MappingOperation,
3351 ) -> bool {
3352 let Ok(occupied) = self.occupied_pte_leaves_overlapping(&[range]) else {
3353 return false;
3354 };
3355 let leaves: Vec<_> = occupied.into_iter().map(|leaf| leaf.range).collect();
3356 if leaves
3357 .iter()
3358 .any(|leaf| !operation.validate_unmap_range(*leaf, &self.pt))
3359 {
3360 return false;
3361 }
3362 let Ok(mut context) = MappingMutationContext::for_rollback(leaves.len()) else {
3363 return false;
3364 };
3365 let detached = leaves
3366 .into_iter()
3367 .all(|leaf| {
3368 operation
3369 .unmap_range(leaf, &mut context, &mut self.pt)
3370 .is_ok()
3371 });
3372 if !context.pte_changed() {
3373 return detached;
3374 }
3375 if crate::mm::flush_tlb_range_sync(range.start, range.size()).is_err() {
3376 return false;
3377 }
3378 unsafe { context.reclaim() };
3381 detached
3382 }
3383
3384 fn detach_current_materialized_ranges(&mut self, ranges: &[VirtAddrRange]) -> StarryResult {
3388 let leaves = self.occupied_pte_leaves_overlapping(ranges)?;
3389 let mut context = MappingMutationContext::for_rollback(leaves.len())?;
3390 let mut operations = Vec::new();
3391 operations
3392 .try_reserve(leaves.len())
3393 .map_err(|_| StarryError::NoMemory)?;
3394 for leaf in leaves {
3395 let operation = self
3396 .vma_root
3397 .lookup_entry(leaf.range.start)
3398 .map(|entry| entry.operation_clone())
3399 .ok_or(StarryError::BadState)?;
3400 operations.push((leaf.range, operation));
3401 }
3402 if operations
3403 .iter()
3404 .any(|(leaf, operation)| !operation.validate_unmap_range(*leaf, &self.pt))
3405 {
3406 return Err(StarryError::BadState);
3407 }
3408 for (leaf, operation) in operations {
3409 if let Err(error) = operation.unmap_range(leaf, &mut context, &mut self.pt) {
3410 for range in ranges {
3411 crate::mm::flush_tlb_range_sync(range.start, range.size())?;
3412 }
3413 unsafe { context.reclaim() };
3416 return Err(error);
3417 }
3418 }
3419 for range in ranges {
3420 crate::mm::flush_tlb_range_sync(range.start, range.size())?;
3421 }
3422 unsafe { context.reclaim() };
3425 Ok(())
3426 }
3427
3428 #[allow(clippy::too_many_arguments)]
3429 fn prepare_mapping_successor(
3430 &self,
3431 range: VirtAddrRange,
3432 permissions: MappingPermissions,
3433 operation: &MappingOperation,
3434 huge_page_advice: HugePageAdvice,
3435 lock_mode: VmaLockMode,
3436 advice_policy: VmaAdvicePolicy,
3437 replace: bool,
3438 ) -> StarryResult<VmaMap> {
3439 let entry = self
3440 .vma_root
3441 .prepare_mapping_entry(
3442 range,
3443 permissions.current,
3444 permissions.reported,
3445 permissions.maximum,
3446 huge_page_advice,
3447 lock_mode,
3448 advice_policy,
3449 operation.clone(),
3450 )
3451 .ok_or(StarryError::BadState)?;
3452 let successor = self.vma_root.with_mapping_entry(entry, replace).ok_or(
3453 if self.vma_root.overlaps(range) && !replace {
3454 StarryError::AlreadyExists
3455 } else {
3456 StarryError::BadState
3457 },
3458 )?;
3459 Ok(successor)
3460 }
3461
3462 fn apply_mapping_pages_unpublished(
3466 &mut self,
3467 range: VirtAddrRange,
3468 permissions: MappingPermissions,
3469 operation: &MappingOperation,
3470 replace: bool,
3471 context: &mut MappingMutationContext,
3472 ) -> StarryResult<PteMaterialization> {
3473 let replaced = if replace {
3474 self.mapping_operation_fragments(range, false)?
3475 } else {
3476 Vec::new()
3477 };
3478 if replaced
3479 .iter()
3480 .any(|(fragment, old)| !old.validate_unmap_range(*fragment, &self.pt))
3481 || (!replace && !operation.validate_map_range(range, &self.pt))
3482 {
3483 return Err(StarryError::BadState);
3484 }
3485 for (fragment, old) in &replaced {
3486 old.unmap_range(*fragment, context, &mut self.pt)?;
3487 }
3488 match operation.map_range(range, permissions.current, &mut self.pt) {
3489 Ok(materialization) => Ok(materialization),
3490 Err(error) => {
3491 if !self.detach_materialized_operation(range, operation) {
3492 self.mutation_gate.mark_needs_repair();
3493 return Err(StarryError::BadState);
3494 }
3495 Err(error)
3496 }
3497 }
3498 }
3499
3500 #[allow(clippy::too_many_arguments)]
3503 fn apply_mapping_unpublished(
3504 &mut self,
3505 range: VirtAddrRange,
3506 permissions: MappingPermissions,
3507 operation: &MappingOperation,
3508 huge_page_advice: HugePageAdvice,
3509 lock_mode: VmaLockMode,
3510 advice_policy: VmaAdvicePolicy,
3511 memlock_limit: Option<MemlockLimit>,
3512 replace: bool,
3513 context: &mut MappingMutationContext,
3514 ) -> StarryResult<PteMaterialization> {
3515 let successor = self.prepare_mapping_successor(
3516 range,
3517 permissions,
3518 operation,
3519 huge_page_advice,
3520 lock_mode,
3521 advice_policy,
3522 replace,
3523 )?;
3524 self.validate_memlock_successor(&successor, memlock_limit)?;
3525 let materialization = self.apply_mapping_pages_unpublished(
3526 range,
3527 permissions,
3528 operation,
3529 replace,
3530 context,
3531 )?;
3532 self.vma_root = Arc::new(successor);
3533 Ok(materialization)
3534 }
3535
3536 fn apply_unmap_pages_unpublished(
3537 &mut self,
3538 range: VirtAddrRange,
3539 context: &mut MappingMutationContext,
3540 ) -> StarryResult {
3541 let operations = self.mapping_operation_fragments(range, false)?;
3542 if operations
3543 .iter()
3544 .any(|(fragment, operation)| !operation.validate_unmap_range(*fragment, &self.pt))
3545 {
3546 return Err(StarryError::BadState);
3547 }
3548 for (fragment, operation) in operations {
3549 operation.unmap_range(fragment, context, &mut self.pt)?;
3550 }
3551 Ok(())
3552 }
3553
3554 fn apply_unmap_unpublished(
3558 &mut self,
3559 range: VirtAddrRange,
3560 context: &mut MappingMutationContext,
3561 ) -> StarryResult {
3562 let successor = self
3563 .vma_root
3564 .without_range(range)
3565 .ok_or(StarryError::BadState)?;
3566 self.apply_unmap_pages_unpublished(range, context)?;
3567 self.vma_root = Arc::new(successor);
3568 Ok(())
3569 }
3570
3571 fn apply_protection_unpublished(
3573 &mut self,
3574 range: VirtAddrRange,
3575 flags: MappingFlags,
3576 reported_flags: MappingFlags,
3577 ) -> StarryResult {
3578 let successor = self
3579 .vma_root
3580 .with_permissions(range, flags, reported_flags)
3581 .ok_or(StarryError::NoMemory)?;
3582 let operations = self.mapping_operation_fragments(range, true)?;
3583 if operations
3584 .iter()
3585 .any(|(fragment, operation)| !operation.validate_protect_range(*fragment, &self.pt))
3586 {
3587 return Err(StarryError::BadState);
3588 }
3589 for (fragment, operation) in operations {
3590 operation.protect_range(fragment, flags, &mut self.pt)?;
3591 }
3592 self.vma_root = Arc::new(successor);
3593 Ok(())
3594 }
3595
3596 fn apply_extend_unpublished(
3597 &mut self,
3598 address: VirtAddr,
3599 additional_size: usize,
3600 memlock_limit: Option<MemlockLimit>,
3601 ) -> StarryResult<(VirtAddrRange, MappingOperation, PteMaterialization)> {
3602 let entry = self
3603 .vma_root
3604 .lookup_entry(address)
3605 .ok_or(StarryError::InvalidInput)?;
3606 let suffix = VirtAddrRange::try_from_start_size(entry.end(), additional_size)
3607 .ok_or(StarryError::InvalidInput)?;
3608 let operation = entry.operation_clone();
3609 let flags = entry.rights();
3610 let successor = self
3611 .vma_root
3612 .with_extended_right(address, additional_size)
3613 .ok_or(StarryError::AlreadyExists)?;
3614 self.validate_memlock_successor(&successor, memlock_limit)?;
3615 if !operation.validate_map_range(suffix, &self.pt) {
3616 return Err(StarryError::BadState);
3617 }
3618 let materialization = match operation.map_range(suffix, flags, &mut self.pt) {
3619 Ok(materialization) => materialization,
3620 Err(error) => {
3621 if !self.detach_materialized_operation(suffix, &operation) {
3622 self.mutation_gate.mark_needs_repair();
3623 return Err(StarryError::BadState);
3624 }
3625 return Err(error);
3626 }
3627 };
3628 self.vma_root = Arc::new(successor);
3629 Ok((suffix, operation, materialization))
3630 }
3631
3632 pub fn map_linear(
3641 &mut self,
3642 start_vaddr: VirtAddr,
3643 start_paddr: PhysAddr,
3644 size: usize,
3645 flags: MappingFlags,
3646 ) -> StarryResult {
3647 self.validate_region(start_vaddr, size)?;
3648 let range = VirtAddrRange::from_start_size(start_vaddr, size);
3649 let preimage = self.capture_mapping_preimage(range)?;
3650 let graph_preimage = self.capture_mapping_graph_snapshot(&[range])?;
3651 let mut mutation = self.prepare_mutation_range(start_vaddr, size);
3652
3653 if !start_paddr.is_aligned_4k() {
3654 return Err(StarryError::InvalidInput);
3655 }
3656 if start_paddr.checked_add(size).is_none() {
3657 return Err(StarryError::InvalidInput);
3658 }
3659
3660 let operation = MappingOperation::new_linear(start_vaddr, start_paddr, false);
3661 let mut page_table_reclaims = MappingMutationContext::for_published_mutation(0)?;
3662 let materialization = match self.apply_mapping_unpublished(
3663 range,
3664 MappingPermissions {
3665 current: flags,
3666 reported: flags,
3667 maximum: flags,
3668 },
3669 &operation,
3670 HugePageAdvice::Default,
3671 VmaLockMode::Unlocked,
3672 VmaAdvicePolicy::default(),
3673 None,
3674 false,
3675 &mut page_table_reclaims,
3676 ) {
3677 Ok(materialization) => materialization,
3678 Err(error) => {
3679 return self.abort_unpublished_mapping_mutation(range, preimage, error);
3680 }
3681 };
3682 mutation.set_vma_delta(VmaDelta {
3683 inserted: 1,
3684 ..VmaDelta::default()
3685 });
3686 mutation.set_pte_delta(PteDelta {
3687 mapped: u32::try_from(size / PAGE_SIZE_4K).unwrap_or(u32::MAX),
3688 ..PteDelta::default()
3689 });
3690 self.vm_stat.on_map((size / PAGE_SIZE_4K) as u64);
3691 if let Err(error) = self.publish_prepared_pte_owners(&operation, range, &materialization) {
3692 return self.abort_unpublished_mapping_mutation(range, preimage, error);
3693 }
3694 if let Err(error) =
3695 self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[range])
3696 {
3697 return self.abort_unpublished_mapping_mutation(range, preimage, error);
3698 }
3699 match self.commit_mutation_classified(mutation) {
3700 Ok(()) => Ok(()),
3701 Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
3702 Err(CommitMutationError::Unpublished(error)) => {
3703 self.abort_unpublished_mapping_mutation(range, preimage, error)
3704 }
3705 }
3706 }
3707
3708 pub fn map(
3709 &mut self,
3710 start: VirtAddr,
3711 size: usize,
3712 flags: MappingFlags,
3713 populate: bool,
3714 backend: MappingOperation,
3715 ) -> StarryResult {
3716 self.map_with_reported_flags(start, size, flags, flags, populate, backend)
3717 }
3718
3719 pub(crate) fn map_outcome(
3726 &mut self,
3727 start: VirtAddr,
3728 size: usize,
3729 flags: MappingFlags,
3730 populate: bool,
3731 backend: MappingOperation,
3732 ) -> StarryResult<AddressSpaceMutationOutcome> {
3733 self.map_with_permissions_mode_classified(
3734 start,
3735 size,
3736 MappingPermissions {
3737 current: flags,
3738 reported: flags,
3739 maximum: flags,
3740 },
3741 populate,
3742 backend,
3743 MappingPublication::new(false),
3744 )
3745 }
3746
3747 pub fn map_with_reported_flags(
3748 &mut self,
3749 start: VirtAddr,
3750 size: usize,
3751 flags: MappingFlags,
3752 reported_flags: MappingFlags,
3753 populate: bool,
3754 backend: MappingOperation,
3755 ) -> StarryResult {
3756 self.map_with_permissions(
3757 start,
3758 size,
3759 MappingPermissions {
3760 current: flags,
3761 reported: reported_flags,
3762 maximum: flags,
3763 },
3764 populate,
3765 backend,
3766 )
3767 }
3768
3769 pub fn map_with_permissions_replace(
3776 &mut self,
3777 start: VirtAddr,
3778 size: usize,
3779 permissions: MappingPermissions,
3780 populate: bool,
3781 backend: MappingOperation,
3782 replace: bool,
3783 ) -> StarryResult {
3784 self.map_with_permissions_mode(
3785 start,
3786 size,
3787 permissions,
3788 populate,
3789 backend,
3790 MappingPublication::new(replace),
3791 )
3792 }
3793
3794 pub(crate) fn map_with_permissions_publication(
3798 &mut self,
3799 start: VirtAddr,
3800 size: usize,
3801 permissions: MappingPermissions,
3802 populate: bool,
3803 backend: MappingOperation,
3804 publication: MappingPublication,
3805 ) -> StarryResult {
3806 self.map_with_permissions_mode(start, size, permissions, populate, backend, publication)
3807 }
3808
3809 pub(crate) fn map_mremap_duplicate(
3813 &mut self,
3814 start: VirtAddr,
3815 size: usize,
3816 permissions: MappingPermissions,
3817 backend: MappingOperation,
3818 publication: MappingPublication,
3819 ) -> StarryResult {
3820 self.map_with_permissions_mode(start, size, permissions, false, backend, publication)
3821 }
3822
3823 pub fn map_with_permissions(
3828 &mut self,
3829 start: VirtAddr,
3830 size: usize,
3831 permissions: MappingPermissions,
3832 populate: bool,
3833 backend: MappingOperation,
3834 ) -> StarryResult {
3835 self.map_with_permissions_mode(
3836 start,
3837 size,
3838 permissions,
3839 populate,
3840 backend,
3841 MappingPublication::new(false),
3842 )
3843 }
3844
3845 fn map_with_permissions_mode(
3846 &mut self,
3847 start: VirtAddr,
3848 size: usize,
3849 permissions: MappingPermissions,
3850 populate: bool,
3851 backend: MappingOperation,
3852 publication: MappingPublication,
3853 ) -> StarryResult {
3854 self.map_with_permissions_mode_classified(
3855 start,
3856 size,
3857 permissions,
3858 populate,
3859 backend,
3860 publication,
3861 )?
3862 .into_result()
3863 }
3864
3865 fn map_with_permissions_mode_classified(
3866 &mut self,
3867 start: VirtAddr,
3868 size: usize,
3869 permissions: MappingPermissions,
3870 populate: bool,
3871 backend: MappingOperation,
3872 publication: MappingPublication,
3873 ) -> StarryResult<AddressSpaceMutationOutcome> {
3874 let MappingPublication {
3875 replace,
3876 huge_page_advice,
3877 lock_mode,
3878 advice_policy,
3879 memlock_limit,
3880 } = publication;
3881 self.validate_region(start, size)?;
3882 if !permissions.maximum.contains(permissions.current) {
3883 return Err(StarryError::PermissionDenied);
3884 }
3885 let range =
3886 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
3887 let mut mutation = prepare_mapping_publication_mutation(
3888 &self.mutation_gate,
3889 self.id,
3890 &self.tlb_targets,
3891 start,
3892 size,
3893 replace,
3894 );
3895 self.mutation_gate
3896 .validate_publish_preconditions(&mutation)
3897 .map_err(Self::map_unpublished_mutation_error)?;
3898 let removed_pages = if replace {
3901 self.vma_root
3902 .iter_entries()
3903 .filter(|entry| entry.start() < range.end && entry.end() > range.start)
3904 .try_fold(0u64, |pages, entry| {
3905 let lo = entry.start().max(range.start);
3906 let hi = entry.end().min(range.end);
3907 let fragment = hi.checked_sub_addr(lo).ok_or(StarryError::InvalidInput)?;
3908 pages
3909 .checked_add((fragment / PAGE_SIZE_4K) as u64)
3910 .ok_or(StarryError::InvalidInput)
3911 })?
3912 } else {
3913 0
3914 };
3915 let mapping_preimage = self.capture_mapping_preimage(range)?;
3916 let graph_preimage = self.capture_mapping_graph_snapshot(&[range])?;
3917 let retire_epoch = mutation
3918 .receipt()
3919 .base_epoch
3920 .checked_next()
3921 .ok_or(StarryError::BadState)?;
3922 let mut retired_owners = replace
3923 .then(|| self.prepare_retired_mapping_owners(range))
3924 .transpose()?;
3925 let mut page_table_reclaims = if replace {
3926 self.prepare_page_table_reclaims(&[range])?
3927 } else {
3928 MappingMutationContext::for_published_mutation(0)?
3929 };
3930
3931 let touched_memfds = if replace {
3936 crate::syscall::memfd_collect_metas_touching_mprotect_range(self, start, size)
3937 } else {
3938 Vec::new()
3939 };
3940
3941 let map_materialization = match self.apply_mapping_unpublished(
3942 range,
3943 permissions,
3944 &backend,
3945 huge_page_advice,
3946 lock_mode,
3947 advice_policy,
3948 memlock_limit,
3949 replace,
3950 &mut page_table_reclaims,
3951 ) {
3952 Ok(materialization) => materialization,
3953 Err(error) => {
3954 if let Some(owners) = retired_owners.take() {
3955 self.park_retired_mapping_owners(retire_epoch, owners);
3956 }
3957 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
3958 return self
3959 .abort_unpublished_parked_mapping_mutation(
3960 range,
3961 mapping_preimage,
3962 replace.then_some(retire_epoch),
3963 error,
3964 )
3965 .map(|()| AddressSpaceMutationOutcome::Complete);
3966 }
3967 };
3968 if let Some(owners) = retired_owners.take() {
3969 self.park_retired_mapping_owners(retire_epoch, owners);
3970 }
3971 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
3972 if removed_pages != 0
3973 && let Err(error) = self.detach_mapping_slots(range)
3974 {
3975 return self
3976 .abort_unpublished_parked_mapping_mutation(
3977 range,
3978 mapping_preimage,
3979 replace.then_some(retire_epoch),
3980 error,
3981 )
3982 .map(|()| AddressSpaceMutationOutcome::Complete);
3983 }
3984 if let Err(error) = self.publish_prepared_pte_owners(&backend, range, &map_materialization)
3985 {
3986 return self
3987 .abort_unpublished_parked_mapping_mutation(
3988 range,
3989 mapping_preimage,
3990 replace.then_some(retire_epoch),
3991 error,
3992 )
3993 .map(|()| AddressSpaceMutationOutcome::Complete);
3994 }
3995 if populate
3996 && let Err(populate_error) = self.apply_populate_area(start, size, permissions.current)
3997 {
3998 return self
3999 .abort_unpublished_parked_mapping_mutation(
4000 range,
4001 mapping_preimage,
4002 replace.then_some(retire_epoch),
4003 populate_error,
4004 )
4005 .map(|()| AddressSpaceMutationOutcome::Complete);
4006 }
4007 mutation.set_vma_delta(VmaDelta {
4008 inserted: 1,
4009 removed: u32::try_from(removed_pages).unwrap_or(u32::MAX),
4010 ..VmaDelta::default()
4011 });
4012 self.vm_stat.on_map((size / PAGE_SIZE_4K) as u64);
4013 if removed_pages != 0 {
4014 self.vm_stat.on_unmap(removed_pages);
4015 }
4016 if let Err(error) =
4017 self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[range])
4018 {
4019 return self
4020 .abort_unpublished_parked_mapping_mutation(
4021 range,
4022 mapping_preimage,
4023 replace.then_some(retire_epoch),
4024 error,
4025 )
4026 .map(|()| AddressSpaceMutationOutcome::Complete);
4027 }
4028 match self.commit_mutation_classified(mutation) {
4029 Ok(()) => {
4030 if replace {
4031 self.release_retired_mapping_owners(retire_epoch);
4032 crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
4033 self,
4034 &touched_memfds,
4035 );
4036 } else {
4037 crate::syscall::memfd_on_after_map(self, start);
4038 }
4039 Ok(AddressSpaceMutationOutcome::Complete)
4040 }
4041 Err(CommitMutationError::PublishedPendingTlb(error)) => {
4042 if replace {
4043 crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
4044 self,
4045 &touched_memfds,
4046 );
4047 } else {
4048 crate::syscall::memfd_on_after_map(self, start);
4049 }
4050 Ok(AddressSpaceMutationOutcome::PublishedPendingTlb(error))
4051 }
4052 Err(CommitMutationError::Unpublished(error)) => self
4053 .abort_unpublished_parked_mapping_mutation(
4054 range,
4055 mapping_preimage,
4056 replace.then_some(retire_epoch),
4057 error,
4058 )
4059 .map(|()| AddressSpaceMutationOutcome::Complete),
4060 }
4061 }
4062
4063 fn apply_populate_area(
4067 &mut self,
4068 mut start: VirtAddr,
4069 size: usize,
4070 access_flags: MappingFlags,
4071 ) -> StarryResult<usize> {
4072 self.validate_region(start, size)?;
4073 let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4074 let mut populated = 0usize;
4075
4076 loop {
4077 let area_end = {
4078 let Some(entry) = self.vma_root.lookup_entry(start) else {
4079 break;
4080 };
4081 let entry_end = entry.end();
4082 let range = VirtAddrRange::new(start, entry_end.min(end));
4083 let flags = entry.rights();
4084 let backend = entry.operation_clone();
4085 let request = PopulateRequest::area(range, backend.page_size())?;
4086 let materialization =
4087 backend.populate(self.id, request, flags, access_flags, &mut self.pt)?;
4088 let publication =
4089 self.publish_prepared_pte_owners(&backend, range, &materialization)?;
4090 populated = populated
4091 .checked_add(publication.satisfied_pages)
4092 .ok_or(StarryError::NoMemory)?;
4093 entry_end
4094 };
4095 start = area_end;
4096 assert!(start.is_aligned_4k());
4097 if start >= end {
4098 break;
4099 }
4100 }
4101
4102 if start < end {
4103 return Err(StarryError::NoMemory);
4105 }
4106
4107 Ok(populated)
4108 }
4109
4110 fn materialized_range_satisfies_access(
4119 &self,
4120 range: VirtAddrRange,
4121 access_flags: MappingFlags,
4122 ) -> bool {
4123 let required = access_flags | MappingFlags::USER;
4124 let mut cursor = range.start;
4125 while cursor < range.end {
4126 let Ok((_, flags, leaf_size)) = self.pt.query(cursor) else {
4127 return false;
4128 };
4129 if leaf_size < PAGE_SIZE_4K || !leaf_size.is_power_of_two() || !flags.contains(required)
4130 {
4131 return false;
4132 }
4133 let Some(leaf_end) = cursor.align_down(leaf_size).checked_add(leaf_size) else {
4134 return false;
4135 };
4136 if leaf_end <= cursor {
4137 return false;
4138 }
4139 cursor = leaf_end.min(range.end);
4140 }
4141 true
4142 }
4143
4144 pub fn populate_area(
4146 &mut self,
4147 start: VirtAddr,
4148 size: usize,
4149 access_flags: MappingFlags,
4150 ) -> StarryResult {
4151 let range =
4152 VirtAddrRange::try_from_start_size(start, size).ok_or(StarryError::InvalidInput)?;
4153 self.validate_region(start, size)?;
4154 if self.can_access_range(start, size, access_flags)
4155 && self.materialized_range_satisfies_access(range, access_flags)
4156 {
4157 return Ok(());
4158 }
4159 let preimage = self.capture_mapping_preimage(range)?;
4160 let graph_preimage = self.capture_mapping_graph_snapshot(&[range])?;
4161 let mut mutation = self.prepare_mutation_range(start, size);
4162 let retire_epoch = mutation
4163 .receipt()
4164 .base_epoch
4165 .checked_next()
4166 .ok_or(StarryError::BadState)?;
4167 let retired_owners = (!graph_preimage.slots.is_empty())
4168 .then(|| self.prepare_retired_mapping_owners(range))
4169 .transpose()?;
4170 let populated = match self.apply_populate_area(start, size, access_flags) {
4171 Ok(populated) => populated,
4172 Err(populate_error) => {
4173 if self.restore_mapping_preimage(range, preimage).is_err() {
4174 return Err(StarryError::BadState);
4175 }
4176 return Err(populate_error);
4177 }
4178 };
4179
4180 mutation.set_pte_delta(PteDelta {
4181 mapped: u32::try_from(populated).unwrap_or(u32::MAX),
4182 ..PteDelta::default()
4183 });
4184 if let Err(error) =
4185 self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[range])
4186 {
4187 return self.abort_unpublished_mapping_mutation(range, preimage, error);
4188 }
4189 if let Some(owners) = retired_owners {
4190 self.park_retired_mapping_owners(retire_epoch, owners);
4191 }
4192 match self.commit_mutation_classified(mutation) {
4193 Ok(()) => {
4194 self.release_retired_mapping_owners(retire_epoch);
4195 Ok(())
4196 }
4197 Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
4198 Err(CommitMutationError::Unpublished(error)) => self
4199 .abort_unpublished_parked_mapping_mutation(
4200 range,
4201 preimage,
4202 Some(retire_epoch),
4203 error,
4204 ),
4205 }
4206 }
4207
4208 pub fn discard_range(&mut self, start: VirtAddr, size: usize) -> StarryResult {
4211 self.validate_region(start, size)?;
4212 let retired_range = VirtAddrRange::from_start_size(start, size);
4213 let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4214
4215 let mut frags: alloc::vec::Vec<(VirtAddrRange, MappingOperation)> = alloc::vec::Vec::new();
4216 frags
4217 .try_reserve(self.vma_root.len())
4218 .map_err(|_| StarryError::NoMemory)?;
4219 let mut covered = start;
4220 for entry in self.vma_root.iter_entries() {
4221 if entry.start() >= end {
4222 break;
4223 }
4224 if entry.end() <= start {
4225 continue;
4226 }
4227 let frag_start = entry.start().max(start);
4228 let frag_end = entry.end().min(end);
4229 if frag_start > covered {
4230 return Err(StarryError::NoMemory);
4231 }
4232 let backend = entry.operation_clone();
4233 backend.validate_discard_fragment(VirtAddrRange::new(frag_start, frag_end))?;
4237 frags.push((VirtAddrRange::new(frag_start, frag_end), backend));
4238 covered = frag_end;
4239 }
4240 if covered < end {
4241 return Err(StarryError::NoMemory);
4242 }
4243
4244 let mut mutation = self.prepare_mutation_range(start, size);
4245 mutation
4246 .try_reserve_tlb_ranges(2)
4247 .map_err(|_| StarryError::NoMemory)?;
4248 let retire_epoch = mutation
4249 .receipt()
4250 .base_epoch
4251 .checked_next()
4252 .ok_or(StarryError::BadState)?;
4253 let splits = self.apply_partial_huge_splits(retired_range)?;
4254 for index in 0..splits.len() {
4255 let split = &splits[index];
4256 let Some(tlb_range) =
4257 TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
4258 else {
4259 return self.abort_unpublished_huge_splits(splits, StarryError::BadState);
4260 };
4261 mutation.add_tlb_range(tlb_range);
4262 }
4263 if frags
4264 .iter()
4265 .any(|(range, backend)| !backend.validate_unmap_range(*range, &self.pt))
4266 {
4267 return self.abort_unpublished_huge_splits(splits, StarryError::OperationNotSupported);
4268 }
4269
4270 let preimage = match self.capture_mapping_preimage(retired_range) {
4271 Ok(preimage) => preimage,
4272 Err(error) => return self.abort_unpublished_huge_splits(splits, error),
4273 };
4274 let retired_owners = match self.prepare_retired_mapping_owners(retired_range) {
4275 Ok(owners) => owners,
4276 Err(error) => {
4277 return self.abort_unpublished_split_mapping_mutation(
4278 retired_range,
4279 preimage,
4280 None,
4281 splits,
4282 error,
4283 );
4284 }
4285 };
4286 let mut page_table_reclaims = match self.prepare_page_table_reclaims(&[retired_range]) {
4287 Ok(reclaims) => reclaims,
4288 Err(error) => {
4289 return self.abort_unpublished_split_mapping_mutation(
4290 retired_range,
4291 preimage,
4292 None,
4293 splits,
4294 error,
4295 );
4296 }
4297 };
4298 let Ok((detached_slots, retired_pages, retired_resident)) =
4299 self.mapping_slot_summary(retired_range)
4300 else {
4301 return self.abort_unpublished_split_mapping_mutation(
4302 retired_range,
4303 preimage,
4304 None,
4305 splits,
4306 StarryError::BadState,
4307 );
4308 };
4309 let split_slots = splits.iter().try_fold(0usize, |slots, split| {
4310 slots.checked_add(split.child_slots.len().saturating_sub(1))
4311 });
4312 let Some(split_slots) = split_slots else {
4313 return self.abort_unpublished_split_mapping_mutation(
4314 retired_range,
4315 preimage,
4316 None,
4317 splits,
4318 StarryError::BadState,
4319 );
4320 };
4321
4322 for (range, backend) in frags {
4323 if let Err(error) =
4324 backend.unmap_range(range, &mut page_table_reclaims, &mut self.pt)
4325 {
4326 self.park_retired_mapping_owners(retire_epoch, retired_owners);
4327 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4328 return self.abort_unpublished_split_mapping_mutation(
4329 retired_range,
4330 preimage,
4331 Some(retire_epoch),
4332 splits,
4333 error,
4334 );
4335 }
4336 }
4337 self.park_retired_mapping_owners(retire_epoch, retired_owners);
4338 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4339 if let Err(error) = self.detach_mapping_slots(retired_range) {
4340 return self.abort_unpublished_split_mapping_mutation(
4341 retired_range,
4342 preimage,
4343 Some(retire_epoch),
4344 splits,
4345 error,
4346 );
4347 }
4348 mutation.set_pte_delta(PteDelta {
4349 unmapped: u32::try_from(retired_pages).unwrap_or(u32::MAX),
4350 ..PteDelta::default()
4351 });
4352 mutation.set_mapping_delta(MappingDelta {
4353 attached: u32::try_from(split_slots).unwrap_or(u32::MAX),
4354 detached: u32::try_from(detached_slots).unwrap_or(u32::MAX),
4355 });
4356 mutation.set_resident_delta(retired_resident.checked_negated_delta()?);
4357
4358 match self.commit_mutation_classified(mutation) {
4359 Ok(()) => {
4360 self.release_retired_mapping_owners(retire_epoch);
4361 Ok(())
4362 }
4363 Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
4364 Err(CommitMutationError::Unpublished(error)) => self
4365 .abort_unpublished_split_mapping_mutation(
4366 retired_range,
4367 preimage,
4368 Some(retire_epoch),
4369 splits,
4370 error,
4371 ),
4372 }
4373 }
4374
4375 pub fn mark_lazy_free(&mut self, start: VirtAddr, size: usize) -> StarryResult {
4383 self.validate_region(start, size)?;
4384 let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4385 let mut covered = start;
4386 for entry in self.vma_root.iter_entries() {
4387 if entry.start() >= end {
4388 break;
4389 }
4390 if entry.end() <= start {
4391 continue;
4392 }
4393 let fragment_start = entry.start().max(start);
4394 if fragment_start > covered {
4395 return Err(StarryError::NoMemory);
4396 }
4397 if !entry.operation().is_private_anonymous() {
4398 return Err(StarryError::InvalidInput);
4399 }
4400 covered = entry.end().min(end);
4401 }
4402 if covered < end {
4403 return Err(StarryError::NoMemory);
4404 }
4405
4406 let range = VirtAddrRange::from_start_size(start, size);
4407 let mut candidates = Vec::new();
4408 candidates
4409 .try_reserve(self.mapping_slots_overlapping(range).count())
4410 .map_err(|_| StarryError::NoMemory)?;
4411 for (_, slot) in self.mapping_slots_overlapping(range) {
4412 if slot.page_order != PageOrder::BASE || slot.page.mapping_refs() != 1 {
4413 continue;
4414 }
4415 match slot.page.state() {
4416 PageState::LazyFree => continue,
4417 PageState::Present => {}
4418 PageState::Reserved
4419 | PageState::Evicting
4420 | PageState::Writeback
4421 | PageState::Retired => return Err(StarryError::ResourceBusy),
4422 }
4423 let (paddr, flags, page_size) = self.pt.query(slot.va)?;
4424 if slot.mapped_paddr() != Some(paddr) || page_size != PAGE_SIZE_4K {
4425 return Err(StarryError::BadState);
4426 }
4427 candidates.push((slot.va, paddr, flags, slot.page.clone()));
4428 }
4429 if candidates.is_empty() {
4430 return Ok(());
4431 }
4432
4433 let mut mutation = self.prepare_mutation_range(start, size);
4434 let pt = &mut self.pt;
4435 let stripes = self.pte_domain.lock_range(range);
4436 let mut protected = 0usize;
4437 for &(va, paddr, flags, _) in &candidates {
4438 if !flags.contains(MappingFlags::WRITE) {
4439 continue;
4440 }
4441 if pt
4444 .remap_page(va, paddr, flags - MappingFlags::WRITE)
4445 .is_err()
4446 {
4447 for &(old_va, old_paddr, old_flags, _) in candidates.iter().rev() {
4448 if old_flags.contains(MappingFlags::WRITE) {
4449 let _ = pt.remap_page(old_va, old_paddr, old_flags);
4451 }
4452 }
4453 return Err(StarryError::BadState);
4454 }
4455 protected += 1;
4456 }
4457 drop(stripes);
4458
4459 for (marked, (_, _, _, page)) in candidates.iter().enumerate() {
4460 if !page.mark_lazy_free() {
4461 for (_, _, _, marked_page) in candidates[..marked].iter().rev() {
4462 let _ = marked_page.clear_lazy_free();
4463 }
4464 let pt = &mut self.pt;
4465 let _stripes = self.pte_domain.lock_range(range);
4466 for &(va, paddr, flags, _) in &candidates {
4467 if flags.contains(MappingFlags::WRITE) {
4468 let _ = pt.remap_page(va, paddr, flags);
4470 }
4471 }
4472 return Err(StarryError::ResourceBusy);
4473 }
4474 }
4475
4476 mutation.set_pte_delta(PteDelta {
4477 protected: u32::try_from(protected).unwrap_or(u32::MAX),
4478 ..PteDelta::default()
4479 });
4480 match self.commit_mutation_classified(mutation) {
4481 Ok(()) => {
4482 lifecycle::request_lazy_free_reclaim();
4483 Ok(())
4484 }
4485 Err(CommitMutationError::PublishedPendingTlb(error)) => {
4486 lifecycle::request_lazy_free_reclaim();
4489 Err(error)
4490 }
4491 Err(CommitMutationError::Unpublished(error)) => {
4492 for (_, _, _, page) in candidates.iter().rev() {
4493 if !page.clear_lazy_free() {
4494 self.mutation_gate.mark_needs_repair();
4495 return Err(StarryError::BadState);
4496 }
4497 }
4498 let pt = &mut self.pt;
4499 let _stripes = self.pte_domain.lock_range(range);
4500 for &(va, paddr, flags, _) in &candidates {
4501 if flags.contains(MappingFlags::WRITE)
4502 && pt.remap_page(va, paddr, flags).is_err()
4503 {
4504 self.mutation_gate.mark_needs_repair();
4505 return Err(StarryError::BadState);
4506 }
4507 }
4508 Err(error)
4509 }
4510 }
4511 }
4512
4513 pub(crate) fn reclaim_lazy_free_pages(&mut self, limit: usize) -> StarryResult<usize> {
4518 if limit == 0 {
4519 return Ok(0);
4520 }
4521 let mut reclaimed = 0;
4522 let mut cursor = None;
4523 while reclaimed < limit {
4524 let eligible = |(key, slot): (&MappingSlotKey, &Arc<MappingSlot>)| {
4525 (slot.page_order == PageOrder::BASE
4526 && slot.page.state() == PageState::LazyFree
4527 && slot.page.mapping_refs() == 1)
4528 .then(|| (*key, slot.page.clone()))
4529 };
4530 let candidate = if let Some(after) = cursor {
4531 self.mapping_slots
4532 .range((
4533 core::ops::Bound::Excluded(after),
4534 core::ops::Bound::Unbounded,
4535 ))
4536 .find_map(eligible)
4537 } else {
4538 self.mapping_slots.iter().find_map(eligible)
4539 };
4540 let Some((key, page)) = candidate else {
4541 break;
4542 };
4543 cursor = Some(key);
4544 let Some(entry) = self.vma_root.lookup_entry(key.va) else {
4545 return Err(StarryError::BadState);
4546 };
4547 if !entry.operation().is_private_anonymous() {
4548 return Err(StarryError::BadState);
4549 }
4550 let still_reclaimable = self.mapping_slots.get(&key).is_some_and(|slot| {
4551 Arc::ptr_eq(&slot.page, &page)
4552 && page.state() == PageState::LazyFree
4553 && page.mapping_refs() == 1
4554 });
4555 if !still_reclaimable {
4556 continue;
4557 }
4558 self.discard_range(key.va, PAGE_SIZE_4K)?;
4559 if !page.rmap.is_empty()
4560 || page.mapping_refs() != 0
4561 || !page.transition(PageState::LazyFree, PageState::Retired)
4562 {
4563 self.mutation_gate.mark_needs_repair();
4564 return Err(StarryError::BadState);
4565 }
4566 reclaimed += 1;
4567 }
4568 Ok(reclaimed)
4569 }
4570
4571 pub fn unmap(&mut self, start: VirtAddr, size: usize) -> StarryResult {
4576 self.unmap_classified(start, size)?.into_result()
4577 }
4578
4579 pub(crate) fn unmap_outcome(
4584 &mut self,
4585 start: VirtAddr,
4586 size: usize,
4587 ) -> StarryResult<AddressSpaceMutationOutcome> {
4588 self.unmap_classified(start, size)
4589 }
4590
4591 fn unmap_classified(
4592 &mut self,
4593 start: VirtAddr,
4594 size: usize,
4595 ) -> StarryResult<AddressSpaceMutationOutcome> {
4596 self.validate_region(start, size)?;
4597 let range = VirtAddrRange::from_start_size(start, size);
4598 let before_vmas = self.vma_root.len();
4599 let memfd_deltas = crate::syscall::memfd_prepare_aspace_unmap_deltas(self, start, size);
4603 let mut mutation = self.prepare_mutation_range(start, size);
4604 let retire_epoch = mutation
4605 .receipt()
4606 .base_epoch
4607 .checked_next()
4608 .ok_or(StarryError::BadState)?;
4609 mutation
4610 .try_reserve_tlb_ranges(2)
4611 .map_err(|_| StarryError::NoMemory)?;
4612 let splits = self.apply_partial_huge_splits(range)?;
4613 for index in 0..splits.len() {
4614 let split = &splits[index];
4615 let Some(tlb_range) =
4616 TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
4617 else {
4618 return self
4619 .abort_unpublished_huge_splits(splits, StarryError::BadState)
4620 .map(|()| AddressSpaceMutationOutcome::Complete);
4621 };
4622 mutation.add_tlb_range(tlb_range);
4623 }
4624 let preimage = match self.capture_mapping_preimage(range) {
4625 Ok(preimage) => preimage,
4626 Err(error) => {
4627 return self
4628 .abort_unpublished_huge_splits(splits, error)
4629 .map(|()| AddressSpaceMutationOutcome::Complete);
4630 }
4631 };
4632 let retired_owners = match self.prepare_retired_mapping_owners(range) {
4633 Ok(owners) => owners,
4634 Err(error) => {
4635 return self
4636 .abort_unpublished_huge_splits(splits, error)
4637 .map(|()| AddressSpaceMutationOutcome::Complete);
4638 }
4639 };
4640 let mut page_table_reclaims = match self.prepare_page_table_reclaims(&[range]) {
4641 Ok(reclaims) => reclaims,
4642 Err(error) => {
4643 return self
4644 .abort_unpublished_split_mapping_mutation(
4645 range,
4646 preimage,
4647 None,
4648 splits,
4649 error,
4650 )
4651 .map(|()| AddressSpaceMutationOutcome::Complete);
4652 }
4653 };
4654 let Ok((detached_slots, detached_resident_pages, detached_resident)) =
4655 self.mapping_slot_summary(range)
4656 else {
4657 return self
4658 .abort_unpublished_huge_splits(splits, StarryError::BadState)
4659 .map(|()| AddressSpaceMutationOutcome::Complete);
4660 };
4661 let split_slots = splits.iter().try_fold(0usize, |slots, split| {
4662 slots.checked_add(split.child_slots.len().saturating_sub(1))
4663 });
4664 let Some(split_slots) = split_slots else {
4665 return self
4666 .abort_unpublished_huge_splits(splits, StarryError::BadState)
4667 .map(|()| AddressSpaceMutationOutcome::Complete);
4668 };
4669
4670 let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
4672 let removed_pages: u64 = self
4673 .vma_root
4674 .iter_entries()
4675 .filter(|entry| entry.start() < end && entry.end() > start)
4676 .map(|entry| {
4677 let lo = entry.start().max(start);
4678 let hi = entry.end().min(end);
4679 ((hi - lo) / PAGE_SIZE_4K) as u64
4680 })
4681 .sum();
4682
4683 if let Err(error) = self.apply_unmap_unpublished(range, &mut page_table_reclaims) {
4684 self.park_retired_mapping_owners(retire_epoch, retired_owners);
4685 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4686 return self
4687 .abort_unpublished_split_mapping_mutation(
4688 range,
4689 preimage,
4690 Some(retire_epoch),
4691 splits,
4692 error,
4693 )
4694 .map(|()| AddressSpaceMutationOutcome::Complete);
4695 }
4696 self.park_retired_mapping_owners(retire_epoch, retired_owners);
4697 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
4698 if let Err(error) = self.detach_mapping_slots(range) {
4699 return self
4700 .abort_unpublished_split_mapping_mutation(
4701 range,
4702 preimage,
4703 Some(retire_epoch),
4704 splits,
4705 error,
4706 )
4707 .map(|()| AddressSpaceMutationOutcome::Complete);
4708 }
4709 mutation.set_pte_delta(PteDelta {
4710 unmapped: u32::try_from(detached_resident_pages).unwrap_or(u32::MAX),
4711 ..PteDelta::default()
4712 });
4713 mutation.set_mapping_delta(MappingDelta {
4714 attached: u32::try_from(split_slots).unwrap_or(u32::MAX),
4715 detached: u32::try_from(detached_slots).unwrap_or(u32::MAX),
4716 });
4717 mutation.set_resident_delta(detached_resident.checked_negated_delta()?);
4718 self.vm_stat.on_unmap(removed_pages);
4719 let after_vmas = self.vma_root.len();
4720 mutation.set_vma_delta(VmaDelta {
4721 removed: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
4722 split: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
4723 ..VmaDelta::default()
4724 });
4725 match self.commit_mutation_classified(mutation) {
4726 Ok(()) => {
4727 self.release_retired_mapping_owners(retire_epoch);
4728 crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
4729 Ok(AddressSpaceMutationOutcome::Complete)
4730 }
4731 Err(CommitMutationError::PublishedPendingTlb(error)) => {
4732 crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
4733 Ok(AddressSpaceMutationOutcome::PublishedPendingTlb(error))
4734 }
4735 Err(CommitMutationError::Unpublished(error)) => self
4736 .abort_unpublished_split_mapping_mutation(
4737 range,
4738 preimage,
4739 Some(retire_epoch),
4740 splits,
4741 error,
4742 )
4743 .map(|()| AddressSpaceMutationOutcome::Complete),
4744 }
4745 }
4746
4747 fn prepare_moved_slots(
4748 &mut self,
4749 moved_pages: &[MovedPage],
4750 target_mapping: MappingId,
4751 ) -> StarryResult<Vec<PreparedMovedSlot>> {
4752 let mut prepared = Vec::new();
4753 prepared
4754 .try_reserve(moved_pages.len())
4755 .map_err(|_| StarryError::NoMemory)?;
4756
4757 for moved in moved_pages {
4758 let source_key = MappingSlotKey {
4759 space_id: self.id,
4760 va: moved.src_va,
4761 };
4762 let target_slot_va = match moved.destination {
4763 MovedPageDestination::SourceOwner => moved.dst_va,
4764 MovedPageDestination::TargetOwner { slot_va } => slot_va,
4765 };
4766 let target_key = MappingSlotKey {
4767 space_id: self.id,
4768 va: target_slot_va,
4769 };
4770 if matches!(moved.destination, MovedPageDestination::SourceOwner)
4771 && prepared.iter().any(|entry| {
4772 matches!(
4773 entry,
4774 PreparedMovedSlot::Relocate {
4775 target_key: existing,
4776 ..
4777 } if *existing == target_key
4778 )
4779 })
4780 {
4781 return Err(StarryError::BadState);
4782 }
4783 let source = self
4784 .mapping_slots
4785 .get(&source_key)
4786 .cloned()
4787 .ok_or(StarryError::BadState)?;
4788 let page_order = moved
4789 .page_size
4790 .trailing_zeros()
4791 .checked_sub(PAGE_SIZE_4K.trailing_zeros())
4792 .and_then(|order| u8::try_from(order).ok())
4793 .map(PageOrder::new)
4794 .ok_or(StarryError::BadState)?;
4795 let frame_start = source.page.frame().paddr().as_usize();
4796 let frame_end = frame_start
4797 .checked_add(source.page.frame().size())
4798 .ok_or(StarryError::BadState)?;
4799 let leaf_start = moved.paddr.as_usize();
4800 let leaf_end = leaf_start
4801 .checked_add(moved.page_size)
4802 .ok_or(StarryError::BadState)?;
4803 if source.state() != SlotState::Present
4804 || source.mm_id != self.id
4805 || source.va != moved.src_va
4806 || source.page_order != page_order
4807 || source.mapped_paddr() != Some(moved.paddr)
4808 || leaf_start < frame_start
4809 || leaf_end > frame_end
4810 || (page_order != PageOrder::BASE && !source.has_huge_split_deposit())
4811 {
4812 return Err(StarryError::BadState);
4813 }
4814
4815 match moved.destination {
4816 MovedPageDestination::SourceOwner => {
4817 let replacement = MappingSlot::new_with_frame_offset(
4818 target_mapping,
4819 self.id,
4820 moved.dst_va,
4821 page_order,
4822 source.page.clone(),
4823 source.frame_offset(),
4824 source.resident_kind(),
4825 )
4826 .ok_or(StarryError::BadState)?;
4827 let replacement = if page_order == PageOrder::BASE {
4828 replacement
4829 } else {
4830 replacement
4831 .attach_huge_split_deposit(self.pt.prepare_huge_split(moved.dst_va)?)
4832 .map_err(|_| StarryError::BadState)?
4833 };
4834 prepared.push(PreparedMovedSlot::Relocate {
4835 source_key,
4836 target_key,
4837 source,
4838 replacement: Arc::new(replacement),
4839 });
4840 }
4841 MovedPageDestination::TargetOwner { .. } => {
4842 prepared.push(PreparedMovedSlot::DetachSource {
4843 source_key,
4844 target_key,
4845 source,
4846 });
4847 }
4848 }
4849 }
4850 Ok(prepared)
4851 }
4852
4853 fn publish_moved_slots(&mut self, prepared: Vec<PreparedMovedSlot>) -> StarryResult {
4854 for entry in prepared {
4855 match entry {
4856 PreparedMovedSlot::Relocate {
4857 source_key,
4858 target_key,
4859 source,
4860 replacement,
4861 } => {
4862 if self.mapping_slots.contains_key(&target_key) {
4863 return Err(StarryError::BadState);
4864 }
4865 let removed = self
4866 .mapping_slots
4867 .remove(&source_key)
4868 .ok_or(StarryError::BadState)?;
4869 if !Arc::ptr_eq(&removed, &source) {
4870 self.mapping_slots.insert(source_key, removed);
4871 return Err(StarryError::BadState);
4872 }
4873 if let Err(error) = source.relocate_to(&replacement) {
4874 let restored = source.state() == SlotState::Present
4875 && self
4876 .mapping_slots
4877 .insert(source_key, source.clone())
4878 .is_none();
4879 if !restored || error == MappingGraphError::RollbackFailed {
4880 self.mutation_gate.mark_needs_repair();
4881 }
4882 return Err(match error {
4883 MappingGraphError::ResourceExhausted => StarryError::NoMemory,
4884 _ => StarryError::BadState,
4885 });
4886 }
4887 if self.mapping_slots.insert(target_key, replacement).is_some() {
4888 self.mutation_gate.mark_needs_repair();
4891 return Err(StarryError::BadState);
4892 }
4893 }
4894 PreparedMovedSlot::DetachSource {
4895 source_key,
4896 target_key,
4897 source,
4898 } => {
4899 let target = self
4900 .mapping_slots
4901 .get(&target_key)
4902 .ok_or(StarryError::BadState)?;
4903 if target.state() != SlotState::Present {
4904 return Err(StarryError::BadState);
4905 }
4906 let removed = self
4907 .mapping_slots
4908 .remove(&source_key)
4909 .ok_or(StarryError::BadState)?;
4910 if !Arc::ptr_eq(&removed, &source) || !removed.detach() {
4911 self.mapping_slots.insert(source_key, removed);
4912 return Err(StarryError::BadState);
4913 }
4914 }
4915 }
4916 }
4917 Ok(())
4918 }
4919
4920 fn apply_move_pages(
4925 &mut self,
4926 src: VirtAddr,
4927 dst: VirtAddr,
4928 size: usize,
4929 ) -> StarryResult<Vec<MovedPage>> {
4930 let move_range =
4931 VirtAddrRange::try_from_start_size(src, size).ok_or(StarryError::InvalidInput)?;
4932 let dst_range =
4933 VirtAddrRange::try_from_start_size(dst, size).ok_or(StarryError::InvalidInput)?;
4934 if move_range.overlaps(dst_range) {
4935 return Err(StarryError::InvalidInput);
4941 }
4942 self.validate_materialized_leaf_boundaries(src, size)?;
4943 let source_slots = self.materialized_slots_overlapping(&[move_range])?;
4944 let mut mapped_pages = alloc::vec::Vec::new();
4945 mapped_pages
4946 .try_reserve(source_slots.len())
4947 .map_err(|_| StarryError::NoMemory)?;
4948 for (key, slot, occupied_leaf) in source_slots {
4949 let offset = key.va.checked_sub_addr(src).ok_or(StarryError::BadState)?;
4950 let dst_va = dst.checked_add(offset).ok_or(StarryError::InvalidInput)?;
4951 let paddr = occupied_leaf.paddr;
4952 let flags = occupied_leaf.flags;
4953 let page_size = occupied_leaf.range.size();
4954 let expected_size = PAGE_SIZE_4K
4955 .checked_shl(slot.page_order.get().into())
4956 .ok_or(StarryError::BadState)?;
4957 if slot.state() != SlotState::Present
4958 || slot.va != key.va
4959 || slot.mm_id != self.id
4960 || slot.mapped_paddr() != Some(paddr)
4961 || page_size != expected_size
4962 {
4963 return Err(StarryError::BadState);
4964 }
4965 if !key.va.is_aligned(page_size) || !dst_va.is_aligned(page_size) {
4966 return Err(StarryError::OperationNotSupported);
4967 }
4968 mapped_pages.push((key.va, dst_va, paddr, flags, page_size));
4969 }
4970
4971 let mut moved_pages = alloc::vec::Vec::new();
4972 moved_pages
4973 .try_reserve(mapped_pages.len())
4974 .map_err(|_| StarryError::NoMemory)?;
4975 let mut move_plans = Vec::<PageTableMovePlan>::new();
4976 move_plans
4977 .try_reserve(mapped_pages.len())
4978 .map_err(|_| StarryError::NoMemory)?;
4979
4980 for &(_, dst_va, _, _, page_size) in &mapped_pages {
4985 match self.pt.query_occupied(dst_va) {
4986 Ok(_) => {}
4987 Err(PagingError::NotMapped) => {
4988 let plan = self.pt.plan_map_page(dst_va, page_size)?;
4989 if let Some(deposit) = plan.prepare_path()? {
4990 let apply_result = {
4991 let _structure = self.pte_domain.lock_structure();
4992 self.pt.try_install_map_path(deposit)
4993 };
4994 if let Err(failure) = apply_result {
4995 let (error, deposit) = failure.into_parts();
4996 drop(deposit);
5000 return Err(error.into());
5001 }
5002 }
5003 }
5004 Err(error) => return Err(error.into()),
5005 }
5006 }
5007
5008 for &(src_va, dst_va, paddr, flags, page_size) in &mapped_pages {
5009 let plan = self.pt.plan_move_page(src_va, dst_va)?;
5010 if plan.source_vaddr() != src_va
5011 || plan.destination_vaddr() != dst_va
5012 || plan.paddr() != paddr
5013 || plan.config() != flags
5014 || plan.page_size() != page_size
5015 {
5016 return Err(StarryError::BadState);
5017 }
5018 let destination = if plan.destination_is_occupied() {
5019 let target_size = plan.destination_page_size();
5020 if target_size < PAGE_SIZE_4K || !target_size.is_power_of_two() {
5021 return Err(StarryError::BadState);
5022 }
5023 MovedPageDestination::TargetOwner {
5024 slot_va: dst_va.align_down(target_size),
5025 }
5026 } else {
5027 MovedPageDestination::SourceOwner
5028 };
5029 moved_pages.push(MovedPage {
5030 src_va,
5031 dst_va,
5032 paddr,
5033 page_size,
5034 destination,
5035 });
5036 move_plans.push(plan);
5037 }
5038
5039 let pte_domain = &self.pte_domain;
5044 let cursor = &mut self.pt;
5045 let apply_result = {
5046 let _structure = pte_domain.lock_structure();
5047 let pte_stripes = pte_domain.lock_ranges(&[move_range, dst_range]);
5048 debug_assert!(!pte_stripes.stripe_indices().is_empty());
5049 cursor.try_move_pages_with(&move_plans)
5050 };
5051 let applied = apply_result?;
5052 if applied != moved_pages.len() {
5053 self.mutation_gate.mark_needs_repair();
5054 return Err(StarryError::BadState);
5055 }
5056 Ok(moved_pages)
5057 }
5058
5059 #[allow(clippy::too_many_arguments)]
5063 pub(crate) fn mremap_move_transaction(
5064 &mut self,
5065 src: VirtAddr,
5066 src_size: usize,
5067 target: VirtAddr,
5068 target_size: usize,
5069 permissions: MappingPermissions,
5070 target_backend: MappingOperation,
5071 huge_page_advice: HugePageAdvice,
5072 lock_mode: VmaLockMode,
5073 advice_policy: VmaAdvicePolicy,
5074 dontunmap: bool,
5075 replace_target: bool,
5076 memlock_limit: Option<MemlockLimit>,
5077 ) -> StarryResult {
5078 self.validate_region(src, src_size)?;
5079 self.validate_region(target, target_size)?;
5080 if !permissions.maximum.contains(permissions.current) {
5081 return Err(StarryError::PermissionDenied);
5082 }
5083 let source_range =
5084 VirtAddrRange::try_from_start_size(src, src_size).ok_or(StarryError::InvalidInput)?;
5085 let target_range = VirtAddrRange::try_from_start_size(target, target_size)
5086 .ok_or(StarryError::InvalidInput)?;
5087 if source_range.overlaps(target_range) {
5088 return Err(StarryError::InvalidInput);
5089 }
5090 let move_size = src_size.min(target_size);
5091 let moved_source_range =
5092 VirtAddrRange::try_from_start_size(src, move_size).ok_or(StarryError::InvalidInput)?;
5093
5094 let source_page_size = self
5095 .vma_root
5096 .lookup_entry(src)
5097 .map(|entry| entry.operation().page_size())
5098 .ok_or(StarryError::BadAddress)?;
5099 let source_replacement =
5100 dontunmap.then(|| MappingOperation::new_alloc(src, source_page_size, ""));
5101 let rollback_ranges = [source_range, target_range];
5102 let tail_range = if src_size > move_size {
5103 Some(
5104 VirtAddrRange::try_from_start_size(
5105 src.checked_add(move_size)
5106 .ok_or(StarryError::InvalidInput)?,
5107 src_size - move_size,
5108 )
5109 .ok_or(StarryError::InvalidInput)?,
5110 )
5111 } else {
5112 None
5113 };
5114
5115 let target_removed_pages = if replace_target {
5116 self.vma_root
5117 .iter_entries()
5118 .filter(|entry| {
5119 entry.start() < target_range.end && entry.end() > target_range.start
5120 })
5121 .try_fold(0u64, |pages, entry| {
5122 let lo = entry.start().max(target_range.start);
5123 let hi = entry.end().min(target_range.end);
5124 let bytes = hi.checked_sub_addr(lo).ok_or(StarryError::BadState)?;
5125 pages
5126 .checked_add((bytes / PAGE_SIZE_4K) as u64)
5127 .ok_or(StarryError::InvalidInput)
5128 })?
5129 } else {
5130 0
5131 };
5132 let target_successor = self.prepare_mapping_successor(
5136 target_range,
5137 permissions,
5138 &target_backend,
5139 huge_page_advice,
5140 lock_mode,
5141 advice_policy,
5142 replace_target,
5143 )?;
5144 let mut final_successor = target_successor.clone();
5145 if dontunmap {
5146 final_successor = final_successor
5147 .without_range(moved_source_range)
5148 .ok_or(StarryError::BadState)?;
5149 let replacement = source_replacement.as_ref().ok_or(StarryError::BadState)?;
5150 let replacement_entry = final_successor
5151 .prepare_mapping_entry(
5152 moved_source_range,
5153 permissions.current,
5154 permissions.reported,
5155 permissions.maximum,
5156 huge_page_advice,
5157 VmaLockMode::Unlocked,
5161 advice_policy,
5162 replacement.clone(),
5163 )
5164 .ok_or(StarryError::BadState)?;
5165 final_successor = final_successor
5166 .with_mapping_entry(replacement_entry, false)
5167 .ok_or(StarryError::BadState)?;
5168 } else {
5169 if let Some(tail) = tail_range {
5170 final_successor = final_successor
5171 .without_range(tail)
5172 .ok_or(StarryError::BadState)?;
5173 }
5174 final_successor = final_successor
5175 .without_range(moved_source_range)
5176 .ok_or(StarryError::BadState)?;
5177 }
5178 self.validate_memlock_successor(&final_successor, memlock_limit)?;
5179 let before_vmas = self.vma_root.len();
5180 let graph_preimage = self.capture_mapping_graph_snapshot(&rollback_ranges)?;
5181
5182 let mut mutation = self.prepare_mutation_range(src, src_size);
5183 mutation
5184 .try_reserve_tlb_ranges(5)
5185 .map_err(|error| match error {
5186 MutationError::ResourceExhausted => StarryError::NoMemory,
5187 _ => StarryError::BadState,
5188 })?;
5189 mutation
5190 .add_tlb_range(TlbRange::new(target, target_size).ok_or(StarryError::InvalidInput)?);
5191 let retire_epoch = mutation
5192 .receipt()
5193 .base_epoch
5194 .checked_next()
5195 .ok_or(StarryError::BadState)?;
5196
5197 let split_ranges = [moved_source_range, target_range];
5203 let splits = self.apply_partial_huge_splits_for_ranges(&split_ranges)?;
5204 for index in 0..splits.len() {
5205 let split = &splits[index];
5206 let Some(tlb_range) =
5207 TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
5208 else {
5209 return self.abort_unpublished_huge_splits(splits, StarryError::BadState);
5210 };
5211 mutation.add_tlb_range(tlb_range);
5212 }
5213
5214 let preimage = match self.capture_mapping_preimage_ranges(&rollback_ranges) {
5215 Ok(preimage) => preimage,
5216 Err(error) => return self.abort_unpublished_huge_splits(splits, error),
5217 };
5218 let mut target_owners = match replace_target
5219 .then(|| self.prepare_retired_mapping_owners(target_range))
5220 .transpose()
5221 {
5222 Ok(owners) => owners,
5223 Err(error) => {
5224 return self.abort_unpublished_split_mapping_mutation_ranges(
5225 &rollback_ranges,
5226 preimage,
5227 None,
5228 splits,
5229 error,
5230 );
5231 }
5232 };
5233 let mut tail_owners = match tail_range
5234 .map(|range| self.prepare_retired_mapping_owners(range))
5235 .transpose()
5236 {
5237 Ok(owners) => owners,
5238 Err(error) => {
5239 return self.abort_unpublished_split_mapping_mutation_ranges(
5240 &rollback_ranges,
5241 preimage,
5242 None,
5243 splits,
5244 error,
5245 );
5246 }
5247 };
5248 let mut page_table_reclaims = if replace_target || tail_range.is_some() {
5249 match self.prepare_page_table_reclaims(&rollback_ranges) {
5250 Ok(reclaims) => reclaims,
5251 Err(error) => {
5252 return self.abort_unpublished_split_mapping_mutation_ranges(
5253 &rollback_ranges,
5254 preimage,
5255 None,
5256 splits,
5257 error,
5258 );
5259 }
5260 }
5261 } else {
5262 MappingMutationContext::for_published_mutation(0)?
5263 };
5264
5265 let mut memfd_deltas = crate::syscall::memfd_prepare_aspace_replace_deltas(
5266 self,
5267 target,
5268 target_size,
5269 permissions.current,
5270 &target_backend,
5271 );
5272 if dontunmap {
5273 if let Some(replacement) = source_replacement.as_ref() {
5274 memfd_deltas.extend(crate::syscall::memfd_prepare_aspace_replace_deltas(
5275 self,
5276 src,
5277 move_size,
5278 permissions.current,
5279 replacement,
5280 ));
5281 }
5282 } else {
5283 memfd_deltas.extend(crate::syscall::memfd_prepare_aspace_unmap_deltas(
5284 self, src, src_size,
5285 ));
5286 }
5287
5288 let apply_result = (|| -> StarryResult<usize> {
5289 let target_materialization = self.apply_mapping_pages_unpublished(
5290 target_range,
5291 permissions,
5292 &target_backend,
5293 replace_target,
5294 &mut page_table_reclaims,
5295 )?;
5296 self.vma_root = Arc::new(target_successor.clone());
5297
5298 let moved_pages = self.apply_move_pages(src, target, move_size)?;
5299 let prepared_moved_slots =
5300 self.prepare_moved_slots(&moved_pages, target_backend.mapping_id())?;
5301 if !dontunmap && let Some(tail) = tail_range {
5302 self.apply_unmap_pages_unpublished(tail, &mut page_table_reclaims)?;
5303 }
5304
5305 if replace_target {
5306 self.detach_mapping_slots(target_range)?;
5307 }
5308 self.publish_prepared_pte_owners(
5309 &target_backend,
5310 target_range,
5311 &target_materialization,
5312 )?;
5313 self.publish_moved_slots(prepared_moved_slots)?;
5314 if !dontunmap && let Some(tail) = tail_range {
5315 self.detach_mapping_slots(tail)?;
5316 }
5317 if self
5318 .mapping_slots_overlapping(moved_source_range)
5319 .next()
5320 .is_some()
5321 {
5322 return Err(StarryError::BadState);
5323 }
5324 self.vma_root = Arc::new(final_successor.clone());
5325
5326 self.vm_stat.on_map((target_size / PAGE_SIZE_4K) as u64);
5327 if target_removed_pages != 0 {
5328 self.vm_stat.on_unmap(target_removed_pages);
5329 }
5330 if !dontunmap {
5331 self.vm_stat.on_unmap((src_size / PAGE_SIZE_4K) as u64);
5332 }
5333 Ok(moved_pages.len())
5334 })();
5335
5336 let moved_leaves = match apply_result {
5337 Ok(moved) => moved,
5338 Err(error) => {
5339 let parked = target_owners.is_some()
5340 || tail_owners.is_some()
5341 || page_table_reclaims.pte_changed();
5342 if let Some(owners) = target_owners.take() {
5343 self.park_retired_mapping_owners(retire_epoch, owners);
5344 }
5345 if let Some(owners) = tail_owners.take() {
5346 self.park_retired_mapping_owners(retire_epoch, owners);
5347 }
5348 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
5349 return self.abort_unpublished_split_mapping_mutation_ranges(
5350 &rollback_ranges,
5351 preimage,
5352 parked.then_some(retire_epoch),
5353 splits,
5354 error,
5355 );
5356 }
5357 };
5358 if let Some(owners) = target_owners.take() {
5359 self.park_retired_mapping_owners(retire_epoch, owners);
5360 }
5361 if let Some(owners) = tail_owners.take() {
5362 self.park_retired_mapping_owners(retire_epoch, owners);
5363 }
5364 self.park_page_table_reclaims(retire_epoch, page_table_reclaims);
5365
5366 let after_vmas = self.vma_root.len();
5367 mutation.set_vma_delta(VmaDelta {
5368 inserted: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
5369 removed: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
5370 ..VmaDelta::default()
5371 });
5372 mutation.set_pte_delta(PteDelta {
5373 mapped: u32::try_from(moved_leaves).unwrap_or(u32::MAX),
5374 unmapped: u32::try_from(moved_leaves).unwrap_or(u32::MAX),
5375 ..PteDelta::default()
5376 });
5377 if let Err(error) =
5378 self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &rollback_ranges)
5379 {
5380 return self.abort_unpublished_split_mapping_mutation_ranges(
5381 &rollback_ranges,
5382 preimage,
5383 Some(retire_epoch),
5384 splits,
5385 error,
5386 );
5387 }
5388
5389 match self.commit_mutation_classified(mutation) {
5390 Ok(()) => {
5391 self.release_retired_mapping_owners(retire_epoch);
5392 crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5393 Ok(())
5394 }
5395 Err(CommitMutationError::PublishedPendingTlb(error)) => {
5396 crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5397 Err(error)
5398 }
5399 Err(CommitMutationError::Unpublished(error)) => self
5400 .abort_unpublished_split_mapping_mutation_ranges(
5401 &rollback_ranges,
5402 preimage,
5403 Some(retire_epoch),
5404 splits,
5405 error,
5406 ),
5407 }
5408 }
5409
5410 pub fn extend_area(&mut self, addr: VirtAddr, additional_size: usize) -> StarryResult {
5412 self.extend_area_with_memlock(addr, additional_size, None)
5413 }
5414
5415 pub(crate) fn extend_area_with_memlock(
5416 &mut self,
5417 addr: VirtAddr,
5418 additional_size: usize,
5419 memlock_limit: Option<MemlockLimit>,
5420 ) -> StarryResult {
5421 if additional_size == 0 {
5422 return Ok(());
5423 }
5424 let entry = self
5425 .vma_root
5426 .lookup_entry(addr)
5427 .ok_or(StarryError::InvalidInput)?;
5428 if !additional_size.is_multiple_of(PAGE_SIZE_4K) {
5429 return Err(StarryError::InvalidInput);
5430 }
5431 let old_end = entry.end();
5432 let grown = VirtAddrRange::try_from_start_size(old_end, additional_size)
5433 .ok_or(StarryError::InvalidInput)?;
5434 let preimage = self.capture_mapping_preimage(grown)?;
5435 let graph_preimage = self.capture_mapping_graph_snapshot(&[grown])?;
5436 let mut mutation = self.prepare_mutation_range(old_end, additional_size);
5437 if entry
5438 .end()
5439 .checked_add(additional_size)
5440 .is_none_or(|new_end| new_end > self.end())
5441 {
5442 return Err(StarryError::NoMemory);
5443 }
5444 let (materialized_range, operation, materialization) =
5445 match self.apply_extend_unpublished(addr, additional_size, memlock_limit) {
5446 Ok(applied) => applied,
5447 Err(error) => {
5448 return self.abort_unpublished_mapping_mutation(grown, preimage, error);
5449 }
5450 };
5451 if materialized_range != grown {
5452 return self.abort_unpublished_mapping_mutation(grown, preimage, StarryError::BadState);
5453 }
5454 self.vm_stat.on_map((additional_size / PAGE_SIZE_4K) as u64);
5455 if let Err(error) = self.publish_prepared_pte_owners(&operation, grown, &materialization) {
5456 return self.abort_unpublished_mapping_mutation(grown, preimage, error);
5457 }
5458 if let Err(error) =
5459 self.set_mapping_graph_receipt_delta(&mut mutation, &graph_preimage, &[grown])
5460 {
5461 return self.abort_unpublished_mapping_mutation(grown, preimage, error);
5462 }
5463 match self.commit_mutation_classified(mutation) {
5464 Ok(()) => Ok(()),
5465 Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
5466 Err(CommitMutationError::Unpublished(error)) => {
5467 self.abort_unpublished_mapping_mutation(grown, preimage, error)
5468 }
5469 }
5470 }
5471
5472 fn process_area_data<F>(&self, start: VirtAddr, size: usize, mut f: F) -> StarryResult
5476 where
5477 F: FnMut(VirtAddr, usize, usize),
5478 {
5479 if size == 0 {
5480 return Ok(());
5481 }
5482 if !self.contains_range(start, size) {
5483 return Err(StarryError::InvalidInput);
5484 }
5485 let end = start.checked_add(size).ok_or(StarryError::InvalidInput)?;
5486 let end_align_up = end
5490 .as_usize()
5491 .checked_add(PAGE_SIZE_4K - 1)
5492 .map(|value| VirtAddr::from_usize(value & !(PAGE_SIZE_4K - 1)))
5493 .ok_or(StarryError::InvalidInput)?;
5494 let page_start = start.align_down_4k();
5495 let pages = PageIter4K::new(page_start, end_align_up).ok_or(StarryError::InvalidInput)?;
5496 let mut copied = 0usize;
5497 for vaddr in pages {
5498 let (paddr, ..) = self.pt.query(vaddr).map_err(|_| StarryError::BadAddress)?;
5499 let page_offset = if vaddr == page_start {
5500 start.align_offset_4k()
5501 } else {
5502 0
5503 };
5504 let copy_size = (PAGE_SIZE_4K - page_offset).min(size - copied);
5505 if copy_size == 0 {
5506 break;
5507 }
5508 let paddr = paddr
5509 .checked_add(page_offset)
5510 .ok_or(StarryError::BadAddress)?;
5511 f(phys_to_virt(paddr), copied, copy_size);
5512 copied = copied
5513 .checked_add(copy_size)
5514 .ok_or(StarryError::InvalidInput)?;
5515 }
5516 (copied == size)
5517 .then_some(())
5518 .ok_or(StarryError::BadAddress)
5519 }
5520
5521 pub fn read(&self, start: VirtAddr, buf: &mut [u8]) -> StarryResult {
5522 self.process_area_data(start, buf.len(), |src, offset, read_size| unsafe {
5523 core::ptr::copy_nonoverlapping(src.as_ptr(), buf.as_mut_ptr().add(offset), read_size);
5524 })
5525 }
5526
5527 pub fn write(&self, start: VirtAddr, buf: &[u8]) -> StarryResult {
5534 self.process_area_data(start, buf.len(), |dst, offset, write_size| unsafe {
5535 core::ptr::copy_nonoverlapping(buf.as_ptr().add(offset), dst.as_mut_ptr(), write_size);
5536 })
5537 }
5538
5539 pub fn sync_modified_text(&self, start: VirtAddr, size: usize) -> StarryResult {
5541 if size == 0 {
5542 return Ok(());
5543 }
5544
5545 self.process_area_data(start, size, |dst, _offset, sync_size| {
5546 let range = ax_cpu::cache::CacheRange::new(dst, sync_size)
5547 .expect("mapped text chunk must not wrap");
5548 unsafe { ax_cpu::cache::clean_dcache_range_to_pou(range) };
5551 })?;
5552 ax_cpu::cache::flush_icache_all();
5553 Ok(())
5554 }
5555
5556 pub fn protect(&mut self, start: VirtAddr, size: usize, flags: MappingFlags) -> StarryResult {
5561 self.protect_with_reported_flags(start, size, flags, flags)
5562 }
5563
5564 pub fn protect_with_reported_flags(
5565 &mut self,
5566 start: VirtAddr,
5567 size: usize,
5568 flags: MappingFlags,
5569 reported_flags: MappingFlags,
5570 ) -> StarryResult {
5571 self.validate_region(start, size)?;
5572 let range = VirtAddrRange::from_start_size(start, size);
5573
5574 start.checked_add(size).ok_or(StarryError::InvalidInput)?;
5579 let mut denied = false;
5583 self.vma_root.for_each_overlapping_entry(range, |entry| {
5584 if entry.max_rights().contains(flags) {
5585 return true;
5586 }
5587 denied = true;
5588 false
5589 });
5590 if denied {
5591 return Err(StarryError::PermissionDenied);
5592 }
5593
5594 let vma_preimage = self.vma_root.clone();
5595 let vm_stat_preimage = self.vm_stat.snapshot();
5596 let before_vmas = self.vma_root.len();
5597 let mut mutation = self.prepare_mutation_range(start, size);
5598 mutation
5599 .try_reserve_tlb_ranges(2)
5600 .map_err(|_| StarryError::NoMemory)?;
5601 let splits = self.apply_partial_huge_splits(range)?;
5602 for split in &splits {
5603 let Some(tlb_range) =
5604 TlbRange::new(split.installed.block_vaddr(), split.installed.block_size())
5605 else {
5606 return self.abort_unpublished_protection(
5607 vma_preimage,
5608 vm_stat_preimage,
5609 &[],
5610 splits,
5611 StarryError::BadState,
5612 );
5613 };
5614 mutation.add_tlb_range(tlb_range);
5615 }
5616 let protection_preimage = match self.capture_protection_leaf_preimage(range) {
5617 Ok(preimage) => preimage,
5618 Err(error) => {
5619 return self.abort_unpublished_protection(
5620 vma_preimage,
5621 vm_stat_preimage,
5622 &[],
5623 splits,
5624 error,
5625 );
5626 }
5627 };
5628 let protected_leaves = protection_preimage.len();
5629 mutation.set_pte_delta(PteDelta {
5630 protected: u32::try_from(protected_leaves).unwrap_or(u32::MAX),
5631 ..PteDelta::default()
5632 });
5633 let split_slots = splits
5634 .iter()
5635 .map(|split| split.child_slots.len().saturating_sub(1))
5636 .sum::<usize>();
5637 mutation.set_mapping_delta(MappingDelta {
5638 attached: u32::try_from(split_slots).unwrap_or(u32::MAX),
5639 ..MappingDelta::default()
5640 });
5641 let touched_memfds =
5642 crate::syscall::memfd_collect_metas_touching_mprotect_range(self, start, size);
5643 if flags.contains(MappingFlags::EXECUTE) {
5644 for leaf in &protection_preimage {
5645 let key = MappingSlotKey {
5646 space_id: self.id,
5647 va: leaf.va,
5648 };
5649 let slot = self
5650 .mapping_slots
5651 .get(&key)
5652 .expect("published executable leaf must retain its mapping owner");
5653 slot.page.prepare_executable_mapping(
5654 leaf.paddr,
5655 leaf.page_size,
5656 flags | (leaf.flags & (MappingFlags::DEVICE | MappingFlags::UNCACHED)),
5657 );
5658 }
5659 }
5660 if let Err(error) = self.apply_protection_unpublished(range, flags, reported_flags) {
5661 return self.abort_unpublished_protection(
5662 vma_preimage,
5663 vm_stat_preimage,
5664 &protection_preimage,
5665 splits,
5666 error,
5667 );
5668 }
5669 let after_vmas = self.vma_root.len();
5670 mutation.set_vma_delta(VmaDelta {
5671 split: u32::try_from(after_vmas.saturating_sub(before_vmas)).unwrap_or(u32::MAX),
5672 merged: u32::try_from(before_vmas.saturating_sub(after_vmas)).unwrap_or(u32::MAX),
5673 ..VmaDelta::default()
5674 });
5675
5676 match self.commit_mutation_classified(mutation) {
5677 Ok(()) => {
5678 crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
5679 self,
5680 &touched_memfds,
5681 );
5682 Ok(())
5683 }
5684 Err(CommitMutationError::PublishedPendingTlb(error)) => {
5685 crate::syscall::memfd_resync_shared_writable_counts_after_mprotect(
5689 self,
5690 &touched_memfds,
5691 );
5692 Err(error)
5693 }
5694 Err(CommitMutationError::Unpublished(error)) => self.abort_unpublished_protection(
5695 vma_preimage,
5696 vm_stat_preimage,
5697 &protection_preimage,
5698 splits,
5699 error,
5700 ),
5701 }
5702 }
5703
5704 fn ensure_quiescent_for_content_clear(&self) -> StarryResult {
5705 if self.tlb_targets.load(core::sync::atomic::Ordering::Acquire) != 0
5706 || self.mutation_gate.pending_count() != 0
5707 || self.pending_retired_mapping_batches() != 0
5708 {
5709 return Err(StarryError::ResourceBusy);
5710 }
5711 Ok(())
5712 }
5713
5714 fn clear_quiescent_contents(&mut self) -> StarryResult {
5719 self.ensure_quiescent_for_content_clear()?;
5720 let range = self.layout.range();
5721 let operations = self.mapping_operation_fragments(range, false)?;
5722 if operations
5723 .iter()
5724 .any(|(fragment, operation)| !operation.validate_unmap_range(*fragment, &self.pt))
5725 {
5726 return Err(StarryError::BadState);
5727 }
5728
5729 let leaf_count = self.occupied_pte_leaves_overlapping(&[range])?.len();
5730 let mut page_table_reclaims =
5731 MappingMutationContext::for_published_mutation(leaf_count)?;
5732 let clear_result = operations
5740 .into_iter()
5741 .try_for_each(|(fragment, operation)| {
5742 operation.unmap_range(fragment, &mut page_table_reclaims, &mut self.pt)
5743 });
5744 unsafe { page_table_reclaims.reclaim() };
5747 if let Err(error) = clear_result {
5748 self.mutation_gate.mark_needs_repair();
5749 return Err(error);
5750 }
5751 let slots = core::mem::take(&mut self.mapping_slots);
5752 for slot in slots.into_values() {
5753 slot.detach();
5754 }
5755 self.resident_watermark.reset();
5756 self.vm_stat.on_clear();
5757 self.vma_root = Arc::new(VmaMap::default());
5758 self.executable_file = None;
5759 self.mutation_gate.clear_repair();
5762 Ok(())
5763 }
5764
5765 pub(crate) fn reset_uninstalled_for_loader(&mut self) -> StarryResult {
5774 self.ensure_quiescent_for_content_clear()?;
5775 let range = self.layout.range();
5776 let memfd_deltas =
5777 crate::syscall::memfd_prepare_aspace_unmap_deltas(self, range.start, range.size());
5778 self.clear_quiescent_contents()?;
5779 self.resident_pages = ResidentPageCounts::default();
5780 self.heap = HeapState::new(USER_HEAP_BASE);
5781 self.executable_data = ExecutableDataLayout::default();
5782 crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5783 Ok(())
5784 }
5785
5786 fn clear_retired_contents(&mut self) -> StarryResult {
5789 self.ensure_quiescent_for_content_clear()?;
5790 let base_epoch = self.vm_epoch();
5791 base_epoch.checked_next().ok_or(StarryError::BadState)?;
5792 let range = self.layout.range();
5793 let removed_vmas = self.vma_root.len();
5794 let detached_slots = self.mapping_slots.len();
5795 let materialized_pages = self
5796 .mapping_slots
5797 .values()
5798 .try_fold(0usize, |pages, slot| {
5799 pages.checked_add(1usize.checked_shl(slot.page_order.get().into())?)
5800 })
5801 .ok_or(StarryError::BadState)?;
5802 let memfd_deltas =
5803 crate::syscall::memfd_prepare_aspace_unmap_deltas(self, range.start, range.size());
5804 let mut mutation = self.prepare_mutation_range(range.start, range.size());
5805 mutation.set_vma_delta(VmaDelta {
5806 removed: u32::try_from(removed_vmas).unwrap_or(u32::MAX),
5807 ..VmaDelta::default()
5808 });
5809 mutation.set_pte_delta(PteDelta {
5810 unmapped: u32::try_from(materialized_pages).unwrap_or(u32::MAX),
5811 ..PteDelta::default()
5812 });
5813 mutation.set_mapping_delta(MappingDelta {
5814 detached: u32::try_from(detached_slots).unwrap_or(u32::MAX),
5815 ..MappingDelta::default()
5816 });
5817 mutation.set_resident_delta(self.resident_pages.checked_negated_delta()?);
5818 self.clear_quiescent_contents()?;
5819 let result = self.commit_mutation(mutation);
5820 if self.vm_epoch() != base_epoch {
5821 crate::syscall::memfd_apply_shared_writable_deltas(&memfd_deltas);
5822 }
5823 result
5824 }
5825
5826 pub(crate) fn try_reclaim_contents(&mut self) -> StarryResult {
5828 if self.tlb_targets.load(core::sync::atomic::Ordering::Acquire) != 0
5834 || self.mutation_gate.pending_count() != 0
5835 || self.pending_retired_mapping_batches() != 0
5836 {
5837 return Err(StarryError::ResourceBusy);
5838 }
5839 self.clear_retired_contents()?;
5840 let epoch = self.vm_epoch();
5841
5842 let targets = self.tlb_targets.load(core::sync::atomic::Ordering::Acquire);
5849 let request = TlbRequest::new(self.id, epoch, targets);
5850 debug_assert!(request.is_complete());
5851 unsafe {
5857 self.pt.detach(|token| token.reclaim());
5858 }
5859 Ok(())
5860 }
5861
5862 pub fn can_access_range(
5867 &self,
5868 start: VirtAddr,
5869 size: usize,
5870 access_flags: MappingFlags,
5871 ) -> bool {
5872 let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
5873 return false;
5874 };
5875 if range.is_empty() {
5876 return false;
5877 }
5878 let mut cursor = range.start;
5882 let mut permitted = false;
5883 self.vma_root.for_each_overlapping(range, |vma| {
5884 if vma.range.end <= cursor {
5885 return true;
5886 }
5887 if vma.range.start > cursor || !vma.rights.contains(access_flags) {
5888 return false;
5889 }
5890 cursor = vma.range.end.min(range.end);
5891 if cursor >= range.end {
5892 permitted = true;
5893 return false;
5894 }
5895 true
5896 });
5897 permitted
5898 }
5899
5900 fn fault_transaction_page_size(
5909 &self,
5910 vaddr: VirtAddr,
5911 vma_range: VirtAddrRange,
5912 policy_size: usize,
5913 ) -> StarryResult<usize> {
5914 match self.pt.query(vaddr) {
5915 Ok((_, _, leaf_size)) => return Ok(leaf_size),
5916 Err(PagingError::NotMapped) => {}
5917 Err(error) => return Err(error.into()),
5918 }
5919 if policy_size == PAGE_SIZE_4K {
5920 return Ok(PAGE_SIZE_4K);
5921 }
5922 if policy_size < PAGE_SIZE_4K
5923 || !policy_size.is_power_of_two()
5924 || !policy_size.is_multiple_of(PAGE_SIZE_4K)
5925 {
5926 return Err(StarryError::BadState);
5927 }
5928
5929 let policy_start = vaddr.align_down(policy_size);
5930 let policy_range = VirtAddrRange::try_from_start_size(policy_start, policy_size)
5931 .ok_or(StarryError::BadState)?;
5932 if !vma_range.contains_range(policy_range) {
5933 return Ok(PAGE_SIZE_4K);
5934 }
5935
5936 if self
5941 .pt
5942 .walk_occupied_range(policy_range.start, policy_range.end)
5943 .next()
5944 .is_some()
5945 {
5946 return Ok(PAGE_SIZE_4K);
5947 }
5948 Ok(policy_size)
5949 }
5950
5951 fn plan_page_fault(
5952 &self,
5953 vaddr: VirtAddr,
5954 access_flags: PageFaultFlags,
5955 thp_mode: TransparentHugePageMode,
5956 ) -> Result<PageFaultPlan, FaultResult> {
5957 if self.mutation_gate.needs_repair() {
5958 return Err(FaultResult::Sigbus(BusCode::ObjErr));
5961 }
5962 if !self.layout.range().contains(vaddr) {
5963 return Err(FaultResult::Unmapped);
5964 }
5965 let access_flags = MappingFlags::from(access_flags);
5966 let Some(entry) = self.vma_root.lookup_entry(vaddr) else {
5967 return Err(FaultResult::Unmapped);
5968 };
5969 let vma = entry.snapshot().clone();
5970 let flags = vma.rights;
5971 if !flags.contains(access_flags) {
5972 return Err(FaultResult::PermissionDenied);
5973 }
5974 let backend = entry.operation_clone();
5975 let Some(policy_size) = vma
5976 .group
5977 .page_policy
5978 .fault_leaf_size(vma.huge_page_advice, thp_mode)
5979 else {
5980 return Err(FaultResult::Unmapped);
5981 };
5982 let page_size = match self.fault_transaction_page_size(vaddr, vma.range, policy_size) {
5983 Ok(page_size) => page_size,
5984 Err(error) => {
5985 warn!("could not classify page-fault leaf for {vaddr:?}: {error}");
5986 return Err(FaultResult::Retry);
5987 }
5988 };
5989 let page_start = vaddr.align_down(page_size);
5990 let Some(range) = VirtAddrRange::try_from_start_size(page_start, page_size) else {
5991 return Err(FaultResult::Unmapped);
5992 };
5993 let fault_fallback = if vma.group.page_policy.permits_fault_fallback() {
5994 FaultFallback::BasePage
5995 } else {
5996 FaultFallback::Forbidden
5997 };
5998 let request = match PopulateRequest::fault(range, page_size, vaddr, fault_fallback) {
5999 Ok(request) => request,
6000 Err(_) => return Err(FaultResult::Unmapped),
6001 };
6002 let preimage = match FaultPteSnapshot::capture(&self.pt, page_start) {
6003 Ok(preimage) => preimage,
6004 Err(error) => {
6005 warn!("could not capture page-fault PTE at {page_start:?}: {error}");
6006 return Err(FaultResult::Retry);
6007 }
6008 };
6009 let map_plans = if preimage == FaultPteSnapshot::NotMapped {
6010 let preferred = match self.pt.plan_map_page(page_start, page_size) {
6011 Ok(plan) => plan,
6012 Err(error) => {
6013 warn!("could not plan page-table path for {page_start:?}: {error}");
6014 return Err(FaultResult::Retry);
6015 }
6016 };
6017 let fallback = if page_size > PAGE_SIZE_4K && fault_fallback == FaultFallback::BasePage
6018 {
6019 let fallback_start = vaddr.align_down_4k();
6020 match self.pt.plan_map_page(fallback_start, PAGE_SIZE_4K) {
6021 Ok(plan) => Some(plan),
6022 Err(error) => {
6023 warn!(
6024 "could not plan fallback page-table path for {fallback_start:?}: \
6025 {error}"
6026 );
6027 return Err(FaultResult::Retry);
6028 }
6029 }
6030 } else {
6031 None
6032 };
6033 Some(PageFaultMapPlans {
6034 preferred,
6035 fallback,
6036 })
6037 } else {
6038 None
6039 };
6040 Ok(PageFaultPlan {
6041 base_epoch: self.vm_epoch(),
6042 space_id: self.id,
6043 vaddr,
6044 range,
6045 vma_flags: flags,
6046 access_flags,
6047 operation: backend,
6048 request,
6049 preimage,
6050 map_plans,
6051 })
6052 }
6053
6054 fn classify_fault_error(file_backed: bool, error: StarryError) -> FaultResult {
6055 if matches!(
6056 error,
6057 StarryError::NoMemory
6058 | StarryError::Paging(PagingError::NoMemory)
6059 | StarryError::Vfs(axfs_ng_vfs::VfsError::NoMemory)
6060 ) {
6061 return FaultResult::NoMemory;
6062 }
6063 if matches!(error, StarryError::ResourceBusy) {
6064 return FaultResult::Retry;
6065 }
6066 if !file_backed {
6067 return FaultResult::Unmapped;
6068 }
6069 match error {
6070 StarryError::ResourceBusy | StarryError::Vfs(axfs_ng_vfs::VfsError::ResourceBusy) => {
6071 FaultResult::Retry
6072 }
6073 StarryError::BadAddress => FaultResult::Sigbus(BusCode::AdrErr),
6074 StarryError::Io | StarryError::Vfs(_) => FaultResult::Sigbus(BusCode::ObjErr),
6075 _ => FaultResult::Unmapped,
6076 }
6077 }
6078
6079 fn prepare_fault_materialization(
6080 plan: &PageFaultPlan,
6081 request: PopulateRequest,
6082 ) -> Result<FaultMaterialization, FaultResult> {
6083 match plan.operation.prepare_fault(
6084 plan.space_id,
6085 request,
6086 plan.vma_flags,
6087 plan.access_flags,
6088 plan.preimage,
6089 ) {
6090 Ok(materialization) => Ok(materialization),
6091 Err(error) => {
6092 warn!(
6093 "failed to prepare page fault for {:?} ({:?}): {error}",
6094 plan.vaddr, plan.vma_flags
6095 );
6096 Err(Self::classify_fault_error(
6097 plan.operation.is_file_backed(),
6098 error,
6099 ))
6100 }
6101 }
6102 }
6103
6104 fn cancel_fault_materialization(
6105 plan: &PageFaultPlan,
6106 materialization: FaultMaterialization,
6107 ) -> Result<(), FaultResult> {
6108 plan.operation
6109 .cancel_prepared_fault_publication(materialization)
6110 .map_err(|error| {
6111 warn!(
6112 "failed to cancel prepared page fault for {:?}: {error}",
6113 plan.vaddr
6114 );
6115 FaultResult::Retry
6116 })
6117 }
6118
6119 fn prepare_page_fault(mut plan: PageFaultPlan) -> Result<PreparedPageFault, FaultResult> {
6120 let mut materialization = Self::prepare_fault_materialization(&plan, plan.request)?;
6121 let installed_owner = materialization.owner().and_then(|owner| {
6122 (owner.transition == PteOwnerTransition::Installed).then_some((
6123 owner.va,
6124 owner.paddr,
6125 owner.page_size,
6126 ))
6127 });
6128 let map_deposit = if let Some((owner_va, owner_paddr, owner_page_size)) = installed_owner {
6129 let Some(plans) = plan.map_plans.take() else {
6130 Self::cancel_fault_materialization(&plan, materialization)?;
6131 return Err(FaultResult::Retry);
6132 };
6133 let PageFaultMapPlans {
6134 preferred,
6135 mut fallback,
6136 } = plans;
6137 let preferred_selected =
6138 preferred.vaddr() == owner_va && preferred.page_size() == owner_page_size;
6139 let fallback_selected = fallback.as_ref().is_some_and(|fallback| {
6140 fallback.vaddr() == owner_va && fallback.page_size() == owner_page_size
6141 });
6142 if !preferred_selected && !fallback_selected {
6143 Self::cancel_fault_materialization(&plan, materialization)?;
6144 return Err(FaultResult::Retry);
6145 }
6146 let Some(flags) = materialization.pte_flags() else {
6147 Self::cancel_fault_materialization(&plan, materialization)?;
6148 return Err(FaultResult::Retry);
6149 };
6150
6151 if fallback_selected {
6152 let Some(fallback_request) = plan.request.into_base_page_fallback() else {
6153 Self::cancel_fault_materialization(&plan, materialization)?;
6154 return Err(FaultResult::Retry);
6155 };
6156 plan.request = fallback_request;
6157 plan.range = fallback_request.range();
6158 let Some(fallback) = fallback.take() else {
6159 Self::cancel_fault_materialization(&plan, materialization)?;
6160 return Err(FaultResult::Retry);
6161 };
6162 match fallback.prepare(owner_paddr, flags) {
6163 Ok(deposit) => Some(deposit),
6164 Err(error) => {
6165 warn!(
6166 "could not prepare fallback page-table path for {owner_va:?}: {error}"
6167 );
6168 Self::cancel_fault_materialization(&plan, materialization)?;
6169 return Err(Self::classify_fault_error(false, error.into()));
6170 }
6171 }
6172 } else {
6173 match preferred.prepare(owner_paddr, flags) {
6174 Ok(deposit) => Some(deposit),
6175 Err(PagingError::NoMemory) if fallback.is_some() => {
6176 Self::cancel_fault_materialization(&plan, materialization)?;
6181 let Some(fallback_request) = plan.request.into_base_page_fallback() else {
6182 return Err(FaultResult::Retry);
6183 };
6184 plan.request = fallback_request;
6185 plan.range = fallback_request.range();
6186 materialization =
6187 Self::prepare_fault_materialization(&plan, fallback_request)?;
6188 let Some(owner) = materialization.owner() else {
6189 Self::cancel_fault_materialization(&plan, materialization)?;
6190 return Err(FaultResult::Retry);
6191 };
6192 let Some(fallback) = fallback.take() else {
6193 Self::cancel_fault_materialization(&plan, materialization)?;
6194 return Err(FaultResult::Retry);
6195 };
6196 if owner.transition != PteOwnerTransition::Installed
6197 || owner.va != fallback.vaddr()
6198 || owner.page_size != fallback.page_size()
6199 {
6200 Self::cancel_fault_materialization(&plan, materialization)?;
6201 return Err(FaultResult::Retry);
6202 }
6203 let Some(flags) = materialization.pte_flags() else {
6204 Self::cancel_fault_materialization(&plan, materialization)?;
6205 return Err(FaultResult::Retry);
6206 };
6207 match fallback.prepare(owner.paddr, flags) {
6208 Ok(deposit) => Some(deposit),
6209 Err(error) => {
6210 warn!(
6211 "could not prepare base-page table path for {:?}: {error}",
6212 owner.va
6213 );
6214 Self::cancel_fault_materialization(&plan, materialization)?;
6215 return Err(Self::classify_fault_error(false, error.into()));
6216 }
6217 }
6218 }
6219 Err(error) => {
6220 warn!("could not prepare page-table path for {owner_va:?}: {error}");
6221 Self::cancel_fault_materialization(&plan, materialization)?;
6222 return Err(Self::classify_fault_error(false, error.into()));
6223 }
6224 }
6225 }
6226 } else {
6227 None
6228 };
6229 Ok(PreparedPageFault {
6230 plan,
6231 materialization,
6232 map_deposit,
6233 })
6234 }
6235
6236 fn page_fault_plan_is_current(&self, plan: &PageFaultPlan) -> bool {
6237 if self.vm_epoch() != plan.base_epoch || !plan.preimage.matches(plan.range.start, &self.pt)
6238 {
6239 return false;
6240 }
6241 self.vma_root.lookup_entry(plan.vaddr).is_some_and(|entry| {
6242 entry.snapshot().rights == plan.vma_flags
6243 && entry.snapshot().range.contains_range(plan.range)
6244 && entry.operation().mapping_id() == plan.operation.mapping_id()
6245 })
6246 }
6247
6248 fn apply_prepared_page_fault(
6249 &mut self,
6250 attempt: &mut PageFaultApplyAttempt,
6251 ) -> PageFaultApplyOutcome {
6252 if !self.page_fault_plan_is_current(&attempt.prepared().plan) {
6253 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6254 }
6255
6256 let (pages, file_backed, vaddr, vma_flags, range, access_flags, fault_preimage) = {
6257 let prepared = attempt.prepared();
6258 (
6259 prepared.materialization.satisfied_pages(),
6260 prepared.plan.operation.is_file_backed(),
6261 prepared.plan.vaddr,
6262 prepared.plan.vma_flags,
6263 prepared.plan.range,
6264 prepared.plan.access_flags,
6265 prepared.plan.preimage,
6266 )
6267 };
6268 if pages == 0 {
6269 let result = if file_backed {
6270 FaultResult::Sigbus(BusCode::AdrErr)
6271 } else {
6272 warn!("no pages prepared for {vaddr:?} ({vma_flags:?})");
6273 FaultResult::Unmapped
6274 };
6275 return PageFaultApplyOutcome::Cancel(result);
6276 }
6277 if attempt.prepared().materialization.owner().is_none() {
6278 return PageFaultApplyOutcome::Cancel(FaultResult::Handled);
6279 }
6280 let (owner_va, owner_paddr, owner_page_size, owner_transition, desired_flags) = {
6281 let prepared = attempt.prepared();
6282 let owner = prepared
6283 .materialization
6284 .owner()
6285 .expect("checked fault owner must remain present");
6286 let Some(desired_flags) = prepared.materialization.pte_flags() else {
6287 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6288 };
6289 (
6290 owner.va,
6291 owner.paddr,
6292 owner.page_size,
6293 owner.transition,
6294 desired_flags,
6295 )
6296 };
6297 let mapping_preimage = match self.capture_mapping_preimage(range) {
6298 Ok(preimage) => preimage,
6299 Err(error) => {
6300 warn!("could not retain page-fault preimage for {vaddr:?}: {error}");
6301 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6302 }
6303 };
6304 let replaces_owner = attempt
6305 .prepared()
6306 .materialization
6307 .owner()
6308 .is_some_and(|owner| owner.transition == PteOwnerTransition::Replaced);
6309 let retired_owners = if replaces_owner {
6310 match self.prepare_retired_mapping_owners(range) {
6311 Ok(owners) => Some(owners),
6312 Err(error) => {
6313 warn!("could not reserve page-fault retire owners for {vaddr:?}: {error}");
6314 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6315 }
6316 }
6317 } else {
6318 None
6319 };
6320 let lazy_free_page = access_flags
6321 .contains(MappingFlags::WRITE)
6322 .then(|| {
6323 self.mapping_slots
6324 .get(&MappingSlotKey {
6325 space_id: self.id,
6326 va: owner_va,
6327 })
6328 .filter(|slot| slot.page.state() == PageState::LazyFree)
6329 .map(|slot| slot.page.clone())
6330 })
6331 .flatten();
6332 let fresh_install = matches!(fault_preimage, FaultPteSnapshot::NotMapped)
6333 && attempt
6334 .prepared()
6335 .materialization
6336 .owner()
6337 .is_some_and(|owner| owner.transition == PteOwnerTransition::Installed);
6338 let mut mutation = if fresh_install {
6339 self.prepare_fresh_pte_mutation_range(range.start, range.size())
6340 } else {
6341 self.prepare_mutation_range(range.start, range.size())
6342 };
6343 if let Some(request) = self.mutation_gate.pending_overlap_request(&mutation) {
6348 return PageFaultApplyOutcome::CancelPendingTlb {
6349 request,
6350 targets: self.tlb_targets(),
6351 };
6352 }
6353 let Some(retire_epoch) = mutation.receipt().base_epoch.checked_next() else {
6354 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6355 };
6356
6357 let mut map_deposit = if owner_transition == PteOwnerTransition::Installed {
6358 match attempt.take_map_deposit() {
6359 Some(deposit) => Some(deposit),
6360 None => return PageFaultApplyOutcome::Cancel(FaultResult::Retry),
6361 }
6362 } else {
6363 None
6364 };
6365 let apply_result = {
6366 let _structure = (owner_transition == PteOwnerTransition::Installed)
6367 .then(|| self.pte_domain.lock_structure());
6368 let _stripe = self.pte_domain.lock_range(range);
6369 let pt = &mut self.pt;
6370 let preimage_matches = fault_preimage.matches(range.start, pt);
6371 let result = if !preimage_matches {
6372 Err(PagingError::stale_map_deposit(owner_va))
6373 } else {
6374 let owner = attempt
6377 .prepared()
6378 .materialization
6379 .owner()
6380 .expect("prepared fault retains its page until PTE publication");
6381 owner
6382 .page
6383 .prepare_executable_mapping(owner_paddr, owner_page_size, desired_flags);
6384 match owner_transition {
6385 PteOwnerTransition::Installed => {
6386 let deposit = map_deposit
6387 .take()
6388 .expect("fresh page fault must retain its map deposit");
6389 match pt.try_map_page_with(deposit) {
6390 Ok(()) => Ok(owner_page_size),
6391 Err(failure) => {
6392 let (error, deposit) = failure.into_parts();
6393 map_deposit = Some(deposit);
6394 Err(error)
6395 }
6396 }
6397 }
6398 PteOwnerTransition::Replaced | PteOwnerTransition::Updated => {
6399 pt.remap_page(owner_va, owner_paddr, desired_flags)
6400 }
6401 }
6402 };
6403 result.and_then(|installed_size| {
6404 (installed_size == owner_page_size)
6405 .then_some(installed_size)
6406 .ok_or(PagingError::NotMapped)
6407 })
6408 };
6409 if let Some(deposit) = map_deposit.take() {
6410 attempt.restore_map_deposit(deposit);
6411 }
6412 if let Err(error) = apply_result {
6413 warn!("could not apply prepared page fault for {vaddr:?}: {error}");
6414 if fault_preimage.matches(range.start, &self.pt) {
6415 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6416 }
6417 if self
6418 .restore_mapping_preimage(range, mapping_preimage)
6419 .is_ok()
6420 {
6421 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6422 } else {
6423 self.mutation_gate.mark_needs_repair();
6424 }
6425 return PageFaultApplyOutcome::NeedsRepair(FaultResult::Retry);
6426 }
6427
6428 mutation.set_pte_delta(PteDelta {
6429 mapped: u32::try_from(pages).unwrap_or(u32::MAX),
6430 ..PteDelta::default()
6431 });
6432 let publication = match self.publish_prepared_fault_owner(
6433 &attempt.prepared().plan.operation,
6434 range,
6435 &attempt.prepared().materialization,
6436 ) {
6437 Ok(publication) => publication,
6438 Err(error) => {
6439 warn!("could not publish prepared page owner for {vaddr:?}: {error}");
6440 if self
6441 .restore_mapping_preimage(range, mapping_preimage)
6442 .is_err()
6443 {
6444 self.mutation_gate.mark_needs_repair();
6445 return PageFaultApplyOutcome::NeedsRepair(FaultResult::Retry);
6446 }
6447 return PageFaultApplyOutcome::Cancel(FaultResult::Retry);
6448 }
6449 };
6450 let PreparedPageFault {
6451 plan: _,
6452 materialization: _,
6453 map_deposit,
6454 } = attempt.take_prepared();
6455 debug_assert!(map_deposit.is_none());
6456 mutation.set_mapping_delta(publication.mapping_delta);
6457 mutation.set_resident_delta(publication.resident_delta);
6458 if let Some(page) = &lazy_free_page
6459 && !page.clear_lazy_free()
6460 {
6461 if self
6462 .restore_mapping_preimage(range, mapping_preimage)
6463 .is_err()
6464 {
6465 self.mutation_gate.mark_needs_repair();
6466 }
6467 return PageFaultApplyOutcome::Complete(FaultResult::Retry);
6468 }
6469 if let Some(owners) = retired_owners {
6470 self.park_retired_mapping_owners(retire_epoch, owners);
6471 }
6472 match self.publish_mutation_classified(mutation) {
6473 Ok(MutationPublication::Complete) => {
6474 self.release_retired_mapping_owners(retire_epoch);
6475 PageFaultApplyOutcome::Complete(FaultResult::Handled)
6476 }
6477 Ok(MutationPublication::PendingTlb) => {
6478 let Some(request) = self.mutation_gate.pending_request(self.id, retire_epoch)
6479 else {
6480 self.mutation_gate.mark_needs_repair();
6481 return PageFaultApplyOutcome::Complete(FaultResult::Retry);
6482 };
6483 PageFaultApplyOutcome::PendingTlb {
6484 request,
6485 targets: self.tlb_targets(),
6486 }
6487 }
6488 Err(CommitMutationError::Unpublished(error)) => {
6489 warn!("page-fault publication for {vaddr:?} failed before publish: {error}");
6490 if self
6491 .restore_mapping_preimage(range, mapping_preimage)
6492 .is_err()
6493 {
6494 self.mutation_gate.mark_needs_repair();
6495 } else {
6496 self.release_retired_mapping_owners(retire_epoch);
6497 if let Some(page) = &lazy_free_page
6498 && !page.mark_lazy_free()
6499 {
6500 self.mutation_gate.mark_needs_repair();
6501 }
6502 }
6503 PageFaultApplyOutcome::Complete(FaultResult::Retry)
6504 }
6505 Err(CommitMutationError::PublishedPendingTlb(error)) => {
6506 warn!("unexpected synchronous TLB result for page fault {vaddr:?}: {error}");
6507 let Some(request) = self.mutation_gate.pending_request(self.id, retire_epoch)
6508 else {
6509 self.mutation_gate.mark_needs_repair();
6510 return PageFaultApplyOutcome::Complete(FaultResult::Retry);
6511 };
6512 PageFaultApplyOutcome::PendingTlb {
6513 request,
6514 targets: self.tlb_targets(),
6515 }
6516 }
6517 }
6518 }
6519
6520 #[cfg(all(test, axtest))]
6523 fn handle_page_fault_result(
6524 &mut self,
6525 vaddr: VirtAddr,
6526 access_flags: PageFaultFlags,
6527 ) -> FaultResult {
6528 let plan =
6529 match self.plan_page_fault(vaddr, access_flags, TransparentHugePageMode::default()) {
6530 Ok(plan) => plan,
6531 Err(result) => return result,
6532 };
6533 let prepared = match Self::prepare_page_fault(plan) {
6534 Ok(prepared) => prepared,
6535 Err(result) => return result,
6536 };
6537 let mut attempt = prepared.into_apply_attempt();
6538 let result = match self.apply_prepared_page_fault(&mut attempt) {
6539 PageFaultApplyOutcome::Complete(result) => result,
6540 PageFaultApplyOutcome::Cancel(result) => {
6541 if attempt.cancel().is_ok() {
6542 result
6543 } else {
6544 FaultResult::Retry
6545 }
6546 }
6547 PageFaultApplyOutcome::NeedsRepair(result) => {
6548 attempt.release_to_repair_state();
6549 result
6550 }
6551 PageFaultApplyOutcome::CancelPendingTlb { request, targets } => {
6552 if attempt.cancel().is_ok()
6553 && Self::flush_tlb_requests(core::slice::from_ref(&request), &targets).is_ok()
6554 {
6555 let _ = self.acknowledge_tlb_requests(core::slice::from_ref(&request));
6556 }
6557 FaultResult::Retry
6558 }
6559 PageFaultApplyOutcome::PendingTlb { request, targets } => {
6560 if Self::flush_tlb_requests(core::slice::from_ref(&request), &targets).is_ok()
6561 && self
6562 .acknowledge_tlb_requests(core::slice::from_ref(&request))
6563 .is_ok()
6564 {
6565 FaultResult::Handled
6566 } else {
6567 FaultResult::Retry
6568 }
6569 }
6570 };
6571 complete_page_fault_with(
6572 matches!(result, FaultResult::Handled),
6573 vaddr,
6574 ax_cpu::mmu::update_mmu_cache,
6575 );
6576 result
6577 }
6578
6579 fn prepare_fork_parent_mutation(&self) -> StarryResult<Option<PreparedForkParentMutation>> {
6585 let mut mutation = self.prepare_mutation();
6586 let mut ptes = Vec::new();
6587 let mut ranges = Vec::new();
6588
6589 for entry in self.vma_root.iter_entries() {
6590 if entry.snapshot().advice_policy.dont_fork() {
6591 continue;
6592 }
6593 if !entry.operation().requires_fork_write_protect() {
6594 continue;
6595 }
6596 let mut range_changed = false;
6597 for leaf in self.occupied_pte_leaves_overlapping(&[entry.range()])? {
6598 let page_size = leaf.range.size();
6599 if page_size < PAGE_SIZE_4K || !page_size.is_power_of_two() {
6600 return Err(StarryError::BadState);
6601 }
6602 let protected_flags = leaf.flags - MappingFlags::WRITE;
6603 if protected_flags != leaf.flags {
6604 ptes.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
6605 ptes.push(ForkParentPteProtection {
6606 va: leaf.range.start,
6607 paddr: leaf.paddr,
6608 page_size,
6609 original_flags: leaf.flags,
6610 protected_flags,
6611 });
6612 range_changed = true;
6613 }
6614 }
6615 if range_changed {
6616 ranges.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
6617 ranges.push(entry.range());
6618 mutation
6619 .try_add_tlb_range(
6620 TlbRange::new(entry.start(), entry.size())
6621 .ok_or(StarryError::InvalidInput)?,
6622 )
6623 .map_err(|error| match error {
6624 MutationError::ResourceExhausted => StarryError::NoMemory,
6625 _ => StarryError::BadState,
6626 })?;
6627 }
6628 }
6629
6630 if ptes.is_empty() {
6631 return Ok(None);
6632 }
6633 mutation.set_pte_delta(PteDelta {
6634 protected: u32::try_from(ptes.len()).unwrap_or(u32::MAX),
6635 ..PteDelta::default()
6636 });
6637 Ok(Some(PreparedForkParentMutation {
6638 mutation,
6639 ptes,
6640 ranges,
6641 }))
6642 }
6643
6644 fn rollback_fork_parent_ptes(
6645 cursor: &mut PageTable,
6646 applied: &[ForkParentPteProtection],
6647 ) -> bool {
6648 let mut complete = true;
6649 for protection in applied.iter().rev() {
6650 let current_matches =
6651 cursor
6652 .query(protection.va)
6653 .is_ok_and(|(paddr, flags, page_size)| {
6654 paddr == protection.paddr
6655 && flags == protection.protected_flags
6656 && page_size == protection.page_size
6657 });
6658 if !current_matches
6659 || cursor
6660 .protect_page(protection.va, protection.original_flags)
6661 .is_err()
6662 {
6663 complete = false;
6664 }
6665 }
6666 complete
6667 }
6668
6669 fn apply_fork_parent_mutation(&mut self, prepared: PreparedForkParentMutation) -> StarryResult {
6672 let PreparedForkParentMutation {
6673 mutation,
6674 ptes,
6675 ranges,
6676 } = prepared;
6677 let pt = &mut self.pt;
6678 let pte_stripes = self.pte_domain.lock_ranges(&ranges);
6679 let cursor = pt;
6683 for (applied, protection) in ptes.iter().enumerate() {
6684 let preimage_matches =
6685 cursor
6686 .query(protection.va)
6687 .is_ok_and(|(paddr, flags, page_size)| {
6688 paddr == protection.paddr
6689 && flags == protection.original_flags
6690 && page_size == protection.page_size
6691 });
6692 if !preimage_matches
6693 || cursor
6694 .protect_page(protection.va, protection.protected_flags)
6695 .is_err()
6696 {
6697 if !Self::rollback_fork_parent_ptes(cursor, &ptes[..applied]) {
6698 self.mutation_gate.mark_needs_repair();
6699 return Err(StarryError::BadState);
6700 }
6701 return Err(StarryError::BadState);
6702 }
6703 }
6704 drop(pte_stripes);
6705
6706 match self.commit_mutation_classified(mutation) {
6712 Ok(()) => Ok(()),
6713 Err(CommitMutationError::PublishedPendingTlb(error)) => Err(error),
6714 Err(CommitMutationError::Unpublished(error)) => {
6715 let pt = &mut self.pt;
6716 let _pte_stripes = self.pte_domain.lock_ranges(&ranges);
6717 let restored = Self::rollback_fork_parent_ptes(pt, &ptes);
6721 if restored {
6722 self.mutation_gate.clear_repair();
6723 Err(error)
6724 } else {
6725 self.mutation_gate.mark_needs_repair();
6726 Err(StarryError::BadState)
6727 }
6728 }
6729 }
6730 }
6731
6732 fn abort_unpublished_clone(child: &mut Self) -> StarryResult {
6733 match child.reset_uninstalled_for_loader() {
6734 Ok(()) => Ok(()),
6735 Err(error)
6736 if child.vma_root.is_empty()
6737 && child.mapping_slots.is_empty()
6738 && child.pending_retired_mapping_batches() == 0 =>
6739 {
6740 warn!(
6745 "unpublished fork child cleared all mappings but could not publish cleanup \
6746 epoch: {error}"
6747 );
6748 Ok(())
6749 }
6750 Err(error) => Err(error),
6751 }
6752 }
6753
6754 pub fn try_clone(&mut self) -> StarryResult<Arc<Mutex<Self>>> {
6764 let parent_mutation = self.prepare_fork_parent_mutation()?;
6768 let new_aspace = Arc::new(Mutex::new(Self::new_with_layout(self.layout)?));
6769
6770 let mut guard = new_aspace.lock_nested(CLONED_ADDR_SPACE_LOCK_SUBCLASS);
6774 guard.heap = self.heap;
6775 guard.executable_data = self.executable_data;
6776 guard.executable_file = self.executable_file.clone();
6777 let mut child_memfd_deltas = Vec::new();
6778 let mut child_vss_pages = 0u64;
6779
6780 let child_preparation = (|| -> StarryResult {
6781 let self_modify = &mut self.pt;
6782 for entry in self.vma_root.iter_entries() {
6783 if entry.snapshot().advice_policy.dont_fork() {
6784 continue;
6785 }
6786 let (new_backend, materialization) = entry.operation().clone_map(
6787 entry.range(),
6788 entry.rights(),
6789 self_modify,
6790 &mut guard.pt,
6791 )?;
6792 let start = entry.start();
6793 child_memfd_deltas.extend(crate::syscall::memfd_prepare_aspace_replace_deltas(
6794 &guard,
6795 start,
6796 entry.size(),
6797 entry.rights(),
6798 &new_backend,
6799 ));
6800
6801 let child_entry = guard
6802 .vma_root
6803 .prepare_mapping_entry(
6804 entry.range(),
6805 entry.rights(),
6806 entry.reported_rights(),
6807 entry.max_rights(),
6808 entry.snapshot().huge_page_advice,
6809 VmaLockMode::Unlocked,
6810 entry.snapshot().advice_policy,
6811 new_backend.clone(),
6812 )
6813 .ok_or(StarryError::BadState)?;
6814 let child_root = guard
6815 .vma_root
6816 .with_mapping_entry(child_entry, false)
6817 .ok_or(StarryError::BadState)?;
6818 guard.vma_root = Arc::new(child_root);
6819 guard.publish_prepared_pte_owners(&new_backend, entry.range(), &materialization)?;
6820 child_vss_pages = child_vss_pages
6821 .checked_add((entry.size() / PAGE_SIZE_4K) as u64)
6822 .ok_or(StarryError::BadState)?;
6823 }
6824
6825 guard.vm_stat.seed_clone(child_vss_pages);
6828 Ok(())
6829 })();
6830
6831 if let Err(error) = child_preparation {
6832 if let Err(cleanup_error) = Self::abort_unpublished_clone(&mut guard) {
6833 warn!(
6834 "fork child preparation failed ({error}); unpublished cleanup also failed \
6835 ({cleanup_error})"
6836 );
6837 return Err(StarryError::BadState);
6838 }
6839 return Err(error);
6840 }
6841
6842 if let Some(parent_mutation) = parent_mutation
6846 && let Err(error) = self.apply_fork_parent_mutation(parent_mutation)
6847 {
6848 if let Err(cleanup_error) = Self::abort_unpublished_clone(&mut guard) {
6849 warn!(
6850 "fork parent publication failed ({error}); unpublished child cleanup also \
6851 failed ({cleanup_error})"
6852 );
6853 return Err(StarryError::BadState);
6854 }
6855 return Err(error);
6856 }
6857
6858 if !guard.vma_root.is_empty() {
6862 let mut child_mutation = guard.prepare_mutation();
6863 child_mutation.set_vma_delta(VmaDelta {
6864 inserted: u32::try_from(guard.vma_root.len()).unwrap_or(u32::MAX),
6865 ..VmaDelta::default()
6866 });
6867 child_mutation.set_pte_delta(PteDelta {
6868 mapped: u32::try_from(guard.mapping_slots.len()).unwrap_or(u32::MAX),
6869 ..PteDelta::default()
6870 });
6871 child_mutation.set_mapping_delta(MappingDelta {
6872 attached: u32::try_from(guard.mapping_slots.len()).unwrap_or(u32::MAX),
6873 ..MappingDelta::default()
6874 });
6875 child_mutation.set_resident_delta(
6876 guard
6877 .resident_counts_from_all_slots()?
6878 .checked_positive_delta()?,
6879 );
6880 if let Err(error) = guard.commit_mutation(child_mutation) {
6881 if let Err(cleanup_error) = Self::abort_unpublished_clone(&mut guard) {
6882 warn!(
6883 "fork child publication failed ({error}); unpublished cleanup also failed \
6884 ({cleanup_error})"
6885 );
6886 return Err(StarryError::BadState);
6887 }
6888 return Err(error);
6889 }
6890 }
6891 crate::syscall::memfd_apply_shared_writable_deltas(&child_memfd_deltas);
6892 drop(guard);
6893
6894 Ok(new_aspace)
6895 }
6896}
6897
6898#[cfg(all(test, not(axtest)))]
6899fn page_fault_completion_updates_only_success_for_test() -> bool {
6900 use core::cell::Cell;
6901
6902 let calls = Cell::new(0);
6903 let observed = Cell::new(VirtAddr::from(0));
6904 let success = complete_page_fault_with(true, VirtAddr::from(0x4567), |vaddr| {
6905 calls.set(calls.get() + 1);
6906 observed.set(vaddr);
6907 });
6908 let rejected = complete_page_fault_with(false, VirtAddr::from(0x89ab), |_| {
6909 calls.set(calls.get() + 1);
6910 });
6911
6912 success && !rejected && calls.get() == 1 && observed.get() == VirtAddr::from(0x4567)
6913}
6914
6915impl fmt::Debug for AddrSpace {
6916 fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
6917 f.debug_struct("AddrSpace")
6918 .field("id", &self.id)
6919 .field("layout", &self.layout)
6920 .field("page_table_root", &self.pt.root_paddr())
6921 .field("vma_root", &self.vma_root)
6922 .field("vm_epoch", &self.vm_epoch())
6923 .finish()
6924 }
6925}
6926
6927impl Drop for AddrSpace {
6928 fn drop(&mut self) {
6929 let has_retired_batches = self.pending_retired_mapping_batches() != 0;
6937 if !self.vma_root.is_empty() || !self.mapping_slots.is_empty() || has_retired_batches {
6938 warn!(
6939 "address space {} dropped before retire/reclaim; mappings intentionally retained",
6940 self.id.get()
6941 );
6942 self.pt.leak();
6948 if has_retired_batches {
6949 let batches = core::mem::take(&mut *self.retired_mapping_batches.lock());
6950 for batch in batches {
6951 core::mem::forget(batch);
6952 }
6953 }
6954 }
6955 }
6956}
6957
6958#[cfg(test)]
6959mod tests {
6960 use alloc::sync::Arc;
6961 use core::sync::atomic::AtomicUsize;
6962
6963 use ax_memory_addr::{PAGE_SIZE_4K, VirtAddr};
6964
6965 use super::{
6966 AddressSpaceId, MutationError, MutationGate, TlbRange, VmEpoch,
6967 prepare_mapping_publication_mutation,
6968 };
6969
6970 #[cfg(all(test, not(axtest)))]
6971 #[test]
6972 fn page_fault_completion_updates_only_success() {
6973 assert!(super::page_fault_completion_updates_only_success_for_test());
6974 }
6975
6976 #[cfg_attr(axtest, axtest::axtest)]
6977 #[cfg_attr(not(axtest), test)]
6978 fn fresh_mapping_publication_has_no_tlb_targets() {
6979 let gate = MutationGate::new();
6980 let id = AddressSpaceId::allocate();
6981 let targets = Arc::new(AtomicUsize::new(0b1110));
6982 let start = VirtAddr::from(0x20_0000);
6983
6984 let fresh =
6985 prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, false);
6986 assert_eq!(fresh.receipt().tlb_obligation.targets(), 0);
6987
6988 let replacement =
6989 prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, true);
6990 assert_eq!(replacement.receipt().tlb_obligation.targets(), 0b1110);
6991 }
6992
6993 #[cfg_attr(axtest, axtest::axtest)]
6994 #[cfg_attr(not(axtest), test)]
6995 fn fresh_mapping_cannot_reuse_range_with_pending_shootdown() {
6996 let gate = MutationGate::new();
6997 let id = AddressSpaceId::allocate();
6998 let targets = Arc::new(AtomicUsize::new(0b1));
6999 let start = VirtAddr::from(0x20_0000);
7000 let mut unmap = gate.begin(id, 0b1);
7001 unmap.add_tlb_range(TlbRange::new(start, PAGE_SIZE_4K).unwrap());
7002 assert_eq!(gate.commit(unmap).unwrap_err(), MutationError::TlbPending);
7003
7004 let nonoverlapping = prepare_mapping_publication_mutation(
7005 &gate,
7006 id,
7007 &targets,
7008 start + PAGE_SIZE_4K * 2,
7009 PAGE_SIZE_4K,
7010 false,
7011 );
7012 gate.validate_publish_preconditions(&nonoverlapping)
7013 .unwrap();
7014 gate.commit(nonoverlapping).unwrap();
7015
7016 let fresh =
7017 prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, false);
7018 assert_eq!(
7019 gate.validate_publish_preconditions(&fresh),
7020 Err(MutationError::PendingTlbOverlap)
7021 );
7022 assert_eq!(
7023 gate.commit(fresh).unwrap_err(),
7024 MutationError::PendingTlbOverlap
7025 );
7026 assert_eq!(gate.current_epoch(), VmEpoch::new(2));
7027
7028 gate.acknowledge(id, VmEpoch::new(1), 0).unwrap().unwrap();
7029 let retry =
7030 prepare_mapping_publication_mutation(&gate, id, &targets, start, PAGE_SIZE_4K, false);
7031 gate.validate_publish_preconditions(&retry).unwrap();
7032 gate.commit(retry).unwrap();
7033 }
7034
7035 #[cfg_attr(axtest, axtest::axtest)]
7036 #[cfg_attr(not(axtest), test)]
7037 fn pending_full_flush_blocks_every_fresh_mapping_range() {
7038 let gate = MutationGate::new();
7039 let id = AddressSpaceId::allocate();
7040 let targets = Arc::new(AtomicUsize::new(0b1));
7041 assert_eq!(
7042 gate.commit(gate.begin(id, 0b1)).unwrap_err(),
7043 MutationError::TlbPending
7044 );
7045
7046 let fresh = prepare_mapping_publication_mutation(
7047 &gate,
7048 id,
7049 &targets,
7050 VirtAddr::from(0x40_0000),
7051 PAGE_SIZE_4K,
7052 false,
7053 );
7054 assert_eq!(
7055 gate.commit(fresh).unwrap_err(),
7056 MutationError::PendingTlbOverlap
7057 );
7058 assert_eq!(gate.current_epoch(), VmEpoch::new(1));
7059 }
7060
7061 #[cfg(axtest)]
7062 fn refault_waits_for_discard_shootdown(full_flush: bool) {
7063 use ax_runtime::hal::trap::PageFaultFlags;
7064
7065 use super::{AddrSpace, FaultResult, MappingFlags, MappingOperation, PagingError};
7066
7067 let start = VirtAddr::from(0x7200_0000);
7068 let mut aspace = AddrSpace::new_empty(start, PAGE_SIZE_4K).unwrap();
7069 let flags = MappingFlags::READ | MappingFlags::WRITE | MappingFlags::USER;
7070 aspace
7071 .map(
7072 start,
7073 PAGE_SIZE_4K,
7074 flags,
7075 true,
7076 MappingOperation::new_alloc(start, PAGE_SIZE_4K, "[discard-refault]"),
7077 )
7078 .unwrap();
7079 aspace.discard_range(start, PAGE_SIZE_4K).unwrap();
7080 let mut discard = aspace.mutation_gate.begin(aspace.id, 1);
7083 if !full_flush {
7084 discard.add_tlb_range(TlbRange::new(start, PAGE_SIZE_4K).unwrap());
7085 }
7086 assert_eq!(
7087 aspace.mutation_gate.commit(discard).unwrap_err(),
7088 MutationError::TlbPending
7089 );
7090 let epoch = aspace.vm_epoch();
7091 let plan = aspace
7092 .plan_page_fault(
7093 start,
7094 PageFaultFlags::READ | PageFaultFlags::USER,
7095 Default::default(),
7096 )
7097 .ok()
7098 .unwrap();
7099 let prepared = AddrSpace::prepare_page_fault(plan).ok().unwrap();
7100 let mut attempt = prepared.into_apply_attempt();
7101 let outcome = aspace.apply_prepared_page_fault(&mut attempt);
7102 let unpublished = matches!(aspace.pt.query(start), Err(PagingError::NotMapped))
7103 && aspace.vm_epoch() == epoch
7104 && aspace.mapping_slots.is_empty()
7105 && attempt.prepared.is_some()
7106 && !aspace.mutation_gate.needs_repair();
7107 drop(outcome);
7108 if attempt.prepared.is_some() {
7109 attempt.cancel().unwrap();
7110 }
7111 aspace
7112 .mutation_gate
7113 .acknowledge(aspace.id, epoch, 0)
7114 .unwrap()
7115 .unwrap();
7116 let retry =
7117 aspace.handle_page_fault_result(start, PageFaultFlags::READ | PageFaultFlags::USER);
7118 let recovered = matches!(retry, FaultResult::Handled) && aspace.pt.query(start).is_ok();
7119 aspace.reset_uninstalled_for_loader().unwrap();
7120 assert!(
7121 unpublished,
7122 "refault must leave the PTE, epoch and owner graph untouched until discard is \
7123 acknowledged"
7124 );
7125 assert!(
7126 recovered,
7127 "acknowledged discard must allow refault to make progress"
7128 );
7129 }
7130
7131 #[cfg(axtest)]
7132 #[axtest::axtest]
7133 fn refault_waits_for_pending_discard_range() {
7134 refault_waits_for_discard_shootdown(false);
7135 }
7136
7137 #[cfg(axtest)]
7138 #[axtest::axtest]
7139 fn refault_waits_for_pending_discard_full_flush() {
7140 refault_waits_for_discard_shootdown(true);
7141 }
7142
7143 #[cfg(axtest)]
7144 #[axtest::axtest]
7145 fn page_table_reclaim_waits_for_last_tlb_acknowledgement() {
7146 use ax_memory_addr::PhysAddr;
7147
7148 use super::{AddrSpace, MappingFlags, RetiredMappingOwners};
7149
7150 let start = VirtAddr::from(0x7300_0000);
7151 let mut aspace = AddrSpace::new_empty(start, PAGE_SIZE_4K).unwrap();
7152 aspace
7153 .pt
7154 .map_page(
7155 start,
7156 PhysAddr::from(0x1000_0000),
7157 PAGE_SIZE_4K,
7158 MappingFlags::READ | MappingFlags::USER,
7159 )
7160 .unwrap();
7161 let (_, _, _, deferred) = aspace.pt.unmap_page_deferred(start).unwrap();
7162 assert!(!deferred.is_empty());
7163
7164 let epoch = VmEpoch::new(1);
7165 let mut owners = RetiredMappingOwners::default();
7166 owners.page_tables.try_reserve(1).unwrap();
7167 owners.page_tables.push(deferred);
7168 aspace.park_retired_mapping_owners(epoch, owners);
7169
7170 let mut mutation = aspace.mutation_gate.begin(aspace.id, 0b11);
7171 mutation.add_tlb_range(TlbRange::new(start, PAGE_SIZE_4K).unwrap());
7172 assert!(matches!(
7173 aspace.mutation_gate.commit(mutation),
7174 Err(MutationError::TlbPending)
7175 ));
7176 assert_eq!(aspace.pending_retired_page_table_reclaims(), 1);
7177
7178 assert!(aspace.acknowledge_tlb(aspace.id, epoch, 0).unwrap().is_empty());
7179 assert_eq!(aspace.pending_retired_page_table_reclaims(), 1);
7180
7181 assert!(aspace.acknowledge_tlb(aspace.id, epoch, 1).unwrap().is_empty());
7182 assert_eq!(aspace.pending_retired_page_table_reclaims(), 0);
7183 }
7184}