use alloc::{
string::{String, ToString},
sync::Arc,
vec::Vec,
};
use ax_alloc::{UsageKind, global_allocator};
use ax_fs_ng::{file::CachedFileIdentity, vfs::CachedFile};
use ax_memory_addr::{DynPageIter, MemoryAddr, PAGE_SIZE_4K, PhysAddr, VirtAddr, VirtAddrRange};
use ax_memory_set::MappingBackend;
use ax_runtime::hal::{
mem::{phys_to_virt, virt_to_phys},
paging::{DeferredPageTableFrames, MappingFlags, PageTable},
};
use crate::{StarryError, StarryResult};
mod cow;
mod file;
mod linear;
mod shared;
pub use self::shared::SharedMemoryObject;
pub use super::accounting::RssKind;
use super::{
AddressSpaceId,
vma::{MappingId, VmaDescriptor},
};
fn mincore_file_visible(location: &axfs_ng_vfs::Location, cred: &crate::task::Cred) -> bool {
use axfs_ng_vfs::NodePermission;
let Ok(metadata) = location.metadata() else {
return false;
};
cred.fsuid == metadata.uid
|| cred.has_cap_fowner()
|| cred.has_cap_dac_override()
|| metadata.mode.contains(if cred.in_group(metadata.gid) {
NodePermission::GROUP_WRITE
} else {
NodePermission::OTHER_WRITE
})
}
pub(super) struct MappingMutationContext {
deferred_page_tables: Vec<DeferredPageTableFrames>,
rollback_cleanups: Vec<RollbackCleanup>,
backend_owners_deferred: bool,
pte_changed: bool,
}
enum RollbackCleanup {
Cow(cow::CowRollbackOwner),
File(file::FileRollbackOwner),
}
impl RollbackCleanup {
fn complete(self) -> StarryResult {
match self {
Self::Cow(owner) => owner.complete(),
Self::File(owner) => owner.complete(),
}
}
}
impl MappingMutationContext {
fn with_capacity(capacity: usize, backend_owners_deferred: bool) -> StarryResult<Self> {
let mut deferred_page_tables = Vec::new();
deferred_page_tables
.try_reserve(capacity)
.map_err(|_| StarryError::NoMemory)?;
let mut rollback_cleanups = Vec::new();
if !backend_owners_deferred {
rollback_cleanups
.try_reserve(capacity)
.map_err(|_| StarryError::NoMemory)?;
}
Ok(Self {
deferred_page_tables,
rollback_cleanups,
backend_owners_deferred,
pte_changed: false,
})
}
pub(super) fn for_published_mutation(capacity: usize) -> StarryResult<Self> {
Self::with_capacity(capacity, true)
}
pub(super) fn for_rollback(capacity: usize) -> StarryResult<Self> {
Self::with_capacity(capacity, false)
}
pub(super) const fn backend_owners_deferred(&self) -> bool {
self.backend_owners_deferred
}
pub(super) fn record_unmap(&mut self, tables: DeferredPageTableFrames) {
self.pte_changed = true;
if tables.is_empty() {
return;
}
let slot_reserved = self.deferred_page_tables.len() < self.deferred_page_tables.capacity();
debug_assert!(
slot_reserved,
"page-table reclaim ownership must be reserved before PTE mutation"
);
if !slot_reserved && let Err(error) = self.deferred_page_tables.try_reserve(1) {
warn!(
"cannot retain detached page-table ownership after PTE mutation; leaking the \
frames: {error}"
);
core::mem::forget(tables);
return;
}
self.deferred_page_tables.push(tables);
}
pub(in crate::mm::aspace::backend) fn defer_cow_cleanup(
&mut self,
owner: cow::CowRollbackOwner,
) {
debug_assert!(!self.backend_owners_deferred);
self.defer_rollback_cleanup(RollbackCleanup::Cow(owner));
}
pub(in crate::mm::aspace::backend) fn defer_file_cleanup(
&mut self,
owner: file::FileRollbackOwner,
) {
debug_assert!(!self.backend_owners_deferred);
self.defer_rollback_cleanup(RollbackCleanup::File(owner));
}
fn defer_rollback_cleanup(&mut self, cleanup: RollbackCleanup) {
let slot_reserved = self.rollback_cleanups.len() < self.rollback_cleanups.capacity();
debug_assert!(
slot_reserved,
"backend rollback ownership must be reserved before PTE mutation"
);
if !slot_reserved && let Err(error) = self.rollback_cleanups.try_reserve(1) {
warn!(
"cannot retain backend rollback ownership after PTE mutation; leaking the owner: \
{error}"
);
core::mem::forget(cleanup);
return;
}
self.rollback_cleanups.push(cleanup);
}
pub(super) fn is_empty(&self) -> bool {
self.deferred_page_tables.is_empty()
}
pub(super) const fn pte_changed(&self) -> bool {
self.pte_changed
}
pub(super) fn into_page_tables(mut self) -> Vec<DeferredPageTableFrames> {
debug_assert!(self.rollback_cleanups.is_empty());
core::mem::take(&mut self.deferred_page_tables)
}
pub(super) unsafe fn reclaim(mut self) {
for tables in core::mem::take(&mut self.deferred_page_tables) {
unsafe { tables.reclaim() };
}
for cleanup in core::mem::take(&mut self.rollback_cleanups) {
if let Err(error) = cleanup.complete() {
warn!("failed to finish mapping rollback ownership: {error}");
}
}
}
}
impl Drop for MappingMutationContext {
fn drop(&mut self) {
if self.pte_changed && !self.rollback_cleanups.is_empty() {
core::mem::forget(core::mem::take(&mut self.rollback_cleanups));
}
}
}
pub(super) fn rollback_live_mapped_pages<I>(
pt: &mut PageTable,
pages: I,
range: VirtAddrRange,
mut context: MappingMutationContext,
) -> bool
where
I: IntoIterator<Item = (VirtAddr, PhysAddr, usize)>,
{
let mut complete = true;
for (vaddr, expected_paddr, expected_size) in pages {
match pt.unmap_page_deferred(vaddr) {
Ok((paddr, _, page_size, deferred)) => {
context.record_unmap(deferred);
if paddr != expected_paddr || page_size != expected_size {
complete = false;
warn!(
"mapping rollback found {paddr:?}/{page_size:#x} instead of \
{expected_paddr:?}/{expected_size:#x} at {vaddr:?}"
);
}
}
Err(error) => {
complete = false;
warn!("mapping rollback could not unmap {vaddr:?}: {error}");
}
}
}
if !context.pte_changed() {
return complete;
}
if let Err(error) = crate::mm::flush_tlb_range_sync(range.start, range.size()) {
warn!("mapping rollback could not invalidate {range:?}: {error}");
return false;
}
unsafe { context.reclaim() };
complete
}
fn divide_page(size: usize, page_size: usize) -> usize {
assert!(size.is_multiple_of(page_size), "unaligned");
size >> page_size.trailing_zeros()
}
pub(crate) fn alloc_frame(zeroed: bool, size: usize) -> StarryResult<PhysAddr> {
let num_pages = size / PAGE_SIZE_4K;
let vaddr = VirtAddr::from(
global_allocator()
.alloc_pages(num_pages, size, UsageKind::VirtMem)
.map_err(|_| StarryError::NoMemory)?,
);
if zeroed {
unsafe { core::ptr::write_bytes(vaddr.as_mut_ptr(), 0, size) };
}
let paddr = virt_to_phys(vaddr);
Ok(paddr)
}
pub(crate) fn dealloc_frame(frame: PhysAddr, align: usize) {
let vaddr = phys_to_virt(frame);
let num_pages = align / PAGE_SIZE_4K;
global_allocator().dealloc_pages(vaddr.as_usize(), num_pages, UsageKind::VirtMem);
}
fn pages_in(range: VirtAddrRange, align: usize) -> StarryResult<DynPageIter<VirtAddr>> {
DynPageIter::new(range.start, range.end, align).ok_or(StarryError::InvalidInput)
}
fn occupied_leaf_ranges(
range: VirtAddrRange,
pt: &PageTable,
) -> StarryResult<Vec<(VirtAddr, usize)>> {
if range.is_empty() || !range.start.is_aligned_4k() || !range.end.is_aligned_4k() {
return Err(StarryError::InvalidInput);
}
let occupied = pt.walk_occupied_range(range.start, range.end);
let (lower, upper) = occupied.size_hint();
let mut leaves = Vec::new();
leaves
.try_reserve(upper.unwrap_or(lower))
.map_err(|_| StarryError::NoMemory)?;
for entry in occupied {
let leaf_size = pt
.mapping_size_for_level(entry.level)
.ok_or(StarryError::BadState)?;
let leaf_end = entry
.vaddr
.checked_add(leaf_size)
.ok_or(StarryError::BadState)?;
if leaf_size < PAGE_SIZE_4K
|| !leaf_size.is_power_of_two()
|| entry.vaddr < range.start
|| leaf_end > range.end
{
return Err(StarryError::OperationNotSupported);
}
leaves.try_reserve(1).map_err(|_| StarryError::NoMemory)?;
leaves.push((entry.vaddr, leaf_size));
}
Ok(leaves)
}
fn validate_occupied_leaf_range(
range: VirtAddrRange,
expected_leaf_size: Option<usize>,
pt: &PageTable,
) -> bool {
occupied_leaf_ranges(range, pt).is_ok_and(|leaves| {
expected_leaf_size
.is_none_or(|expected| leaves.iter().all(|(_, leaf_size)| *leaf_size == expected))
})
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum PteOwnerTransition {
Installed,
Replaced,
Updated,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum ProviderPublication {
Complete,
Pending,
}
pub(super) struct PreparedPteOwner {
pub va: VirtAddr,
pub paddr: PhysAddr,
pub page_size: usize,
pub page: Arc<super::objects::PageObject>,
pub resident_kind: Option<RssKind>,
pub transition: PteOwnerTransition,
pub provider_publication: ProviderPublication,
}
impl PreparedPteOwner {
pub(super) fn installed(
va: VirtAddr,
paddr: PhysAddr,
page_size: usize,
page: Arc<super::objects::PageObject>,
resident_kind: Option<RssKind>,
provider_publication: ProviderPublication,
) -> Self {
Self {
va,
paddr,
page_size,
page,
resident_kind,
transition: PteOwnerTransition::Installed,
provider_publication,
}
}
pub(super) fn replaced(
va: VirtAddr,
paddr: PhysAddr,
page_size: usize,
page: Arc<super::objects::PageObject>,
resident_kind: Option<RssKind>,
provider_publication: ProviderPublication,
) -> Self {
Self {
va,
paddr,
page_size,
page,
resident_kind,
transition: PteOwnerTransition::Replaced,
provider_publication,
}
}
pub(super) fn updated(
va: VirtAddr,
paddr: PhysAddr,
page_size: usize,
page: Arc<super::objects::PageObject>,
resident_kind: Option<RssKind>,
) -> Self {
Self {
va,
paddr,
page_size,
page,
resident_kind,
transition: PteOwnerTransition::Updated,
provider_publication: ProviderPublication::Complete,
}
}
}
#[derive(Default)]
pub(super) struct PteMaterialization {
satisfied_pages: usize,
owners: Vec<PreparedPteOwner>,
}
#[derive(Clone, Copy, Debug)]
pub(super) struct PopulateRequest {
range: VirtAddrRange,
preferred_leaf_size: usize,
fault_address: Option<VirtAddr>,
fallback: FaultFallback,
}
#[derive(Clone, Copy, Debug, Eq, PartialEq)]
pub(super) enum FaultFallback {
Forbidden,
BasePage,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum FaultPteSnapshot {
NotMapped,
Mapped {
paddr: PhysAddr,
flags: MappingFlags,
page_size: usize,
},
}
impl FaultPteSnapshot {
pub(super) fn capture(pt: &PageTable, va: VirtAddr) -> StarryResult<Self> {
match pt.query(va) {
Ok((paddr, flags, page_size)) => Ok(Self::Mapped {
paddr,
flags,
page_size,
}),
Err(ax_runtime::hal::paging::PagingError::NotMapped) => Ok(Self::NotMapped),
Err(error) => Err(error.into()),
}
}
pub(super) fn matches(self, va: VirtAddr, pt: &PageTable) -> bool {
match (self, pt.query(va)) {
(Self::NotMapped, Err(ax_runtime::hal::paging::PagingError::NotMapped)) => true,
(
Self::Mapped {
paddr,
flags,
page_size,
},
Ok((current_paddr, current_flags, current_size)),
) => paddr == current_paddr && flags == current_flags && page_size == current_size,
_ => false,
}
}
}
pub(super) struct FaultMaterialization {
satisfied_pages: usize,
owner: Option<PreparedPteOwner>,
pte_flags: Option<MappingFlags>,
}
impl FaultMaterialization {
pub(super) const fn empty() -> Self {
Self {
satisfied_pages: 0,
owner: None,
pte_flags: None,
}
}
pub(super) const fn satisfied(satisfied_pages: usize) -> Self {
Self {
satisfied_pages,
owner: None,
pte_flags: None,
}
}
pub(super) const fn with_owner(
satisfied_pages: usize,
owner: PreparedPteOwner,
pte_flags: MappingFlags,
) -> Self {
Self {
satisfied_pages,
owner: Some(owner),
pte_flags: Some(pte_flags),
}
}
pub(super) const fn satisfied_pages(&self) -> usize {
self.satisfied_pages
}
pub(super) const fn owner(&self) -> Option<&PreparedPteOwner> {
self.owner.as_ref()
}
pub(super) const fn pte_flags(&self) -> Option<MappingFlags> {
self.pte_flags
}
fn into_owner(self) -> Option<PreparedPteOwner> {
self.owner
}
}
impl PopulateRequest {
pub(super) fn area(range: VirtAddrRange, preferred_leaf_size: usize) -> StarryResult<Self> {
Self::new(range, preferred_leaf_size, None, FaultFallback::Forbidden)
}
pub(super) fn fault(
range: VirtAddrRange,
preferred_leaf_size: usize,
fault_address: VirtAddr,
fallback: FaultFallback,
) -> StarryResult<Self> {
Self::new(range, preferred_leaf_size, Some(fault_address), fallback)
}
fn new(
range: VirtAddrRange,
preferred_leaf_size: usize,
fault_address: Option<VirtAddr>,
fallback: FaultFallback,
) -> StarryResult<Self> {
if range.is_empty()
|| preferred_leaf_size < PAGE_SIZE_4K
|| !preferred_leaf_size.is_power_of_two()
|| !preferred_leaf_size.is_multiple_of(PAGE_SIZE_4K)
|| fault_address.is_some_and(|address| !range.contains(address))
{
return Err(StarryError::InvalidInput);
}
Ok(Self {
range,
preferred_leaf_size,
fault_address,
fallback,
})
}
pub(super) const fn range(self) -> VirtAddrRange {
self.range
}
pub(super) const fn preferred_leaf_size(self) -> usize {
self.preferred_leaf_size
}
pub(super) const fn fault_address(self) -> Option<VirtAddr> {
self.fault_address
}
pub(super) const fn fallback(self) -> FaultFallback {
self.fallback
}
pub(super) fn into_base_page_fallback(self) -> Option<Self> {
if self.fallback != FaultFallback::BasePage || self.preferred_leaf_size <= PAGE_SIZE_4K {
return None;
}
let fault_address = self.fault_address?;
let range =
VirtAddrRange::try_from_start_size(fault_address.align_down_4k(), PAGE_SIZE_4K)?;
Self::fault(range, PAGE_SIZE_4K, fault_address, FaultFallback::Forbidden).ok()
}
}
impl PteMaterialization {
pub(super) const fn empty() -> Self {
Self {
satisfied_pages: 0,
owners: Vec::new(),
}
}
pub(super) fn with_capacity(capacity: usize) -> StarryResult<Self> {
let mut owners = Vec::new();
owners
.try_reserve(capacity)
.map_err(|_| StarryError::NoMemory)?;
Ok(Self {
satisfied_pages: 0,
owners,
})
}
pub(super) const fn satisfied_pages(&self) -> usize {
self.satisfied_pages
}
pub(super) fn owners(&self) -> &[PreparedPteOwner] {
&self.owners
}
pub(super) fn set_satisfied_pages(&mut self, pages: usize) {
self.satisfied_pages = pages;
}
pub(super) fn increment_satisfied(&mut self, pages: usize) -> StarryResult {
self.satisfied_pages = self
.satisfied_pages
.checked_add(pages)
.ok_or(StarryError::NoMemory)?;
Ok(())
}
pub(super) fn push(&mut self, owner: PreparedPteOwner) {
self.owners.push(owner);
}
pub(super) fn append(&mut self, mut other: Self) -> StarryResult {
self.satisfied_pages = self
.satisfied_pages
.checked_add(other.satisfied_pages)
.ok_or(StarryError::NoMemory)?;
self.owners
.try_reserve(other.owners.len())
.map_err(|_| StarryError::NoMemory)?;
self.owners.append(&mut other.owners);
Ok(())
}
}
pub(super) trait MappingExecution {
fn page_size(&self) -> usize;
fn vma_descriptor(&self, area_start: VirtAddr) -> VmaDescriptor;
fn map(
&self,
range: VirtAddrRange,
flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult<PteMaterialization>;
fn validate_map(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
if range.is_empty()
|| !range.start.is_aligned(self.page_size())
|| !range.end.is_aligned(self.page_size())
{
return false;
}
pt.walk_occupied_range(range.start, range.end)
.next()
.is_none()
}
fn unmap(
&self,
range: VirtAddrRange,
context: &mut MappingMutationContext,
pt: &mut PageTable,
) -> StarryResult;
fn validate_unmap(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
validate_occupied_leaf_range(range, Some(self.page_size()), pt)
}
fn validate_protect(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
validate_occupied_leaf_range(range, Some(self.page_size()), pt)
}
fn on_protect(
&self,
_range: VirtAddrRange,
_new_flags: MappingFlags,
_pt: &mut PageTable,
) -> StarryResult {
Ok(())
}
fn populate(
&self,
_space_id: AddressSpaceId,
_request: PopulateRequest,
_flags: MappingFlags,
_access_flags: MappingFlags,
_pt: &mut PageTable,
) -> StarryResult<PteMaterialization> {
Ok(PteMaterialization::empty())
}
fn prepare_fault(
&self,
_space_id: AddressSpaceId,
_request: PopulateRequest,
_flags: MappingFlags,
_access_flags: MappingFlags,
_preimage: FaultPteSnapshot,
) -> StarryResult<FaultMaterialization> {
Ok(FaultMaterialization::empty())
}
fn clone_map(
&self,
range: VirtAddrRange,
flags: MappingFlags,
old_pt: &mut PageTable,
new_pt: &mut PageTable,
) -> StarryResult<(MappingOperation, PteMaterialization)>;
fn split(&mut self, align_diff: usize) -> Option<MappingOperation>;
fn shrink_left(&mut self, _shrink_size: usize) -> bool;
fn shrink_right(&mut self, _shrink_size: usize) -> bool;
}
#[derive(Clone)]
pub struct MappingOperation {
kind: MappingOperationKind,
write_access: Option<Arc<MappingWriteAccess>>,
}
struct MappingWriteAccess {
lease: crate::sync::Mutex<Option<Arc<ax_fs_ng::file::WriteAccess>>>,
}
#[derive(Clone)]
enum MappingOperationKind {
Linear(linear::LinearBackend),
Cow(cow::CowBackend),
Shared(shared::SharedBackend),
File(file::FileBackend),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
pub enum SharedFutexRegion {
SharedMemory(MappingId),
File(CachedFileIdentity),
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SharedFutexIdentity {
region: SharedFutexRegion,
offset: usize,
}
impl SharedFutexIdentity {
const fn shared_memory(mapping: MappingId, offset: usize) -> Self {
Self {
region: SharedFutexRegion::SharedMemory(mapping),
offset,
}
}
const fn file(cache: CachedFileIdentity, offset: usize) -> Self {
Self {
region: SharedFutexRegion::File(cache),
offset,
}
}
pub const fn region(self) -> SharedFutexRegion {
self.region
}
pub const fn offset(self) -> usize {
self.offset
}
}
#[derive(Debug, Clone)]
pub struct MappingFileInfo {
pub path: String,
pub offset: Option<u64>,
pub inode: Option<u64>,
pub dev: Option<u64>,
pub shared: bool,
}
#[derive(Clone)]
pub(crate) struct SharedFileMappingLease {
backend: file::FileBackend,
}
impl SharedFileMappingLease {
pub(crate) fn check_flags(&self, flags: MappingFlags) -> StarryResult {
self.backend.check_flags(flags)
}
pub(crate) fn cache(&self) -> &CachedFile {
self.backend.cache()
}
pub(crate) fn cache_location(&self) -> &axfs_ng_vfs::Location {
self.backend.cache_location()
}
pub(crate) fn file_offset_at(&self, address: VirtAddr) -> u64 {
self.backend.file_offset_at(address)
}
pub(crate) fn writeback_range(&self, start: VirtAddr, end: VirtAddr) -> StarryResult {
self.backend.writeback_range(start, end)
}
pub(crate) fn pageout_range(
&self,
start: VirtAddr,
end: VirtAddr,
) -> StarryResult<ax_fs_ng::file::CachePageoutResult> {
self.backend.pageout_range(start, end)
}
}
pub(crate) struct ResidentLeafRestore<'a> {
pub va: VirtAddr,
pub paddr: PhysAddr,
pub page_size: usize,
pub flags: MappingFlags,
pub page: Option<&'a Arc<super::objects::PageObject>>,
}
impl MappingOperation {
pub(crate) fn with_write_access(
mut self,
access: Option<Arc<ax_fs_ng::file::WriteAccess>>,
) -> Self {
self.write_access = access.map(|lease| {
Arc::new(MappingWriteAccess {
lease: crate::sync::Mutex::new(Some(lease)),
})
});
self
}
pub(super) fn take_write_access(&self) -> Option<Arc<ax_fs_ng::file::WriteAccess>> {
self.write_access.as_ref()?.lease.lock().take()
}
fn from_linear(backend: linear::LinearBackend) -> Self {
Self {
kind: MappingOperationKind::Linear(backend),
write_access: None,
}
}
fn from_cow(backend: cow::CowBackend) -> Self {
Self {
kind: MappingOperationKind::Cow(backend),
write_access: None,
}
}
fn from_shared(backend: shared::SharedBackend) -> Self {
Self {
kind: MappingOperationKind::Shared(backend),
write_access: None,
}
}
fn from_file(backend: file::FileBackend) -> Self {
Self {
kind: MappingOperationKind::File(backend),
write_access: None,
}
}
pub fn page_size(&self) -> usize {
MappingExecution::page_size(self)
}
pub(crate) fn validate_map_range(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
MappingExecution::validate_map(self, range, pt)
}
pub(super) fn map_range(
&self,
range: VirtAddrRange,
flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult<PteMaterialization> {
MappingExecution::map(self, range, flags, pt)
}
pub(crate) fn validate_protect_range(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
MappingExecution::validate_protect(self, range, pt)
}
pub(crate) fn protect_range(
&self,
range: VirtAddrRange,
new_flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult {
MappingExecution::on_protect(self, range, new_flags, pt)?;
let pte_flags = match &self.kind {
MappingOperationKind::Cow(cow) => cow.pte_flags_for_protect(new_flags),
_ => new_flags,
};
let leaves = occupied_leaf_ranges(range, pt)?;
for (va, expected_size) in leaves {
if pt.protect_page(va, pte_flags)? != expected_size {
return Err(StarryError::BadState);
}
}
Ok(())
}
pub(crate) fn fragment(
&self,
source: VirtAddrRange,
fragment: VirtAddrRange,
) -> StarryResult<Self> {
if fragment.is_empty() || fragment.start < source.start || fragment.end > source.end {
return Err(StarryError::InvalidInput);
}
let mut operation = self.clone();
if fragment.start > source.start {
let offset = fragment
.start
.checked_sub_addr(source.start)
.ok_or(StarryError::InvalidInput)?;
operation =
MappingExecution::split(&mut operation, offset).ok_or(StarryError::BadState)?;
}
if fragment.end < source.end {
let shrink = source
.end
.checked_sub_addr(fragment.end)
.ok_or(StarryError::InvalidInput)?;
if !MappingExecution::shrink_right(&mut operation, shrink) {
return Err(StarryError::BadState);
}
}
Ok(operation)
}
pub(crate) fn shared_file_lease(&self) -> Option<SharedFileMappingLease> {
match &self.kind {
MappingOperationKind::File(backend) if backend.is_shared_file_map() => {
Some(SharedFileMappingLease {
backend: backend.clone(),
})
}
_ => None,
}
}
pub(crate) fn shared_memory_object(&self) -> Option<Arc<SharedMemoryObject>> {
match &self.kind {
MappingOperationKind::Shared(backend) => Some(backend.object().clone()),
_ => None,
}
}
pub(crate) fn mremap_alignment(&self) -> usize {
match &self.kind {
MappingOperationKind::Shared(shared) => shared.mapping_alignment(),
MappingOperationKind::Linear(_)
| MappingOperationKind::Cow(_)
| MappingOperationKind::File(_) => PAGE_SIZE_4K,
}
}
pub(crate) fn supports_mremap_dontunmap(&self) -> bool {
matches!(&self.kind, MappingOperationKind::Cow(cow) if cow.is_anonymous())
}
pub(crate) fn is_linear(&self) -> bool {
matches!(&self.kind, MappingOperationKind::Linear(_))
}
pub(crate) fn is_private_anonymous(&self) -> bool {
self.supports_mremap_dontunmap()
}
pub(crate) fn maximum_mapping_flags(&self, current: MappingFlags) -> MappingFlags {
let permission_bits =
MappingFlags::READ | MappingFlags::WRITE | MappingFlags::EXECUTE | MappingFlags::USER;
let attributes = current - permission_bits;
match &self.kind {
MappingOperationKind::Cow(_) | MappingOperationKind::Shared(_) => {
attributes | permission_bits
}
MappingOperationKind::File(file) => {
let mut maximum =
attributes | MappingFlags::READ | MappingFlags::EXECUTE | MappingFlags::USER;
if file.check_flags(MappingFlags::WRITE).is_ok() {
maximum |= MappingFlags::WRITE;
}
maximum
}
MappingOperationKind::Linear(_) => current,
}
}
pub(crate) fn check_mprotect_flags(&self, flags: MappingFlags) -> StarryResult {
match &self.kind {
MappingOperationKind::File(file) => file.check_flags(flags),
_ => Ok(()),
}
}
pub(crate) fn validate_discard_fragment(&self, range: VirtAddrRange) -> StarryResult {
match &self.kind {
MappingOperationKind::Linear(_) => Err(StarryError::InvalidInput),
MappingOperationKind::Shared(_)
if !range.start.is_aligned(self.page_size())
|| !range.end.is_aligned(self.page_size()) =>
{
Err(StarryError::OperationNotSupported)
}
MappingOperationKind::Cow(_)
| MappingOperationKind::Shared(_)
| MappingOperationKind::File(_) => Ok(()),
}
}
pub(crate) fn pin_file_cache_owner_for_mapping(
&self,
address: VirtAddr,
paddr: PhysAddr,
) -> StarryResult<Option<ax_fs_ng::file::CachedPagePin>> {
match &self.kind {
MappingOperationKind::File(file) => {
file.pin_cache_owner_for_mapping(address, paddr).map(Some)
}
_ => Ok(None),
}
}
pub(super) fn populate(
&self,
space_id: AddressSpaceId,
request: PopulateRequest,
flags: MappingFlags,
access_flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult<PteMaterialization> {
MappingExecution::populate(self, space_id, request, flags, access_flags, pt)
}
pub(super) fn prepare_fault(
&self,
space_id: AddressSpaceId,
request: PopulateRequest,
flags: MappingFlags,
access_flags: MappingFlags,
preimage: FaultPteSnapshot,
) -> StarryResult<FaultMaterialization> {
MappingExecution::prepare_fault(self, space_id, request, flags, access_flags, preimage)
}
pub(super) fn clone_map(
&self,
range: VirtAddrRange,
flags: MappingFlags,
old_pt: &mut PageTable,
new_pt: &mut PageTable,
) -> StarryResult<(Self, PteMaterialization)> {
MappingExecution::clone_map(self, range, flags, old_pt, new_pt)
}
pub(crate) fn validate_unmap_range(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
MappingExecution::validate_unmap(self, range, pt)
}
pub(super) fn unmap_range(
&self,
range: VirtAddrRange,
context: &mut MappingMutationContext,
pt: &mut PageTable,
) -> StarryResult {
MappingExecution::unmap(self, range, context, pt)
}
pub(crate) fn shared_futex_identity(&self, address: VirtAddr) -> Option<SharedFutexIdentity> {
match &self.kind {
MappingOperationKind::Shared(shared) => shared.shared_futex_identity(address),
MappingOperationKind::File(file) => file.shared_futex_identity(address),
MappingOperationKind::Linear(_) | MappingOperationKind::Cow(_) => None,
}
}
pub(crate) const fn requires_fork_write_protect(&self) -> bool {
matches!(&self.kind, MappingOperationKind::Cow(_))
}
pub(crate) fn mapping_id(&self) -> super::vma::MappingId {
match &self.kind {
MappingOperationKind::Linear(backend) => backend.mapping_id(),
MappingOperationKind::Cow(backend) => backend.mapping_id(),
MappingOperationKind::Shared(backend) => backend.object().mapping_id(),
MappingOperationKind::File(backend) => backend.mapping_id(),
}
}
pub(crate) fn finish_page_publication(
&self,
va: VirtAddr,
page: &Arc<super::objects::PageObject>,
) -> StarryResult {
match &self.kind {
MappingOperationKind::Cow(cow) => cow.publish_page_object(page),
MappingOperationKind::File(file) => file.finish_page_publication(va, page),
MappingOperationKind::Linear(_) | MappingOperationKind::Shared(_) => Ok(()),
}
}
pub(super) fn cancel_prepared_fault_publication(
&self,
materialization: FaultMaterialization,
) -> StarryResult {
let Some(owner) = materialization.into_owner() else {
return Ok(());
};
if owner.provider_publication != ProviderPublication::Pending {
return Ok(());
}
match &self.kind {
MappingOperationKind::Cow(cow) => cow.cancel_page_publication(&owner.page),
MappingOperationKind::File(file) => file.cancel_page_publication(owner.va, &owner.page),
MappingOperationKind::Linear(_) | MappingOperationKind::Shared(_) => Ok(()),
}
}
pub(crate) fn restore_resident_preimage(
&self,
leaf: ResidentLeafRestore<'_>,
pt: &mut PageTable,
) -> StarryResult {
let ResidentLeafRestore {
va,
paddr,
page_size,
flags,
page,
} = leaf;
match &self.kind {
MappingOperationKind::Cow(cow) => {
let page = page.ok_or(StarryError::BadState)?;
cow.restore_page_identity(page)?;
}
MappingOperationKind::File(file) => {
let page = page.ok_or(StarryError::BadState)?;
file.ensure_page_identity(va, page)?;
}
MappingOperationKind::Linear(_) | MappingOperationKind::Shared(_) => {}
}
if let Err(error) = pt.map_page(va, paddr, page_size, flags) {
return Err(error.into());
}
Ok(())
}
pub fn is_file_backed(&self) -> bool {
match &self.kind {
MappingOperationKind::Cow(backend) => !backend.is_anonymous(),
MappingOperationKind::File(_) => true,
MappingOperationKind::Linear(_) | MappingOperationKind::Shared(_) => false,
}
}
pub(crate) fn mincore_resident(&self, va: VirtAddr, cred: &crate::task::Cred) -> bool {
match &self.kind {
MappingOperationKind::Cow(cow) => {
cow.mincore_location()
.is_some_and(|location| !mincore_file_visible(location, cred))
|| cow.page_cache_resident(va)
}
MappingOperationKind::File(file) => {
!mincore_file_visible(file.mincore_location(), cred) || file.page_cache_resident(va)
}
MappingOperationKind::Shared(shared) => shared.page_cache_resident(va),
MappingOperationKind::Linear(_) => true,
}
}
pub(crate) fn vma_descriptor(&self, area_start: VirtAddr) -> VmaDescriptor {
MappingExecution::vma_descriptor(self, area_start)
}
pub fn file_info(&self) -> StarryResult<MappingFileInfo> {
match &self.kind {
MappingOperationKind::Cow(b) => b.file_info(),
MappingOperationKind::Linear(b) => Ok(MappingFileInfo {
path: "".to_string(),
offset: None,
inode: None,
dev: None,
shared: b.is_shared(),
}),
MappingOperationKind::Shared(_) => Ok(MappingFileInfo {
path: "".to_string(),
offset: None,
inode: None,
dev: None,
shared: true,
}),
MappingOperationKind::File(b) => b.file_info(),
}
}
pub fn relocated(&self, new_start: VirtAddr, src_offset: usize) -> StarryResult<Self> {
let adjusted = new_start
.as_usize()
.checked_sub(src_offset)
.map(VirtAddr::from)
.ok_or(StarryError::InvalidInput)?;
let mut operation = match &self.kind {
MappingOperationKind::Cow(cb) => Self::from_cow(cb.with_start(adjusted)),
MappingOperationKind::Shared(sb) => Self::from_shared(sb.with_start(adjusted)),
MappingOperationKind::Linear(_) => return Err(StarryError::OperationNotSupported),
MappingOperationKind::File(fb) => Self::from_file(fb.with_start(adjusted)?),
};
operation.write_access = self.write_access.clone();
Ok(operation)
}
pub fn resized(&self, size: usize) -> StarryResult<Self> {
if size == 0 {
return Err(StarryError::InvalidInput);
}
let mut operation = match &self.kind {
MappingOperationKind::Cow(cow) => Self::from_cow(cow.for_extent(size)?),
MappingOperationKind::Shared(shared) => Self::from_shared(shared.with_size(size)?),
MappingOperationKind::Linear(_) => return Err(StarryError::OperationNotSupported),
MappingOperationKind::File(file) => Self::from_file(file.clone()),
};
operation.write_access = self.write_access.clone();
Ok(operation)
}
}
impl MappingExecution for MappingOperation {
fn page_size(&self) -> usize {
match &self.kind {
MappingOperationKind::Linear(backend) => MappingExecution::page_size(backend),
MappingOperationKind::Cow(backend) => MappingExecution::page_size(backend),
MappingOperationKind::Shared(backend) => MappingExecution::page_size(backend),
MappingOperationKind::File(backend) => MappingExecution::page_size(backend),
}
}
fn vma_descriptor(&self, area_start: VirtAddr) -> VmaDescriptor {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::vma_descriptor(backend, area_start)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::vma_descriptor(backend, area_start)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::vma_descriptor(backend, area_start)
}
MappingOperationKind::File(backend) => {
MappingExecution::vma_descriptor(backend, area_start)
}
}
}
fn map(
&self,
range: VirtAddrRange,
flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult<PteMaterialization> {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::map(backend, range, flags, pt)
}
MappingOperationKind::Cow(backend) => MappingExecution::map(backend, range, flags, pt),
MappingOperationKind::Shared(backend) => {
MappingExecution::map(backend, range, flags, pt)
}
MappingOperationKind::File(backend) => MappingExecution::map(backend, range, flags, pt),
}
}
fn validate_map(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::validate_map(backend, range, pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::validate_map(backend, range, pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::validate_map(backend, range, pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::validate_map(backend, range, pt)
}
}
}
fn unmap(
&self,
range: VirtAddrRange,
context: &mut MappingMutationContext,
pt: &mut PageTable,
) -> StarryResult {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::unmap(backend, range, context, pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::unmap(backend, range, context, pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::unmap(backend, range, context, pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::unmap(backend, range, context, pt)
}
}
}
fn validate_unmap(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::validate_unmap(backend, range, pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::validate_unmap(backend, range, pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::validate_unmap(backend, range, pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::validate_unmap(backend, range, pt)
}
}
}
fn validate_protect(&self, range: VirtAddrRange, pt: &PageTable) -> bool {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::validate_protect(backend, range, pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::validate_protect(backend, range, pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::validate_protect(backend, range, pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::validate_protect(backend, range, pt)
}
}
}
fn on_protect(
&self,
range: VirtAddrRange,
new_flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::on_protect(backend, range, new_flags, pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::on_protect(backend, range, new_flags, pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::on_protect(backend, range, new_flags, pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::on_protect(backend, range, new_flags, pt)
}
}
}
fn populate(
&self,
space_id: AddressSpaceId,
request: PopulateRequest,
flags: MappingFlags,
access_flags: MappingFlags,
pt: &mut PageTable,
) -> StarryResult<PteMaterialization> {
match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::populate(backend, space_id, request, flags, access_flags, pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::populate(backend, space_id, request, flags, access_flags, pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::populate(backend, space_id, request, flags, access_flags, pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::populate(backend, space_id, request, flags, access_flags, pt)
}
}
}
fn prepare_fault(
&self,
space_id: AddressSpaceId,
request: PopulateRequest,
flags: MappingFlags,
access_flags: MappingFlags,
preimage: FaultPteSnapshot,
) -> StarryResult<FaultMaterialization> {
match &self.kind {
MappingOperationKind::Linear(backend) => MappingExecution::prepare_fault(
backend,
space_id,
request,
flags,
access_flags,
preimage,
),
MappingOperationKind::Cow(backend) => MappingExecution::prepare_fault(
backend,
space_id,
request,
flags,
access_flags,
preimage,
),
MappingOperationKind::Shared(backend) => MappingExecution::prepare_fault(
backend,
space_id,
request,
flags,
access_flags,
preimage,
),
MappingOperationKind::File(backend) => MappingExecution::prepare_fault(
backend,
space_id,
request,
flags,
access_flags,
preimage,
),
}
}
fn clone_map(
&self,
range: VirtAddrRange,
flags: MappingFlags,
old_pt: &mut PageTable,
new_pt: &mut PageTable,
) -> StarryResult<(MappingOperation, PteMaterialization)> {
let (operation, materialization) = match &self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::clone_map(backend, range, flags, old_pt, new_pt)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::clone_map(backend, range, flags, old_pt, new_pt)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::clone_map(backend, range, flags, old_pt, new_pt)
}
MappingOperationKind::File(backend) => {
MappingExecution::clone_map(backend, range, flags, old_pt, new_pt)
}
}?;
let write_access = self
.write_access
.as_ref()
.and_then(|access| access.lease.lock().clone());
Ok((operation.with_write_access(write_access), materialization))
}
fn split(&mut self, align_diff: usize) -> Option<MappingOperation> {
let mut operation = match &mut self.kind {
MappingOperationKind::Linear(backend) => MappingExecution::split(backend, align_diff),
MappingOperationKind::Cow(backend) => MappingExecution::split(backend, align_diff),
MappingOperationKind::Shared(backend) => MappingExecution::split(backend, align_diff),
MappingOperationKind::File(backend) => MappingExecution::split(backend, align_diff),
}?;
operation.write_access = self.write_access.clone();
Some(operation)
}
fn shrink_left(&mut self, shrink_size: usize) -> bool {
match &mut self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::shrink_left(backend, shrink_size)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::shrink_left(backend, shrink_size)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::shrink_left(backend, shrink_size)
}
MappingOperationKind::File(backend) => {
MappingExecution::shrink_left(backend, shrink_size)
}
}
}
fn shrink_right(&mut self, shrink_size: usize) -> bool {
match &mut self.kind {
MappingOperationKind::Linear(backend) => {
MappingExecution::shrink_right(backend, shrink_size)
}
MappingOperationKind::Cow(backend) => {
MappingExecution::shrink_right(backend, shrink_size)
}
MappingOperationKind::Shared(backend) => {
MappingExecution::shrink_right(backend, shrink_size)
}
MappingOperationKind::File(backend) => {
MappingExecution::shrink_right(backend, shrink_size)
}
}
}
}
impl MappingBackend for MappingOperation {
type Addr = VirtAddr;
type Flags = MappingFlags;
type MutationContext = ();
type PageTable = PageTable;
fn map(
&self,
start: VirtAddr,
size: usize,
flags: MappingFlags,
_context: &mut (),
pt: &mut PageTable,
) -> bool {
let range = VirtAddrRange::from_start_size(start, size);
if let Err(err) = MappingExecution::map(self, range, flags, pt) {
warn!("Failed to map area: {:?}", err);
false
} else {
true
}
}
fn validate_map(
&self,
start: VirtAddr,
size: usize,
_flags: MappingFlags,
pt: &PageTable,
) -> bool {
let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
return false;
};
MappingExecution::validate_map(self, range, pt)
}
fn unmap(
&self,
start: VirtAddr,
size: usize,
_context: &mut (),
pt: &mut PageTable,
) -> bool {
let range = VirtAddrRange::from_start_size(start, size);
let capacity = match occupied_leaf_ranges(range, pt) {
Ok(leaves) => leaves.len(),
Err(err) => {
warn!("Failed to prepare area unmap: {err:?}");
return false;
}
};
let mut context = match MappingMutationContext::for_rollback(capacity) {
Ok(context) => context,
Err(err) => {
warn!("Failed to reserve area-unmap ownership: {err:?}");
return false;
}
};
let result = MappingExecution::unmap(self, range, &mut context, pt);
if context.pte_changed() {
if let Err(err) = crate::mm::flush_tlb_range_sync(start, size) {
warn!("Failed to confirm area unmap: {err:?}");
return false;
}
unsafe { context.reclaim() };
}
if let Err(err) = result {
warn!("Failed to unmap area: {:?}", err);
false
} else {
true
}
}
fn validate_unmap(&self, start: VirtAddr, size: usize, pt: &PageTable) -> bool {
let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
return false;
};
MappingExecution::validate_unmap(self, range, pt)
}
fn protect(
&self,
start: Self::Addr,
size: usize,
new_flags: Self::Flags,
_context: &mut (),
pt: &mut Self::PageTable,
) -> bool {
let range = VirtAddrRange::from_start_size(start, size);
if let Err(err) = MappingExecution::on_protect(self, range, new_flags, pt) {
warn!("Failed to protect area: {:?}", err);
return false;
}
let pte_flags = match &self.kind {
MappingOperationKind::Cow(c) => c.pte_flags_for_protect(new_flags),
_ => new_flags,
};
pt.protect_region(start, size, pte_flags).is_ok()
}
fn validate_protect(
&self,
start: VirtAddr,
size: usize,
_new_flags: MappingFlags,
pt: &PageTable,
) -> bool {
let Some(range) = VirtAddrRange::try_from_start_size(start, size) else {
return false;
};
MappingExecution::validate_protect(self, range, pt)
}
fn split(&mut self, align_diff: usize) -> Option<Self> {
MappingExecution::split(self, align_diff)
}
fn shrink_left(&mut self, shrink_size: usize) -> bool {
MappingExecution::shrink_left(self, shrink_size)
}
fn shrink_right(&mut self, shrink_size: usize) -> bool {
MappingExecution::shrink_right(self, shrink_size)
}
}