#[cfg_attr(target_arch = "x86_64", path = "arch/amd64/vmem.rs")]
#[cfg_attr(target_arch = "aarch64", path = "arch/aarch64/vmem.rs")]
mod arch;
pub use arch::PAGE_SIZE;
pub use arch::{PAGE_PRESENT, PAGE_TABLE_SIZE, PTE_ADDR_MASK, PageTableEntry, PhysAddr, VirtAddr};
pub const PAGE_TABLE_ENTRIES_PER_TABLE: usize =
PAGE_TABLE_SIZE / core::mem::size_of::<PageTableEntry>();
#[cfg(target_arch = "aarch64")]
pub use arch::ATTR_INDEX_NORMAL;
#[inline(always)]
pub(crate) fn bits<const HIGH_BIT: u8, const LOW_BIT: u8>(x: u64) -> u64 {
(x & ((1 << (HIGH_BIT + 1)) - 1)) >> LOW_BIT
}
pub(in crate::vmem) unsafe fn write_entry_updating<
Op: TableOps,
P: UpdateParent<
Op,
TableMoveInfo = <Op::TableMovability as TableMovabilityBase<Op>>::TableMoveInfo,
>,
>(
op: &Op,
parent: P,
addr: Op::TableAddr,
entry: u64,
) {
#[allow(clippy::useless_conversion)]
if let Some(again) = unsafe { op.write_entry(addr, entry as PageTableEntry) } {
parent.update_parent(op, again);
}
}
pub trait UpdateParent<Op: TableReadOps + ?Sized>: Copy {
type TableMoveInfo;
type ChildType: UpdateParent<Op, TableMoveInfo = Self::TableMoveInfo>;
fn update_parent(self, op: &Op, new_ptr: Self::TableMoveInfo);
fn for_child_at_entry(self, entry_ptr: Op::TableAddr) -> Self::ChildType;
}
#[derive(Copy, Clone)]
pub struct UpdateParentNone {}
impl<Op: TableReadOps> UpdateParent<Op> for UpdateParentNone {
type TableMoveInfo = Void;
type ChildType = Self;
fn update_parent(self, _op: &Op, impossible: Void) {
match impossible {}
}
fn for_child_at_entry(self, _entry_ptr: Op::TableAddr) -> Self {
self
}
}
#[allow(unused)] pub(in crate::vmem) struct UpdateParentTable<Op: TableOps, P: UpdateParent<Op>> {
pub(in crate::vmem) parent: P,
pub(in crate::vmem) entry_ptr: Op::TableAddr,
}
impl<Op: TableOps, P: UpdateParent<Op>> Clone for UpdateParentTable<Op, P> {
fn clone(&self) -> Self {
*self
}
}
impl<Op: TableOps, P: UpdateParent<Op>> Copy for UpdateParentTable<Op, P> {}
impl<Op: TableOps, P: UpdateParent<Op>> UpdateParentTable<Op, P> {
#[allow(unused)] pub(in crate::vmem) fn new(parent: P, entry_ptr: Op::TableAddr) -> Self {
UpdateParentTable { parent, entry_ptr }
}
}
#[derive(Copy, Clone)]
pub struct UpdateParentRoot {}
pub(in crate::vmem) struct MapRequest<Op: TableReadOps, P: UpdateParent<Op>> {
pub table_base: Op::TableAddr,
pub vmin: u64,
pub len: u64,
pub update_parent: P,
}
pub(in crate::vmem) struct MapResponse<Op: TableReadOps, P: UpdateParent<Op>> {
pub entry_ptr: Op::TableAddr,
pub vmin: u64,
pub len: u64,
pub update_parent: P,
}
pub(in crate::vmem) struct ModifyPteIterator<
const HIGH_BIT: u8,
const LOW_BIT: u8,
Op: TableReadOps,
P: UpdateParent<Op>,
> {
request: MapRequest<Op, P>,
n: u64,
}
impl<const HIGH_BIT: u8, const LOW_BIT: u8, Op: TableReadOps, P: UpdateParent<Op>> Iterator
for ModifyPteIterator<HIGH_BIT, LOW_BIT, Op, P>
{
type Item = MapResponse<Op, P>;
fn next(&mut self) -> Option<Self::Item> {
let lower_bits_mask = (1u64 << LOW_BIT) - 1;
let next_vmin = if self.n == 0 {
self.request.vmin
} else {
let aligned_min = self.request.vmin & !lower_bits_mask;
aligned_min.checked_add(self.n << LOW_BIT)?
};
if next_vmin >= self.request.vmin + self.request.len {
return None;
}
let pte_index = bits::<HIGH_BIT, LOW_BIT>(next_vmin);
let entry_ptr = Op::entry_addr(
self.request.table_base,
pte_index * core::mem::size_of::<PageTableEntry>() as u64,
);
let len_from_here = self.request.len - (next_vmin - self.request.vmin);
let max_len = (1u64 << LOW_BIT) - (next_vmin & lower_bits_mask);
let next_len = core::cmp::min(len_from_here, max_len);
self.n += 1;
Some(MapResponse {
entry_ptr,
vmin: next_vmin,
len: next_len,
update_parent: self.request.update_parent,
})
}
}
pub(in crate::vmem) fn modify_ptes<
const HIGH_BIT: u8,
const LOW_BIT: u8,
Op: TableReadOps,
P: UpdateParent<Op>,
>(
r: MapRequest<Op, P>,
) -> ModifyPteIterator<HIGH_BIT, LOW_BIT, Op, P> {
ModifyPteIterator { request: r, n: 0 }
}
pub trait TableReadOps {
type TableAddr: Copy;
fn entry_addr(addr: Self::TableAddr, entry_offset: u64) -> Self::TableAddr;
unsafe fn read_entry(&self, addr: Self::TableAddr) -> PageTableEntry;
fn to_phys(addr: Self::TableAddr) -> PhysAddr;
fn from_phys(addr: PhysAddr) -> Self::TableAddr;
fn root_table(&self) -> Self::TableAddr;
}
pub enum Void {}
pub struct MayMoveTable {}
pub struct MayNotMoveTable {}
mod sealed {
use super::{MayMoveTable, MayNotMoveTable, TableReadOps, Void};
pub trait TableMovabilityBase<Op: TableReadOps + ?Sized> {
type TableMoveInfo;
}
impl<Op: TableReadOps> TableMovabilityBase<Op> for MayMoveTable {
type TableMoveInfo = Op::TableAddr;
}
impl<Op: TableReadOps> TableMovabilityBase<Op> for MayNotMoveTable {
type TableMoveInfo = Void;
}
}
use sealed::*;
#[allow(private_bounds)] pub trait TableMovability<Op: TableReadOps + ?Sized>:
TableMovabilityBase<Op>
+ arch::TableMovability<Op, <Self as TableMovabilityBase<Op>>::TableMoveInfo>
{
}
impl<
Op: TableReadOps,
T: TableMovabilityBase<Op>
+ arch::TableMovability<Op, <Self as TableMovabilityBase<Op>>::TableMoveInfo>,
> TableMovability<Op> for T
{
}
pub trait TableOps: TableReadOps {
type TableMovability: TableMovability<Self>;
unsafe fn alloc_table(&self) -> Self::TableAddr;
unsafe fn write_entry(
&self,
addr: Self::TableAddr,
entry: PageTableEntry,
) -> Option<<Self::TableMovability as TableMovabilityBase<Self>>::TableMoveInfo>;
unsafe fn update_root(
&self,
new_root: <Self::TableMovability as TableMovabilityBase<Self>>::TableMoveInfo,
);
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub struct BasicMapping {
pub readable: bool,
pub writable: bool,
pub executable: bool,
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub struct CowMapping {
pub readable: bool,
pub executable: bool,
}
#[derive(Debug, PartialEq, Clone, Copy)]
pub enum MappingKind {
Unmapped,
ZeroInit(BasicMapping),
Basic(BasicMapping),
Cow(CowMapping),
}
#[derive(Debug, Clone, Copy)]
pub struct Mapping {
pub phys_base: u64,
pub virt_base: u64,
pub len: u64,
pub kind: MappingKind,
}
pub use arch::map;
pub use arch::modify_mapping;
pub use arch::virt_to_phys;
pub type SpaceId = u64;
#[derive(Debug, Clone, Copy)]
pub struct SpaceReferenceMapping {
pub depth: usize,
pub space: SpaceId,
pub our_va: u64,
pub their_va: u64,
}
#[derive(Debug)]
pub enum SpaceAwareMapping {
ThisSpace(Mapping),
AnotherSpace(SpaceReferenceMapping),
}
pub(crate) fn push_space_mapping(
mappings: &mut alloc::vec::Vec<SpaceAwareMapping>,
mapping: SpaceAwareMapping,
) {
if let SpaceAwareMapping::ThisSpace(next) = &mapping
&& let Some(SpaceAwareMapping::ThisSpace(previous)) = mappings.last_mut()
&& previous.kind == next.kind
&& previous.phys_base.checked_add(previous.len) == Some(next.phys_base)
&& previous.virt_base.checked_add(previous.len) == Some(next.virt_base)
&& let Some(len) = previous.len.checked_add(next.len)
{
previous.len = len;
return;
}
mappings.push(mapping);
}
pub use arch::space_aware_map;
pub use arch::walk_va_spaces;
#[cfg(test)]
mod tests {
use super::*;
fn basic_mapping(phys_base: u64, virt_base: u64, len: u64) -> SpaceAwareMapping {
SpaceAwareMapping::ThisSpace(Mapping {
phys_base,
virt_base,
len,
kind: MappingKind::Basic(BasicMapping {
readable: true,
writable: false,
executable: true,
}),
})
}
#[test]
fn push_space_mapping_coalesces_adjacent_mappings() {
let mut mappings = alloc::vec![basic_mapping(0x1000, 0x2000, PAGE_SIZE as u64)];
push_space_mapping(
&mut mappings,
basic_mapping(
0x1000 + PAGE_SIZE as u64,
0x2000 + PAGE_SIZE as u64,
PAGE_SIZE as u64,
),
);
assert_eq!(mappings.len(), 1);
let SpaceAwareMapping::ThisSpace(mapping) = &mappings[0] else {
panic!("expected a local mapping");
};
assert_eq!(mapping.len, 2 * PAGE_SIZE as u64);
}
#[test]
fn push_space_mapping_preserves_non_adjacent_mappings() {
for next in [
basic_mapping(0x3000, 0x3000, PAGE_SIZE as u64),
basic_mapping(0x2000, 0x4000, PAGE_SIZE as u64),
SpaceAwareMapping::ThisSpace(Mapping {
phys_base: 0x2000,
virt_base: 0x3000,
len: PAGE_SIZE as u64,
kind: MappingKind::Cow(CowMapping {
readable: true,
executable: true,
}),
}),
] {
let mut mappings = alloc::vec![basic_mapping(0x1000, 0x2000, PAGE_SIZE as u64)];
push_space_mapping(&mut mappings, next);
assert_eq!(mappings.len(), 2);
}
}
#[test]
fn push_space_mapping_preserves_space_reference_boundaries() {
let reference = SpaceAwareMapping::AnotherSpace(SpaceReferenceMapping {
depth: 1,
space: 0x1000,
our_va: 0x2000,
their_va: 0x2000,
});
let mut mappings = alloc::vec![basic_mapping(0x1000, 0x2000, PAGE_SIZE as u64)];
push_space_mapping(&mut mappings, reference);
push_space_mapping(
&mut mappings,
basic_mapping(
0x1000 + PAGE_SIZE as u64,
0x2000 + PAGE_SIZE as u64,
PAGE_SIZE as u64,
),
);
assert_eq!(mappings.len(), 3);
}
#[test]
fn push_space_mapping_preserves_overflowing_mappings() {
let mut mappings = alloc::vec![basic_mapping(0, 0, u64::MAX)];
push_space_mapping(&mut mappings, basic_mapping(u64::MAX, u64::MAX, 1));
assert_eq!(mappings.len(), 2);
}
}