#![cfg(target_arch = "x86_64")]
use super::*;
use crate::registers::control::Cr3;
use crate::structures::paging::PageTableIndex;
use crate::structures::paging::{
frame_alloc::FrameAllocator,
page::NotGiantPageSize,
page_table::{FrameError, PageTable, PageTableEntry, PageTableFlags},
Page, PageSize, PhysFrame, Size1GiB, Size2MiB, Size4KiB,
};
use crate::VirtAddr;
#[derive(Debug)]
pub struct RecursivePageTable<'a> {
p4: &'a mut PageTable,
recursive_index: PageTableIndex,
}
impl<'a> RecursivePageTable<'a> {
#[inline]
pub fn new(table: &'a mut PageTable) -> Result<Self, ()> {
let page = Page::containing_address(VirtAddr::new(table as *const _ as u64));
let recursive_index = page.p4_index();
if page.p3_index() != recursive_index
|| page.p2_index() != recursive_index
|| page.p1_index() != recursive_index
{
return Err(());
}
if Ok(Cr3::read().0) != table[recursive_index].frame() {
return Err(());
}
Ok(RecursivePageTable {
p4: table,
recursive_index,
})
}
#[inline]
pub unsafe fn new_unchecked(table: &'a mut PageTable, recursive_index: PageTableIndex) -> Self {
RecursivePageTable {
p4: table,
recursive_index,
}
}
unsafe fn create_next_table<'b, A, S: PageSize>(
entry: &'b mut PageTableEntry,
next_table_page: Page,
allocator: &mut A,
) -> Result<&'b mut PageTable, MapToError<S>>
where
A: FrameAllocator<Size4KiB>,
{
fn inner<'b, A, S: PageSize>(
entry: &'b mut PageTableEntry,
next_table_page: Page,
allocator: &mut A,
) -> Result<&'b mut PageTable, MapToError<S>>
where
A: FrameAllocator<Size4KiB>,
{
use crate::structures::paging::PageTableFlags as Flags;
let created;
if entry.is_unused() {
if let Some(frame) = allocator.allocate_frame() {
entry.set_frame(frame, Flags::PRESENT | Flags::WRITABLE);
created = true;
} else {
return Err(MapToError::FrameAllocationFailed);
}
} else {
created = false;
}
if entry.flags().contains(Flags::HUGE_PAGE) {
return Err(MapToError::ParentEntryHugePage);
}
let page_table_ptr = next_table_page.start_address().as_mut_ptr();
let page_table: &mut PageTable = unsafe { &mut *(page_table_ptr) };
if created {
page_table.zero();
}
Ok(page_table)
}
inner(entry, next_table_page, allocator)
}
fn map_to_1gib<A>(
&mut self,
page: Page<Size1GiB>,
frame: PhysFrame<Size1GiB>,
flags: PageTableFlags,
allocator: &mut A,
) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
where
A: FrameAllocator<Size4KiB>,
{
use crate::structures::paging::PageTableFlags as Flags;
let p4 = &mut self.p4;
let p3_page = p3_page(page, self.recursive_index);
let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? };
if !p3[page.p3_index()].is_unused() {
return Err(MapToError::PageAlreadyMapped(frame));
}
p3[page.p3_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE);
Ok(MapperFlush::new(page))
}
fn map_to_2mib<A>(
&mut self,
page: Page<Size2MiB>,
frame: PhysFrame<Size2MiB>,
flags: PageTableFlags,
allocator: &mut A,
) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
where
A: FrameAllocator<Size4KiB>,
{
use crate::structures::paging::PageTableFlags as Flags;
let p4 = &mut self.p4;
let p3_page = p3_page(page, self.recursive_index);
let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? };
let p2_page = p2_page(page, self.recursive_index);
let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? };
if !p2[page.p2_index()].is_unused() {
return Err(MapToError::PageAlreadyMapped(frame));
}
p2[page.p2_index()].set_addr(frame.start_address(), flags | Flags::HUGE_PAGE);
Ok(MapperFlush::new(page))
}
fn map_to_4kib<A>(
&mut self,
page: Page<Size4KiB>,
frame: PhysFrame<Size4KiB>,
flags: PageTableFlags,
allocator: &mut A,
) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
where
A: FrameAllocator<Size4KiB>,
{
let p4 = &mut self.p4;
let p3_page = p3_page(page, self.recursive_index);
let p3 = unsafe { Self::create_next_table(&mut p4[page.p4_index()], p3_page, allocator)? };
let p2_page = p2_page(page, self.recursive_index);
let p2 = unsafe { Self::create_next_table(&mut p3[page.p3_index()], p2_page, allocator)? };
let p1_page = p1_page(page, self.recursive_index);
let p1 = unsafe { Self::create_next_table(&mut p2[page.p2_index()], p1_page, allocator)? };
if !p1[page.p1_index()].is_unused() {
return Err(MapToError::PageAlreadyMapped(frame));
}
p1[page.p1_index()].set_frame(frame, flags);
Ok(MapperFlush::new(page))
}
}
impl<'a> Mapper<Size1GiB> for RecursivePageTable<'a> {
#[inline]
unsafe fn map_to<A>(
&mut self,
page: Page<Size1GiB>,
frame: PhysFrame<Size1GiB>,
flags: PageTableFlags,
allocator: &mut A,
) -> Result<MapperFlush<Size1GiB>, MapToError<Size1GiB>>
where
A: FrameAllocator<Size4KiB>,
{
self.map_to_1gib(page, frame, flags, allocator)
}
fn unmap(
&mut self,
page: Page<Size1GiB>,
) -> Result<(PhysFrame<Size1GiB>, MapperFlush<Size1GiB>), UnmapError> {
let p4 = &mut self.p4;
let p4_entry = &p4[page.p4_index()];
p4_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
let p3_entry = &mut p3[page.p3_index()];
let flags = p3_entry.flags();
if !flags.contains(PageTableFlags::PRESENT) {
return Err(UnmapError::PageNotMapped);
}
if !flags.contains(PageTableFlags::HUGE_PAGE) {
return Err(UnmapError::ParentEntryHugePage);
}
let frame = PhysFrame::from_start_address(p3_entry.addr())
.map_err(|()| UnmapError::InvalidFrameAddress(p3_entry.addr()))?;
p3_entry.set_unused();
Ok((frame, MapperFlush::new(page)))
}
#[allow(unused_unsafe)]
unsafe fn update_flags(
&mut self,
page: Page<Size1GiB>,
flags: PageTableFlags,
) -> Result<MapperFlush<Size1GiB>, FlagUpdateError> {
use crate::structures::paging::PageTableFlags as Flags;
let p4 = &mut self.p4;
if p4[page.p4_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
if p3[page.p3_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
p3[page.p3_index()].set_flags(flags | Flags::HUGE_PAGE);
Ok(MapperFlush::new(page))
}
fn translate_page(&self, page: Page<Size1GiB>) -> Result<PhysFrame<Size1GiB>, TranslateError> {
let p4 = &self.p4;
if p4[page.p4_index()].is_unused() {
return Err(TranslateError::PageNotMapped);
}
let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
let p3_entry = &p3[page.p3_index()];
if p3_entry.is_unused() {
return Err(TranslateError::PageNotMapped);
}
PhysFrame::from_start_address(p3_entry.addr())
.map_err(|()| TranslateError::InvalidFrameAddress(p3_entry.addr()))
}
}
impl<'a> Mapper<Size2MiB> for RecursivePageTable<'a> {
#[inline]
unsafe fn map_to<A>(
&mut self,
page: Page<Size2MiB>,
frame: PhysFrame<Size2MiB>,
flags: PageTableFlags,
allocator: &mut A,
) -> Result<MapperFlush<Size2MiB>, MapToError<Size2MiB>>
where
A: FrameAllocator<Size4KiB>,
{
self.map_to_2mib(page, frame, flags, allocator)
}
fn unmap(
&mut self,
page: Page<Size2MiB>,
) -> Result<(PhysFrame<Size2MiB>, MapperFlush<Size2MiB>), UnmapError> {
let p4 = &mut self.p4;
let p4_entry = &p4[page.p4_index()];
p4_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
let p3_entry = &p3[page.p3_index()];
p3_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
let p2_entry = &mut p2[page.p2_index()];
let flags = p2_entry.flags();
if !flags.contains(PageTableFlags::PRESENT) {
return Err(UnmapError::PageNotMapped);
}
if !flags.contains(PageTableFlags::HUGE_PAGE) {
return Err(UnmapError::ParentEntryHugePage);
}
let frame = PhysFrame::from_start_address(p2_entry.addr())
.map_err(|()| UnmapError::InvalidFrameAddress(p2_entry.addr()))?;
p2_entry.set_unused();
Ok((frame, MapperFlush::new(page)))
}
#[allow(unused_unsafe)]
unsafe fn update_flags(
&mut self,
page: Page<Size2MiB>,
flags: PageTableFlags,
) -> Result<MapperFlush<Size2MiB>, FlagUpdateError> {
use crate::structures::paging::PageTableFlags as Flags;
let p4 = &mut self.p4;
if p4[page.p4_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
if p3[page.p3_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
if p2[page.p2_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
p2[page.p2_index()].set_flags(flags | Flags::HUGE_PAGE);
Ok(MapperFlush::new(page))
}
fn translate_page(&self, page: Page<Size2MiB>) -> Result<PhysFrame<Size2MiB>, TranslateError> {
let p4 = &self.p4;
if p4[page.p4_index()].is_unused() {
return Err(TranslateError::PageNotMapped);
}
let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
let p3_entry = &p3[page.p3_index()];
if p3_entry.is_unused() {
return Err(TranslateError::PageNotMapped);
}
let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
let p2_entry = &p2[page.p2_index()];
if p2_entry.is_unused() {
return Err(TranslateError::PageNotMapped);
}
PhysFrame::from_start_address(p2_entry.addr())
.map_err(|()| TranslateError::InvalidFrameAddress(p2_entry.addr()))
}
}
impl<'a> Mapper<Size4KiB> for RecursivePageTable<'a> {
#[inline]
unsafe fn map_to<A>(
&mut self,
page: Page<Size4KiB>,
frame: PhysFrame<Size4KiB>,
flags: PageTableFlags,
allocator: &mut A,
) -> Result<MapperFlush<Size4KiB>, MapToError<Size4KiB>>
where
A: FrameAllocator<Size4KiB>,
{
self.map_to_4kib(page, frame, flags, allocator)
}
fn unmap(
&mut self,
page: Page<Size4KiB>,
) -> Result<(PhysFrame<Size4KiB>, MapperFlush<Size4KiB>), UnmapError> {
let p4 = &mut self.p4;
let p4_entry = &p4[page.p4_index()];
p4_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
let p3_entry = &p3[page.p3_index()];
p3_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
let p2_entry = &p2[page.p2_index()];
p2_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) };
let p1_entry = &mut p1[page.p1_index()];
let frame = p1_entry.frame().map_err(|err| match err {
FrameError::FrameNotPresent => UnmapError::PageNotMapped,
FrameError::HugeFrame => UnmapError::ParentEntryHugePage,
})?;
p1_entry.set_unused();
Ok((frame, MapperFlush::new(page)))
}
#[allow(unused_unsafe)]
unsafe fn update_flags(
&mut self,
page: Page<Size4KiB>,
flags: PageTableFlags,
) -> Result<MapperFlush<Size4KiB>, FlagUpdateError> {
let p4 = &mut self.p4;
if p4[page.p4_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
let p3 = unsafe { &mut *(p3_ptr(page, self.recursive_index)) };
if p3[page.p3_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
let p2 = unsafe { &mut *(p2_ptr(page, self.recursive_index)) };
if p2[page.p2_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
let p1 = unsafe { &mut *(p1_ptr(page, self.recursive_index)) };
if p1[page.p1_index()].is_unused() {
return Err(FlagUpdateError::PageNotMapped);
}
p1[page.p1_index()].set_flags(flags);
Ok(MapperFlush::new(page))
}
fn translate_page(&self, page: Page<Size4KiB>) -> Result<PhysFrame<Size4KiB>, TranslateError> {
let p4 = &self.p4;
if p4[page.p4_index()].is_unused() {
return Err(TranslateError::PageNotMapped);
}
let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
let p3_entry = &p3[page.p3_index()];
if p3_entry.is_unused() {
return Err(TranslateError::PageNotMapped);
}
let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
let p2_entry = &p2[page.p2_index()];
if p2_entry.is_unused() {
return Err(TranslateError::PageNotMapped);
}
let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) };
let p1_entry = &p1[page.p1_index()];
if p1_entry.is_unused() {
return Err(TranslateError::PageNotMapped);
}
PhysFrame::from_start_address(p1_entry.addr())
.map_err(|()| TranslateError::InvalidFrameAddress(p1_entry.addr()))
}
}
impl<'a> MapperAllSizes for RecursivePageTable<'a> {
#[allow(clippy::inconsistent_digit_grouping)]
fn translate(&self, addr: VirtAddr) -> TranslateResult {
let page = Page::containing_address(addr);
let p4 = &self.p4;
let p4_entry = &p4[addr.p4_index()];
if p4_entry.is_unused() {
return TranslateResult::PageNotMapped;
}
if p4_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
panic!("level 4 entry has huge page bit set")
}
let p3 = unsafe { &*(p3_ptr(page, self.recursive_index)) };
let p3_entry = &p3[addr.p3_index()];
if p3_entry.is_unused() {
return TranslateResult::PageNotMapped;
}
if p3_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
let frame = PhysFrame::containing_address(p3[addr.p3_index()].addr());
let offset = addr.as_u64() & 0o_777_777_7777;
return TranslateResult::Frame1GiB { frame, offset };
}
let p2 = unsafe { &*(p2_ptr(page, self.recursive_index)) };
let p2_entry = &p2[addr.p2_index()];
if p2_entry.is_unused() {
return TranslateResult::PageNotMapped;
}
if p2_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
let frame = PhysFrame::containing_address(p2[addr.p2_index()].addr());
let offset = addr.as_u64() & 0o_777_7777;
return TranslateResult::Frame2MiB { frame, offset };
}
let p1 = unsafe { &*(p1_ptr(page, self.recursive_index)) };
let p1_entry = &p1[addr.p1_index()];
if p1_entry.is_unused() {
return TranslateResult::PageNotMapped;
}
if p1_entry.flags().contains(PageTableFlags::HUGE_PAGE) {
panic!("level 1 entry has huge page bit set")
}
let frame = match PhysFrame::from_start_address(p1_entry.addr()) {
Ok(frame) => frame,
Err(()) => return TranslateResult::InvalidFrameAddress(p1_entry.addr()),
};
let offset = u64::from(addr.page_offset());
TranslateResult::Frame4KiB { frame, offset }
}
}
#[inline]
fn p3_ptr<S: PageSize>(page: Page<S>, recursive_index: PageTableIndex) -> *mut PageTable {
p3_page(page, recursive_index).start_address().as_mut_ptr()
}
#[inline]
fn p3_page<S: PageSize>(page: Page<S>, recursive_index: PageTableIndex) -> Page {
Page::from_page_table_indices(
recursive_index,
recursive_index,
recursive_index,
page.p4_index(),
)
}
#[inline]
fn p2_ptr<S: NotGiantPageSize>(page: Page<S>, recursive_index: PageTableIndex) -> *mut PageTable {
p2_page(page, recursive_index).start_address().as_mut_ptr()
}
#[inline]
fn p2_page<S: NotGiantPageSize>(page: Page<S>, recursive_index: PageTableIndex) -> Page {
Page::from_page_table_indices(
recursive_index,
recursive_index,
page.p4_index(),
page.p3_index(),
)
}
#[inline]
fn p1_ptr(page: Page<Size4KiB>, recursive_index: PageTableIndex) -> *mut PageTable {
p1_page(page, recursive_index).start_address().as_mut_ptr()
}
#[inline]
fn p1_page(page: Page<Size4KiB>, recursive_index: PageTableIndex) -> Page {
Page::from_page_table_indices(
recursive_index,
page.p4_index(),
page.p3_index(),
page.p2_index(),
)
}