Skip to main content

hyperlight_common/arch/amd64/
vmem.rs

1// SPDX-License-Identifier: Apache-2.0
2// Copyright 2025 The Hyperlight Authors.
3
4//! x86-64 4-level page table manipulation code.
5//!
6//! This module implements page table setup for x86-64 long mode using 4-level paging:
7//! - PML4 (Page Map Level 4) - bits 47:39 - 512 entries, each covering 512GB
8//! - PDPT (Page Directory Pointer Table) - bits 38:30 - 512 entries, each covering 1GB
9//! - PD (Page Directory) - bits 29:21 - 512 entries, each covering 2MB
10//! - PT (Page Table) - bits 20:12 - 512 entries, each covering 4KB pages
11//!
12//! The code uses an iterator-based approach to walk the page table hierarchy,
13//! allocating intermediate tables as needed and setting appropriate flags on leaf PTEs
14
15use crate::vmem::{
16    BasicMapping, CowMapping, MapRequest, MapResponse, Mapping, MappingKind, TableMovabilityBase,
17    TableOps, TableReadOps, UpdateParent, UpdateParentNone, UpdateParentTable, Void, modify_ptes,
18    write_entry_updating,
19};
20
21#[derive(Copy, Clone)]
22pub(in crate::vmem) struct UpdateParentRoot {}
23
24/// Read a PTE and return it (widened to u64) if the present bit is
25/// set. The amd64 "present" encoding is a single bit (bit 0); other
26/// architectures may need richer semantics, which is why this lives
27/// per-arch rather than in the common module.
28///
29/// # Safety
30/// `entry_ptr` must point to a valid page table entry.
31#[inline(always)]
32#[allow(clippy::useless_conversion)]
33pub(super) unsafe fn read_pte_if_present<Op: TableReadOps>(
34    op: &Op,
35    entry_ptr: Op::TableAddr,
36) -> Option<u64> {
37    let pte: u64 = unsafe { op.read_entry(entry_ptr) }.into();
38    if (pte & PAGE_PRESENT) != 0 {
39        Some(pte)
40    } else {
41        None
42    }
43}
44
45/// Require that a PTE is present and descend to the next-level table.
46///
47/// # Safety
48/// `op` must provide valid page table memory.
49pub(super) unsafe fn require_pte_exist<Op: TableReadOps, P: UpdateParent<Op>>(
50    op: &Op,
51    x: MapResponse<Op, P>,
52) -> Option<MapRequest<Op, P::ChildType>>
53where
54    P::ChildType: UpdateParent<Op>,
55{
56    unsafe { read_pte_if_present(op, x.entry_ptr) }.map(|pte| MapRequest {
57        #[allow(clippy::unnecessary_cast)]
58        table_base: Op::from_phys((pte & PTE_ADDR_MASK) as PhysAddr),
59        vmin: x.vmin,
60        len: x.len,
61        update_parent: x.update_parent.for_child_at_entry(x.entry_ptr),
62    })
63}
64
65// Paging Flags
66//
67// See the following links explaining paging:
68//
69// * Intel® 64 and IA-32 Architectures Software Developer’s Manual, Volume 3A: System Programming Guide, Part 1
70//  - Chapter 5 "Paging"
71//
72// https://cdrdv2.intel.com/v1/dl/getContent/671200
73//
74// * AMD64 Architecture Programmer’s Manual, Volume 2: System Programming, Section 5.3: Long-Mode Page Translation
75//
76// https://docs.amd.com/v/u/en-US/24593_3.43
77//
78// Or if you prefer something less formal:
79//
80// * Very basic description: https://stackoverflow.com/a/26945892
81// * More in-depth descriptions: https://wiki.osdev.org/Paging
82//
83
84/// Page is Present
85pub const PAGE_PRESENT: u64 = 1;
86/// Page is Read/Write (if not set page is read only so long as the WP bit in CR0 is set to 1 - which it is in Hyperlight)
87const PAGE_RW: u64 = 1 << 1;
88/// Execute Disable (if this bit is set then data in the page cannot be executed)`
89const PAGE_NX: u64 = 1 << 63;
90/// Mask to extract the physical address from a PTE (bits 51:12)
91/// This masks out the lower 12 flag bits AND the upper bits including NX (bit 63)
92pub const PTE_ADDR_MASK: u64 = 0x000F_FFFF_FFFF_F000;
93const PAGE_USER_ACCESS_DISABLED: u64 = 0 << 2; // U/S bit not set - supervisor mode only (no code runs in user mode for now)
94const PAGE_DIRTY_SET: u64 = 1 << 6; // D - dirty bit
95const PAGE_ACCESSED_SET: u64 = 1 << 5; // A - accessed bit
96const PAGE_CACHE_ENABLED: u64 = 0 << 4; // PCD - page cache disable bit not set (caching enabled)
97const PAGE_WRITE_BACK: u64 = 0 << 3; // PWT - page write-through bit not set (write-back caching)
98const PAGE_PAT_WB: u64 = 0 << 7; // PAT - page attribute table index bit (0 for write-back memory when PCD=0, PWT=0)
99
100// We use various patterns of the available-for-software-use bits to
101// represent certain special mappings.
102const PTE_AVL_MASK: u64 = 0x0000_0000_0000_0E00;
103const PAGE_AVL_COW: u64 = 1 << 9;
104
105const INVALID_KIND_MIN_BIT: u8 = 2;
106const INVALID_KIND_MAX_BIT: u8 = 2;
107const INVALID_KIND_ZERO_INIT: u8 = 0b1;
108const INVALID_KIND_ZERO_INIT_PERM_READABLE_BIT: u8 = 8;
109const INVALID_KIND_ZERO_INIT_PERM_WRITABLE_BIT: u8 = 9;
110const INVALID_KIND_ZERO_INIT_PERM_EXECUTABLE_BIT: u8 = 10;
111
112/// Returns PAGE_RW if writable is true, 0 otherwise
113#[inline(always)]
114const fn page_rw_flag(writable: bool) -> u64 {
115    if writable { PAGE_RW } else { 0 }
116}
117
118/// Returns PAGE_NX if executable is false (NX = No Execute), 0 otherwise
119#[inline(always)]
120const fn page_nx_flag(executable: bool) -> u64 {
121    if executable { 0 } else { PAGE_NX }
122}
123
124/// Helper function to generate a page table entry that points to another table
125#[allow(clippy::identity_op)]
126#[allow(clippy::precedence)]
127fn pte_for_table<Op: TableOps>(table_addr: Op::TableAddr) -> u64 {
128    Op::to_phys(table_addr) |
129        PAGE_ACCESSED_SET | // prevent the CPU writing to the access flag
130        PAGE_CACHE_ENABLED | // leave caching enabled
131        PAGE_WRITE_BACK | // use write-back caching
132        PAGE_USER_ACCESS_DISABLED |// dont allow user access (no code runs in user mode for now)
133        PAGE_RW | // R/W - we don't use block-level permissions
134        PAGE_PRESENT // P   - this entry is present
135}
136
137/// This trait is used to select appropriate implementations of
138/// [`UpdateParent`] to be used, depending on whether a particular
139/// implementation needs the ability to move tables.
140pub(in crate::vmem) trait TableMovability<Op: TableReadOps + ?Sized, TableMoveInfo> {
141    type RootUpdateParent: UpdateParent<Op, TableMoveInfo = TableMoveInfo>;
142    fn root_update_parent() -> Self::RootUpdateParent;
143}
144impl<Op: TableOps<TableMovability = crate::vmem::MayMoveTable>> TableMovability<Op, Op::TableAddr>
145    for crate::vmem::MayMoveTable
146{
147    type RootUpdateParent = UpdateParentRoot;
148    fn root_update_parent() -> Self::RootUpdateParent {
149        UpdateParentRoot {}
150    }
151}
152impl<Op: TableReadOps> TableMovability<Op, Void> for crate::vmem::MayNotMoveTable {
153    type RootUpdateParent = UpdateParentNone;
154    fn root_update_parent() -> Self::RootUpdateParent {
155        UpdateParentNone {}
156    }
157}
158
159impl<
160    Op: TableOps<TableMovability = crate::vmem::MayMoveTable>,
161    P: UpdateParent<Op, TableMoveInfo = Op::TableAddr>,
162> UpdateParent<Op> for UpdateParentTable<Op, P>
163{
164    type TableMoveInfo = Op::TableAddr;
165    type ChildType = UpdateParentTable<Op, Self>;
166    fn update_parent(self, op: &Op, new_ptr: Op::TableAddr) {
167        let pte = pte_for_table::<Op>(new_ptr);
168        unsafe {
169            write_entry_updating(op, self.parent, self.entry_ptr, pte);
170        }
171    }
172    fn for_child_at_entry(self, entry_ptr: Op::TableAddr) -> Self::ChildType {
173        Self::ChildType::new(self, entry_ptr)
174    }
175}
176
177impl<Op: TableOps<TableMovability = crate::vmem::MayMoveTable>> UpdateParent<Op>
178    for UpdateParentRoot
179{
180    type TableMoveInfo = Op::TableAddr;
181    type ChildType = UpdateParentTable<Op, Self>;
182    fn update_parent(self, op: &Op, new_ptr: Op::TableAddr) {
183        unsafe {
184            op.update_root(new_ptr);
185        }
186    }
187    fn for_child_at_entry(self, entry_ptr: Op::TableAddr) -> Self::ChildType {
188        Self::ChildType::new(self, entry_ptr)
189    }
190}
191
192/// Page-mapping callback to allocate a next-level page table if necessary.
193/// # Safety
194/// This function modifies page table data structures, and should not be called concurrently
195/// with any other operations that modify the page tables.
196unsafe fn alloc_pte_if_needed<
197    Op: TableOps,
198    P: UpdateParent<
199            Op,
200            TableMoveInfo = <Op::TableMovability as TableMovabilityBase<Op>>::TableMoveInfo,
201        >,
202>(
203    op: &Op,
204    x: MapResponse<Op, P>,
205) -> MapRequest<Op, P::ChildType>
206where
207    P::ChildType: UpdateParent<Op>,
208{
209    let new_update_parent = x.update_parent.for_child_at_entry(x.entry_ptr);
210    if let Some(pte) = unsafe { read_pte_if_present(op, x.entry_ptr) } {
211        return MapRequest {
212            table_base: Op::from_phys(pte & PTE_ADDR_MASK),
213            vmin: x.vmin,
214            len: x.len,
215            update_parent: new_update_parent,
216        };
217    }
218
219    let page_addr = unsafe { op.alloc_table() };
220
221    let pte = pte_for_table::<Op>(page_addr);
222    unsafe {
223        write_entry_updating(op, x.update_parent, x.entry_ptr, pte);
224    };
225    MapRequest {
226        table_base: page_addr,
227        vmin: x.vmin,
228        len: x.len,
229        update_parent: new_update_parent,
230    }
231}
232
233/// Map a normal memory page
234/// # Safety
235/// This function modifies page table data structures, and should not be called concurrently
236/// with any other operations that modify the page tables.
237#[allow(clippy::identity_op)]
238#[allow(clippy::precedence)]
239unsafe fn map_page<
240    Op: TableOps,
241    P: UpdateParent<
242            Op,
243            TableMoveInfo = <Op::TableMovability as TableMovabilityBase<Op>>::TableMoveInfo,
244        >,
245>(
246    op: &Op,
247    mapping: &Mapping,
248    r: MapResponse<Op, P>,
249) {
250    let pte = match &mapping.kind {
251        MappingKind::Basic(bm) =>
252        // TODO: Support not readable
253        // NOTE: On x86-64, there is no separate "readable" bit in the page table entry.
254        // This means that pages cannot be made write-only or execute-only without also being readable.
255        // All pages that are mapped as writable or executable are also implicitly readable.
256        // If support for "not readable" mappings is required in the future, it would need to be
257        // implemented using additional mechanisms (e.g., page-fault handling or memory protection keys),
258        // but for now, this architectural limitation is accepted.
259        {
260            (mapping.phys_base + (r.vmin - mapping.virt_base)) |
261                page_nx_flag(bm.executable) | // NX - no execute unless allowed
262                PAGE_PAT_WB | // PAT index bit for write-back memory
263                PAGE_DIRTY_SET | // prevent the CPU writing to the dirty bit
264                PAGE_ACCESSED_SET | // prevent the CPU writing to the access flag
265                PAGE_CACHE_ENABLED | // leave caching enabled
266                PAGE_WRITE_BACK | // use write-back caching
267                PAGE_USER_ACCESS_DISABLED | // dont allow user access (no code runs in user mode for now)
268                page_rw_flag(bm.writable) | // R/W - set if writable
269                PAGE_PRESENT // P   - this entry is present
270        }
271        MappingKind::Cow(cm) => {
272            (mapping.phys_base + (r.vmin - mapping.virt_base)) |
273                page_nx_flag(cm.executable) | // NX - no execute unless allowed
274                PAGE_AVL_COW |
275                PAGE_PAT_WB | // PAT index bit for write-back memory
276                PAGE_DIRTY_SET | // prevent the CPU writing to the dirty bit
277                PAGE_ACCESSED_SET | // prevent the CPU writing to the access flag
278                PAGE_CACHE_ENABLED | // leave caching enabled
279                PAGE_WRITE_BACK | // use write-back caching
280                PAGE_USER_ACCESS_DISABLED | // dont allow user access (no code runs in user mode for now)
281                0 | // R/W - Cow page is never writable
282                PAGE_PRESENT // P   - this entry is present
283        }
284        MappingKind::Unmapped => 0,
285        MappingKind::ZeroInit(bm) => {
286            let mut pte = (INVALID_KIND_ZERO_INIT as u64) << INVALID_KIND_MIN_BIT;
287            if bm.readable {
288                pte |= 1 << INVALID_KIND_ZERO_INIT_PERM_READABLE_BIT
289            }
290            if bm.writable {
291                pte |= 1 << INVALID_KIND_ZERO_INIT_PERM_WRITABLE_BIT
292            }
293            if bm.executable {
294                pte |= 1 << INVALID_KIND_ZERO_INIT_PERM_EXECUTABLE_BIT
295            }
296            pte
297        }
298    };
299    unsafe {
300        write_entry_updating(op, r.update_parent, r.entry_ptr, pte);
301    }
302}
303
304// There are no notable architecture-specific safety considerations
305// here, and the general conditions are documented in the
306// architecture-independent re-export in vmem.rs
307
308/// Maps a contiguous virtual address range to physical memory.
309///
310/// This function walks the 4-level page table hierarchy (PML4 → PDPT → PD → PT),
311/// allocating intermediate tables as needed via `alloc_pte_if_needed`, and finally
312/// writing the leaf page table entries with the requested permissions via `map_page`.
313///
314/// The iterator chain processes each level:
315/// 1. PML4 (47:39) - allocate PDPT if needed
316/// 2. PDPT (38:30) - allocate PD if needed
317/// 3. PD (29:21) - allocate PT if needed
318/// 4. PT (20:12) - write final PTE with physical address and flags
319///
320/// Multi-space page-table walking on amd64: walks each root
321/// independently and emits all leaves as `ThisSpace`. Aliased
322/// intermediate-table detection is not implemented here because no
323/// current embedder exercises that pattern on amd64.
324#[allow(clippy::missing_safety_doc)]
325pub unsafe fn walk_va_spaces<Op: TableReadOps>(
326    op: &Op,
327    roots: &[Op::TableAddr],
328    address: u64,
329    len: u64,
330) -> ::alloc::vec::Vec<(
331    crate::vmem::SpaceId,
332    ::alloc::vec::Vec<crate::vmem::SpaceAwareMapping>,
333)> {
334    use ::alloc::vec::Vec;
335
336    let mut out: Vec<(crate::vmem::SpaceId, Vec<crate::vmem::SpaceAwareMapping>)> =
337        Vec::with_capacity(roots.len());
338
339    let addr = address & ((1u64 << VA_BITS) - 1);
340    let vmin = addr & !(PAGE_SIZE as u64 - 1);
341    let vmax = core::cmp::min(addr + len, 1u64 << VA_BITS);
342
343    for &root in roots {
344        #[allow(clippy::unnecessary_cast)]
345        let root_id: crate::vmem::SpaceId = Op::to_phys(root) as u64;
346        let mut mappings: Vec<crate::vmem::SpaceAwareMapping> = Vec::new();
347
348        let iter = modify_ptes::<47, 39, Op, _>(MapRequest {
349            table_base: root,
350            vmin,
351            len: vmax.saturating_sub(vmin),
352            update_parent: UpdateParentNone {},
353        })
354        .filter_map(|r| unsafe { require_pte_exist(op, r) })
355        .flat_map(modify_ptes::<38, 30, Op, _>)
356        .filter_map(|r| unsafe { require_pte_exist(op, r) })
357        .flat_map(modify_ptes::<29, 21, Op, _>)
358        .filter_map(|r| unsafe { require_pte_exist(op, r) })
359        .flat_map(modify_ptes::<20, 12, Op, _>);
360
361        for r in iter {
362            if let Some(mapping) = unsafe { read_and_decode_pte(op, &r) } {
363                crate::vmem::push_space_mapping(
364                    &mut mappings,
365                    crate::vmem::SpaceAwareMapping::ThisSpace(mapping),
366                );
367            }
368        }
369
370        out.push((root_id, mappings));
371    }
372
373    out
374}
375
376/// See [`walk_va_spaces`]: amd64 never emits `AnotherSpace`, so this
377/// is unreachable in practice. It silently no-ops (rather than
378/// panicking) to keep the architecture-independent re-export usable.
379#[allow(clippy::missing_safety_doc)]
380pub unsafe fn space_aware_map<Op: TableOps>(
381    _op: &Op,
382    _ref_map: crate::vmem::SpaceReferenceMapping,
383    _built_roots: &::alloc::collections::BTreeMap<crate::vmem::SpaceId, Op::TableAddr>,
384) {
385}
386
387#[allow(clippy::missing_safety_doc)]
388pub unsafe fn map<Op: TableOps>(op: &Op, mapping: Mapping) {
389    modify_ptes::<47, 39, Op, _>(MapRequest {
390        table_base: op.root_table(),
391        vmin: mapping.virt_base,
392        len: mapping.len,
393        update_parent: Op::TableMovability::root_update_parent(),
394    })
395    .map(|r| unsafe { alloc_pte_if_needed(op, r) })
396    .flat_map(modify_ptes::<38, 30, Op, _>)
397    .map(|r| unsafe { alloc_pte_if_needed(op, r) })
398    .flat_map(modify_ptes::<29, 21, Op, _>)
399    .map(|r| unsafe { alloc_pte_if_needed(op, r) })
400    .flat_map(modify_ptes::<20, 12, Op, _>)
401    .map(|r| unsafe { map_page(op, &mapping, r) })
402    .for_each(drop);
403}
404
405#[allow(clippy::missing_safety_doc)]
406pub unsafe fn modify_mapping<Op: TableOps>(
407    op: &Op,
408    vmin: VirtAddr,
409    len: u64,
410    mut f: impl FnMut(VirtAddr, Mapping) -> MappingKind,
411) {
412    modify_ptes::<47, 39, Op, _>(MapRequest {
413        table_base: op.root_table(),
414        vmin,
415        len,
416        update_parent: Op::TableMovability::root_update_parent(),
417    })
418    .map(|r| unsafe { alloc_pte_if_needed(op, r) })
419    .flat_map(modify_ptes::<38, 30, Op, _>)
420    .map(|r| unsafe { alloc_pte_if_needed(op, r) })
421    .flat_map(modify_ptes::<29, 21, Op, _>)
422    .map(|r| unsafe { alloc_pte_if_needed(op, r) })
423    .flat_map(modify_ptes::<20, 12, Op, _>)
424    .map(|r| unsafe {
425        let mut mapping = read_and_decode_pte(op, &r).unwrap_or(Mapping {
426            phys_base: 0,
427            virt_base: r.vmin,
428            len: PAGE_SIZE as u64,
429            kind: MappingKind::Unmapped,
430        });
431        mapping.kind = f(r.vmin, mapping);
432        map_page(op, &mapping, r);
433    })
434    .for_each(drop);
435}
436
437unsafe fn read_and_decode_pte<Op: TableReadOps, P: UpdateParent<Op>>(
438    op: &Op,
439    r: &MapResponse<Op, P>,
440) -> Option<Mapping> {
441    // Re-do the sign extension
442    let sgn_bit = r.vmin >> (VA_BITS - 1);
443    let sgn_bits = 0u64.wrapping_sub(sgn_bit) << VA_BITS;
444    let virt_addr = sgn_bits | r.vmin;
445
446    let Some(pte) = (unsafe { read_pte_if_present(op, r.entry_ptr) }) else {
447        let pte: u64 = unsafe { op.read_entry(r.entry_ptr) };
448        let ik = crate::vmem::bits::<INVALID_KIND_MAX_BIT, INVALID_KIND_MIN_BIT>(pte);
449        return match ik as u8 {
450            INVALID_KIND_ZERO_INIT => Some(Mapping {
451                phys_base: 0,
452                virt_base: virt_addr,
453                len: PAGE_SIZE as u64,
454                kind: MappingKind::ZeroInit(BasicMapping {
455                    readable: pte & (1 << INVALID_KIND_ZERO_INIT_PERM_READABLE_BIT) != 0,
456                    writable: pte & (1 << INVALID_KIND_ZERO_INIT_PERM_WRITABLE_BIT) != 0,
457                    executable: pte & (1 << INVALID_KIND_ZERO_INIT_PERM_EXECUTABLE_BIT) != 0,
458                }),
459            }),
460            _ => None,
461        };
462    };
463    let phys_addr = pte & PTE_ADDR_MASK;
464
465    let executable = (pte & PAGE_NX) == 0;
466    let avl = pte & PTE_AVL_MASK;
467    let kind = if avl == PAGE_AVL_COW {
468        MappingKind::Cow(CowMapping {
469            readable: true,
470            executable,
471        })
472    } else {
473        MappingKind::Basic(BasicMapping {
474            readable: true,
475            writable: (pte & PAGE_RW) != 0,
476            executable,
477        })
478    };
479    Some(Mapping {
480        phys_base: phys_addr,
481        virt_base: virt_addr,
482        len: PAGE_SIZE as u64,
483        kind,
484    })
485}
486
487// There are no notable architecture-specific safety considerations
488// here, and the general conditions are documented in the
489// architecture-independent re-export in vmem.rs
490
491/// Translates a virtual address range to the physical address pages
492/// that back it by walking the page tables.
493///
494/// Returns an iterator with an entry for each mapped page that
495/// intersects the given range.
496///
497/// This takes AsRef<Op> + Copy so that on targets where the
498/// operations have little state (e.g. the guest) the operations state
499/// can be copied into the closure(s) in the iterator, allowing for a
500/// nicer result lifetime.  On targets like the
501/// building-an-original-snapshot portion of the host, where the
502/// operations structure owns a large buffer, a reference can instead
503/// be passed.
504#[allow(clippy::missing_safety_doc)]
505pub unsafe fn virt_to_phys<'a, Op: TableReadOps + 'a>(
506    op: impl core::convert::AsRef<Op> + Copy + 'a,
507    address: u64,
508    len: u64,
509) -> impl Iterator<Item = Mapping> + 'a {
510    // Undo sign-extension
511    let addr = address & ((1u64 << VA_BITS) - 1);
512    // Mask off any sub-page bits
513    let vmin = addr & !(PAGE_SIZE as u64 - 1);
514    // Calculate the maximum virtual address we need to look at based on the starting
515    // address and length ensuring we don't go past the end of the address space
516    let vmax = core::cmp::min(addr + len, 1u64 << VA_BITS);
517    modify_ptes::<47, 39, Op, _>(MapRequest {
518        table_base: op.as_ref().root_table(),
519        vmin,
520        len: vmax - vmin,
521        update_parent: UpdateParentNone {},
522    })
523    .filter_map(move |r| unsafe { require_pte_exist(op.as_ref(), r) })
524    .flat_map(modify_ptes::<38, 30, Op, _>)
525    .filter_map(move |r| unsafe { require_pte_exist(op.as_ref(), r) })
526    .flat_map(modify_ptes::<29, 21, Op, _>)
527    .filter_map(move |r| unsafe { require_pte_exist(op.as_ref(), r) })
528    .flat_map(modify_ptes::<20, 12, Op, _>)
529    .filter_map(move |r| unsafe { read_and_decode_pte(op.as_ref(), &r) })
530}
531
532const VA_BITS: usize = 48; // We use 48-bit virtual addresses at the moment.
533
534pub const PAGE_SIZE: usize = 4096;
535pub const PAGE_TABLE_SIZE: usize = 4096;
536pub type PageTableEntry = u64;
537pub type VirtAddr = u64;
538pub type PhysAddr = u64;
539
540#[cfg(test)]
541mod tests {
542    use alloc::vec;
543    use alloc::vec::Vec;
544    use core::cell::RefCell;
545
546    use super::*;
547    use crate::vmem::{
548        BasicMapping, Mapping, MappingKind, MayNotMoveTable, PAGE_TABLE_ENTRIES_PER_TABLE,
549        TableOps, TableReadOps, Void, bits,
550    };
551
552    /// A mock TableOps implementation for testing that stores page tables in memory
553    /// needed because the `GuestPageTableBuffer` is in hyperlight_host which would cause a circular dependency
554    struct MockTableOps {
555        tables: RefCell<Vec<[u64; PAGE_TABLE_ENTRIES_PER_TABLE]>>,
556    }
557
558    // for virt_to_phys
559    impl core::convert::AsRef<MockTableOps> for MockTableOps {
560        fn as_ref(&self) -> &Self {
561            self
562        }
563    }
564
565    impl MockTableOps {
566        fn new() -> Self {
567            // Start with one table (the root/PML4)
568            Self {
569                tables: RefCell::new(vec![[0u64; PAGE_TABLE_ENTRIES_PER_TABLE]]),
570            }
571        }
572
573        fn table_count(&self) -> usize {
574            self.tables.borrow().len()
575        }
576
577        fn get_entry(&self, table_idx: usize, entry_idx: usize) -> u64 {
578            self.tables.borrow()[table_idx][entry_idx]
579        }
580    }
581
582    impl TableReadOps for MockTableOps {
583        type TableAddr = (usize, usize); // (table_index, entry_index)
584
585        fn entry_addr(addr: Self::TableAddr, entry_offset: u64) -> Self::TableAddr {
586            // Convert to physical address, add offset, convert back
587            let phys = Self::to_phys(addr) + entry_offset;
588            Self::from_phys(phys)
589        }
590
591        unsafe fn read_entry(&self, addr: Self::TableAddr) -> u64 {
592            self.tables.borrow()[addr.0][addr.1]
593        }
594
595        fn to_phys(addr: Self::TableAddr) -> PhysAddr {
596            // Each table is 4KB, entries are 8 bytes
597            (addr.0 as u64 * PAGE_TABLE_SIZE as u64) + (addr.1 as u64 * 8)
598        }
599
600        fn from_phys(addr: PhysAddr) -> Self::TableAddr {
601            let table_idx = (addr / PAGE_TABLE_SIZE as u64) as usize;
602            let entry_idx = ((addr % PAGE_TABLE_SIZE as u64) / 8) as usize;
603            (table_idx, entry_idx)
604        }
605
606        fn root_table(&self) -> Self::TableAddr {
607            (0, 0)
608        }
609    }
610
611    impl TableOps for MockTableOps {
612        type TableMovability = MayNotMoveTable;
613
614        unsafe fn alloc_table(&self) -> Self::TableAddr {
615            let mut tables = self.tables.borrow_mut();
616            let idx = tables.len();
617            tables.push([0u64; PAGE_TABLE_ENTRIES_PER_TABLE]);
618            (idx, 0)
619        }
620
621        unsafe fn write_entry(&self, addr: Self::TableAddr, entry: u64) -> Option<Void> {
622            self.tables.borrow_mut()[addr.0][addr.1] = entry;
623            None
624        }
625
626        unsafe fn update_root(&self, impossible: Void) {
627            match impossible {}
628        }
629    }
630
631    // ==================== bits() function tests ====================
632
633    #[test]
634    fn test_bits_extracts_pml4_index() {
635        // PML4 uses bits 47:39
636        // Address 0x0000_0080_0000_0000 should have PML4 index 1
637        let addr: u64 = 0x0000_0080_0000_0000;
638        assert_eq!(bits::<47, 39>(addr), 1);
639    }
640
641    #[test]
642    fn test_bits_extracts_pdpt_index() {
643        // PDPT uses bits 38:30
644        // Address with PDPT index 1: bit 30 set = 0x4000_0000 (1GB)
645        let addr: u64 = 0x4000_0000;
646        assert_eq!(bits::<38, 30>(addr), 1);
647    }
648
649    #[test]
650    fn test_bits_extracts_pd_index() {
651        // PD uses bits 29:21
652        // Address 0x0000_0000_0020_0000 (2MB) should have PD index 1
653        let addr: u64 = 0x0000_0000_0020_0000;
654        assert_eq!(bits::<29, 21>(addr), 1);
655    }
656
657    #[test]
658    fn test_bits_extracts_pt_index() {
659        // PT uses bits 20:12
660        // Address 0x0000_0000_0000_1000 (4KB) should have PT index 1
661        let addr: u64 = 0x0000_0000_0000_1000;
662        assert_eq!(bits::<20, 12>(addr), 1);
663    }
664
665    #[test]
666    fn test_bits_max_index() {
667        // Maximum 9-bit index is 511
668        // PML4 index 511 = bits 47:39 all set = 0x0000_FF80_0000_0000
669        let addr: u64 = 0x0000_FF80_0000_0000;
670        assert_eq!(bits::<47, 39>(addr), 511);
671    }
672
673    // ==================== PTE flag tests ====================
674
675    #[test]
676    fn test_page_rw_flag_writable() {
677        assert_eq!(page_rw_flag(true), PAGE_RW);
678    }
679
680    #[test]
681    fn test_page_rw_flag_readonly() {
682        assert_eq!(page_rw_flag(false), 0);
683    }
684
685    #[test]
686    fn test_page_nx_flag_executable() {
687        assert_eq!(page_nx_flag(true), 0); // Executable = no NX bit
688    }
689
690    #[test]
691    fn test_page_nx_flag_not_executable() {
692        assert_eq!(page_nx_flag(false), PAGE_NX);
693    }
694
695    // ==================== map() function tests ====================
696
697    #[test]
698    fn test_map_single_page() {
699        let ops = MockTableOps::new();
700        let mapping = Mapping {
701            phys_base: 0x1000,
702            virt_base: 0x1000,
703            len: PAGE_SIZE as u64,
704            kind: MappingKind::Basic(BasicMapping {
705                readable: true,
706                writable: true,
707                executable: false,
708            }),
709        };
710
711        unsafe { map(&ops, mapping) };
712
713        // Should have allocated: PML4(exists) + PDPT + PD + PT = 4 tables
714        assert_eq!(ops.table_count(), 4);
715
716        // Check PML4 entry 0 points to PDPT (table 1) with correct flags
717        let pml4_entry = ops.get_entry(0, 0);
718        assert_ne!(pml4_entry & PAGE_PRESENT, 0, "PML4 entry should be present");
719        assert_ne!(pml4_entry & PAGE_RW, 0, "PML4 entry should be writable");
720
721        // Check the leaf PTE has correct flags
722        // PT is table 3, entry 1 (for virt_base 0x1000)
723        let pte = ops.get_entry(3, 1);
724        assert_ne!(pte & PAGE_PRESENT, 0, "PTE should be present");
725        assert_ne!(pte & PAGE_RW, 0, "PTE should be writable");
726        assert_ne!(pte & PAGE_NX, 0, "PTE should have NX set (not executable)");
727        assert_eq!(pte & PTE_ADDR_MASK, 0x1000, "PTE should map to phys 0x1000");
728    }
729
730    #[test]
731    fn test_map_executable_page() {
732        let ops = MockTableOps::new();
733        let mapping = Mapping {
734            phys_base: 0x2000,
735            virt_base: 0x2000,
736            len: PAGE_SIZE as u64,
737            kind: MappingKind::Basic(BasicMapping {
738                readable: true,
739                writable: false,
740                executable: true,
741            }),
742        };
743
744        unsafe { map(&ops, mapping) };
745
746        // PT is table 3, entry 2 (for virt_base 0x2000)
747        let pte = ops.get_entry(3, 2);
748        assert_ne!(pte & PAGE_PRESENT, 0, "PTE should be present");
749        assert_eq!(pte & PAGE_RW, 0, "PTE should be read-only");
750        assert_eq!(pte & PAGE_NX, 0, "PTE should NOT have NX set (executable)");
751    }
752
753    #[test]
754    fn test_map_multiple_pages() {
755        let ops = MockTableOps::new();
756        let mapping = Mapping {
757            phys_base: 0x10000,
758            virt_base: 0x10000,
759            len: 4 * PAGE_SIZE as u64, // 4 pages = 16KB
760            kind: MappingKind::Basic(BasicMapping {
761                readable: true,
762                writable: true,
763                executable: false,
764            }),
765        };
766
767        unsafe { map(&ops, mapping) };
768
769        // Check all 4 PTEs are present
770        for i in 0..4 {
771            let entry_idx = 16 + i; // 0x10000 / 0x1000 = 16
772            let pte = ops.get_entry(3, entry_idx);
773            assert_ne!(pte & PAGE_PRESENT, 0, "PTE {} should be present", i);
774            let expected_phys = 0x10000 + (i as u64 * PAGE_SIZE as u64);
775            assert_eq!(
776                pte & PTE_ADDR_MASK,
777                expected_phys,
778                "PTE {} should map to correct phys addr",
779                i
780            );
781        }
782    }
783
784    #[test]
785    fn test_map_reuses_existing_tables() {
786        let ops = MockTableOps::new();
787
788        // Map first region
789        let mapping1 = Mapping {
790            phys_base: 0x1000,
791            virt_base: 0x1000,
792            len: PAGE_SIZE as u64,
793            kind: MappingKind::Basic(BasicMapping {
794                readable: true,
795                writable: true,
796                executable: false,
797            }),
798        };
799        unsafe { map(&ops, mapping1) };
800        let tables_after_first = ops.table_count();
801
802        // Map second region in same PT (different page)
803        let mapping2 = Mapping {
804            phys_base: 0x5000,
805            virt_base: 0x5000,
806            len: PAGE_SIZE as u64,
807            kind: MappingKind::Basic(BasicMapping {
808                readable: true,
809                writable: true,
810                executable: false,
811            }),
812        };
813        unsafe { map(&ops, mapping2) };
814
815        // Should NOT allocate new tables (reuses existing hierarchy)
816        assert_eq!(
817            ops.table_count(),
818            tables_after_first,
819            "Should reuse existing page tables"
820        );
821    }
822
823    // ==================== virt_to_phys() tests ====================
824
825    #[test]
826    fn test_virt_to_phys_mapped_address() {
827        let ops = MockTableOps::new();
828        let mapping = Mapping {
829            phys_base: 0x1000,
830            virt_base: 0x1000,
831            len: PAGE_SIZE as u64,
832            kind: MappingKind::Basic(BasicMapping {
833                readable: true,
834                writable: true,
835                executable: false,
836            }),
837        };
838
839        unsafe { map(&ops, mapping) };
840
841        let result = unsafe { virt_to_phys(&ops, 0x1000, 1).next() };
842        assert!(result.is_some(), "Should find mapped address");
843        let mapping = result.unwrap();
844        assert_eq!(mapping.phys_base, 0x1000);
845    }
846
847    #[test]
848    fn test_virt_to_phys_unaligned_virt() {
849        let ops = MockTableOps::new();
850        let mapping = Mapping {
851            phys_base: 0x1000,
852            virt_base: 0x1000,
853            len: PAGE_SIZE as u64,
854            kind: MappingKind::Basic(BasicMapping {
855                readable: true,
856                writable: true,
857                executable: false,
858            }),
859        };
860
861        unsafe { map(&ops, mapping) };
862
863        let result = unsafe { virt_to_phys(&ops, 0x1234, 1).next() };
864        assert!(result.is_some(), "Should find mapped address");
865        let mapping = result.unwrap();
866        assert_eq!(mapping.phys_base, 0x1000);
867    }
868
869    #[test]
870    fn test_virt_to_phys_unaligned_virt_and_across_pages_len() {
871        let ops = MockTableOps::new();
872        let mapping = Mapping {
873            phys_base: 0x1000,
874            virt_base: 0x1000,
875            len: 2 * PAGE_SIZE as u64, // 2 page
876            kind: MappingKind::Basic(BasicMapping {
877                readable: true,
878                writable: true,
879                executable: false,
880            }),
881        };
882
883        unsafe { map(&ops, mapping) };
884
885        let mappings = unsafe { virt_to_phys(&ops, 0x1F00, 0x300).collect::<Vec<_>>() };
886        assert_eq!(mappings.len(), 2, "Should return 2 mappings for 2 pages");
887        assert_eq!(mappings[0].phys_base, 0x1000);
888        assert_eq!(mappings[1].phys_base, 0x2000);
889    }
890
891    #[test]
892    fn test_virt_to_phys_unaligned_virt_and_multiple_page_len() {
893        let ops = MockTableOps::new();
894        let mapping = Mapping {
895            phys_base: 0x1000,
896            virt_base: 0x1000,
897            len: PAGE_SIZE as u64 * 2 + 0x200, // 2 page + 512 bytes
898            kind: MappingKind::Basic(BasicMapping {
899                readable: true,
900                writable: true,
901                executable: false,
902            }),
903        };
904
905        unsafe { map(&ops, mapping) };
906
907        let mappings =
908            unsafe { virt_to_phys(&ops, 0x1234, PAGE_SIZE as u64 * 2 + 0x10).collect::<Vec<_>>() };
909        assert_eq!(mappings.len(), 3, "Should return 3 mappings for 3 pages");
910        assert_eq!(mappings[0].phys_base, 0x1000);
911        assert_eq!(mappings[1].phys_base, 0x2000);
912        assert_eq!(mappings[2].phys_base, 0x3000);
913    }
914
915    #[test]
916    fn test_virt_to_phys_perms() {
917        let test = |kind| {
918            let ops = MockTableOps::new();
919            let mapping = Mapping {
920                phys_base: 0x1000,
921                virt_base: 0x1000,
922                len: PAGE_SIZE as u64,
923                kind,
924            };
925            unsafe { map(&ops, mapping) };
926            let result = unsafe { virt_to_phys(&ops, 0x1000, 1).next() };
927            let mapping = result.unwrap();
928            assert_eq!(mapping.kind, kind);
929        };
930        test(MappingKind::Basic(BasicMapping {
931            readable: true,
932            writable: false,
933            executable: false,
934        }));
935        test(MappingKind::Basic(BasicMapping {
936            readable: true,
937            writable: false,
938            executable: true,
939        }));
940        test(MappingKind::Basic(BasicMapping {
941            readable: true,
942            writable: true,
943            executable: false,
944        }));
945        test(MappingKind::Basic(BasicMapping {
946            readable: true,
947            writable: true,
948            executable: true,
949        }));
950        test(MappingKind::Cow(CowMapping {
951            readable: true,
952            executable: false,
953        }));
954        test(MappingKind::Cow(CowMapping {
955            readable: true,
956            executable: true,
957        }));
958    }
959
960    #[test]
961    fn test_virt_to_phys_unmapped_address() {
962        let ops = MockTableOps::new();
963        // Don't map anything
964
965        let result = unsafe { virt_to_phys(&ops, 0x1000, 1).next() };
966        assert!(result.is_none(), "Should return None for unmapped address");
967    }
968
969    #[test]
970    fn test_virt_to_phys_partially_mapped() {
971        let ops = MockTableOps::new();
972        let mapping = Mapping {
973            phys_base: 0x1000,
974            virt_base: 0x1000,
975            len: PAGE_SIZE as u64,
976            kind: MappingKind::Basic(BasicMapping {
977                readable: true,
978                writable: true,
979                executable: false,
980            }),
981        };
982
983        unsafe { map(&ops, mapping) };
984
985        // Query an address in a different PT entry (unmapped)
986        let result = unsafe { virt_to_phys(&ops, 0x5000, 1).next() };
987        assert!(
988            result.is_none(),
989            "Should return None for unmapped address in same PT"
990        );
991    }
992
993    // ==================== ModifyPteIterator tests ====================
994
995    #[test]
996    fn test_modify_pte_iterator_single_page() {
997        let ops = MockTableOps::new();
998        let request = MapRequest {
999            table_base: ops.root_table(),
1000            vmin: 0x1000,
1001            len: PAGE_SIZE as u64,
1002            update_parent: UpdateParentNone {},
1003        };
1004
1005        let responses: Vec<_> = modify_ptes::<20, 12, MockTableOps, _>(request).collect();
1006        assert_eq!(responses.len(), 1, "Single page should yield one response");
1007        assert_eq!(responses[0].vmin, 0x1000);
1008        assert_eq!(responses[0].len, PAGE_SIZE as u64);
1009    }
1010
1011    #[test]
1012    fn test_modify_pte_iterator_multiple_pages() {
1013        let ops = MockTableOps::new();
1014        let request = MapRequest {
1015            table_base: ops.root_table(),
1016            vmin: 0x1000,
1017            len: 3 * PAGE_SIZE as u64,
1018            update_parent: UpdateParentNone {},
1019        };
1020
1021        let responses: Vec<_> = modify_ptes::<20, 12, MockTableOps, _>(request).collect();
1022        assert_eq!(responses.len(), 3, "3 pages should yield 3 responses");
1023    }
1024
1025    #[test]
1026    fn test_modify_pte_iterator_zero_length() {
1027        let ops = MockTableOps::new();
1028        let request = MapRequest {
1029            table_base: ops.root_table(),
1030            vmin: 0x1000,
1031            len: 0,
1032            update_parent: UpdateParentNone {},
1033        };
1034
1035        let responses: Vec<_> = modify_ptes::<20, 12, MockTableOps, _>(request).collect();
1036        assert_eq!(responses.len(), 0, "Zero length should yield no responses");
1037    }
1038
1039    #[test]
1040    fn test_modify_pte_iterator_unaligned_start() {
1041        let ops = MockTableOps::new();
1042        // Start at 0x1800 (mid-page), map 0x1000 bytes
1043        // Should cover 0x1800-0x1FFF (first page) and 0x2000-0x27FF (second page)
1044        let request = MapRequest {
1045            table_base: ops.root_table(),
1046            vmin: 0x1800,
1047            len: 0x1000,
1048            update_parent: UpdateParentNone {},
1049        };
1050
1051        let responses: Vec<_> = modify_ptes::<20, 12, MockTableOps, _>(request).collect();
1052        assert_eq!(
1053            responses.len(),
1054            2,
1055            "Unaligned mapping spanning 2 pages should yield 2 responses"
1056        );
1057        assert_eq!(responses[0].vmin, 0x1800);
1058        assert_eq!(responses[0].len, 0x800); // Remaining in first page
1059        assert_eq!(responses[1].vmin, 0x2000);
1060        assert_eq!(responses[1].len, 0x800); // Continuing in second page
1061    }
1062
1063    // ==================== TableOps entry_addr tests ====================
1064
1065    #[test]
1066    fn test_entry_addr_from_table_base() {
1067        // entry_addr is called with a table base (entry_index = 0) and a byte offset
1068        // offset = entry_index * 8, so offset 40 means entry 5
1069        let result = MockTableOps::entry_addr((2, 0), 40);
1070        assert_eq!(result, (2, 5), "Should return (table 2, entry 5)");
1071    }
1072
1073    #[test]
1074    fn test_entry_addr_with_nonzero_base_entry() {
1075        // Even though entry_addr is typically called with entry_index=0,
1076        // it should handle non-zero base correctly by adding the offset
1077        // Base: table 1, entry 10 (phys = 1*4096 + 10*8 = 4176)
1078        // Offset: 16 bytes (2 entries)
1079        // Result phys: 4176 + 16 = 4192 = 1*4096 + 12*8 → (1, 12)
1080        let result = MockTableOps::entry_addr((1, 10), 16);
1081        assert_eq!(result, (1, 12), "Should add offset to base entry");
1082    }
1083
1084    #[test]
1085    fn test_to_phys_from_phys_roundtrip() {
1086        // Verify to_phys and from_phys are inverses
1087        let addr = (3, 42);
1088        let phys = MockTableOps::to_phys(addr);
1089        let back = MockTableOps::from_phys(phys);
1090        assert_eq!(back, addr, "to_phys/from_phys should roundtrip");
1091    }
1092}