use core::ops::Range;
use byte_unit::{Byte, UnitType};
use kernutil::StaticCell;
pub use kernutil::memory::{MemoryDescriptor, MemoryType, PageTableInfo};
use num_align::NumAlign;
use ranges_ext::*;
pub mod mmu;
pub(crate) mod ram;
pub(crate) mod region;
pub use page_table_generic::*;
use crate::{ArchTrait, DCacheOp, arch::Arch, smp::percpu_range};
pub const KB: usize = 1024;
pub const MB: usize = 1024 * KB;
pub const GB: usize = 1024 * MB;
pub const KIMAGE_MAP_ALIGN: usize = 2 * MB;
static mut VM_LOAD_OFFSET: isize = 0;
static MEMORY_MAP: StaticCell<MemoryMap> = StaticCell::new(MemoryMap::new());
static mut KIMAGE_START: Option<PhysAddr> = None;
static mut KIMAGE_END: PhysAddr = PhysAddr::new(0);
const MEMORY_MAP_CAPACITY: usize = 512;
pub type MemoryMap = heapless::Vec<MemoryDescriptor, MEMORY_MAP_CAPACITY>;
pub(crate) fn setup_entry(
kernel_start: PhysAddr,
kernel_end: PhysAddr,
kernel_start_link: VirtAddr,
) {
unsafe {
KIMAGE_START = Some(kernel_start);
KIMAGE_END = kernel_end.raw().align_up(KIMAGE_MAP_ALIGN).into();
VM_LOAD_OFFSET = kernel_start.raw() as isize - kernel_start_link.raw() as isize;
}
}
pub fn stack_size() -> usize {
unsafe extern "C" {
fn STACK_SIZE();
}
STACK_SIZE as *const () as usize
}
pub fn vm_load_offset() -> isize {
unsafe { VM_LOAD_OFFSET }
}
pub fn __va(paddr: usize) -> *mut u8 {
crate::arch::Arch::_va(paddr)
}
pub fn __io(paddr: usize) -> *mut u8 {
crate::arch::Arch::_io(paddr)
}
pub fn __percpu(paddr: usize) -> *mut u8 {
crate::arch::Arch::_percpu(paddr)
}
pub(crate) fn __kimage_va(paddr: usize) -> *mut u8 {
(paddr as isize - vm_load_offset()) as usize as *mut u8
}
pub(crate) fn __kimage_va_to_pa(vaddr: *const u8) -> usize {
(vaddr as usize as isize + vm_load_offset()) as usize
}
pub fn memory_map() -> &'static [MemoryDescriptor] {
MEMORY_MAP.as_slice()
}
pub fn dcache_range(op: DCacheOp, addr: *const u8, size: usize) {
Arch::dcache_range(op, addr as _, size);
}
pub fn phys_to_virt(paddr: usize) -> *mut u8 {
if mmu::is_kernel_relocated() {
if kimage_range().contains(&paddr) {
__kimage_va(paddr)
} else if percpu_range().contains(&paddr) {
__percpu(paddr)
} else {
__va(paddr)
}
} else if cfg!(target_arch = "loongarch64") {
__va(paddr)
} else {
paddr as *mut u8
}
}
pub fn virt_to_phys(vaddr: *const u8) -> usize {
crate::arch::Arch::virt_to_phys(vaddr)
}
pub(crate) fn _fixmap_io(paddr: usize) -> *mut u8 {
if mmu::is_kernel_relocated() {
__io(paddr)
} else {
paddr as *mut u8
}
}
pub(crate) fn early_init() {
crate::fdt::init_memory_map();
let kernel_range = kimage_range();
add_memory_descriptor(MemoryDescriptor {
physical_start: kernel_range.start,
size_in_bytes: kernel_range.end - kernel_range.start,
memory_type: MemoryType::KImage,
})
.unwrap_or_else(|err| {
panic!("failed to add KImage memory descriptor {kernel_range:#x?}: {err:?}")
});
reserve_arch_early_ranges();
unsafe { MEMORY_MAP.update(|m| m.sort_by_key(|a| a.physical_start)) };
print_memory_map();
let mut free_range = None;
for desc in memory_map().iter() {
if desc.memory_type == MemoryType::Free && desc.size_in_bytes > 8 * MB {
free_range = Some(desc.physical_start..(desc.physical_start + desc.size_in_bytes));
break;
}
}
ram::init(free_range.expect("No free memory"));
crate::fdt::save_fdt();
crate::smp::alloc_percpu();
}
fn reserve_arch_early_ranges() {
#[cfg(target_arch = "x86_64")]
{
let tramp = crate::arch::power::AP_TRAMPOLINE_PADDR;
let desc =
MemoryDescriptor::new_aligned(tramp, page_size(), MemoryType::Reserved, page_size());
match add_memory_descriptor(desc) {
Ok(()) => {}
Err(RangeError::Conflict { existing, .. })
if existing.memory_type != MemoryType::Free =>
{
}
Err(err) => panic!("failed to reserve x86 AP trampoline: {err:?}"),
}
}
}
pub(crate) fn kimage_range() -> core::ops::Range<usize> {
unsafe {
let Some(start) = KIMAGE_START else {
panic!("Kernel image start is not set");
};
let end = KIMAGE_END;
start.raw()..end.raw()
}
}
pub fn page_size() -> usize {
unsafe extern "C" {
static PAGE_SIZE: usize;
}
core::ptr::addr_of!(PAGE_SIZE) as usize
}
pub(crate) fn memory_map_setup() {
let ram_range = ram::used_range();
if !ram_range.is_empty() {
let desc = MemoryDescriptor::new_with_range(ram_range, MemoryType::Reserved);
add_memory_descriptor(desc).unwrap();
}
if let Some(desc) = crate::console::debug_to_memory_desc() {
add_memory_descriptor(desc).unwrap();
}
}
pub fn print_memory_map() {
println!("Memory Map:");
unsafe { MEMORY_MAP.update(|m| m.sort_by_key(|m| m.physical_start)) };
for desc in memory_map().iter() {
let fmt = Byte::from(desc.size_in_bytes).get_appropriate_unit(UnitType::Binary);
println!(
" {} {:>#016x} - {:>#016x} ({:#.2})",
desc.memory_type,
desc.physical_start,
desc.physical_start + desc.size_in_bytes,
fmt
);
}
}
pub(crate) fn add_memory_descriptor(
desc: MemoryDescriptor,
) -> Result<(), RangeError<MemoryDescriptor>> {
unsafe { MEMORY_MAP.update(|mem| mem.merge_add(desc)) }
}
pub fn kernel_space() -> Range<usize> {
Arch::kernel_space()
}