use alloc::{
boxed::Box,
ffi::CString,
string::{String, ToString},
vec::Vec,
};
use core::{ffi::CStr, fmt::Display};
use bitflags::bitflags;
use goblin::elf::{Elf, SectionHeader};
use kmod_tools::Module;
use crate::{ModuleErr, Result, arch::ModuleArchSpecific, module::ModuleInfo};
bitflags! {
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SectionPerm: u8 {
const READ = 0b001;
const WRITE = 0b010;
const EXECUTE = 0b100;
}
}
impl Display for SectionPerm {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
let mut perms = String::new();
if self.contains(SectionPerm::READ) {
perms.push('R');
}
if self.contains(SectionPerm::WRITE) {
perms.push('W');
}
if self.contains(SectionPerm::EXECUTE) {
perms.push('X');
}
write!(f, "{}", perms)
}
}
impl SectionPerm {
pub fn from_elf_flags(sh_flags: u64) -> Self {
let mut perms = SectionPerm::empty();
if (sh_flags & goblin::elf::section_header::SHF_ALLOC as u64) != 0 {
perms |= SectionPerm::READ;
}
if (sh_flags & goblin::elf::section_header::SHF_WRITE as u64) != 0 {
perms |= SectionPerm::WRITE;
}
if (sh_flags & goblin::elf::section_header::SHF_EXECINSTR as u64) != 0 {
perms |= SectionPerm::EXECUTE;
}
perms
}
}
pub trait SectionMemOps: Send + Sync {
fn as_ptr(&self) -> *const u8;
fn as_mut_ptr(&mut self) -> *mut u8;
fn change_perms(&mut self, perms: SectionPerm) -> bool;
}
pub trait KernelModuleHelper {
fn vmalloc(size: usize) -> Box<dyn SectionMemOps>;
fn resolve_symbol(name: &str) -> Option<usize>;
fn flsuh_cache(_addr: usize, _size: usize) {
}
}
pub struct ModuleLoader<'a, H: KernelModuleHelper> {
elf: Elf<'a>,
elf_data: &'a [u8],
__helper: core::marker::PhantomData<H>,
}
struct SectionPages {
name: String,
addr: Box<dyn SectionMemOps>,
size: usize,
perms: SectionPerm,
}
pub struct ModuleOwner<H: KernelModuleHelper> {
module_info: ModuleInfo,
pages: Vec<SectionPages>,
name: String,
module: Module,
#[allow(unused)]
pub(crate) arch: ModuleArchSpecific,
_helper: core::marker::PhantomData<H>,
}
impl<H: KernelModuleHelper> ModuleOwner<H> {
pub fn name(&self) -> &str {
&self.name
}
pub(crate) fn set_name(&mut self, name: &str) {
self.name = name.to_string();
}
pub fn call_init(&mut self) -> Result<i32> {
if let Some(init_fn) = self.module.take_init_fn() {
let result = unsafe { init_fn() };
Ok(result)
} else {
log::warn!("The init function can only be called once.");
Err(ModuleErr::EINVAL)
}
}
pub fn call_exit(&mut self) {
if let Some(exit_fn) = self.module.take_exit_fn() {
log::warn!("Calling module exit function...");
unsafe {
exit_fn();
}
} else {
log::warn!("The exit function can only be called once.");
}
}
}
const fn align_up(addr: usize, align: usize) -> usize {
(addr + align - 1) & !(align - 1)
}
const SKIP_SECTIONS: &[&str] = &[".note", ".modinfo", "__version"];
pub(crate) struct ModuleLoadInfo {
pub(crate) syms: Vec<(goblin::elf::sym::Sym, String)>,
}
impl<'a, H: KernelModuleHelper> ModuleLoader<'a, H> {
pub fn new(elf_data: &'a [u8]) -> Result<Self> {
let elf = Elf::parse(elf_data).map_err(|_| ModuleErr::ENOEXEC)?;
if !elf.is_64 {
return Err(ModuleErr::ENOEXEC);
}
Ok(ModuleLoader {
elf,
elf_data,
__helper: core::marker::PhantomData,
})
}
fn module_sig_check(&self) -> bool {
true
}
fn elf_validity_cache_copy(&self) -> Result<ModuleOwner<H>> {
if self.elf.header.e_type != goblin::elf::header::ET_REL {
log::error!(
"Invalid ELF type: {}, expected ET_REL",
self.elf.header.e_type
);
return Err(ModuleErr::ENOEXEC);
}
elf_check_arch(&self.elf)?;
if self.elf.header.e_shstrndx == goblin::elf::section_header::SHN_UNDEF as _
|| self.elf.header.e_shstrndx as usize >= self.elf.section_headers.len()
{
log::error!(
"Invalid ELF section name index: {} || e_shstrndx ({}) >= e_shnum ({})",
self.elf.header.e_shstrndx,
self.elf.header.e_shstrndx,
self.elf.section_headers.len()
);
return Err(ModuleErr::ENOEXEC);
}
if self.elf.shdr_strtab.len() == 0 {
log::error!("ELF section name string table is empty");
return Err(ModuleErr::ENOEXEC);
}
if self.elf.section_headers[0].sh_type != goblin::elf::section_header::SHT_NULL
|| self.elf.section_headers[0].sh_size != 0
|| self.elf.section_headers[0].sh_addr != 0
{
log::error!(
"ELF Spec violation: section 0 type({})!=SH_NULL or non-zero len or addr",
self.elf.section_headers[0].sh_type
);
return Err(ModuleErr::ENOEXEC);
}
let mut num_sym_secs = 0;
let mut num_mod_secs = 0;
let mut num_info_secs = 0;
let mut info_idx = 0;
let mut mod_idx = 0;
for (idx, shdr) in self.elf.section_headers.iter().enumerate() {
let ty = shdr.sh_type;
match ty {
goblin::elf::section_header::SHT_NULL | goblin::elf::section_header::SHT_NOBITS => {
continue;
}
goblin::elf::section_header::SHT_SYMTAB => {
if shdr.sh_link == goblin::elf::section_header::SHN_UNDEF
|| shdr.sh_link as usize >= self.elf.section_headers.len()
{
log::error!(
"Invalid ELF sh_link!=SHN_UNDEF({}) or (sh_link({}) >= hdr->e_shnum({})",
shdr.sh_link,
shdr.sh_link,
self.elf.section_headers.len()
);
return Err(ModuleErr::ENOEXEC);
}
num_sym_secs += 1;
}
_ => {
let shdr_name = self
.elf
.shdr_strtab
.get_at(shdr.sh_name)
.ok_or(ModuleErr::ENOEXEC)?;
if shdr_name == ".gnu.linkonce.this_module" {
num_mod_secs += 1;
mod_idx = idx;
} else if shdr_name == ".modinfo" {
num_info_secs += 1;
info_idx = idx;
}
if shdr.sh_flags == goblin::elf::section_header::SHF_ALLOC as _ {
let _name = self
.elf
.shdr_strtab
.get_at(shdr.sh_name)
.ok_or(ModuleErr::ENOEXEC)?;
}
}
}
}
let mut owner = None;
if num_info_secs > 1 {
log::error!("Only one .modinfo section must exist.");
return Err(ModuleErr::ENOEXEC);
} else if num_info_secs == 1 {
owner = Some(self.pre_read_modinfo(info_idx)?);
if let Some(ref o) = owner {
log::error!("Module({:?}) info: {:?}", o.name(), o.module_info);
}
}
let mut owner = owner.ok_or(ModuleErr::ENOEXEC)?;
let module_name = owner.name();
if num_sym_secs != 1 {
log::error!("{}: module has no symbols (stripped?)", module_name);
return Err(ModuleErr::ENOEXEC);
}
if num_mod_secs != 1 {
log::error!(
"{}: Only one .gnu.linkonce.this_module section must exist.",
module_name
);
return Err(ModuleErr::ENOEXEC);
}
let this_module_shdr = &self.elf.section_headers[mod_idx];
if this_module_shdr.sh_type == goblin::elf::section_header::SHT_NOBITS {
log::error!(
"{}: .gnu.linkonce.this_module section must have a size set",
module_name
);
return Err(ModuleErr::ENOEXEC);
}
if this_module_shdr.sh_flags & goblin::elf::section_header::SHF_ALLOC as u64 == 0 {
log::error!(
"{}: .gnu.linkonce.this_module section size must match the kernel's built struct module size at run time",
module_name
);
return Err(ModuleErr::ENOEXEC);
}
if owner.name().is_empty() {
self.pre_read_this_module(mod_idx, &mut owner)?;
}
Ok(owner)
}
pub fn load_module(mut self, args: CString) -> Result<ModuleOwner<H>> {
if !self.module_sig_check() {
log::error!("Module signature check failed");
return Err(ModuleErr::ENOEXEC);
}
let mut owner = self.elf_validity_cache_copy()?;
self.layout_and_allocate(&mut owner)?;
let load_info = self.simplify_symbols(&owner)?;
self.apply_relocations(load_info, &mut owner)?;
self.post_read_this_module(&mut owner)?;
self.find_module_sections(&mut owner)?;
self.complete_formation(&mut owner)?;
self.parse_args(&mut owner, args)?;
log::error!("Module({:?}) loaded successfully!", owner.name());
Ok(owner)
}
fn parse_args(&self, owner: &mut ModuleOwner<H>, args: CString) -> Result<()> {
let name = owner.name().to_string();
let kparams = owner.module.params_mut();
let after_dashes = crate::param::parse_args(&name, args, kparams, i16::MIN, i16::MAX)?;
if !after_dashes.is_empty() {
log::warn!(
"[{}]: parameters '{}' after '--' ignored",
name,
after_dashes.to_str().unwrap_or("<invalid UTF-8>")
);
}
Ok(())
}
fn find_section(&self, name: &str) -> Result<&SectionHeader> {
for shdr in &self.elf.section_headers {
let sec_name = self
.elf
.shdr_strtab
.get_at(shdr.sh_name)
.ok_or(ModuleErr::ENOEXEC)?;
if sec_name == name {
return Ok(shdr);
}
}
log::error!("Section '{}' not found", name);
Err(ModuleErr::ENOEXEC)
}
fn pre_read_modinfo(&self, info_idx: usize) -> Result<ModuleOwner<H>> {
let modinfo_shdr = &self.elf.section_headers[info_idx];
let file_offset = modinfo_shdr.sh_offset as usize;
let size = modinfo_shdr.sh_size as usize;
let mut modinfo_data = &self.elf_data[file_offset..file_offset + size];
let mut module_info = ModuleInfo::new();
log::info!("Reading .modinfo section (size: {:#x})", size);
loop {
if modinfo_data.is_empty() {
break;
}
let cstr = CStr::from_bytes_until_nul(modinfo_data)
.map_err(|_| ModuleErr::EINVAL)
.unwrap();
let str_slice = cstr.to_str().map_err(|_| ModuleErr::EINVAL)?;
modinfo_data = &modinfo_data[cstr.to_bytes_with_nul().len()..];
let mut split = str_slice.splitn(2, '=');
let key = split.next().ok_or(ModuleErr::EINVAL)?.to_string();
let value = split.next().ok_or(ModuleErr::EINVAL)?.to_string();
module_info.add_kv(key, value);
}
let name = module_info
.get("name")
.map(|s| s.to_string())
.unwrap_or_else(|| "".to_string());
Ok(ModuleOwner {
name,
module_info,
pages: Vec::new(),
module: Module::default(),
arch: ModuleArchSpecific::default(),
_helper: core::marker::PhantomData,
})
}
fn pre_read_this_module(&self, idx: usize, owner: &mut ModuleOwner<H>) -> Result<()> {
let this_module_shdr = &self.elf.section_headers[idx];
let size = this_module_shdr.sh_size as usize;
if size != core::mem::size_of::<Module>() {
log::error!(
"Invalid .gnu.linkonce.this_module section size: {}, expected: {}",
size,
core::mem::size_of::<Module>()
);
return Err(ModuleErr::ENOEXEC);
}
let modinfo_data = this_module_shdr.sh_addr as *mut u8;
let module = unsafe { core::ptr::read(modinfo_data as *const Module) };
let name = module.name();
owner.set_name(name);
Ok(())
}
fn post_read_this_module(&mut self, owner: &mut ModuleOwner<H>) -> Result<()> {
let this_module_shdr = self.find_section(".gnu.linkonce.this_module")?;
let modinfo_data = this_module_shdr.sh_addr as *mut u8;
let module = unsafe { core::ptr::read(modinfo_data as *const Module) };
let init_fn = module.init_fn();
let exit_fn = module.exit_fn();
log::error!(
"Module init_fn: {:?}, exit_fn: {:?}",
init_fn.map(|f| f as *const ()),
exit_fn.map(|f| f as *const ())
);
owner.module = module;
Ok(())
}
fn section_objs(&self, name: &str, object_size: usize) -> Result<(usize, *const u8)> {
let section = self
.find_section(name)
.unwrap_or(&self.elf.section_headers[0]); let num = section.sh_size as usize / object_size;
let addr = section.sh_addr as *const u8;
Ok((num, addr))
}
fn find_module_sections(&self, owner: &mut ModuleOwner<H>) -> Result<()> {
let (num_kparams, kparam_addr) =
self.section_objs("__param", size_of::<kmod_tools::kernel_param>())?;
let raw_module = owner.module.raw_mod();
raw_module.kp = kparam_addr as *mut kmod_tools::kernel_param;
raw_module.num_kp = num_kparams as _;
Ok(())
}
fn complete_formation(&self, owner: &mut ModuleOwner<H>) -> Result<()> {
for page in &mut owner.pages {
if !page.addr.change_perms(page.perms) {
log::error!(
"Failed to change permissions of section '{}' to {}",
page.name,
page.perms
);
return Err(ModuleErr::EINVAL);
}
H::flsuh_cache(page.addr.as_ptr() as usize, page.size);
}
Ok(())
}
fn layout_and_allocate(&mut self, owner: &mut ModuleOwner<H>) -> Result<()> {
#[cfg(feature = "module-sections")]
crate::arch::module_frob_arch_sections(&mut self.elf, owner)?;
for shdr in self.elf.section_headers.iter_mut() {
let sec_name = self
.elf
.shdr_strtab
.get_at(shdr.sh_name)
.unwrap_or("<unknown>");
if (shdr.sh_flags & goblin::elf::section_header::SHF_ALLOC as u64) == 0 {
log::debug!("Skipping non-allocatable section '{}'", sec_name);
continue;
}
if SKIP_SECTIONS.iter().any(|&s| sec_name.starts_with(s)) {
log::warn!("Skipping section '{}' in skip list", sec_name);
continue;
}
let file_offset = shdr.sh_offset as usize;
let size = shdr.sh_size as usize;
let perms = SectionPerm::from_elf_flags(shdr.sh_flags);
if size == 0 {
log::error!("Skipping zero-size section '{}'", sec_name);
continue;
}
let aligned_size = align_up(size, 4096);
let mut addr = H::vmalloc(aligned_size);
if addr.as_ptr().is_null() {
return Err(ModuleErr::ENOSPC);
}
let raw_addr = addr.as_ptr() as u64;
if shdr.sh_type != goblin::elf::section_header::SHT_NOBITS {
let section_data = &self.elf_data[file_offset..file_offset + size];
unsafe {
core::ptr::copy_nonoverlapping(section_data.as_ptr(), addr.as_mut_ptr(), size);
}
}
owner.pages.push(SectionPages {
name: sec_name.to_string(),
addr,
size: aligned_size,
perms,
});
shdr.sh_addr = raw_addr;
}
for page in &owner.pages {
log::error!(
"Allocated section '{:>26}' at {:p} [{}] ({:8<#x})",
page.name,
page.addr.as_ptr(),
page.perms,
page.size
);
}
Ok(())
}
fn simplify_symbols(&self, owner: &ModuleOwner<H>) -> Result<ModuleLoadInfo> {
let mut loadinfo = ModuleLoadInfo { syms: Vec::new() };
for (idx, sym) in self.elf.syms.iter().enumerate() {
if idx == 0 {
loadinfo.syms.push((sym, "".to_string()));
continue;
}
let sym_name = self
.elf
.strtab
.get_at(sym.st_name)
.unwrap_or("<unknown>")
.to_string();
let sym_value = sym.st_value;
let sym_size = sym.st_size;
log::debug!(
"Symbol: ('{}') [{}] Value: 0x{:016x} Size: {}",
sym_name,
sym_section_to_str(sym.st_shndx as _),
sym_value,
sym_size
);
let mut updated_sym = sym;
match sym.st_shndx as _ {
goblin::elf::section_header::SHN_UNDEF => {
let sym_address = H::resolve_symbol(&sym_name);
if let Some(addr) = sym_address {
log::error!(
" -> Resolved undefined symbol '{}' ({}) to address 0x{:016x}",
sym_name,
sym_bind_to_str(sym.st_bind()),
addr
);
updated_sym.st_value = addr as u64;
} else {
if sym.st_bind() == goblin::elf::sym::STB_WEAK {
log::warn!(
" -> Unresolved weak symbol '{}' ({})",
sym_name,
sym_bind_to_str(sym.st_bind())
);
} else {
log::warn!(
" -> Unresolved symbol '{}' ({})",
sym_name,
sym_bind_to_str(sym.st_bind())
);
return Err(ModuleErr::ENOENT);
}
}
}
goblin::elf::section_header::SHN_ABS => {
log::debug!("Absolute symbol: {} 0x{:x}", sym_name, sym_value);
}
goblin::elf::section_header::SHN_COMMON => {
log::debug!("Common symbol: {}", sym_name);
log::warn!("{:?}: please compile with -fno-common", owner.name());
return Err(ModuleErr::ENOEXEC);
}
ty => {
let secbase = self.elf.section_headers[ty as usize].sh_addr;
updated_sym.st_value = sym.st_value.wrapping_add(secbase);
log::trace!(
" -> Defined symbol '{}' in section {} at address 0x{:016x} (base: 0x{:016x} + offset: 0x{:016x})",
sym_name,
ty,
updated_sym.st_value,
secbase,
sym.st_value
);
}
}
loadinfo.syms.push((updated_sym, sym_name));
}
Ok(loadinfo)
}
fn apply_relocations(
&self,
load_info: ModuleLoadInfo,
owner: &mut ModuleOwner<H>,
) -> Result<()> {
for shdr in self.elf.section_headers.iter() {
let infosec = shdr.sh_info;
let sec_name = self
.elf
.shdr_strtab
.get_at(shdr.sh_name)
.ok_or(ModuleErr::ENOEXEC)?;
if infosec >= self.elf.section_headers.len() as u32 {
continue;
}
if self.elf.section_headers[infosec as usize].sh_flags
& goblin::elf::section_header::SHF_ALLOC as u64
== 0
{
continue;
}
if shdr.sh_type != goblin::elf::section_header::SHT_RELA {
continue;
}
let to_section = &self.elf.section_headers[infosec as usize];
let to_sec_name = self
.elf
.shdr_strtab
.get_at(to_section.sh_name)
.ok_or(ModuleErr::ENOEXEC)?;
let rela_entries = shdr.sh_size as usize / shdr.sh_entsize as usize;
log::error!(
"Applying relocations for section '{}' to '{}', {} entries",
sec_name,
to_sec_name,
rela_entries
);
let offset = shdr.sh_offset as usize;
debug_assert!(shdr.sh_entsize == 24);
let data_buf = &self.elf_data[offset..offset + shdr.sh_size as usize];
let rela_list = unsafe {
goblin::elf64::reloc::from_raw_rela(data_buf.as_ptr() as _, shdr.sh_size as usize)
};
crate::arch::ArchRelocate::apply_relocate_add(
rela_list,
shdr,
&self.elf.section_headers,
&load_info,
owner,
)?;
}
Ok(())
}
}
const fn sym_bind_to_str(bind: u8) -> &'static str {
match bind {
goblin::elf::sym::STB_LOCAL => "LOCAL",
goblin::elf::sym::STB_GLOBAL => "GLOBAL",
goblin::elf::sym::STB_WEAK => "WEAK",
_ => "UNKNOWN",
}
}
const fn sym_section_to_str(shndx: u32) -> &'static str {
match shndx {
goblin::elf::section_header::SHN_UNDEF => "UNDEF(0)",
goblin::elf::section_header::SHN_LORESERVE => "LORESERVE(0xff00)",
goblin::elf::section_header::SHN_HIPROC => "HIPROC(0xff1f)",
goblin::elf::section_header::SHN_ABS => "ABS(0xfff1)",
goblin::elf::section_header::SHN_COMMON => "COMMON(0xfff2)",
goblin::elf::section_header::SHN_HIRESERVE => "HIRESERVE(0xffff)",
_ => "OTHER",
}
}
fn elf_check_arch(elf: &goblin::elf::Elf) -> Result<()> {
if elf.header.e_machine != goblin::elf::header::EM_AARCH64
&& elf.header.e_machine != goblin::elf::header::EM_X86_64
&& elf.header.e_machine != goblin::elf::header::EM_RISCV
&& elf.header.e_machine != goblin::elf::header::EM_LOONGARCH
{
log::error!(
"Invalid ELF machine: {}, expected AARCH64({}), X86_64({}), RISC-V({}), LOONGARCH({})",
elf.header.e_machine,
goblin::elf::header::EM_AARCH64,
goblin::elf::header::EM_X86_64,
goblin::elf::header::EM_RISCV,
goblin::elf::header::EM_LOONGARCH
);
return Err(ModuleErr::ENOEXEC);
}
Ok(())
}