use super::layout::{MemoryLayoutPlan, SectionId};
use super::plan::{LinkPlan, ModuleId};
use crate::{
LinkerError, Result,
elf::{ElfPhdr, ElfProgramType},
entity::SecondaryMap,
image::{RawDynamic, ScannedDynamic},
input::PathBuf,
loader::ImageBuilder,
memory::{HostRegion, ImageMemory, RegionAccess, VmAddr, VmOffset},
os::Mmap,
relocation::{RelocationArch, RelocationValueProvider},
runtime::{CodeExecutor, NativeCodeExecutor},
segment::ElfSegments,
sync::Arc,
tls::TlsResolver,
};
use alloc::{boxed::Box, vec::Vec};
use core::ptr::NonNull;
mod arena;
mod rewrite;
use arena::MappedArenaMap;
pub(crate) use rewrite::GotPltTarget;
use rewrite::RuntimeMetadataRewriter;
pub(crate) struct RuntimeModuleMemory<R: RegionAccess = HostRegion> {
sections: Box<[RuntimeSectionMemory]>,
segments: ElfSegments<R>,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
#[repr(transparent)]
struct RuntimeOffset(usize);
impl RuntimeOffset {
#[inline]
const fn new(offset: usize) -> Self {
Self(offset)
}
#[inline]
const fn get(self) -> usize {
self.0
}
#[inline]
fn checked_add(self, delta: usize) -> Option<Self> {
self.0.checked_add(delta).map(Self)
}
}
pub(crate) struct MappedRuntimeMemory<R: RegionAccess = HostRegion> {
arenas: MappedArenaMap<R>,
modules: SecondaryMap<ModuleId, RuntimeModuleMemory<R>>,
}
#[derive(Clone, Copy)]
struct RuntimeSectionMemory {
section: SectionId,
source_address: VmOffset,
runtime_offset: RuntimeOffset,
size: usize,
}
impl RuntimeSectionMemory {
fn source_offset(self, source_address: VmOffset) -> Option<usize> {
let offset = source_address
.checked_offset_from(self.source_address)?
.get();
if self.size == 0 {
return (offset == 0).then_some(0);
}
(offset < self.size).then_some(offset)
}
fn runtime_offset(self, source_address: VmOffset) -> Option<RuntimeOffset> {
self.source_offset(source_address)
.and_then(|offset| self.runtime_offset.checked_add(offset))
}
}
impl<R: RegionAccess> RuntimeModuleMemory<R> {
fn build(
module_id: ModuleId,
layout: &MemoryLayoutPlan,
mapped_section_arenas: &MappedArenaMap<R>,
) -> Result<Self> {
let module = layout.module(module_id);
let region = mapped_section_arenas.region();
let mut placed_sections = Vec::with_capacity(module.alloc_sections().len());
for section_id in module.alloc_sections().iter().copied() {
let Some(placement) = layout.placement(section_id) else {
continue;
};
let metadata = layout.section(section_id);
let arena = mapped_section_arenas
.get(placement.arena())
.ok_or_else(|| {
LinkerError::runtime_memory("arena-backed module referenced an unmapped arena")
})?;
let actual_address = arena.address(®ion, placement.offset()).ok_or_else(|| {
LinkerError::runtime_memory("arena-backed module section address overflowed")
})?;
placed_sections.push((
section_id,
VmOffset::new(metadata.source_address()),
actual_address,
metadata.size(),
));
}
let Some(base) = placed_sections
.iter()
.map(|(_, _, actual_address, _)| *actual_address)
.min()
else {
return Err(LinkerError::runtime_memory(
"arena-backed module does not own any alloc sections",
)
.into());
};
let mut mapped_ranges = Vec::with_capacity(placed_sections.len());
let mut runtime_sections = Vec::with_capacity(placed_sections.len());
for (section, source_address, actual_address, size) in &placed_sections {
let runtime_offset = actual_address.checked_sub(base).ok_or_else(|| {
LinkerError::runtime_memory("arena-backed module address precedes runtime base")
})?;
let runtime_offset = RuntimeOffset::new(runtime_offset);
mapped_ranges.push((runtime_offset.get(), *size));
runtime_sections.push(RuntimeSectionMemory {
section: *section,
source_address: *source_address,
runtime_offset,
size: *size,
});
}
Ok(RuntimeModuleMemory {
sections: runtime_sections.into_boxed_slice(),
segments: ElfSegments::from_ranges(region, VmAddr::new(base), mapped_ranges),
})
}
fn remap_source_to_runtime_offset(&self, source_address: VmOffset) -> Option<RuntimeOffset> {
self.sections
.iter()
.copied()
.find_map(|section| section.runtime_offset(source_address))
}
}
impl<R: RegionAccess> ImageMemory for RuntimeModuleMemory<R> {
#[inline]
fn base(&self) -> VmAddr {
self.segments.base()
}
#[inline]
fn host_ptr(&self, addr: VmAddr) -> Option<NonNull<u8>> {
self.segments.host_ptr(addr)
}
#[inline]
fn host_ptr_range(&self, addr: VmAddr, len: usize) -> Option<NonNull<u8>> {
self.segments.host_ptr_range(addr, len)
}
#[inline]
fn read_bytes(&self, addr: VmAddr, dst: &mut [u8]) -> Result<()> {
self.segments.read_bytes(addr, dst)
}
#[inline]
fn write_bytes(&self, addr: VmAddr, src: &[u8]) -> Result<()> {
self.segments.write_bytes(addr, src)
}
}
impl<R: RegionAccess> MappedRuntimeMemory<R> {
pub(crate) fn map<K, Arch, Tls, M>(
mapper: &M,
plan: &LinkPlan<K, Arch, Tls>,
) -> Result<Option<Self>>
where
K: Clone + Ord,
Arch: RelocationArch,
Tls: TlsResolver<Arch>,
M: Mmap<Region = R> + ?Sized,
{
let Some(arenas) = MappedArenaMap::map_plan(mapper, plan)? else {
return Ok(None);
};
Ok(Some(Self {
arenas,
modules: SecondaryMap::default(),
}))
}
pub(crate) fn build_module(&mut self, id: ModuleId, layout: &MemoryLayoutPlan) -> Result<()> {
let runtime = RuntimeModuleMemory::build(id, layout, &self.arenas)?;
let res = self.modules.insert(id, runtime);
debug_assert!(
res.is_none(),
"module runtime memory was built more than once"
);
Ok(())
}
pub(crate) fn repair_module<K, Arch, Tls>(
&mut self,
id: ModuleId,
plan: &mut LinkPlan<K, Arch, Tls>,
) -> Result<()>
where
K: Clone + Ord,
Arch: RelocationArch + RelocationValueProvider + GotPltTarget,
Tls: TlsResolver<Arch>,
crate::elf::ElfRelType<Arch>: crate::ByteRepr,
{
let runtime = self.modules.get(id).ok_or_else(|| {
LinkerError::runtime_memory("section-region module runtime memory was not cached")
})?;
let mut rewriter = RuntimeMetadataRewriter::<_, Arch, R, Tls>::new(id, plan, runtime);
rewriter.rewrite()
}
pub(crate) fn populate<K, Arch, Tls>(&mut self, plan: &mut LinkPlan<K, Arch, Tls>) -> Result<()>
where
K: Clone + Ord,
Arch: RelocationArch,
Tls: TlsResolver<Arch>,
{
self.arenas.populate(plan)
}
pub(crate) fn protect(&self) -> Result<()> {
self.arenas.protect()
}
pub(crate) fn take_module(&mut self, module_id: ModuleId) -> Result<RuntimeModuleMemory<R>> {
self.modules
.remove(module_id)
.ok_or_else(|| {
LinkerError::runtime_memory("section-region planned load is missing runtime memory")
})
.map_err(Into::into)
}
}
pub(crate) fn build_arena_raw_dynamic<D, Tls, Arch, R>(
scanned: ScannedDynamic<Arch>,
runtime: RuntimeModuleMemory<R>,
force_static_tls: bool,
) -> Result<RawDynamic<D, Arch, R, Tls>>
where
D: Default + 'static,
Tls: TlsResolver<Arch>,
Arch: RelocationArch,
R: RegionAccess,
{
let mut phdrs = scanned.phdrs().to_vec();
remap_runtime_phdrs::<Arch, R>(&runtime, &mut phdrs)?;
let original_entry = scanned.ehdr().e_entry();
let entry = runtime
.remap_source_to_runtime_offset(VmOffset::new(original_entry))
.map(|offset| runtime.segments.base() + VmOffset::new(offset.get()))
.unwrap_or_else(|| runtime.segments.base() + VmOffset::new(original_entry));
let path = PathBuf::from(scanned.path());
let builder = ImageBuilder::<Tls, D, Arch, R>::new(
runtime.segments,
path,
None,
entry,
force_static_tls,
1,
D::default(),
Arc::from(Box::new(NativeCodeExecutor) as Box<dyn CodeExecutor<Arch>>),
);
builder.build_dynamic(&phdrs)
}
fn remap_runtime_phdrs<Arch, R>(
runtime: &RuntimeModuleMemory<R>,
phdrs: &mut [ElfPhdr<Arch::Layout>],
) -> Result<()>
where
Arch: RelocationArch,
R: RegionAccess,
{
for phdr in phdrs {
let p_type = phdr.program_type();
if p_type == ElfProgramType::GNU_RELRO {
phdr.set_program_type(ElfProgramType::NULL);
continue;
}
let Some(offset) = runtime.remap_source_to_runtime_offset(phdr.p_vaddr()) else {
if p_type == ElfProgramType::PHDR {
phdr.set_program_type(ElfProgramType::NULL);
continue;
}
if matches!(
p_type,
ElfProgramType::DYNAMIC
| ElfProgramType::GNU_EH_FRAME
| ElfProgramType::TLS
| ElfProgramType::INTERP
) {
return Err(LinkerError::runtime_memory("failed to remap program header").into());
}
continue;
};
phdr.set_p_vaddr(VmOffset::new(offset.get()));
}
Ok(())
}