use rayon::prelude::*;
use rustc_hash::{FxHashMap, FxHashSet};
use self::{
addr::{definition_addr, resolve_addresses},
exports::collect_exports,
got::{fill_got, got_slot, plt_slot},
resolver::{SymFlags, SymRows},
};
use crate::{
buildid::BuildId,
dynamic::{DynConfig, DynamicPlan, LinkMode},
elf::constants::{SHN_UNDEF, STB_WEAK},
error::{Error, Result},
input::InputFile,
linker::Context,
output::OutKind,
reloc::Target,
startstop::Bounds,
symbol::{SymbolId, SymbolKind},
tls::TlsBlock,
util::PAGE,
};
mod addr;
mod exports;
mod got;
mod place;
mod resolver;
mod scan;
pub mod sect;
pub use exports::{CopyName, CopySlot, Export};
pub use got::{GotKey, GotKind, GotOwner};
pub use resolver::FileResolver;
pub use sect::Sect;
const BASE: u64 = 0x0040_0000;
const SHARED_BASE: u64 = 0;
pub(crate) const fn max_page_size(target: Target) -> u64 {
match target {
Target::AArch64 => 0x1_0000,
Target::X86_64 | Target::Riscv64 => PAGE,
}
}
const EHDR_SIZE: u64 = 64;
const PHDR_SIZE: u64 = 56;
const GOT_ENTRY: u64 = 8;
#[derive(Copy, Clone, Debug, Default)]
pub struct Region {
pub offset: u64,
pub vaddr: u64,
pub size: u64,
}
impl Region {
pub const fn end(self) -> u64 {
self.offset + self.size
}
}
pub struct Layout {
entry: u64,
phdr_count: u16,
pub target: Target,
pub e_flags: u32,
pub mode: LinkMode,
pub pie: bool,
pub stack_size: u64,
base: u64,
init: Region,
fini: Region,
text: Region,
rodata: Region,
eh_frame: Region,
eh_frame_hdr: Region,
got: Region,
data_rel_ro: Region,
data: Region,
bss: Region,
tdata: Region,
tbss: Region,
preinit_array: Region,
init_array: Region,
fini_array: Region,
pub tls_align: u64,
pub interp: Vec<u8>,
interp_region: Region,
pub note: Region,
pub build_id: Region,
plt: Region,
got_plt: Region,
rela_plt: Region,
shndx: [u16; sect::TABLE.len()],
measured_align: [u64; sect::TABLE.len()],
start_stop: Vec<Bounds>,
sec_vaddr: Vec<Vec<u64>>,
sec_shndx: Vec<Vec<u16>>,
sym: Vec<SymRows>,
pub got_values: Vec<u64>,
tls_index: u64,
pub ifunc: Vec<SymbolId>,
ifunc_resolvers: Vec<u64>,
ifunc_index: FxHashMap<SymbolId, u64>,
pub got_keys: Vec<GotKey>,
pub needs_sym_sizes: bool,
pub abs_candidates: Vec<Vec<u16>>,
pub exports: Vec<Export>,
pub referenced: FxHashSet<SymbolId>,
got_index: FxHashMap<GotKey, u32>,
pub plt_keys: Vec<GotKey>,
plt_index: FxHashMap<GotKey, u32>,
pub copy_slots: Vec<CopySlot>,
pub canonical_plt: FxHashSet<SymbolId>,
pub has_absolute_refs: bool,
pub dynamic: Option<DynamicPlan>,
pub plt_plan: Option<crate::plt::PltPlan>,
}
impl Layout {
pub const fn entry(&self) -> u64 {
self.entry
}
pub const fn phdr_count(&self) -> u16 {
self.phdr_count
}
pub const fn mode(&self) -> LinkMode {
self.mode
}
pub const fn is_pie(&self) -> bool {
self.pie
}
pub const fn base(&self) -> u64 {
self.base
}
pub fn has_interp(&self) -> bool {
!self.interp.is_empty()
}
pub fn copy_bindings(&self) -> impl Iterator<Item = (SymbolId, u64)> {
self.copy_slots.iter().flat_map(|slot| {
slot.names
.iter()
.filter_map(move |n| Some((n.id?, slot.addr)))
})
}
pub fn region(&self, sect: Sect) -> Region {
match sect {
Sect::Interp => self.interp_region,
Sect::Note => self.note,
Sect::BuildId => self.build_id,
Sect::Init => self.init,
Sect::Fini => self.fini,
Sect::Text => self.text,
Sect::Plt => self.plt,
Sect::Rodata => self.rodata,
Sect::EhFrame => self.eh_frame,
Sect::EhFrameHdr => self.eh_frame_hdr,
Sect::RelaPlt => self.rela_plt,
Sect::GotPlt => self.got_plt,
Sect::Got => self.got,
Sect::DataRelRo => self.data_rel_ro,
Sect::Data => self.data,
Sect::PreinitArray => self.preinit_array,
Sect::InitArray => self.init_array,
Sect::FiniArray => self.fini_array,
Sect::Tdata => self.tdata,
Sect::Tbss => self.tbss,
Sect::Bss => self.bss,
dynamic => self
.dynamic
.as_ref()
.map_or_else(Region::default, |p| p.regions.get(dynamic)),
}
}
pub fn shndx(&self, sect: Sect) -> u16 {
self.shndx.get(sect as usize).copied().unwrap_or_default()
}
pub fn run_bounds(&self, run: usize) -> Bounds {
self.start_stop.get(run).copied().unwrap_or_default()
}
pub(crate) fn set_shndx(&mut self, sect: Sect, index: u16) {
if let Some(slot) = self.shndx.get_mut(sect as usize) {
*slot = index;
}
}
pub fn sect_align(&self, sect: Sect, spec: §::Spec) -> u64 {
if spec.align != 0 {
return spec.align;
}
self.measured_align
.get(sect as usize)
.copied()
.unwrap_or(1)
.max(1)
}
pub(crate) fn set_measured_align(&mut self, sect: Sect, align: u64) {
if let Some(slot) = self.measured_align.get_mut(sect as usize) {
*slot = align.max(1);
}
}
pub fn plt_entry_addr(&self, sym: SymbolId) -> Option<u64> {
let key = GotKey::addr(sym);
let &idx = self.plt_index.get(&key)?;
let spec = self.target.plt_spec();
let entry_off = spec
.header_size
.wrapping_add(u64::from(idx) * spec.entry_size);
Some(self.plt.vaddr.wrapping_add(entry_off))
}
pub fn section_vaddr(&self, file: usize, section: u16) -> Option<u64> {
let v = self
.sec_vaddr
.get(file)
.and_then(|f| f.get(usize::from(section)))?;
(*v != UNPLACED).then_some(*v)
}
pub fn section_shndx(&self, file: usize, section: u16) -> Option<u16> {
self.sec_shndx
.get(file)
.and_then(|f| f.get(usize::from(section)).copied())
.filter(|&s| s != 0)
}
pub fn resolver(&self, file: usize) -> FileResolver<'_> {
let row = self.sym.get(file);
FileResolver {
addr: row.map_or(&[][..], |r| &r.addr),
got: row.map_or(&[][..], |r| &r.got),
plt: row.map_or(&[][..], |r| &r.plt),
tls_got: row.map_or(&[][..], |r| &r.tls_got),
flags: row.map_or(&[][..], |r| &r.flags),
size: row.map_or(&[][..], |r| &r.size),
tls_index: self.tls_index,
tls: self.tls_block().map(|b| (self.target, b)),
got_base: self.got_base(),
exec: self.mode != LinkMode::Shared,
}
}
pub fn got_base(&self) -> u64 {
if self.got.size == 0 {
return self.base();
}
self.got.vaddr
}
pub fn is_ifunc(&self, id: SymbolId) -> bool {
self.ifunc_index.contains_key(&id)
}
pub fn ifunc_resolver(&self, id: SymbolId) -> u64 {
self.ifunc_index.get(&id).copied().unwrap_or_default()
}
fn index_ifuncs(&mut self) {
self.ifunc_index.clear();
self.ifunc_index.reserve(self.ifunc.len());
for (&id, &resolver) in self.ifunc.iter().zip(&self.ifunc_resolvers) {
self.ifunc_index.insert(id, resolver);
}
}
fn placed(&self) -> impl Iterator<Item = Sect> {
sect::TABLE
.iter()
.map(|(s, _)| *s)
.filter(|s| self.region(*s).size != 0)
}
pub fn first_placed(&self) -> Option<Sect> {
self.placed().next()
}
pub fn last_placed(&self) -> Option<Sect> {
self.placed().last()
}
pub fn last_file_backed(&self) -> Option<Sect> {
self.placed().filter(|s| sect::file_backed(*s)).last()
}
pub fn last_executable(&self) -> Option<Sect> {
sect::TABLE
.iter()
.filter(|(_, spec)| sect::executable(spec))
.map(|(s, _)| *s)
.rfind(|s| self.region(*s).size != 0)
}
pub fn shndx_at(&self, vaddr: u64) -> u16 {
for sect in sect::CONTENT {
let r = self.region(sect);
if vaddr >= r.vaddr && vaddr < r.vaddr.wrapping_add(r.size) {
return self.shndx(sect);
}
}
0
}
pub fn tls_block(&self) -> Option<TlsBlock> {
let (vaddr, filesz) = if self.tdata.size != 0 {
(self.tdata.vaddr, self.tdata.size)
} else if self.tbss.size != 0 {
(self.tbss.vaddr, 0)
} else {
return None;
};
let mem_end = self
.tbss
.vaddr
.wrapping_add(self.tbss.size)
.max(self.tdata.vaddr.wrapping_add(self.tdata.size));
let offset = if self.tdata.size != 0 {
self.tdata.offset
} else {
self.tbss.offset
};
Some(TlsBlock {
offset,
vaddr,
filesz,
memsz: mem_end.wrapping_sub(vaddr),
align: self.tls_align.max(1),
})
}
}
pub fn build(
ctx: &Context<'_>,
target: Target,
mode: LinkMode,
entry: &[u8],
config: &DynConfig<'_>,
) -> Result<Layout> {
let mut layout = empty_layout(ctx, target, mode, config)?;
let (rel, common_off) =
place::measure_sections(ctx, scratch_map(ctx, 0, section_count)?);
let eh_frame_hdr_size = hdr_size_for(ctx, rel.size(OutKind::EhFrame));
let sym_id = &ctx.sym_id;
scan::scan_relocations(
ctx,
target,
mode,
sym_id,
config.relax,
&mut layout,
)?;
let any_weak_undef = ctx.files.iter().any(has_weak_undef);
let size_rows = !layout.got_index.is_empty()
|| !layout.plt_index.is_empty()
|| layout.needs_sym_sizes
|| config.relax
|| any_weak_undef;
fix_base(mode, config, &mut layout)?;
let mut rows = std::mem::take(&mut layout.sym);
let (sized, probe) = rayon::join(
|| {
if size_rows {
size_sym_slot_tables(ctx, config.relax, &layout, &mut rows)
} else {
Ok(())
}
},
|| {
if mode.is_dynamic() {
DynamicPlan::probe(ctx, &layout, target, mode, config).map(Some)
} else {
Ok(None)
}
},
);
layout.sym = rows;
sized?;
layout.dynamic = probe?;
place::place_regions(
ctx,
&rel,
mode,
place::Sizes {
eh_frame_hdr: eh_frame_hdr_size,
build_id: config.build_id.map_or(0, BuildId::note_size),
},
&mut layout,
);
place::measure_runs(ctx, &mut layout);
place::assign_shndx(ctx, &mut layout);
apply_folding(ctx, &mut layout);
place::stamp_debug_offsets(ctx, &mut layout);
resolve_addresses(ctx, target, &common_off, sym_id, &mut layout)?;
layout.index_ifuncs();
layout.entry = entry_addr(ctx, &layout, entry, mode)?;
Ok(layout)
}
fn empty_layout(
ctx: &Context<'_>,
target: Target,
mode: LinkMode,
config: &DynConfig<'_>,
) -> Result<Layout> {
let sec_vaddr: Vec<Vec<u64>> = scratch_map(ctx, UNPLACED, section_count)?;
let sec_shndx: Vec<Vec<u16>> = scratch_map(ctx, 0, section_count)?;
let sym: Vec<SymRows> = ctx
.files
.par_iter()
.map(|input| {
Ok(SymRows {
addr: vec![0u64; symbol_count(input)?],
got: Vec::new(),
plt: Vec::new(),
tls_got: Vec::new(),
size: Vec::new(),
flags: Vec::new(),
})
})
.collect::<Result<_>>()?;
let interp = place::interp_bytes(config, mode);
Ok(Layout {
needs_sym_sizes: false,
abs_candidates: Vec::new(),
entry: 0,
phdr_count: 0,
target,
e_flags: crate::linker::derive_eflags(ctx, target)?,
mode,
pie: mode != LinkMode::Static,
stack_size: config.z.stack_size.unwrap_or(0),
base: if mode == LinkMode::Static {
BASE
} else {
SHARED_BASE
},
init: Region::default(),
fini: Region::default(),
text: Region::default(),
rodata: Region::default(),
eh_frame: Region::default(),
eh_frame_hdr: Region::default(),
got: Region::default(),
data_rel_ro: Region::default(),
data: Region::default(),
bss: Region::default(),
tdata: Region::default(),
tbss: Region::default(),
preinit_array: Region::default(),
init_array: Region::default(),
fini_array: Region::default(),
tls_align: 1,
interp,
interp_region: Region::default(),
note: Region::default(),
build_id: Region::default(),
shndx: [0; sect::TABLE.len()],
measured_align: [1; sect::TABLE.len()],
start_stop: Vec::new(),
sec_vaddr,
sec_shndx,
sym,
got_values: Vec::new(),
ifunc: Vec::new(),
ifunc_resolvers: Vec::new(),
ifunc_index: FxHashMap::default(),
tls_index: 0,
got_keys: Vec::new(),
exports: Vec::new(),
referenced: FxHashSet::default(),
got_index: FxHashMap::default(),
plt_keys: Vec::new(),
plt_index: FxHashMap::default(),
copy_slots: Vec::new(),
canonical_plt: FxHashSet::default(),
has_absolute_refs: false,
plt: Region::default(),
got_plt: Region::default(),
rela_plt: Region::default(),
dynamic: None,
plt_plan: None,
})
}
fn hdr_size_for(ctx: &Context<'_>, eh_frame_size: u64) -> u64 {
if eh_frame_size == 0 {
return 0;
}
crate::ehframe::hdr_size(ctx.eh.live_fdes())
}
fn fix_base_if_needed(mode: LinkMode, layout: &mut Layout) {
if mode == LinkMode::DynExec
&& (!layout.copy_slots.is_empty() || layout.has_absolute_refs)
{
layout.pie = false;
layout.base = BASE;
}
}
fn fix_base(
mode: LinkMode,
config: &DynConfig<'_>,
layout: &mut Layout,
) -> Result<()> {
let forced = mode == LinkMode::DynExec
&& (!layout.copy_slots.is_empty() || layout.has_absolute_refs);
match config.pie {
Some(true) if forced => {
return Err(Error::CommandLine(
"-pie was given, but an input holds a non-PIC absolute \
reference whose value this link has to fix, so the image \
cannot be loaded at a random base"
.into(),
));
}
Some(true) => {
layout.pie = true;
layout.base = SHARED_BASE;
}
Some(false) => {
layout.pie = false;
layout.base = BASE;
}
None => fix_base_if_needed(mode, layout),
}
Ok(())
}
pub(crate) const UNPLACED: u64 = u64::MAX;
fn scratch_map<T: Copy + Send + Sync>(
ctx: &Context<'_>,
fill: T,
row_len: impl Fn(&InputFile<'_>) -> Result<usize> + Sync,
) -> Result<Vec<Vec<T>>> {
ctx.files
.par_iter()
.map(|input| Ok(vec![fill; row_len(input)?]))
.collect()
}
fn section_count(input: &InputFile<'_>) -> Result<usize> {
Ok(input.object()?.sections().len())
}
fn symbol_count(input: &InputFile<'_>) -> Result<usize> {
Ok(input.symbol_table()?.map_or(0, |st| st.syms.len()))
}
fn has_weak_undef(input: &InputFile<'_>) -> bool {
input.symbol_table().is_ok_and(|table| {
table.is_some_and(|st| {
st.syms
.iter()
.any(|s| s.bind() == STB_WEAK && s.st_shndx.get() == SHN_UNDEF)
})
})
}
fn size_sym_slot_tables(
ctx: &Context<'_>,
relax: bool,
layout: &Layout,
rows: &mut [SymRows],
) -> Result<()> {
let need_got = !layout.got_index.is_empty();
let need_plt = !layout.plt_index.is_empty();
let need_tls = layout.got_keys.iter().any(|key| key.kind != GotKind::Addr);
let need_size = layout.needs_sym_sizes;
rows.par_iter_mut().zip(ctx.files.par_iter()).try_for_each(
|(row, input)| {
let n = symbol_count(input)?;
if need_size {
row.size.resize(n, 0);
if let Ok(Some(symtab)) = input.symbol_table() {
for (slot, sym) in row.size.iter_mut().zip(symtab.syms) {
*slot = sym.st_size.get();
}
}
}
if need_got {
row.got.resize(n, 0);
}
if need_got || relax || has_weak_undef(input) {
row.flags.resize(n, SymFlags::default());
}
if need_plt {
row.plt.resize(n, 0);
}
if need_tls {
row.tls_got.resize(n, 0);
}
Ok(())
},
)
}
fn apply_folding(ctx: &Context<'_>, layout: &mut Layout) {
for ((f_file, f_sec), (r_file, r_sec)) in ctx.folding.iter() {
let Some(&rep_vaddr) = layout
.sec_vaddr
.get(r_file)
.and_then(|f| f.get(usize::from(r_sec)))
else {
continue;
};
let rep_shndx = layout
.sec_shndx
.get(r_file)
.and_then(|f| f.get(usize::from(r_sec)).copied())
.unwrap_or(0);
if let Some(f) = layout.sec_vaddr.get_mut(f_file)
&& let Some(slot) = f.get_mut(usize::from(f_sec))
{
*slot = rep_vaddr;
}
if let Some(f) = layout.sec_shndx.get_mut(f_file)
&& let Some(slot) = f.get_mut(usize::from(f_sec))
{
*slot = rep_shndx;
}
}
}
fn entry_addr(
ctx: &Context<'_>,
layout: &Layout,
entry: &[u8],
mode: LinkMode,
) -> Result<u64> {
let addr = ctx
.symbols
.find(entry)
.and_then(|id| ctx.symbols.symbol(id))
.and_then(|sym| match &sym.kind {
SymbolKind::Defined(def) => {
Some(definition_addr(&ctx.merge, layout, def))
}
_ => None,
});
match addr {
Some(addr) => Ok(addr),
None if mode == LinkMode::Shared || entry.is_empty() => Ok(0),
None => Err(Error::UndefinedEntry(
String::from_utf8_lossy(entry).into_owned(),
)),
}
}