use std::ops::Range;
use rayon::prelude::*;
use crate::{
ehframe::{EhMember, Piece},
elf::{
Rela64, SymbolTable,
constants::{
SHF_ALLOC, SHN_LORESERVE, SHN_UNDEF, SHT_NOBITS, STB_LOCAL,
},
},
error::{Error, Result},
layout::{FileResolver, Layout},
linker::Context,
output::OutKind,
reloc::{
RelExpr, RelaxSite, Target, apply_target, relax_covers_target,
relax_lead_target, relax_required_target, relax_span_target,
relax_spans_type, relax_target, relax_trail_target, riscv, spec_target,
},
symbol::SymbolId,
};
pub(super) fn copy_sections(
ctx: &Context<'_>,
target: Target,
layout: &Layout,
base: u64,
relax: bool,
image: &mut [u8],
) -> Result<()> {
let mut items = copy_work_items(ctx, layout, base);
let discarded = files_with_discarded_sections(ctx, layout);
for item in &mut items {
item.has_discarded = discarded.get(item.file).copied().unwrap_or(false);
}
let mut slices = split_member_slices(image, &items)?;
slices
.par_iter_mut()
.enumerate()
.map(|(i, slice)| {
copy_member_into(ctx, target, layout, relax, items[i], slice)
})
.collect::<Result<Vec<()>>>()?;
Ok(())
}
#[derive(Copy, Clone)]
pub(super) struct CopyItem {
pub(super) file: usize,
pub(super) section: u16,
pub(super) start: usize,
pub(super) len: usize,
pub(super) vaddr: u64,
pub(super) tombstone: Option<u64>,
pub(super) has_discarded: bool,
}
fn copy_work_items(
ctx: &Context<'_>,
layout: &Layout,
base: u64,
) -> Vec<CopyItem> {
let mut out = Vec::new();
for kind in OutKind::FILE_ORDER {
let Some(section) = ctx.outputs.section(kind) else {
continue;
};
out.extend(section.members.iter().filter_map(|m| {
let len = usize::try_from(m.size).unwrap_or(0);
let vaddr = layout.section_vaddr(m.file, m.section)?;
(len != 0).then(|| CopyItem {
tombstone: None,
has_discarded: false,
file: m.file,
section: m.section,
start: usize::try_from(vaddr.wrapping_sub(base))
.unwrap_or(usize::MAX),
len,
vaddr,
})
}));
}
out
}
fn files_with_discarded_sections(
ctx: &Context<'_>,
layout: &Layout,
) -> Vec<bool> {
(0..ctx.files.len())
.map(|file| {
let Some(input) = ctx.files.get(file) else {
return false;
};
let Ok(obj) = input.object() else {
return false;
};
obj.sections().iter().enumerate().any(|(i, s)| {
let shndx = u16::try_from(i).unwrap_or(u16::MAX);
s.sh_flags.get() & SHF_ALLOC != 0
&& s.sh_type.get() != SHT_NOBITS
&& s.sh_size.get() != 0
&& !ctx.merge.is_merged(file, shndx)
&& layout.section_vaddr(file, shndx).is_none()
})
})
.collect()
}
pub(super) fn split_member_slices<'a>(
image: &'a mut [u8],
items: &[CopyItem],
) -> Result<Vec<&'a mut [u8]>> {
let mut out: Vec<&'a mut [u8]> = Vec::with_capacity(items.len());
let mut cursor = 0usize;
let mut remaining = image;
for item in items {
if item.start < cursor {
return Err(Error::OutOfRange("member placement"));
}
let gap = item.start - cursor;
if gap > 0 {
if gap > remaining.len() {
return Err(Error::OutOfRange("member placement"));
}
let (_, right) = remaining.split_at_mut(gap);
remaining = right;
cursor += gap;
}
if item.len > remaining.len() {
return Err(Error::OutOfRange("member copy"));
}
let (chunk, right) = remaining.split_at_mut(item.len);
out.push(chunk);
remaining = right;
cursor += item.len;
}
Ok(out)
}
pub(super) fn copy_member_into(
ctx: &Context<'_>,
target: Target,
layout: &Layout,
relax: bool,
item: CopyItem,
slice: &mut [u8],
) -> Result<()> {
let input = ctx
.files
.get(item.file)
.ok_or(Error::OutOfRange("input file index"))?;
let obj = input.object()?;
let Some(shdr) = obj.sections().get(usize::from(item.section)) else {
return Ok(());
};
let pool = ctx.merge.carried_bytes(item.file, item.section);
let data = match pool {
Some(bytes) => bytes,
None => obj.section_data(shdr)?,
};
let split = ctx.eh.member(item.file, item.section);
if let Some(member) = split {
copy_pieces(member, layout, item.vaddr, data, slice)?;
} else {
if slice.len() < data.len() {
return Err(Error::OutOfRange("section copy"));
}
slice[..data.len()].copy_from_slice(data);
}
let Some(entries) = input.relocations(item.section)? else {
return Ok(());
};
if pool.is_some() && !entries.is_empty() {
return Err(Error::Format("relocations in a merged section"));
}
let resolver = layout.resolver(item.file);
let rewritten = if target == Target::Riscv64 {
riscv::rewrite_pcrel_pairs(entries, item.vaddr, &resolver)?
} else {
None
};
Patch {
ctx,
layout,
target,
relax,
item,
entries: rewritten.as_deref().unwrap_or(entries),
symtab: if ctx.merge.file_has_merges(item.file)
|| item.tombstone.is_some()
|| item.has_discarded
{
input.symbol_table()?
} else {
None
},
split,
}
.apply_all(&resolver, slice)
}
struct Patch<'a, 'i> {
ctx: &'a Context<'i>,
layout: &'a Layout,
target: Target,
relax: bool,
item: CopyItem,
entries: &'a [Rela64],
symtab: Option<SymbolTable<'a>>,
split: Option<&'a EhMember>,
}
impl Patch<'_, '_> {
fn site(&self, r: &Rela64) -> Option<(u64, i64)> {
let slot = match self.split {
None => r.r_offset.get(),
Some(member) => {
let piece = member.piece_at(r.r_offset.get())?;
let out_off = piece.out_off?;
out_off
.wrapping_add(r.r_offset.get().wrapping_sub(piece.in_off))
}
};
let addend = if self.ctx.merge.names_merged(self.item.file, r.sym()) {
self.symtab.as_ref().map_or_else(
|| r.r_addend.get(),
|symtab| self.ctx.merge.addend(symtab, self.item.file, r),
)
} else {
r.r_addend.get()
};
Some((slot, addend))
}
fn consumed(
&self,
resolver: &FileResolver<'_>,
slice: &[u8],
out: &mut Vec<Consumed>,
) {
for r in self.entries {
if !relax_spans_type(self.target, r.r_type()) {
continue;
}
let Some((slot, addend)) = self.site(r) else {
continue;
};
let site = RelaxSite {
r_type: r.r_type(),
sym: sym_of(r),
addend,
place: self.item.vaddr.wrapping_add(slot),
data: slice,
slot,
};
if let Some(range) = relax_span_target(self.target, site, resolver)
{
out.push(Consumed { range, slot });
}
}
}
fn apply_all(
&self,
resolver: &FileResolver<'_>,
slice: &mut [u8],
) -> Result<()> {
let mut consumed = Vec::new();
self.consumed(resolver, slice, &mut consumed);
for r in self.entries {
let Some((slot, addend)) = self.site(r) else {
continue;
};
if consumed.iter().any(|c| c.swallows(slot)) {
continue;
}
self.apply_entry(
RelocApply {
r,
addend,
target: self.target,
relax: self.relax,
vaddr: self.item.vaddr,
out_off: slot,
},
resolver,
slice,
)?;
}
Ok(())
}
fn apply_entry(
&self,
site: RelocApply<'_>,
resolver: &FileResolver<'_>,
slice: &mut [u8],
) -> Result<()> {
let r_type = site.r.r_type();
let spec = spec_target(site.target, r_type)?;
if spec.expr == RelExpr::None {
return Ok(());
}
let width = spec.write.width();
if spec.expr == RelExpr::Abs
&& let Some(value) = self.tombstone(site.r)
{
return write_tombstone(site.out_off, width, value, slice);
}
self.check_not_discarded(site.r)?;
let sym = sym_of(site.r);
let required =
relax_required_target(site.target, r_type, sym, resolver);
if site.relax || required {
match try_rewrite(site, width, resolver, slice)? {
Rewrite::Done => return Ok(()),
Rewrite::Declined(why) if required => {
return Err(self.unlowered(site.r, why));
}
Rewrite::Declined(_) => {}
}
}
let slot_off = usize::try_from(site.out_off).unwrap_or(usize::MAX);
let Some(slot) =
slice.get_mut(slot_off..slot_off.saturating_add(width))
else {
return Err(Error::OutOfRange("relocation slot"));
};
let place = site.vaddr.wrapping_add(site.out_off);
apply_target(
site.target,
r_type,
sym,
site.addend,
place,
resolver,
slot,
)
}
fn tombstone(&self, r: &Rela64) -> Option<u64> {
let value = self.item.tombstone?;
let symtab = self.symtab.as_ref()?;
let sym = symtab.syms.get(r.sym() as usize)?;
if sym.bind() != STB_LOCAL {
return None;
}
let shndx = sym.st_shndx.get();
if shndx == SHN_UNDEF || shndx >= SHN_LORESERVE {
return None;
}
if self.ctx.merge.is_merged(self.item.file, shndx) {
return None;
}
self.layout
.section_vaddr(self.item.file, shndx)
.is_none()
.then_some(value)
}
fn check_not_discarded(&self, r: &Rela64) -> Result<()> {
if !self.item.has_discarded {
return Ok(());
}
let Some(symtab) = self.symtab.as_ref() else {
return Ok(());
};
let Some(sym) = symtab.syms.get(r.sym() as usize) else {
return Ok(());
};
if sym.bind() != STB_LOCAL {
return Ok(());
}
let shndx = sym.st_shndx.get();
if shndx == SHN_UNDEF
|| shndx >= SHN_LORESERVE
|| self.ctx.merge.is_merged(self.item.file, shndx)
|| self.layout.section_vaddr(self.item.file, shndx).is_some()
{
return Ok(());
}
Err(Error::UndefinedReference(
String::from_utf8_lossy(symtab.name(sym)).into_owned(),
))
}
fn unlowered(&self, r: &Rela64, reason: &'static str) -> Error {
Error::unresolvable_tls(
String::from_utf8_lossy(self.symbol_name(r)).into_owned(),
reason,
)
}
fn symbol_name(&self, r: &Rela64) -> &[u8] {
let Some(input) = self.ctx.files.get(self.item.file) else {
return &[];
};
let Ok(Some(symtab)) = input.symbol_table() else {
return &[];
};
symtab
.syms
.get(r.sym() as usize)
.map_or(&[] as &[u8], |sym| symtab.name(sym))
}
}
#[derive(Clone)]
struct Consumed {
range: Range<u64>,
slot: u64,
}
impl Consumed {
fn swallows(&self, slot: u64) -> bool {
slot != self.slot && self.range.contains(&slot)
}
}
fn sym_of(r: &Rela64) -> Option<SymbolId> {
match r.sym() {
0 => None,
idx => Some(SymbolId(usize::try_from(idx).unwrap_or(0))),
}
}
fn copy_pieces(
member: &EhMember,
layout: &Layout,
vaddr: u64,
data: &[u8],
slice: &mut [u8],
) -> Result<()> {
let mut end = 0usize;
for piece in member.pieces() {
let Some(out_off) = piece.out_off else {
continue;
};
let src_start = usize::try_from(piece.in_off).unwrap_or(usize::MAX);
let len = usize::try_from(piece.len).unwrap_or(0);
let dst_start = usize::try_from(out_off).unwrap_or(usize::MAX);
let src = data
.get(src_start..src_start.saturating_add(len))
.ok_or(Error::OutOfRange("eh_frame record"))?;
let dst = slice
.get_mut(dst_start..dst_start.saturating_add(len))
.ok_or(Error::OutOfRange("eh_frame record slot"))?;
dst.copy_from_slice(src);
end = end.max(dst_start.saturating_add(len));
if piece.fde {
rewrite_cie_pointer(member, layout, vaddr, *piece, src, dst)?;
}
}
if let Some(tail) = slice.get_mut(end..) {
tail.fill(0);
}
Ok(())
}
fn rewrite_cie_pointer(
member: &EhMember,
layout: &Layout,
vaddr: u64,
piece: Piece,
src: &[u8],
dst: &mut [u8],
) -> Result<()> {
let field: [u8; 4] = src
.get(4..8)
.and_then(|b| b.try_into().ok())
.ok_or(Error::OutOfRange("FDE CIE pointer"))?;
let old = u64::from(u32::from_le_bytes(field));
let cie_in = piece
.in_off
.wrapping_add(4)
.checked_sub(old)
.ok_or(Error::Format("FDE CIE pointer"))?;
let cie_addr = if let Some(out_off) =
member.piece_at(cie_in).and_then(|c| c.out_off)
{
vaddr.wrapping_add(out_off)
} else {
let at = member
.cie_redirect(cie_in)
.ok_or(Error::Format("FDE names a dropped CIE"))?;
layout
.section_vaddr(at.file, at.section)
.ok_or(Error::Format("FDE names a dropped CIE"))?
.wrapping_add(at.out_off)
};
let field_addr = vaddr
.wrapping_add(piece.out_off.unwrap_or_default())
.wrapping_add(4);
let value = u32::try_from(field_addr.wrapping_sub(cie_addr))
.map_err(|_| Error::OutOfRange("FDE CIE pointer"))?;
let slot = dst
.get_mut(4..8)
.ok_or(Error::OutOfRange("FDE CIE pointer"))?;
slot.copy_from_slice(&value.to_le_bytes());
Ok(())
}
#[derive(Copy, Clone)]
struct RelocApply<'a> {
r: &'a Rela64,
addend: i64,
target: Target,
relax: bool,
vaddr: u64,
out_off: u64,
}
fn write_tombstone(
off: u64,
width: usize,
value: u64,
slice: &mut [u8],
) -> Result<()> {
let at = usize::try_from(off).unwrap_or(usize::MAX);
let Some(slot) = slice.get_mut(at..at.saturating_add(width)) else {
return Err(Error::OutOfRange("relocation slot"));
};
let bytes = value.to_le_bytes();
let Some(src) = bytes.get(..width) else {
return Err(Error::OutOfRange("tombstone width"));
};
slot.copy_from_slice(src);
Ok(())
}
enum Rewrite {
Done,
Declined(&'static str),
}
const NOT_LOWERABLE: &str = "the instruction sequence around the reference is not one this linker \
knows how to rewrite into a form an executable can resolve";
const NO_WINDOW: &str = "the instruction sequence around the reference runs past the bounds of \
the section holding it";
fn try_rewrite(
site: RelocApply<'_>,
width: usize,
resolver: &FileResolver<'_>,
slice: &mut [u8],
) -> Result<Rewrite> {
let r_type = site.r.r_type();
let lead = relax_lead_target(site.target, r_type);
let slot_off = usize::try_from(site.out_off).unwrap_or(usize::MAX);
if !relax_covers_target(site.target, r_type) || slot_off < lead {
return Ok(Rewrite::Declined(NO_WINDOW));
}
let start = slot_off - lead;
let slot_end = slot_off.saturating_add(width);
let after = slice.get(slot_end..).unwrap_or(&[]);
let slot = slice.get(slot_off..slot_end).unwrap_or(&[]);
let end = slot_end.saturating_add(relax_trail_target(
site.target,
r_type,
slot,
after,
));
let Some(window) = slice.get_mut(start..end) else {
return Ok(Rewrite::Declined(NO_WINDOW));
};
let place = site.vaddr.wrapping_add(site.out_off);
let done = relax_target(
site.target,
r_type,
sym_of(site.r),
site.addend,
place,
resolver,
window,
)?;
Ok(if done {
Rewrite::Done
} else {
Rewrite::Declined(NOT_LOWERABLE)
})
}