1use smallvec::SmallVec;
15
16use crate::{
17 AsmError,
18 core::{
19 arch_traits::Arch,
20 buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
21 relax::OffsetMap,
22 },
23};
24
25#[cfg(feature = "jit")]
26use crate::core::jit_allocator::{JitAllocator, Span};
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
31pub struct PatchableJump {
32 offset: CodeOffset,
33 kind: LabelUse,
34}
35
36impl PatchableJump {
37 pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
38 Self { offset, kind }
39 }
40
41 pub(crate) const fn invalid(kind: LabelUse) -> Self {
44 Self::new(u32::MAX, kind)
45 }
46}
47
48#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
51pub struct DataLabel {
52 offset: CodeOffset,
53 size: u8,
54 encoding: DataEncoding,
55}
56
57#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
58pub(crate) enum DataEncoding {
59 #[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
61 Raw,
62 #[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
65 A64MovWide,
66}
67
68impl DataLabel {
69 pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
70 Self {
71 offset,
72 size,
73 encoding,
74 }
75 }
76
77 pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
78 Self::new(u32::MAX, size, encoding)
79 }
80
81 pub const fn size(self) -> u8 {
83 self.size
84 }
85}
86
87#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
89pub struct PatchableRegion {
90 offset: CodeOffset,
91 size: CodeOffset,
92 arch: Arch,
93}
94
95impl PatchableRegion {
96 pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
97 Self { offset, size, arch }
98 }
99
100 pub const fn size(self) -> CodeOffset {
101 self.size
102 }
103}
104
105#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
107pub struct CodeLocationJump {
108 offset: CodeOffset,
109 kind: LabelUse,
110}
111
112impl CodeLocationJump {
113 pub const fn offset(self) -> CodeOffset {
115 self.offset
116 }
117
118 pub const fn kind(self) -> LabelUse {
119 self.kind
120 }
121}
122
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
125pub struct CodeLocationData {
126 offset: CodeOffset,
127 size: u8,
128 encoding: DataEncoding,
129}
130
131impl CodeLocationData {
132 pub const fn offset(self) -> CodeOffset {
134 self.offset
135 }
136
137 pub const fn size(self) -> u8 {
138 self.size
139 }
140}
141
142#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
144pub struct CodeLocationRegion {
145 offset: CodeOffset,
146 size: CodeOffset,
147 arch: Arch,
148}
149
150impl CodeLocationRegion {
151 pub const fn offset(self) -> CodeOffset {
152 self.offset
153 }
154
155 pub const fn size(self) -> CodeOffset {
156 self.size
157 }
158}
159
160mod sealed {
161 pub trait Sealed {}
162 impl Sealed for super::PatchableJump {}
163 impl Sealed for super::DataLabel {}
164 impl Sealed for super::PatchableRegion {}
165}
166
167pub trait PatchMark: Copy + sealed::Sealed {
169 type Location: Copy;
170
171 #[doc(hidden)]
172 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> Self::Location;
173}
174
175fn place(base: CodeOffset, layout: &OffsetMap, offset: CodeOffset) -> CodeOffset {
178 if offset == u32::MAX {
179 return offset;
180 }
181 base.saturating_add(layout.map(offset))
182}
183
184impl PatchMark for PatchableJump {
185 type Location = CodeLocationJump;
186
187 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationJump {
188 CodeLocationJump {
189 offset: place(self.offset),
190 kind: self.kind,
191 }
192 }
193}
194
195impl PatchMark for DataLabel {
196 type Location = CodeLocationData;
197
198 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationData {
199 CodeLocationData {
200 offset: place(self.offset),
201 size: self.size,
202 encoding: self.encoding,
203 }
204 }
205}
206
207impl PatchMark for PatchableRegion {
208 type Location = CodeLocationRegion;
209
210 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationRegion {
211 CodeLocationRegion {
212 offset: place(self.offset),
213 size: self.size,
214 arch: self.arch,
215 }
216 }
217}
218
219impl CodeBufferFinalized {
220 pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
226 debug_assert_eq!(self.section_bases.len(), 1, "linked images need `location_in`");
227 mark.locate(|offset| place(self.section_bases[0], &self.layout_maps[0], offset))
228 }
229
230 pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
234 let base = *self.section_bases.get(section)?;
235 let layout = self.layout_maps.get(section)?;
236 Some(mark.locate(|offset| place(base, layout, offset)))
237 }
238}
239
240fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
241 let start = offset as usize;
242 let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
243 if end > len {
244 return Err(AsmError::InvalidState);
245 }
246 Ok(start..end)
247}
248
249fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
250 if !kind.can_reach_delta(delta) {
251 return Err(AsmError::TooLarge);
252 }
253 kind.patch_with_addend(bytes, 0, 0, delta);
254 Ok(())
255}
256
257pub fn repatch_jump(
259 code: &mut [u8],
260 jump: CodeLocationJump,
261 target: CodeOffset,
262) -> Result<(), AsmError> {
263 let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
264 let delta = i64::from(target) - i64::from(jump.offset);
265 relink(&mut code[range], jump.kind, delta)
266}
267
268#[cfg(feature = "jit")]
275pub unsafe fn repatch_jump_span(
276 jit_allocator: &mut JitAllocator,
277 span: &mut Span,
278 jump: CodeLocationJump,
279 target: *const u8,
280) -> Result<(), AsmError> {
281 let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
282 let at = span.rx().addr() + range.start;
283 let delta = i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
284 if !jump.kind.can_reach_delta(delta) {
285 return Err(AsmError::TooLarge);
286 }
287 unsafe {
290 jit_allocator.write(span, |span| {
291 let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
292 jump.kind.patch_with_addend(bytes, 0, 0, delta);
293 })
294 }
295}
296
297fn encode_value(
298 current: &[u8],
299 data: CodeLocationData,
300 value: u64,
301) -> Result<SmallVec<[u8; 16]>, AsmError> {
302 let size = data.size as usize;
303 match data.encoding {
304 DataEncoding::Raw => {
305 if size < 8 && value >> (size * 8) != 0 {
306 return Err(AsmError::TooLarge);
307 }
308 Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
309 }
310 DataEncoding::A64MovWide => {
311 #[cfg(feature = "aarch64")]
312 {
313 let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
316 let is_64bit = movz >> 31 == 1;
317 if !is_64bit && value > u64::from(u32::MAX) {
318 return Err(AsmError::TooLarge);
319 }
320 Ok(crate::aarch64::encode_patchable_mov_imm(
321 movz & 0x1f,
322 is_64bit,
323 value,
324 ))
325 }
326 #[cfg(not(feature = "aarch64"))]
327 {
328 let _ = (current, value);
329 Err(AsmError::InvalidArch)
330 }
331 }
332 }
333}
334
335fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
336 match data.encoding {
337 DataEncoding::Raw => {
338 let mut value = [0u8; 8];
339 value[..bytes.len()].copy_from_slice(bytes);
340 u64::from_le_bytes(value)
341 }
342 DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
343 let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
344 let imm16 = u64::from((insn >> 5) & 0xffff);
345 let hw = (insn >> 21) & 3;
346 value | (imm16 << (16 * hw))
347 }),
348 }
349}
350
351pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
354 let range = field(code.len(), data.offset, data.size as usize)?;
355 let encoded = encode_value(&code[range.clone()], data, value)?;
356 code[range].copy_from_slice(&encoded);
357 Ok(())
358}
359
360pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
362 let range = field(code.len(), data.offset, data.size as usize)?;
363 Ok(decode_value(&code[range], data))
364}
365
366#[cfg(feature = "jit")]
373pub unsafe fn repatch_value_span(
374 jit_allocator: &mut JitAllocator,
375 span: &mut Span,
376 data: CodeLocationData,
377 value: u64,
378) -> Result<(), AsmError> {
379 let range = field(span.size(), data.offset, data.size as usize)?;
380 let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
382 let encoded = encode_value(current, data, value)?;
383 unsafe {
385 jit_allocator.write(span, |span| {
386 span.rw()
387 .add(range.start)
388 .copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
389 })
390 }
391}
392
393#[cfg(feature = "jit")]
399pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
400 let range = field(span.size(), data.offset, data.size as usize)?;
401 let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
403 Ok(decode_value(bytes, data))
404}
405
406fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
407 if new_bytes.len() > region.size as usize {
408 return Err(AsmError::TooLarge);
409 }
410 if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
411 return Err(AsmError::InvalidArgument);
412 }
413 Ok(())
414}
415
416pub fn rewrite_region(
418 code: &mut [u8],
419 region: CodeLocationRegion,
420 new_bytes: &[u8],
421) -> Result<(), AsmError> {
422 check_region_payload(region, new_bytes)?;
423 let range = field(code.len(), region.offset, region.size as usize)?;
424 let block = &mut code[range];
425 block[..new_bytes.len()].copy_from_slice(new_bytes);
426 fill_with_nops(region.arch, &mut block[new_bytes.len()..])
427}
428
429#[cfg(feature = "jit")]
437pub unsafe fn rewrite_region_span(
438 jit_allocator: &mut JitAllocator,
439 span: &mut Span,
440 region: CodeLocationRegion,
441 new_bytes: &[u8],
442) -> Result<(), AsmError> {
443 check_region_payload(region, new_bytes)?;
444 let range = field(span.size(), region.offset, region.size as usize)?;
445 let mut result = Ok(());
446 unsafe {
449 jit_allocator.write(span, |span| {
450 let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
451 block[..new_bytes.len()].copy_from_slice(new_bytes);
452 result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
453 })?;
454 }
455 result
456}
457
458pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
459 match arch {
460 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
461 _ => 1,
462 }
463}
464
465pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
466 let pattern: &[u8] = match arch {
467 Arch::X86 | Arch::X64 => &[0x90],
468 Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
469 Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
470 _ => return Err(AsmError::InvalidArgument),
471 };
472
473 if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
474 return Err(AsmError::InvalidArgument);
475 }
476
477 for chunk in buffer.chunks_mut(pattern.len()) {
478 chunk.copy_from_slice(pattern);
479 }
480
481 Ok(())
482}