1use smallvec::SmallVec;
15
16use crate::{
17 AsmError,
18 core::{
19 arch_traits::Arch,
20 buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
21 },
22};
23
24#[cfg(feature = "jit")]
25use crate::core::jit_allocator::{JitAllocator, Span};
26
27#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
30pub struct PatchableJump {
31 offset: CodeOffset,
32 kind: LabelUse,
33}
34
35impl PatchableJump {
36 pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
37 Self { offset, kind }
38 }
39
40 pub(crate) const fn invalid(kind: LabelUse) -> Self {
43 Self::new(u32::MAX, kind)
44 }
45}
46
47#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
50pub struct DataLabel {
51 offset: CodeOffset,
52 size: u8,
53 encoding: DataEncoding,
54}
55
56#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
57pub(crate) enum DataEncoding {
58 #[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
60 Raw,
61 #[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
64 A64MovWide,
65}
66
67impl DataLabel {
68 pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
69 Self {
70 offset,
71 size,
72 encoding,
73 }
74 }
75
76 pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
77 Self::new(u32::MAX, size, encoding)
78 }
79
80 pub const fn size(self) -> u8 {
82 self.size
83 }
84}
85
86#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
88pub struct PatchableRegion {
89 offset: CodeOffset,
90 size: CodeOffset,
91 arch: Arch,
92}
93
94impl PatchableRegion {
95 pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
96 Self { offset, size, arch }
97 }
98
99 pub const fn size(self) -> CodeOffset {
100 self.size
101 }
102}
103
104#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
106pub struct CodeLocationJump {
107 offset: CodeOffset,
108 kind: LabelUse,
109}
110
111impl CodeLocationJump {
112 pub const fn offset(self) -> CodeOffset {
114 self.offset
115 }
116
117 pub const fn kind(self) -> LabelUse {
118 self.kind
119 }
120}
121
122#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
124pub struct CodeLocationData {
125 offset: CodeOffset,
126 size: u8,
127 encoding: DataEncoding,
128}
129
130impl CodeLocationData {
131 pub const fn offset(self) -> CodeOffset {
133 self.offset
134 }
135
136 pub const fn size(self) -> u8 {
137 self.size
138 }
139}
140
141#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
143pub struct CodeLocationRegion {
144 offset: CodeOffset,
145 size: CodeOffset,
146 arch: Arch,
147}
148
149impl CodeLocationRegion {
150 pub const fn offset(self) -> CodeOffset {
151 self.offset
152 }
153
154 pub const fn size(self) -> CodeOffset {
155 self.size
156 }
157}
158
159mod sealed {
160 pub trait Sealed {}
161 impl Sealed for super::PatchableJump {}
162 impl Sealed for super::DataLabel {}
163 impl Sealed for super::PatchableRegion {}
164}
165
166pub trait PatchMark: Copy + sealed::Sealed {
168 type Location: Copy;
169
170 #[doc(hidden)]
171 fn locate(self, base: CodeOffset) -> Self::Location;
172}
173
174fn place(base: CodeOffset, offset: CodeOffset) -> CodeOffset {
177 if offset == u32::MAX {
178 return offset;
179 }
180 base.saturating_add(offset)
181}
182
183impl PatchMark for PatchableJump {
184 type Location = CodeLocationJump;
185
186 fn locate(self, base: CodeOffset) -> CodeLocationJump {
187 CodeLocationJump {
188 offset: place(base, self.offset),
189 kind: self.kind,
190 }
191 }
192}
193
194impl PatchMark for DataLabel {
195 type Location = CodeLocationData;
196
197 fn locate(self, base: CodeOffset) -> CodeLocationData {
198 CodeLocationData {
199 offset: place(base, self.offset),
200 size: self.size,
201 encoding: self.encoding,
202 }
203 }
204}
205
206impl PatchMark for PatchableRegion {
207 type Location = CodeLocationRegion;
208
209 fn locate(self, base: CodeOffset) -> CodeLocationRegion {
210 CodeLocationRegion {
211 offset: place(base, self.offset),
212 size: self.size,
213 arch: self.arch,
214 }
215 }
216}
217
218impl CodeBufferFinalized {
219 pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
225 debug_assert_eq!(self.section_bases.len(), 1, "linked images need `location_in`");
226 mark.locate(self.section_bases[0])
227 }
228
229 pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
233 Some(mark.locate(*self.section_bases.get(section)?))
234 }
235}
236
237fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
238 let start = offset as usize;
239 let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
240 if end > len {
241 return Err(AsmError::InvalidState);
242 }
243 Ok(start..end)
244}
245
246fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
247 if !kind.can_reach_delta(delta) {
248 return Err(AsmError::TooLarge);
249 }
250 kind.patch_with_addend(bytes, 0, 0, delta);
251 Ok(())
252}
253
254pub fn repatch_jump(
256 code: &mut [u8],
257 jump: CodeLocationJump,
258 target: CodeOffset,
259) -> Result<(), AsmError> {
260 let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
261 let delta = i64::from(target) - i64::from(jump.offset);
262 relink(&mut code[range], jump.kind, delta)
263}
264
265#[cfg(feature = "jit")]
272pub unsafe fn repatch_jump_span(
273 jit_allocator: &mut JitAllocator,
274 span: &mut Span,
275 jump: CodeLocationJump,
276 target: *const u8,
277) -> Result<(), AsmError> {
278 let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
279 let at = span.rx().addr() + range.start;
280 let delta = i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
281 if !jump.kind.can_reach_delta(delta) {
282 return Err(AsmError::TooLarge);
283 }
284 unsafe {
287 jit_allocator.write(span, |span| {
288 let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
289 jump.kind.patch_with_addend(bytes, 0, 0, delta);
290 })
291 }
292}
293
294fn encode_value(
295 current: &[u8],
296 data: CodeLocationData,
297 value: u64,
298) -> Result<SmallVec<[u8; 16]>, AsmError> {
299 let size = data.size as usize;
300 match data.encoding {
301 DataEncoding::Raw => {
302 if size < 8 && value >> (size * 8) != 0 {
303 return Err(AsmError::TooLarge);
304 }
305 Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
306 }
307 DataEncoding::A64MovWide => {
308 #[cfg(feature = "aarch64")]
309 {
310 let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
313 let is_64bit = movz >> 31 == 1;
314 if !is_64bit && value > u64::from(u32::MAX) {
315 return Err(AsmError::TooLarge);
316 }
317 Ok(crate::aarch64::encode_patchable_mov_imm(
318 movz & 0x1f,
319 is_64bit,
320 value,
321 ))
322 }
323 #[cfg(not(feature = "aarch64"))]
324 {
325 let _ = (current, value);
326 Err(AsmError::InvalidArch)
327 }
328 }
329 }
330}
331
332fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
333 match data.encoding {
334 DataEncoding::Raw => {
335 let mut value = [0u8; 8];
336 value[..bytes.len()].copy_from_slice(bytes);
337 u64::from_le_bytes(value)
338 }
339 DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
340 let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
341 let imm16 = u64::from((insn >> 5) & 0xffff);
342 let hw = (insn >> 21) & 3;
343 value | (imm16 << (16 * hw))
344 }),
345 }
346}
347
348pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
351 let range = field(code.len(), data.offset, data.size as usize)?;
352 let encoded = encode_value(&code[range.clone()], data, value)?;
353 code[range].copy_from_slice(&encoded);
354 Ok(())
355}
356
357pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
359 let range = field(code.len(), data.offset, data.size as usize)?;
360 Ok(decode_value(&code[range], data))
361}
362
363#[cfg(feature = "jit")]
370pub unsafe fn repatch_value_span(
371 jit_allocator: &mut JitAllocator,
372 span: &mut Span,
373 data: CodeLocationData,
374 value: u64,
375) -> Result<(), AsmError> {
376 let range = field(span.size(), data.offset, data.size as usize)?;
377 let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
379 let encoded = encode_value(current, data, value)?;
380 unsafe {
382 jit_allocator.write(span, |span| {
383 span.rw()
384 .add(range.start)
385 .copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
386 })
387 }
388}
389
390#[cfg(feature = "jit")]
396pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
397 let range = field(span.size(), data.offset, data.size as usize)?;
398 let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
400 Ok(decode_value(bytes, data))
401}
402
403fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
404 if new_bytes.len() > region.size as usize {
405 return Err(AsmError::TooLarge);
406 }
407 if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
408 return Err(AsmError::InvalidArgument);
409 }
410 Ok(())
411}
412
413pub fn rewrite_region(
415 code: &mut [u8],
416 region: CodeLocationRegion,
417 new_bytes: &[u8],
418) -> Result<(), AsmError> {
419 check_region_payload(region, new_bytes)?;
420 let range = field(code.len(), region.offset, region.size as usize)?;
421 let block = &mut code[range];
422 block[..new_bytes.len()].copy_from_slice(new_bytes);
423 fill_with_nops(region.arch, &mut block[new_bytes.len()..])
424}
425
426#[cfg(feature = "jit")]
434pub unsafe fn rewrite_region_span(
435 jit_allocator: &mut JitAllocator,
436 span: &mut Span,
437 region: CodeLocationRegion,
438 new_bytes: &[u8],
439) -> Result<(), AsmError> {
440 check_region_payload(region, new_bytes)?;
441 let range = field(span.size(), region.offset, region.size as usize)?;
442 let mut result = Ok(());
443 unsafe {
446 jit_allocator.write(span, |span| {
447 let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
448 block[..new_bytes.len()].copy_from_slice(new_bytes);
449 result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
450 })?;
451 }
452 result
453}
454
455pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
456 match arch {
457 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
458 _ => 1,
459 }
460}
461
462pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
463 let pattern: &[u8] = match arch {
464 Arch::X86 | Arch::X64 => &[0x90],
465 Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
466 Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
467 _ => return Err(AsmError::InvalidArgument),
468 };
469
470 if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
471 return Err(AsmError::InvalidArgument);
472 }
473
474 for chunk in buffer.chunks_mut(pattern.len()) {
475 chunk.copy_from_slice(pattern);
476 }
477
478 Ok(())
479}