1use smallvec::SmallVec;
15
16use crate::{
17 AsmError,
18 core::{
19 arch_traits::Arch,
20 buffer::{CodeBufferFinalized, CodeOffset, LabelUse},
21 relax::OffsetMap,
22 },
23};
24
25#[cfg(feature = "jit")]
26use crate::core::jit_allocator::{JitAllocator, Span};
27
28#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
31pub struct PatchableJump {
32 offset: CodeOffset,
33 kind: LabelUse,
34}
35
36impl PatchableJump {
37 pub(crate) const fn new(offset: CodeOffset, kind: LabelUse) -> Self {
38 Self { offset, kind }
39 }
40
41 pub(crate) const fn invalid(kind: LabelUse) -> Self {
44 Self::new(u32::MAX, kind)
45 }
46}
47
48#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
51pub struct DataLabel {
52 offset: CodeOffset,
53 size: u8,
54 encoding: DataEncoding,
55}
56
57#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
58pub(crate) enum DataEncoding {
59 #[cfg_attr(not(any(feature = "x86", feature = "riscv")), allow(dead_code))]
61 Raw,
62 #[cfg_attr(not(feature = "aarch64"), allow(dead_code))]
65 A64MovWide,
66}
67
68impl DataLabel {
69 pub(crate) const fn new(offset: CodeOffset, size: u8, encoding: DataEncoding) -> Self {
70 Self {
71 offset,
72 size,
73 encoding,
74 }
75 }
76
77 pub(crate) const fn invalid(size: u8, encoding: DataEncoding) -> Self {
78 Self::new(u32::MAX, size, encoding)
79 }
80
81 pub const fn size(self) -> u8 {
83 self.size
84 }
85}
86
87#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
89pub struct PatchableRegion {
90 offset: CodeOffset,
91 size: CodeOffset,
92 arch: Arch,
93}
94
95impl PatchableRegion {
96 pub(crate) const fn new(offset: CodeOffset, size: CodeOffset, arch: Arch) -> Self {
97 Self { offset, size, arch }
98 }
99
100 pub const fn size(self) -> CodeOffset {
101 self.size
102 }
103}
104
105#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
107pub struct CodeLocationJump {
108 offset: CodeOffset,
109 kind: LabelUse,
110}
111
112impl CodeLocationJump {
113 pub const fn offset(self) -> CodeOffset {
115 self.offset
116 }
117
118 pub const fn kind(self) -> LabelUse {
119 self.kind
120 }
121}
122
123#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
125pub struct CodeLocationData {
126 offset: CodeOffset,
127 size: u8,
128 encoding: DataEncoding,
129}
130
131impl CodeLocationData {
132 pub const fn offset(self) -> CodeOffset {
134 self.offset
135 }
136
137 pub const fn size(self) -> u8 {
138 self.size
139 }
140}
141
142#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
144pub struct CodeLocationRegion {
145 offset: CodeOffset,
146 size: CodeOffset,
147 arch: Arch,
148}
149
150impl CodeLocationRegion {
151 pub const fn offset(self) -> CodeOffset {
152 self.offset
153 }
154
155 pub const fn size(self) -> CodeOffset {
156 self.size
157 }
158}
159
160mod sealed {
161 pub trait Sealed {}
162 impl Sealed for super::PatchableJump {}
163 impl Sealed for super::DataLabel {}
164 impl Sealed for super::PatchableRegion {}
165}
166
167pub trait PatchMark: Copy + sealed::Sealed {
169 type Location: Copy;
170
171 #[doc(hidden)]
172 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> Self::Location;
173}
174
175fn place(base: CodeOffset, layout: &OffsetMap, offset: CodeOffset) -> CodeOffset {
178 if offset == u32::MAX {
179 return offset;
180 }
181 base.saturating_add(layout.map(offset))
182}
183
184impl PatchMark for PatchableJump {
185 type Location = CodeLocationJump;
186
187 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationJump {
188 CodeLocationJump {
189 offset: place(self.offset),
190 kind: self.kind,
191 }
192 }
193}
194
195impl PatchMark for DataLabel {
196 type Location = CodeLocationData;
197
198 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationData {
199 CodeLocationData {
200 offset: place(self.offset),
201 size: self.size,
202 encoding: self.encoding,
203 }
204 }
205}
206
207impl PatchMark for PatchableRegion {
208 type Location = CodeLocationRegion;
209
210 fn locate(self, place: impl Fn(CodeOffset) -> CodeOffset) -> CodeLocationRegion {
211 CodeLocationRegion {
212 offset: place(self.offset),
213 size: self.size,
214 arch: self.arch,
215 }
216 }
217}
218
219impl CodeBufferFinalized {
220 pub fn location_of<M: PatchMark>(&self, mark: M) -> M::Location {
226 debug_assert_eq!(
227 self.section_bases.len(),
228 1,
229 "linked images need `location_in`"
230 );
231 mark.locate(|offset| place(self.section_bases[0], &self.layout_maps[0], offset))
232 }
233
234 pub fn location_in<M: PatchMark>(&self, section: usize, mark: M) -> Option<M::Location> {
238 let base = *self.section_bases.get(section)?;
239 let layout = self.layout_maps.get(section)?;
240 Some(mark.locate(|offset| place(base, layout, offset)))
241 }
242}
243
244fn field(len: usize, offset: CodeOffset, size: usize) -> Result<core::ops::Range<usize>, AsmError> {
245 let start = offset as usize;
246 let end = start.checked_add(size).ok_or(AsmError::InvalidState)?;
247 if end > len {
248 return Err(AsmError::InvalidState);
249 }
250 Ok(start..end)
251}
252
253fn relink(bytes: &mut [u8], kind: LabelUse, delta: i64) -> Result<(), AsmError> {
254 if !kind.can_reach_delta(delta) {
255 return Err(AsmError::TooLarge);
256 }
257 kind.patch_with_addend(bytes, 0, 0, delta);
258 Ok(())
259}
260
261pub fn repatch_jump(
263 code: &mut [u8],
264 jump: CodeLocationJump,
265 target: CodeOffset,
266) -> Result<(), AsmError> {
267 let range = field(code.len(), jump.offset, jump.kind.patch_size())?;
268 let delta = i64::from(target) - i64::from(jump.offset);
269 relink(&mut code[range], jump.kind, delta)
270}
271
272#[cfg(feature = "jit")]
279pub unsafe fn repatch_jump_span(
280 jit_allocator: &mut JitAllocator,
281 span: &mut Span,
282 jump: CodeLocationJump,
283 target: *const u8,
284) -> Result<(), AsmError> {
285 let range = field(span.size(), jump.offset, jump.kind.patch_size())?;
286 let at = span.rx().addr() + range.start;
287 let delta =
288 i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
289 if !jump.kind.can_reach_delta(delta) {
290 return Err(AsmError::TooLarge);
291 }
292 unsafe {
295 jit_allocator.write(span, |span| {
296 let bytes = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
297 jump.kind.patch_with_addend(bytes, 0, 0, delta);
298 })
299 }
300}
301
302fn encode_value(
303 current: &[u8],
304 data: CodeLocationData,
305 value: u64,
306) -> Result<SmallVec<[u8; 16]>, AsmError> {
307 let size = data.size as usize;
308 match data.encoding {
309 DataEncoding::Raw => {
310 if size < 8 && value >> (size * 8) != 0 {
311 return Err(AsmError::TooLarge);
312 }
313 Ok(SmallVec::from_slice(&value.to_le_bytes()[..size]))
314 }
315 DataEncoding::A64MovWide => {
316 #[cfg(feature = "aarch64")]
317 {
318 let movz = u32::from_le_bytes([current[0], current[1], current[2], current[3]]);
321 let is_64bit = movz >> 31 == 1;
322 if !is_64bit && value > u64::from(u32::MAX) {
323 return Err(AsmError::TooLarge);
324 }
325 Ok(crate::aarch64::encode_patchable_mov_imm(
326 movz & 0x1f,
327 is_64bit,
328 value,
329 ))
330 }
331 #[cfg(not(feature = "aarch64"))]
332 {
333 let _ = (current, value);
334 Err(AsmError::InvalidArch)
335 }
336 }
337 }
338}
339
340fn decode_value(bytes: &[u8], data: CodeLocationData) -> u64 {
341 match data.encoding {
342 DataEncoding::Raw => {
343 let mut value = [0u8; 8];
344 value[..bytes.len()].copy_from_slice(bytes);
345 u64::from_le_bytes(value)
346 }
347 DataEncoding::A64MovWide => bytes.chunks_exact(4).fold(0, |value, insn| {
348 let insn = u32::from_le_bytes([insn[0], insn[1], insn[2], insn[3]]);
349 let imm16 = u64::from((insn >> 5) & 0xffff);
350 let hw = (insn >> 21) & 3;
351 value | (imm16 << (16 * hw))
352 }),
353 }
354}
355
356#[cfg(feature = "jit")]
371pub unsafe fn repatch_jump_span_mt_safe(
372 span: &Span,
373 jump: CodeLocationJump,
374 target: *const u8,
375) -> Result<(), AsmError> {
376 use core::sync::atomic::{AtomicU32, Ordering};
377
378 use crate::util::virtual_memory::{flush_instruction_cache, with_jit_write_access};
379
380 if jump.kind.patch_size() != 4 {
381 return Err(AsmError::InvalidArgument);
382 }
383 let range = field(span.size(), jump.offset, 4)?;
384 let at = span.rx().addr() + range.start;
385 if at % 4 != 0 {
386 return Err(AsmError::UnalignedPatch);
387 }
388 let delta =
389 i64::try_from(target.addr() as i128 - at as i128).map_err(|_| AsmError::TooLarge)?;
390 if !jump.kind.can_reach_delta(delta) {
391 return Err(AsmError::TooLarge);
392 }
393 let mut word = [0u8; 4];
401 unsafe { word.copy_from_slice(core::slice::from_raw_parts(span.rx().add(range.start), 4)) };
402 jump.kind.patch_with_addend(&mut word, 0, 0, delta);
403 let disp = u32::from_le_bytes(word);
404 unsafe {
406 with_jit_write_access(|| {
407 AtomicU32::from_ptr(span.rw().add(range.start) as *mut u32)
408 .store(disp, Ordering::Release);
409 });
410 flush_instruction_cache(span.rx().add(range.start), 4)?;
411 }
412 Ok(())
413}
414
415pub fn repatch_value(code: &mut [u8], data: CodeLocationData, value: u64) -> Result<(), AsmError> {
418 let range = field(code.len(), data.offset, data.size as usize)?;
419 let encoded = encode_value(&code[range.clone()], data, value)?;
420 code[range].copy_from_slice(&encoded);
421 Ok(())
422}
423
424pub fn read_value(code: &[u8], data: CodeLocationData) -> Result<u64, AsmError> {
426 let range = field(code.len(), data.offset, data.size as usize)?;
427 Ok(decode_value(&code[range], data))
428}
429
430#[cfg(feature = "jit")]
437pub unsafe fn repatch_value_span(
438 jit_allocator: &mut JitAllocator,
439 span: &mut Span,
440 data: CodeLocationData,
441 value: u64,
442) -> Result<(), AsmError> {
443 let range = field(span.size(), data.offset, data.size as usize)?;
444 let current = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
446 let encoded = encode_value(current, data, value)?;
447 unsafe {
449 jit_allocator.write(span, |span| {
450 span.rw()
451 .add(range.start)
452 .copy_from_nonoverlapping(encoded.as_ptr(), encoded.len());
453 })
454 }
455}
456
457#[cfg(feature = "jit")]
463pub unsafe fn read_value_span(span: &Span, data: CodeLocationData) -> Result<u64, AsmError> {
464 let range = field(span.size(), data.offset, data.size as usize)?;
465 let bytes = unsafe { core::slice::from_raw_parts(span.rx().add(range.start), range.len()) };
467 Ok(decode_value(bytes, data))
468}
469
470fn check_region_payload(region: CodeLocationRegion, new_bytes: &[u8]) -> Result<(), AsmError> {
471 if new_bytes.len() > region.size as usize {
472 return Err(AsmError::TooLarge);
473 }
474 if new_bytes.len() % minimum_patch_alignment(region.arch) as usize != 0 {
475 return Err(AsmError::InvalidArgument);
476 }
477 Ok(())
478}
479
480pub fn rewrite_region(
482 code: &mut [u8],
483 region: CodeLocationRegion,
484 new_bytes: &[u8],
485) -> Result<(), AsmError> {
486 check_region_payload(region, new_bytes)?;
487 let range = field(code.len(), region.offset, region.size as usize)?;
488 let block = &mut code[range];
489 block[..new_bytes.len()].copy_from_slice(new_bytes);
490 fill_with_nops(region.arch, &mut block[new_bytes.len()..])
491}
492
493#[cfg(feature = "jit")]
501pub unsafe fn rewrite_region_span(
502 jit_allocator: &mut JitAllocator,
503 span: &mut Span,
504 region: CodeLocationRegion,
505 new_bytes: &[u8],
506) -> Result<(), AsmError> {
507 check_region_payload(region, new_bytes)?;
508 let range = field(span.size(), region.offset, region.size as usize)?;
509 let mut result = Ok(());
510 unsafe {
513 jit_allocator.write(span, |span| {
514 let block = core::slice::from_raw_parts_mut(span.rw().add(range.start), range.len());
515 block[..new_bytes.len()].copy_from_slice(new_bytes);
516 result = fill_with_nops(region.arch, &mut block[new_bytes.len()..]);
517 })?;
518 }
519 result
520}
521
522pub fn minimum_patch_alignment(arch: Arch) -> CodeOffset {
523 match arch {
524 Arch::AArch64 | Arch::RISCV32 | Arch::RISCV64 => 4,
525 _ => 1,
526 }
527}
528
529pub fn fill_with_nops(arch: Arch, buffer: &mut [u8]) -> Result<(), AsmError> {
530 let pattern: &[u8] = match arch {
531 Arch::X86 | Arch::X64 => &[0x90],
532 Arch::AArch64 => &[0x1f, 0x20, 0x03, 0xd5],
533 Arch::RISCV32 | Arch::RISCV64 => &[0x13, 0x00, 0x00, 0x00],
534 _ => return Err(AsmError::InvalidArgument),
535 };
536
537 if pattern.len() > 1 && buffer.len() % pattern.len() != 0 {
538 return Err(AsmError::InvalidArgument);
539 }
540
541 for chunk in buffer.chunks_mut(pattern.len()) {
542 chunk.copy_from_slice(pattern);
543 }
544
545 Ok(())
546}