use std::ops::{Deref, DerefMut, Range};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, RwLock};
use bitvec::{bitvec, vec::BitVec};
use crate::backend::ISA;
use super::compiled_code::CompiledCode;
pub struct CodeBuffer {
data: &'static mut [u8],
}
impl CodeBuffer {
fn new(data: &'static mut [u8]) -> Self {
Self { data }
}
pub fn start(&self) -> *const u8 {
self.data.as_ptr()
}
}
impl Deref for CodeBuffer {
type Target = [u8];
fn deref(&self) -> &Self::Target {
self.data
}
}
impl DerefMut for CodeBuffer {
fn deref_mut(&mut self) -> &mut Self::Target {
self.data
}
}
pub(crate) struct CodeSpace {
data: *mut [u8],
alloc_bits: RwLock<BitVec>,
cursor: AtomicUsize,
}
unsafe impl Send for CodeSpace {}
unsafe impl Sync for CodeSpace {}
impl CodeSpace {
const HEADER_SIZE: usize = std::mem::size_of::<*const dyn CompiledCode>();
pub fn new(capacity: usize) -> Self {
let mb_mask: usize = (1 << 20) - 1;
let capacity = (capacity.next_power_of_two() + mb_mask) & !mb_mask;
let ptr = unsafe {
libc::mmap(
0 as _,
capacity,
libc::PROT_READ | libc::PROT_WRITE | libc::PROT_EXEC,
libc::MAP_PRIVATE | libc::MAP_ANONYMOUS,
0,
0,
)
};
let slice = unsafe { std::slice::from_raw_parts_mut(ptr as *mut u8, capacity) };
Self {
data: slice,
alloc_bits: RwLock::new(bitvec![0; capacity >> 3]),
cursor: AtomicUsize::new(0),
}
}
pub fn range(&self) -> Range<*const u8> {
unsafe { (*self.data).as_ptr_range() }
}
fn alloc_raw_code_buffer(&self, size: usize) -> &'static mut [u8] {
let align_mask: usize = (1 << 4) - 1;
let header = Self::HEADER_SIZE;
let size = size + header;
let aligned_size = (size + align_mask) & !align_mask;
let cursor = self.cursor.fetch_add(aligned_size, Ordering::SeqCst);
let mut alloc_bits = self.alloc_bits.write().unwrap();
alloc_bits.set(cursor >> 3, true);
let data = unsafe { &mut *self.data };
&mut data[cursor + header..cursor + size]
}
pub(crate) fn alloc_code_buffer<Isa: ISA>(&self, src: &[u8]) -> CodeBuffer {
let code = self.alloc_raw_code_buffer(src.len());
unsafe {
std::ptr::copy_nonoverlapping(&src[0], &mut code[0], src.len());
}
Isa::flush_cache(&code[0], code.len());
CodeBuffer::new(code)
}
pub(crate) fn register_code_object(
&self,
code: impl CompiledCode + 'static,
) -> Arc<dyn CompiledCode> {
let boxed_code: Arc<dyn CompiledCode> = Arc::new(code);
let code_ptr = boxed_code.as_ref() as *const dyn CompiledCode;
unsafe {
let header_slot = boxed_code.code_buffer().start().sub(Self::HEADER_SIZE)
as *mut *const dyn CompiledCode;
*header_slot = code_ptr;
}
boxed_code
}
pub fn get_code_by_interior_pointer(&self, addr: *const u8) -> Option<&dyn CompiledCode> {
let addr = (addr as usize & !((1 << 3) - 1)) as *const u8;
let alloc_bits = self.alloc_bits.read().unwrap();
let mut cursor = addr;
while cursor >= self.range().start {
if alloc_bits[(cursor as usize - self.range().start as usize) >> 3] {
let ptr = unsafe { *(cursor as *const *const dyn CompiledCode) };
return Some(unsafe { &*ptr });
}
cursor = unsafe { cursor.sub(8) };
}
None
}
}