pub use copypatch::{patch_branch26, patch_x86_rel32};
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
))]
pub use copypatch::ExecBuf;
pub mod stencils {
include!(concat!(env!("OUT_DIR"), "/weavy_stencils.rs"));
}
pub const NATIVE_COPY_PATCH_AVAILABLE: bool = cfg!(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
));
#[repr(C)]
pub struct HostCallInfo {
pub info: *const (),
pub call: unsafe extern "C" fn(cx: *mut (), info: *const ()) -> bool,
}
#[repr(C)]
pub struct HostCallCtx<C> {
pub prog: *const u64,
pub inner: *mut C,
}
impl<C> HostCallCtx<C> {
#[must_use]
#[inline]
pub fn new(prog: *const u64, inner: &mut C) -> Self {
Self { prog, inner }
}
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Chain {
pub entry: usize,
pub prog_index: usize,
}
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct ProgSlot {
pub prog_index: usize,
pub slot: usize,
}
#[derive(Debug, Default)]
pub struct StencilLayout {
code: Vec<u8>,
progs: Vec<Vec<u64>>,
stencil_count: usize,
}
impl StencilLayout {
#[must_use]
pub fn new() -> Self {
Self::default()
}
pub fn start_chain(&mut self) -> Chain {
let entry = self.code.len();
let prog_index = self.progs.len();
self.progs.push(Vec::new());
Chain { entry, prog_index }
}
pub fn emit_stencil(&mut self, stencil: &[u8]) -> usize {
let start = self.code.len();
self.code.extend_from_slice(stencil);
self.stencil_count += 1;
start
}
#[must_use]
pub fn code_len(&self) -> usize {
self.code.len()
}
pub fn patch_branch26(&mut self, site: usize, target: usize) {
patch_branch26(&mut self.code, site, target);
}
pub fn patch_x86_rel32(&mut self, site: usize, target: usize) {
patch_x86_rel32(&mut self.code, site, target);
}
pub fn patch_continuation(&mut self, site: usize, target: usize) {
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
{
return self.patch_branch26(site, target);
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
{
return self.patch_x86_rel32(site, target);
}
#[allow(unreachable_code)]
{
let _ = (site, target);
panic!("native copy-and-patch is not available for this target");
}
}
pub fn push_prog_word(&mut self, prog_index: usize, value: u64) {
self.progs[prog_index].push(value);
}
pub fn prog_mut(&mut self, prog_index: usize) -> &mut Vec<u64> {
&mut self.progs[prog_index]
}
pub fn reserve_prog_slot(&mut self, prog_index: usize) -> ProgSlot {
let slot = self.progs[prog_index].len();
self.progs[prog_index].push(0);
ProgSlot { prog_index, slot }
}
pub fn fill_prog_slot(&mut self, slot: ProgSlot, value: u64) {
self.progs[slot.prog_index][slot.slot] = value;
}
pub fn emit_hostcall(&mut self, chain: Chain, info: *const HostCallInfo) -> usize {
self.push_prog_word(chain.prog_index, info as u64);
self.emit_stencil(stencils::HOSTCALL)
}
pub fn emit_done(&mut self) -> usize {
self.emit_stencil(stencils::DONE)
}
pub fn patch_hostcall_continuation(&mut self, hostcall_start: usize, target: usize) {
for &rel in stencils::HOSTCALL_CONT {
self.patch_continuation(hostcall_start + rel, target);
}
}
#[must_use]
pub fn prog(&self, prog_index: usize) -> &[u64] {
&self.progs[prog_index]
}
#[must_use]
pub fn code(&self) -> &[u8] {
&self.code
}
#[must_use]
pub fn stencil_count(&self) -> usize {
self.stencil_count
}
#[must_use]
pub fn into_parts(self) -> (Vec<u8>, Vec<Vec<u64>>, usize) {
(self.code, self.progs, self.stencil_count)
}
}
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
))]
pub struct NativeProgram {
buf: ExecBuf,
progs: Vec<Vec<u64>>,
entry: Chain,
stencil_count: usize,
}
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
))]
impl NativeProgram {
#[must_use]
#[inline]
pub fn new(layout: StencilLayout, entry: Chain) -> Self {
let (code, progs, stencil_count) = layout.into_parts();
Self {
buf: ExecBuf::new(&code),
progs,
entry,
stencil_count,
}
}
#[must_use]
#[inline]
pub fn code_ptr(&self) -> *const u8 {
self.buf.as_ptr()
}
#[must_use]
#[inline]
pub unsafe fn entry_fn<C>(&self) -> unsafe extern "C" fn(*mut C) {
unsafe { self.chain_fn(self.entry.entry) }
}
#[must_use]
#[inline]
pub unsafe fn chain_fn<C>(&self, entry: usize) -> unsafe extern "C" fn(*mut C) {
unsafe {
core::mem::transmute::<*const u8, unsafe extern "C" fn(*mut C)>(
self.code_ptr().add(entry),
)
}
}
#[must_use]
#[inline]
pub fn entry_prog_index(&self) -> usize {
self.entry.prog_index
}
#[must_use]
#[inline]
pub fn entry_prog(&self) -> *const u64 {
self.prog_ptr(self.entry_prog_index())
}
#[must_use]
#[inline]
pub fn prog_ptr(&self, prog_index: usize) -> *const u64 {
self.progs[prog_index].as_ptr()
}
#[inline]
pub fn fill_prog_word(&mut self, prog_index: usize, slot: usize, value: u64) {
self.progs[prog_index][slot] = value;
}
#[inline]
pub fn fill_prog_slot(&mut self, slot: ProgSlot, value: u64) {
self.fill_prog_word(slot.prog_index, slot.slot, value);
}
#[must_use]
#[inline]
pub fn chain_count(&self) -> usize {
self.progs.len()
}
#[must_use]
#[inline]
pub fn stencil_count(&self) -> usize {
self.stencil_count
}
#[must_use]
#[inline]
pub fn prog_slot_count(&self) -> usize {
self.progs.iter().map(Vec::len).sum()
}
}
#[cfg(test)]
mod tests {
use super::StencilLayout;
#[test]
fn layout_tracks_chains_stencils_and_program_slots() {
let mut layout = StencilLayout::new();
let root = layout.start_chain();
let first = layout.emit_stencil(&[1, 2, 3, 4]);
layout.push_prog_word(root.prog_index, 7);
let slot = layout.reserve_prog_slot(root.prog_index);
layout.fill_prog_slot(slot, 11);
let child = layout.start_chain();
let second = layout.emit_stencil(&[5, 6]);
assert_eq!(root.entry, 0);
assert_eq!(root.prog_index, 0);
assert_eq!(first, 0);
assert_eq!(child.entry, 4);
assert_eq!(child.prog_index, 1);
assert_eq!(second, 4);
assert_eq!(layout.code(), &[1, 2, 3, 4, 5, 6]);
assert_eq!(layout.prog(root.prog_index), &[7, 11]);
assert_eq!(layout.stencil_count(), 2);
}
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
))]
fn ret_stencil() -> &'static [u8] {
#[cfg(all(target_os = "macos", target_arch = "aarch64"))]
{
&[0xc0, 0x03, 0x5f, 0xd6]
}
#[cfg(all(target_os = "linux", target_arch = "x86_64"))]
{
&[0xc3]
}
}
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
))]
#[test]
fn native_program_owns_executable_code_and_program_slots() {
use super::NativeProgram;
let mut layout = StencilLayout::new();
let root = layout.start_chain();
layout.emit_stencil(ret_stencil());
layout.push_prog_word(root.prog_index, 7);
let slot = layout.reserve_prog_slot(root.prog_index);
let mut native = NativeProgram::new(layout, root);
native.fill_prog_slot(slot, 11);
assert_eq!(native.chain_count(), 1);
assert_eq!(native.stencil_count(), 1);
assert_eq!(native.prog_slot_count(), 2);
assert!(!native.entry_prog().is_null());
let entry = unsafe { native.entry_fn::<u8>() };
let mut ctx = 0u8;
unsafe { entry(&mut ctx) };
}
#[cfg(any(
all(target_os = "macos", target_arch = "aarch64"),
all(target_os = "linux", target_arch = "x86_64")
))]
#[test]
fn shared_hostcall_stencil_runs_consumer_intrinsic() {
use super::{HostCallCtx, HostCallInfo, NativeProgram};
struct State {
value: u64,
}
struct Info {
add: u64,
}
unsafe extern "C" fn add(cx: *mut (), info: *const ()) -> bool {
let state = unsafe { &mut *cx.cast::<State>() };
let info = unsafe { &*info.cast::<Info>() };
state.value += info.add;
true
}
let infos = [Info { add: 41 }];
let calls = [HostCallInfo {
info: core::ptr::from_ref(&infos[0]).cast(),
call: add,
}];
let mut layout = StencilLayout::new();
let root = layout.start_chain();
let hostcall = layout.emit_hostcall(root, core::ptr::from_ref(&calls[0]));
let done = layout.emit_done();
layout.patch_hostcall_continuation(hostcall, done);
let native = NativeProgram::new(layout, root);
let mut state = State { value: 1 };
let mut cx = HostCallCtx::new(native.entry_prog(), &mut state);
let entry = unsafe { native.entry_fn::<HostCallCtx<State>>() };
unsafe {
entry(&mut cx);
}
assert_eq!(state.value, 42);
}
}