#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
use core::arch::{asm, naked_asm};
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
use std::ptr::{self, NonNull};
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
use crate::context::{FiberCore, fiber_root};
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
pub(crate) const FRAME_LEN: usize = 32;
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
const SLOT_RETURN: usize = 0;
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
const SLOT_MXCSR: usize = 8;
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
const SLOT_X87_CONTROL: usize = 12;
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
const SLOT_RBX: usize = 16;
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
#[inline(always)]
pub(crate) unsafe fn context_resume(
save_current: *mut usize,
restore: usize,
transfer: usize,
) -> usize {
let returned;
unsafe {
asm!(
"push rbp",
"push rbx",
"sub rsp, 8",
"stmxcsr [rsp]",
"fnstcw [rsp + 4]",
"call qword ptr [rsi]",
"add rsp, 8",
"pop rbx",
"pop rbp",
in("rdi") save_current,
in("rsi") restore,
inlateout("rdx") transfer => returned,
lateout("r12") _,
lateout("r13") _,
lateout("r14") _,
lateout("r15") _,
clobber_abi("sysv64"),
);
}
returned
}
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
#[inline(always)]
pub(crate) unsafe fn context_suspend(
save_current: *mut usize,
restore: usize,
transfer: usize,
) -> usize {
let returned;
unsafe {
asm!(
"push rbp",
"push rbx",
"sub rsp, 8",
"stmxcsr [rsp]",
"fnstcw [rsp + 4]",
"lea rax, [rip + 4f]",
"push rax",
"mov eax, [rsp + 8]",
"movzx ecx, word ptr [rsp + 12]",
"mov [rdi], rsp",
"cmp eax, [rsi + 8]",
"je 2f",
"ldmxcsr [rsi + 8]",
"2:",
"cmp cx, [rsi + 12]",
"je 3f",
"fldcw [rsi + 12]",
"3:",
"mov rsp, rsi",
"ret",
"4:",
"mov [rdi], rsp",
"mov eax, [rsp + 8]",
"movzx ecx, word ptr [rsp + 12]",
"cmp eax, [rsi + 8]",
"je 5f",
"ldmxcsr [rsi + 8]",
"5:",
"cmp cx, [rsi + 12]",
"je 6f",
"fldcw [rsi + 12]",
"6:",
"lea rsp, [rsi + 16]",
"pop rbx",
"pop rbp",
in("rdi") save_current,
in("rsi") restore,
inlateout("rdx") transfer => returned,
lateout("r12") _,
lateout("r13") _,
lateout("r14") _,
lateout("r15") _,
clobber_abi("sysv64"),
);
}
returned
}
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
#[unsafe(naked)]
pub(crate) unsafe extern "C" fn context_finish(restore: usize, transfer: usize) -> ! {
naked_asm!(
"sub rsp, 8",
"stmxcsr [rsp]",
"fnstcw [rsp + 4]",
"mov eax, [rsp]",
"movzx ecx, word ptr [rsp + 4]",
"mov rsp, rdi",
"cmp eax, [rdi + 8]",
"je 2f",
"ldmxcsr [rdi + 8]",
"2:",
"cmp cx, [rdi + 12]",
"je 3f",
"fldcw [rdi + 12]",
"3:",
"mov rdx, rsi",
"mov rsp, rdi",
"ret",
)
}
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
#[unsafe(naked)]
unsafe extern "C" fn trampoline() -> ! {
naked_asm!(
"mov [rdi], rsp",
"mov eax, [rsp + 8]",
"movzx ecx, word ptr [rsp + 12]",
"cmp eax, [rsi + 8]",
"je 2f",
"ldmxcsr [rsi + 8]",
"2:",
"cmp cx, [rsi + 12]",
"je 3f",
"fldcw [rsi + 12]",
"3:",
"lea rsp, [rsi + 16]",
"pop rbx",
"pop rbp",
"mov rdi, rbx",
"xor ebp, ebp",
"push 0",
"jmp {root}",
"ud2",
root = sym fiber_root,
)
}
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
pub(crate) unsafe fn init_frame(frame_top: usize, core: NonNull<FiberCore>) -> usize {
let sp = frame_top - FRAME_LEN;
let frame = sp as *mut u8;
unsafe {
ptr::write_bytes(frame, 0, FRAME_LEN);
frame
.add(SLOT_RBX)
.cast::<usize>()
.write(core.as_ptr() as usize);
frame
.add(SLOT_RETURN)
.cast::<usize>()
.write(trampoline as *const () as usize);
frame.add(SLOT_MXCSR).cast::<u32>().write(mxcsr());
frame
.add(SLOT_X87_CONTROL)
.cast::<u16>()
.write(x87_control_word());
}
sp
}
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
pub(crate) fn mxcsr() -> u32 {
let mut value: u32 = 0;
unsafe {
asm!("stmxcsr [{}]", in(reg) &raw mut value, options(nostack, preserves_flags));
}
value
}
#[cfg(all(target_arch = "x86_64", target_os = "linux"))]
pub(crate) fn x87_control_word() -> u16 {
let mut value: u16 = 0;
unsafe {
asm!("fnstcw [{}]", in(reg) &raw mut value, options(nostack, preserves_flags));
}
value
}
#[cfg(test)]
#[path = "x86_64_sysv_test.rs"]
mod x86_64_sysv_test;