use core::ffi::c_void;
#[repr(C)]
#[derive(Clone, Copy, Debug)]
pub struct VaListTag {
pub gp_offset: u32,
pub fp_offset: u32,
pub overflow_arg_area: *mut c_void,
pub reg_save_area: *mut c_void,
}
const GP_AREA_END: u32 = 48;
const SSE_AREA_END: u32 = 176;
impl VaListTag {
#[inline]
pub unsafe fn arg<T: VaArg>(&mut self) -> T {
unsafe { T::va_arg(self) }
}
#[inline]
unsafe fn next_gp_slot(&mut self) -> *mut c_void {
if self.gp_offset < GP_AREA_END {
let slot = unsafe { self.reg_save_area.byte_add(self.gp_offset as usize) };
self.gp_offset += 8;
slot
} else {
unsafe { self.next_overflow_slot() }
}
}
#[inline]
unsafe fn next_sse_slot(&mut self) -> *mut c_void {
if self.fp_offset < SSE_AREA_END {
let slot = unsafe { self.reg_save_area.byte_add(self.fp_offset as usize) };
self.fp_offset += 16;
slot
} else {
unsafe { self.next_overflow_slot() }
}
}
#[inline]
unsafe fn next_overflow_slot(&mut self) -> *mut c_void {
let slot = self.overflow_arg_area;
self.overflow_arg_area = unsafe { slot.byte_add(8) };
slot
}
}
pub trait VaArg: Copy {
unsafe fn va_arg(tag: &mut VaListTag) -> Self;
}
macro_rules! gp_va_arg {
($($int:ty),*) => {$(
impl VaArg for $int {
#[inline]
unsafe fn va_arg(tag: &mut VaListTag) -> Self {
unsafe { tag.next_gp_slot().cast::<Self>().read() }
}
}
)*};
}
gp_va_arg!(i32, u32, i64, u64, isize, usize);
impl<T> VaArg for *const T {
#[inline]
unsafe fn va_arg(tag: &mut VaListTag) -> Self {
unsafe { tag.next_gp_slot().cast::<Self>().read() }
}
}
impl<T> VaArg for *mut T {
#[inline]
unsafe fn va_arg(tag: &mut VaListTag) -> Self {
unsafe { tag.next_gp_slot().cast::<Self>().read() }
}
}
impl VaArg for f64 {
#[inline]
unsafe fn va_arg(tag: &mut VaListTag) -> Self {
unsafe { tag.next_sse_slot().cast::<f64>().read() }
}
}
#[macro_export]
macro_rules! vararg_entry {
($(#[$attr:meta])* unsafe extern "C" fn $name:ident(
$a1:ident: $t1:ty, ...
) $(-> $ret:ty)? => $imp:path) => {
$crate::__vararg_entry_emit! {
[$(#[$attr])*] $name, ($a1: $t1), ($($ret)?), $imp,
gp = 8, tag_reg = "rsi"
}
};
($(#[$attr:meta])* unsafe extern "C" fn $name:ident(
$a1:ident: $t1:ty, $a2:ident: $t2:ty, ...
) $(-> $ret:ty)? => $imp:path) => {
$crate::__vararg_entry_emit! {
[$(#[$attr])*] $name, ($a1: $t1, $a2: $t2), ($($ret)?), $imp,
gp = 16, tag_reg = "rdx"
}
};
($(#[$attr:meta])* unsafe extern "C" fn $name:ident(
$a1:ident: $t1:ty, $a2:ident: $t2:ty, $a3:ident: $t3:ty, ...
) $(-> $ret:ty)? => $imp:path) => {
$crate::__vararg_entry_emit! {
[$(#[$attr])*] $name, ($a1: $t1, $a2: $t2, $a3: $t3), ($($ret)?), $imp,
gp = 24, tag_reg = "rcx"
}
};
($(#[$attr:meta])* unsafe extern "C" fn $name:ident(
$a1:ident: $t1:ty, $a2:ident: $t2:ty, $a3:ident: $t3:ty, $a4:ident: $t4:ty, ...
) $(-> $ret:ty)? => $imp:path) => {
$crate::__vararg_entry_emit! {
[$(#[$attr])*] $name, ($a1: $t1, $a2: $t2, $a3: $t3, $a4: $t4), ($($ret)?), $imp,
gp = 32, tag_reg = "r8"
}
};
}
#[macro_export]
#[doc(hidden)]
macro_rules! __vararg_entry_emit {
(
[$($attr:tt)*] $name:ident, ($($arg:ident: $ty:ty),*), ($($ret:ty)?), $imp:path,
gp = $gp:literal, tag_reg = $tag_reg:literal
) => {
const _: unsafe extern "C" fn($($ty,)* *mut $crate::va::VaListTag) $(-> $ret)? = $imp;
$($attr)*
#[unsafe(naked)]
unsafe extern "C" fn $name($($arg: $ty),*) $(-> $ret)? {
::core::arch::naked_asm!(
// The callee half of the variadic protocol (psABI 3.5.7).
"sub rsp, 200",
"mov [rsp], rdi",
"mov [rsp + 8], rsi",
"mov [rsp + 16], rdx",
"mov [rsp + 24], rcx",
"mov [rsp + 32], r8",
"mov [rsp + 40], r9",
"test al, al",
"je 2f",
"movaps [rsp + 48], xmm0",
"movaps [rsp + 64], xmm1",
"movaps [rsp + 80], xmm2",
"movaps [rsp + 96], xmm3",
"movaps [rsp + 112], xmm4",
"movaps [rsp + 128], xmm5",
"movaps [rsp + 144], xmm6",
"movaps [rsp + 160], xmm7",
"2:",
concat!("mov dword ptr [rsp + 176], ", $gp),
"mov dword ptr [rsp + 180], 48",
"lea rax, [rsp + 208]",
"mov [rsp + 184], rax",
"mov [rsp + 192], rsp",
concat!("lea ", $tag_reg, ", [rsp + 176]"),
"call {imp}",
// The return value rides through rax (or xmm0) untouched.
"add rsp, 200",
"ret",
imp = sym $imp,
)
}
};
}
// Path-based imports for the crate's own callers and for the taproot
// cdylib crate; `#[macro_export]` also places both at the crate root,
// which is the `$crate` path the expansion rides on.
#[doc(hidden)]
pub use __vararg_entry_emit;
pub use vararg_entry;
#[cfg(test)]
mod tests {
// `vararg_entry!` needs no import: `macro_rules!` scoping already
// carries it through the rest of the file.
use super::VaListTag;
use libc::{c_char, c_int, c_long, c_uint, c_void};
// The implementation halves: ordinary `extern "C"` functions that
// receive the named arguments plus the tag the naked entry built.
unsafe extern "C" fn sum_longs_impl(count: c_int, tag: *mut VaListTag) -> c_long {
// SAFETY: the entry hands us a live tag; each fetch below matches
// one `c_long` the test call sites pass.
unsafe {
let tag = &mut *tag;
let mut acc: c_long = 0;
for _ in 0..count {
acc = acc * 10 + tag.arg::<c_long>();
}
acc
}
}
unsafe extern "C" fn grab_impl(out: *mut u64, count: c_int, tag: *mut VaListTag) -> c_int {
// SAFETY: the fetch script mirrors the call site in
// `widths_and_pointers_recover_exactly`, slot for slot, and `out`
// has room for all eight recovered values.
unsafe {
let tag = &mut *tag;
*out.add(0) = tag.arg::<c_int>() as i64 as u64;
*out.add(1) = u64::from(tag.arg::<c_uint>());
*out.add(2) = tag.arg::<u64>();
*out.add(3) = tag.arg::<*mut u8>() as u64;
*out.add(4) = tag.arg::<usize>() as u64;
*out.add(5) = tag.arg::<i64>() as u64;
*out.add(6) = tag.arg::<*const c_void>() as u64;
*out.add(7) = tag.arg::<isize>() as u64;
}
count
}
unsafe extern "C" fn fgrab_impl(
out: *mut f64,
count: c_int,
check: c_int,
tag: *mut VaListTag,
) -> c_int {
// SAFETY: the callers pass `count` doubles and an `out` with room
// for them.
unsafe {
let tag = &mut *tag;
for i in 0..count {
*out.add(i as usize) = tag.arg::<f64>();
}
}
check
}
unsafe extern "C" fn named4_impl(
a: c_int,
b: c_int,
c: c_int,
d: c_int,
tag: *mut VaListTag,
) -> c_long {
// SAFETY: the caller passes three `c_long`s after the four named
// ints.
let (v1, v2, v3) = unsafe {
let tag = &mut *tag;
(
tag.arg::<c_long>(),
tag.arg::<c_long>(),
tag.arg::<c_long>(),
)
};
c_long::from(a)
+ 2 * c_long::from(b)
+ 3 * c_long::from(c)
+ 4 * c_long::from(d)
+ 1_000 * v1
+ 1_000_000 * v2
+ 1_000_000_000 * v3
}
unsafe extern "C" fn mixed_impl(iout: *mut i64, fout: *mut f64, tag: *mut VaListTag) -> c_int {
// SAFETY: the fetch script mirrors the call site in
// `mixed_classes_share_one_overflow_cursor`: four register ints,
// eight register doubles, then ints five and six and the ninth
// double off the shared overflow cursor, in call order.
unsafe {
let tag = &mut *tag;
for i in 0..4 {
*iout.add(i) = tag.arg::<i64>();
}
for i in 0..8 {
*fout.add(i) = tag.arg::<f64>();
}
*iout.add(4) = tag.arg::<i64>();
*iout.add(5) = tag.arg::<i64>();
*fout.add(8) = tag.arg::<f64>();
}
0
}
unsafe extern "C" fn execle_shape_impl(out: *mut *const c_char, tag: *mut VaListTag) -> c_int {
// SAFETY: the fetch script is the execle loop shape its call site
// mirrors: `*const c_char`s up to and including a null terminator,
// then one trailing envp-style pointer, and `out` has room for all
// of them.
unsafe {
let tag = &mut *tag;
let mut count = 0;
loop {
let ptr = tag.arg::<*const c_char>();
*out.add(count) = ptr;
count += 1;
if ptr.is_null() {
break;
}
}
*out.add(count) = tag.arg::<*const *const c_char>().cast();
count as c_int
}
}
// The generated naked entries, one per named-argument count 1..=4.
vararg_entry! {
#[no_mangle]
unsafe extern "C" fn __taproot_va_test_sum_longs(count: c_int, ...) -> c_long
=> sum_longs_impl
}
vararg_entry! {
#[no_mangle]
unsafe extern "C" fn __taproot_va_test_grab(out: *mut u64, count: c_int, ...) -> c_int
=> grab_impl
}
vararg_entry! {
#[no_mangle]
unsafe extern "C" fn __taproot_va_test_fgrab(
out: *mut f64,
count: c_int,
check: c_int,
...
) -> c_int => fgrab_impl
}
vararg_entry! {
#[no_mangle]
unsafe extern "C" fn __taproot_va_test_named4(
a: c_int,
b: c_int,
c: c_int,
d: c_int,
...
) -> c_long => named4_impl
}
vararg_entry! {
#[no_mangle]
unsafe extern "C" fn __taproot_va_test_mixed(iout: *mut i64, fout: *mut f64, ...) -> c_int
=> mixed_impl
}
vararg_entry! {
#[no_mangle]
unsafe extern "C" fn __taproot_va_test_execle_shape(
out: *mut *const c_char,
...
) -> c_int => execle_shape_impl
}
/// The C-side view of the entries above. Living in a child module
/// keeps these declarations from colliding with the definitions' item
/// names; the linker pairs them up by symbol. Calling through these
/// makes the tests real-ABI round trips: rustc performs the caller
/// half of the variadic protocol (register assignment, AL, stack
/// slots), and the naked entries must undo it exactly.
mod decl {
use libc::{c_char, c_int, c_long};
unsafe extern "C" {
pub fn __taproot_va_test_sum_longs(count: c_int, ...) -> c_long;
pub fn __taproot_va_test_grab(out: *mut u64, count: c_int, ...) -> c_int;
pub fn __taproot_va_test_fgrab(
out: *mut f64,
count: c_int,
check: c_int,
...
) -> c_int;
pub fn __taproot_va_test_named4(
a: c_int,
b: c_int,
c: c_int,
d: c_int,
...
) -> c_long;
pub fn __taproot_va_test_mixed(iout: *mut i64, fout: *mut f64, ...) -> c_int;
pub fn __taproot_va_test_execle_shape(out: *mut *const c_char, ...) -> c_int;
}
}
#[test]
fn ints_cross_into_the_overflow_area() {
// One named plus eight variadic longs: five ride rsi..r9, and the
// last three come off the caller's stack through
// `overflow_arg_area`. The positional fold proves both the values
// and their order.
let got = unsafe {
decl::__taproot_va_test_sum_longs(
8,
1 as c_long,
2 as c_long,
3 as c_long,
4 as c_long,
5 as c_long,
6 as c_long,
7 as c_long,
8 as c_long,
)
};
assert_eq!(got, 12_345_678);
}
#[test]
fn no_fp_call_walks_registers_only() {
// No doubles anywhere, so rustc sets AL = 0 and the entry must
// skip the SSE spill entirely.
let got = unsafe { decl::__taproot_va_test_sum_longs(2, 40 as c_long, 2 as c_long) };
assert_eq!(got, 402);
}
#[test]
fn widths_and_pointers_recover_exactly() {
let mut in_reg: u8 = 0;
let on_stack: u16 = 0;
let preg: *mut u8 = &mut in_reg;
let pstack: *const c_void = (&on_stack as *const u16).cast();
let mut out = [0_u64; 8];
// Two named plus eight variadic: slots one to four ride rdx, rcx,
// r8 and r9; five to eight cross into the overflow area, with a
// pointer landing on each side of the boundary.
let echoed = unsafe {
decl::__taproot_va_test_grab(
out.as_mut_ptr(),
8,
-7_i32,
0xDEAD_BEEF_u32,
0x8000_0000_0000_0001_u64,
preg,
usize::MAX - 41,
i64::MIN + 2,
pstack,
isize::MIN + 9,
)
};
assert_eq!(echoed, 8);
assert_eq!(out[0], -7_i64 as u64);
assert_eq!(out[1], 0xDEAD_BEEF);
assert_eq!(out[2], 0x8000_0000_0000_0001);
assert_eq!(out[3], preg as u64);
assert_eq!(out[4], (usize::MAX - 41) as u64);
assert_eq!(out[5], (i64::MIN + 2) as u64);
assert_eq!(out[6], pstack as u64);
assert_eq!(out[7], (isize::MIN + 9) as u64);
}
#[test]
fn f64s_ride_xmm0_to_7_then_spill() {
const DOUBLES: [f64; 10] = [
0.5,
-3.25,
1.0e300,
f64::MIN_POSITIVE,
-0.0,
6.022e23,
9_007_199_254_740_992.0,
-1.0,
9.75,
1_234.5,
];
let mut out = [0.0_f64; 10];
// Three named ints, then ten doubles: xmm0..7 carry eight of them
// (rustc sets AL = 8) and the last two arrive through the
// overflow area.
let echoed = unsafe {
decl::__taproot_va_test_fgrab(
out.as_mut_ptr(),
10,
0x5AFE,
DOUBLES[0],
DOUBLES[1],
DOUBLES[2],
DOUBLES[3],
DOUBLES[4],
DOUBLES[5],
DOUBLES[6],
DOUBLES[7],
DOUBLES[8],
DOUBLES[9],
)
};
assert_eq!(echoed, 0x5AFE);
// Bit equality, so the negative zero must survive too.
for (got, want) in out.iter().zip(DOUBLES) {
assert_eq!(got.to_bits(), want.to_bits());
}
}
#[test]
fn f64s_partial_register_walk() {
let mut out = [0.0_f64; 3];
// AL = 3: the entry still spills all eight SSE registers, but the
// walker only reads back the three that carry arguments.
let echoed =
unsafe { decl::__taproot_va_test_fgrab(out.as_mut_ptr(), 3, 7, 1.5, 2.5, -8.125) };
assert_eq!(echoed, 7);
assert_eq!(out, [1.5, 2.5, -8.125]);
}
#[test]
fn four_named_args_start_the_walk_at_32() {
// Four named ints occupy rdi..rcx, so gp_offset opens at 32: the
// first two variadic longs sit in r8 and r9, the third on the
// stack.
let got = unsafe {
decl::__taproot_va_test_named4(10, 20, 30, 40, 7 as c_long, 8 as c_long, 9 as c_long)
};
assert_eq!(got, 9_008_007_300);
}
#[test]
fn mixed_classes_share_one_overflow_cursor() {
let mut iout = [0_i64; 6];
let mut fout = [0.0_f64; 9];
// Six GP arguments fill rdi..r9 (two named pointers plus four
// variadic ints), so ints five and six go to the stack. Eight
// doubles fill xmm0..7, so the ninth goes to the stack after
// them. The overflow area therefore reads back, in call order:
// 55, 66, 8.5.
let rc = unsafe {
decl::__taproot_va_test_mixed(
iout.as_mut_ptr(),
fout.as_mut_ptr(),
11_i64,
22_i64,
33_i64,
44_i64,
0.5,
1.5,
2.5,
3.5,
4.5,
5.5,
6.5,
7.5,
55_i64,
66_i64,
8.5,
)
};
assert_eq!(rc, 0);
assert_eq!(iout, [11, 22, 33, 44, 55, 66]);
assert_eq!(fout, [0.5, 1.5, 2.5, 3.5, 4.5, 5.5, 6.5, 7.5, 8.5]);
}
#[test]
fn the_execle_walk_collects_until_null_then_envp() {
// The execl/execle loop shape: one named argument, then `*const
// c_char`s until a null terminator, then (execle only) one more
// pointer. Six strings push the walk off the registers: rsi..r9
// carry the first five, and the sixth, the null and the envp
// pointer all come off the overflow area.
let argv = [
c"sh".as_ptr(),
c"-c".as_ptr(),
c"one".as_ptr(),
c"two".as_ptr(),
c"three".as_ptr(),
c"four".as_ptr(),
];
let env = [c"X=1".as_ptr(), core::ptr::null()];
let envp: *const *const c_char = env.as_ptr();
let mut out = [core::ptr::null::<c_char>(); 8];
let walked = unsafe {
decl::__taproot_va_test_execle_shape(
out.as_mut_ptr(),
argv[0],
argv[1],
argv[2],
argv[3],
argv[4],
argv[5],
core::ptr::null::<c_char>(),
envp,
)
};
assert_eq!(walked, 7);
assert_eq!(&out[..6], &argv);
assert!(out[6].is_null());
assert_eq!(out[7], envp.cast::<c_char>());
}
}