use std::arch::global_asm;
#[repr(u32)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Mode {
Reference = 0,
Reload = 1,
ForcedExit = 2,
}
#[repr(u32)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum Kind {
XrstorInit = 0,
XrstorHeaderSet = 1,
WrpkruZero = 2,
Sigreturn = 3,
Vzeroupper = 4,
LegacyPxor = 5,
Vex128Zero = 6,
Hi16Zero = 7,
PxorVzeroupperHi16 = 8,
X87FullInit = 9,
X87DirtyFullInit = 10,
CompactedXrstorInit = 11,
FxrstorInit = 12,
OpmaskZero = 13,
VzeroallAll = 14,
}
#[repr(C)]
#[derive(Debug)]
pub struct ArmCtx {
pub kind: u32, pub component: u32, pub mxcsr: u32, pub mode: u32, pub xcr0: u64, pub save_area: *mut u8, pub image: *const u8, pub fd_to_peer: i32, pub fd_from_peer: i32, pub tgid: i32, pub tid: i32, pub ones: *const u8, pub pkru_out: u32, pub avx512: u32, pub cpuid_cycles: u64, pub seq: u8, pub reply: u8, pub pkru: u8, pub _pad: u8, pub status: u32, pub saved_pkru: u32, pub saved_mxcsr: u32, }
#[repr(u32)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum ArmStatus {
Ok = 0,
WriteFailed = 1,
ReadFailed = 2,
WrongReply = 3,
TgkillFailed = 4,
}
impl ArmStatus {
pub fn from_raw(raw: u32) -> Option<Self> {
Some(match raw {
0 => ArmStatus::Ok,
1 => ArmStatus::WriteFailed,
2 => ArmStatus::ReadFailed,
3 => ArmStatus::WrongReply,
4 => ArmStatus::TgkillFailed,
_ => return None,
})
}
}
#[repr(u32)]
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub enum PeerState {
DirtyPkruDiffers = 0,
DirtyWrpkruZero = 1,
XrstorInitAll = 2,
DirtyPkruHeaderLoaded = 3,
DirtyPkruSigreturn = 4,
InitValuesAllHeaderBits = 5,
}
#[repr(C)]
#[derive(Debug)]
pub struct PeerCtx {
pub state: u32, pub fd_in: i32, pub fd_out: i32, pub avx512: u32, pub image: *const u8, pub xcr0: u64, pub ones: *const u8, pub tgid: i32, pub tid: i32, pub expected_cpu: u32, pub wrong_cpu: u32, pub status: u32, pub rounds: u32, pub byte: u8, pub reply: u8, pub pkru: u8, pub _pad: u8, pub cpu_out: u32, pub saved_pkru: u32, pub saved_mxcsr: u32, }
pub const PEER_EXIT: u8 = 0xff;
unsafe extern "C" {
pub fn hermit_xinuse_arm(ctx: *mut ArmCtx) -> u32;
pub fn hermit_xinuse_peer_loop(ctx: *mut PeerCtx) -> u32;
pub fn hermit_xinuse_save_restore_cycles(area: *mut u8, xcr0: u64) -> u64;
pub fn hermit_xinuse_cpuid_cycles() -> u64;
pub fn hermit_xinuse_syscall_cycles() -> u64;
}
global_asm!(
r#"
.macro XINUSE_DIRTY ctxreg, avxoff, onesoff
mov rsi, [\ctxreg + \onesoff]
cmp dword ptr [\ctxreg + \avxoff], 0
je 2f
.irp r,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
vmovdqu32 zmm\r, [rsi]
.endr
.irp k,1,2,3,4,5,6,7
kxnorq k\k, k\k, k\k
.endr
kxorq k0, k0, k0
jmp 3f
2:
.irp r,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
vmovdqu ymm\r, [rsi]
.endr
3:
fninit
.rept 6
fld1
.endr
fldpi
fldl2e
.endm
// WRPKRU and RDPKRU raise #UD unless the kernel enables protection keys
// (CR4.PKE), so both run only when the context's PKRU byte is set; without it
// the PKRU value read is 0.
.macro XINUSE_WRPKRU value, ctxreg, flagoff
cmp byte ptr [\ctxreg + \flagoff], 0
je 91f
mov eax, \value
xor ecx, ecx
xor edx, edx
wrpkru
91:
.endm
.macro XINUSE_RDPKRU ctxreg, flagoff
xor eax, eax
cmp byte ptr [\ctxreg + \flagoff], 0
je 92f
xor ecx, ecx
rdpkru
92:
.endm
.macro XINUSE_RESTORE_PKRU ctxreg, flagoff, savedoff
cmp byte ptr [\ctxreg + \flagoff], 0
je 93f
mov eax, [\ctxreg + \savedoff]
xor ecx, ecx
xor edx, edx
wrpkru
93:
.endm
.macro XINUSE_XRSTOR_MASK ctxreg, imgoff, maskreg
mov rax, \maskreg
mov rdx, rax
shr rdx, 32
mov r8, [\ctxreg + \imgoff]
xrstor [r8]
.endm
"#
);
global_asm!(
r#"
.text
.globl hermit_xinuse_arm
.type hermit_xinuse_arm,@function
hermit_xinuse_arm:
push rbx
push r12
push r13
mov r12, rdi
XINUSE_RDPKRU r12, 82
mov [r12 + 88], eax
stmxcsr dword ptr [r12 + 92]
XINUSE_WRPKRU 0x55555554, r12, 82
XINUSE_DIRTY r12, 68, 56
ldmxcsr dword ptr [r12 + 8]
mov eax, [r12 + 0]
cmp eax, 0
je .Larm_xrstor
cmp eax, 1
je .Larm_xrstor
cmp eax, 11
je .Larm_xrstor
cmp eax, 2
je .Larm_wrpkru
cmp eax, 3
je .Larm_sigreturn
cmp eax, 4
je .Larm_vzeroupper
cmp eax, 5
je .Larm_legacy_pxor
cmp eax, 6
je .Larm_vex128
cmp eax, 7
je .Larm_hi16
cmp eax, 8
je .Larm_mix
cmp eax, 9
je .Larm_x87_full
cmp eax, 10
je .Larm_x87_dirty_full
cmp eax, 12
je .Larm_fxrstor
cmp eax, 13
je .Larm_opmask
cmp eax, 14
je .Larm_vzeroall
jmp .Larm_mode
.Larm_xrstor:
mov ecx, [r12 + 4]
mov r13, 1
shl r13, cl
XINUSE_XRSTOR_MASK r12, 32, r13
jmp .Larm_mode
.Larm_wrpkru:
XINUSE_WRPKRU 0, r12, 82
jmp .Larm_mode
.Larm_sigreturn:
XINUSE_WRPKRU 0, r12, 82
mov eax, 234
movsxd rdi, dword ptr [r12 + 48]
movsxd rsi, dword ptr [r12 + 52]
mov edx, 10
syscall
test rax, rax
jz .Larm_mode
mov dword ptr [r12 + 84], 4
jmp .Larm_finish
.Larm_vzeroupper:
vzeroupper
jmp .Larm_mode
.Larm_legacy_pxor:
.irp r,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
pxor xmm\r, xmm\r
.endr
jmp .Larm_mode
.Larm_vex128:
.irp r,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
vpxor xmm\r, xmm\r, xmm\r
.endr
jmp .Larm_mode
.Larm_hi16:
.irp r,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
vpxord zmm\r, zmm\r, zmm\r
.endr
jmp .Larm_mode
.Larm_mix:
.irp r,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15
pxor xmm\r, xmm\r
.endr
vzeroupper
.irp r,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
vpxord zmm\r, zmm\r, zmm\r
.endr
jmp .Larm_mode
.Larm_x87_dirty_full:
fld1
fldpi
.Larm_x87_full:
fninit
.rept 8
fldz
.endr
fninit
jmp .Larm_mode
.Larm_fxrstor:
mov r8, [r12 + 32]
fxrstor [r8]
jmp .Larm_mode
.Larm_opmask:
.irp k,0,1,2,3,4,5,6,7
kxorq k\k, k\k, k\k
.endr
jmp .Larm_mode
.Larm_vzeroall:
vzeroall
cmp dword ptr [r12 + 68], 0
je .Larm_mode
.irp r,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31
vpxord zmm\r, zmm\r, zmm\r
.endr
.Larm_mode:
mov eax, [r12 + 12]
cmp eax, 1
je .Larm_reload
cmp eax, 2
je .Larm_exit
jmp .Larm_save
.Larm_reload:
mov eax, 1
movsxd rdi, dword ptr [r12 + 40]
lea rsi, [r12 + 80]
mov edx, 1
syscall
cmp rax, 1
je 4f
mov dword ptr [r12 + 84], 1
jmp .Larm_finish
4:
xor eax, eax
movsxd rdi, dword ptr [r12 + 44]
lea rsi, [r12 + 81]
mov edx, 1
syscall
cmp rax, 1
je 5f
mov dword ptr [r12 + 84], 2
jmp .Larm_finish
5:
movzx eax, byte ptr [r12 + 80]
inc al
cmp al, byte ptr [r12 + 81]
je .Larm_save
mov dword ptr [r12 + 84], 3
jmp .Larm_finish
.Larm_exit:
rdtscp
shl rdx, 32
or rax, rdx
mov r13, rax
xor eax, eax
xor ecx, ecx
cpuid
rdtscp
shl rdx, 32
or rax, rdx
sub rax, r13
mov [r12 + 72], rax
.Larm_save:
mov rax, [r12 + 16]
mov rdx, rax
shr rdx, 32
mov r8, [r12 + 24]
xsave [r8]
XINUSE_RDPKRU r12, 82
mov [r12 + 64], eax
.Larm_finish:
XINUSE_RESTORE_PKRU r12, 82, 88
ldmxcsr dword ptr [r12 + 92]
fninit
vzeroupper
mov eax, [r12 + 84]
pop r13
pop r12
pop rbx
ret
.size hermit_xinuse_arm, .-hermit_xinuse_arm
.globl hermit_xinuse_peer_loop
.type hermit_xinuse_peer_loop,@function
hermit_xinuse_peer_loop:
push rbx
push r12
push r13
mov r12, rdi
XINUSE_RDPKRU r12, 66
mov [r12 + 72], eax
stmxcsr dword ptr [r12 + 76]
.Lpeer_round:
xor eax, eax
movsxd rdi, dword ptr [r12 + 4]
lea rsi, [r12 + 64]
mov edx, 1
syscall
cmp rax, 1
je 6f
mov dword ptr [r12 + 56], 2
jmp .Lpeer_finish
6:
cmp byte ptr [r12 + 64], 0xff
je .Lpeer_finish
mov eax, [r12 + 0]
cmp eax, 0
je .Lpeer_dirty_differs
cmp eax, 1
je .Lpeer_dirty_wrpkru
cmp eax, 2
je .Lpeer_xrstor_all
cmp eax, 3
je .Lpeer_header_loaded
cmp eax, 4
je .Lpeer_sigreturn
jmp .Lpeer_init_values
.Lpeer_dirty_differs:
XINUSE_DIRTY r12, 12, 32
XINUSE_WRPKRU 0x55555550, r12, 66
jmp .Lpeer_reply
.Lpeer_dirty_wrpkru:
XINUSE_DIRTY r12, 12, 32
XINUSE_WRPKRU 0, r12, 66
jmp .Lpeer_reply
.Lpeer_xrstor_all:
XINUSE_WRPKRU 0x55555550, r12, 66
mov r13, [r12 + 24]
XINUSE_XRSTOR_MASK r12, 16, r13
jmp .Lpeer_reply
.Lpeer_header_loaded:
XINUSE_DIRTY r12, 12, 32
mov r13, 0x200
XINUSE_XRSTOR_MASK r12, 16, r13
jmp .Lpeer_reply
.Lpeer_sigreturn:
XINUSE_DIRTY r12, 12, 32
XINUSE_WRPKRU 0, r12, 66
mov eax, 234
movsxd rdi, dword ptr [r12 + 40]
movsxd rsi, dword ptr [r12 + 44]
mov edx, 10
syscall
test rax, rax
jz .Lpeer_reply
mov dword ptr [r12 + 56], 4
jmp .Lpeer_finish
.Lpeer_init_values:
mov r13, [r12 + 24]
XINUSE_XRSTOR_MASK r12, 16, r13
.Lpeer_reply:
mov eax, 309
lea rdi, [r12 + 68]
xor esi, esi
xor edx, edx
syscall
mov eax, [r12 + 68]
cmp eax, [r12 + 48]
je 7f
inc dword ptr [r12 + 52]
7:
inc dword ptr [r12 + 60]
movzx eax, byte ptr [r12 + 64]
inc al
mov [r12 + 65], al
mov eax, 1
movsxd rdi, dword ptr [r12 + 8]
lea rsi, [r12 + 65]
mov edx, 1
syscall
cmp rax, 1
je .Lpeer_round
mov dword ptr [r12 + 56], 1
.Lpeer_finish:
XINUSE_RESTORE_PKRU r12, 66, 72
ldmxcsr dword ptr [r12 + 76]
fninit
vzeroupper
mov eax, [r12 + 56]
pop r13
pop r12
pop rbx
ret
.size hermit_xinuse_peer_loop, .-hermit_xinuse_peer_loop
.globl hermit_xinuse_save_restore_cycles
.type hermit_xinuse_save_restore_cycles,@function
hermit_xinuse_save_restore_cycles:
push rbx
mov r8, rdi
mov r9, rsi
rdtscp
shl rdx, 32
or rax, rdx
mov r10, rax
mov rax, r9
mov rdx, r9
shr rdx, 32
xsave [r8]
mov rax, r9
mov rdx, r9
shr rdx, 32
xrstor [r8]
rdtscp
shl rdx, 32
or rax, rdx
sub rax, r10
vzeroupper
pop rbx
ret
.size hermit_xinuse_save_restore_cycles, .-hermit_xinuse_save_restore_cycles
.globl hermit_xinuse_cpuid_cycles
.type hermit_xinuse_cpuid_cycles,@function
hermit_xinuse_cpuid_cycles:
push rbx
rdtscp
shl rdx, 32
or rax, rdx
mov r8, rax
xor eax, eax
xor ecx, ecx
cpuid
rdtscp
shl rdx, 32
or rax, rdx
sub rax, r8
pop rbx
ret
.size hermit_xinuse_cpuid_cycles, .-hermit_xinuse_cpuid_cycles
.globl hermit_xinuse_syscall_cycles
.type hermit_xinuse_syscall_cycles,@function
hermit_xinuse_syscall_cycles:
rdtscp
shl rdx, 32
or rax, rdx
mov r8, rax
mov eax, 110
syscall
rdtscp
shl rdx, 32
or rax, rdx
sub rax, r8
ret
.size hermit_xinuse_syscall_cycles, .-hermit_xinuse_syscall_cycles
"#
);
#[cfg(test)]
mod tests {
use std::mem::offset_of;
use super::*;
#[test]
fn arm_ctx_layout_matches_the_assembly() {
assert_eq!(offset_of!(ArmCtx, kind), 0);
assert_eq!(offset_of!(ArmCtx, component), 4);
assert_eq!(offset_of!(ArmCtx, mxcsr), 8);
assert_eq!(offset_of!(ArmCtx, mode), 12);
assert_eq!(offset_of!(ArmCtx, xcr0), 16);
assert_eq!(offset_of!(ArmCtx, save_area), 24);
assert_eq!(offset_of!(ArmCtx, image), 32);
assert_eq!(offset_of!(ArmCtx, fd_to_peer), 40);
assert_eq!(offset_of!(ArmCtx, fd_from_peer), 44);
assert_eq!(offset_of!(ArmCtx, tgid), 48);
assert_eq!(offset_of!(ArmCtx, tid), 52);
assert_eq!(offset_of!(ArmCtx, ones), 56);
assert_eq!(offset_of!(ArmCtx, pkru_out), 64);
assert_eq!(offset_of!(ArmCtx, avx512), 68);
assert_eq!(offset_of!(ArmCtx, cpuid_cycles), 72);
assert_eq!(offset_of!(ArmCtx, seq), 80);
assert_eq!(offset_of!(ArmCtx, pkru), 82);
assert_eq!(offset_of!(ArmCtx, reply), 81);
assert_eq!(offset_of!(ArmCtx, status), 84);
assert_eq!(offset_of!(ArmCtx, saved_pkru), 88);
assert_eq!(offset_of!(ArmCtx, saved_mxcsr), 92);
}
#[test]
fn peer_ctx_layout_matches_the_assembly() {
assert_eq!(offset_of!(PeerCtx, state), 0);
assert_eq!(offset_of!(PeerCtx, fd_in), 4);
assert_eq!(offset_of!(PeerCtx, fd_out), 8);
assert_eq!(offset_of!(PeerCtx, avx512), 12);
assert_eq!(offset_of!(PeerCtx, image), 16);
assert_eq!(offset_of!(PeerCtx, xcr0), 24);
assert_eq!(offset_of!(PeerCtx, ones), 32);
assert_eq!(offset_of!(PeerCtx, tgid), 40);
assert_eq!(offset_of!(PeerCtx, tid), 44);
assert_eq!(offset_of!(PeerCtx, expected_cpu), 48);
assert_eq!(offset_of!(PeerCtx, wrong_cpu), 52);
assert_eq!(offset_of!(PeerCtx, status), 56);
assert_eq!(offset_of!(PeerCtx, rounds), 60);
assert_eq!(offset_of!(PeerCtx, byte), 64);
assert_eq!(offset_of!(PeerCtx, reply), 65);
assert_eq!(offset_of!(PeerCtx, pkru), 66);
assert_eq!(offset_of!(PeerCtx, cpu_out), 68);
assert_eq!(offset_of!(PeerCtx, saved_pkru), 72);
assert_eq!(offset_of!(PeerCtx, saved_mxcsr), 76);
}
}