1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
b0VIM 9.0 �q�j�{ . mkeeter cfarm14 ~mkeeter/fidget/fidget-jit/src/x86_64/grad_slice.rs utf-8
U3210 #"! U tp Q h R i � k # ^ � @ � �
b d ad
f Q � � | < � � � � t ; �
�
�
H
D
� q # � � 9 �
�
O
� e � { - � � C � � Y � y x R * � � � d Y
� � � k X W ; � � � � � Z " � � � � w l f e } ); ; vmovups Rx(reg(dst_reg)), [rsp + sp_offset] dynasm!(self.0.ops .unwrap(); .try_into() + STACK_SIZE_LOWER as u32) let sp_offset: i32 = (self.0.stack_pos(src_mem) assert!((dst_reg as usize) < REGISTER_LIMIT); fn build_load(&mut self, dst_reg: u8, src_mem: u32) { } Self(out) ); ; jz ->X // jump to the exit if we're done, otherwise fallthrough ; test rdx, rdx ; ->L: // The loop returns here, and we check whether to keep looping ; xor rcx, rcx // set the array offset (rcx) to 0 dynasm!(out.ops out.prepare_stack(slot_count, STACK_SIZE_UPPER + STACK_SIZE_LOWER); ); ; mov rbp, rsp ; push rbp dynasm!(out.ops let mut out = AssemblerData::new(mmap); fn init(mmap: Mmap, slot_count: usize) -> Self { type Data = Grad; impl Assembler for GradSliceAssembler { #[expect(clippy::useless_conversion)] const STACK_SIZE_LOWER: usize = 0xc0; // Positions relative to `rsp` const STACK_SIZE_UPPER: usize = 0x20; // Positions relative to `rbp` /// ``` /// | 0x00 | xmm4 | | /// | 0x10 | xmm5 | | /// | 0x20 | xmm6 | | /// | 0x30 | xmm7 | | /// | 0x40 | xmm8 | | /// | 0x50 | xmm9 | | /// | 0x60 | xmm10 | | /// | 0x70 | xmm11 | | /// | 0x80 | xmm12 | | /// | 0x90 | xmm13 | | /// | 0xa0 | xmm14 | calls are placed here, then restored | /// | 0xb0 | xmm15 | Caller-saved registers during functions | /// |----------|--------------|---------------------------------------------| /// | ... | ... | Register spills live up here | /// |----------|--------------|---------------------------------------------| /// | -0x20 | `rcx` | | /// | -0x18 | `rdx` | previous values on the stack | /// | -0x10 | `rsi` | as temporary storage so must preserve their | /// | -0x08 | `rdi` | During functions calls, we use these | /// |----------|--------------|---------------------------------------------| /// | 0x00 | `rbp` | Previous value for base pointer | /// |----------|--------------|---------------------------------------------| /// | Position | Value | Notes | /// ```text /// /// The stack is configured as follows /// /// During evaluation, `rcx` is used to track offset within `vars`. /// /// | `count` | `rdx` | `u64` | /// | `out` | `rsi` | `*const *mut [f32; 4]` | /// | `vars` | `rdi` | `*const *const [f32; 4]` | /// |------------|----------|--------------------------| /// | Variable | Register | Type | /// /// Registers are passed in as follows: /// /// Implementation for the gradient slice assembler on `x86_64` use fidget_core::types::Grad; use dynasmrt::{DynasmApi, DynasmError, DynasmLabelApi, dynasm}; }; grad_slice::GradSliceAssembler, mmap::Mmap, reg, Assembler, AssemblerData, IMM_REG, OFFSET, REGISTER_LIMIT, use crate::{ ad � � � � � \ 3 � � � a 7 6 �
�
�
�
�
E
� � � } } ); ; vpunpcklqdq Rx(reg(out_reg)), xmm0, xmm1 // Collect the 4x floats into the out register ; mov rcx, [rbp - 0x20] ; mov rdx, [rbp - 0x18] ; mov rsi, [rbp - 0x10] ; mov rdi, [rbp - 0x8] // Restore X/Y/Z pointers ; vmovups xmm15, [rsp + 0xb0] ; vmovups xmm14, [rsp + 0xa0] ; vmovups xmm13, [rsp + 0x90] ; vmovups xmm12, [rsp + 0x80] ; vmovups xmm11, [rsp + 0x70] ; vmovups xmm10, [rsp + 0x60] ; vmovups xmm9, [rsp + 0x50] ; vmovups xmm8, [rsp + 0x40] ; vmovups xmm7, [rsp + 0x30] ; vmovups xmm6, [rsp + 0x20] ; vmovups xmm5, [rsp + 0x10] ad � � @ � � � � \ H 4 � � � x w ^ M �
�
�
�
�
D
/
.
� � � � � � [ U ( � � � c X 3 - �
�
�
^
H
G
� � � � � � � � t ^ K 6 ! � � � � � K ) { f: extern "sysv64" fn(Grad) -> Grad, arg ) { f: ex ) { ) { f: extern "sysv64" fn(Grad) -> Grad, arg_r ) { f: extern "sysv64" fn(Grad) -> Grad, arg_reg: u8, out_reg: u8, &mut self, fn call_fn_unary( impl GradSliceAssembler { #[expect(clippy::useless_conversion)] } } self.0.finalize() ); ; -> X: // Finalization code, which happens after all evaluation is complete ; jmp ->L ; add rcx, 16 // input is array is Grad (f32 x 4) ; sub rdx, 1 // we process one element at a time dynasm!(self.0.ops fn finalize(mut self) -> Result<Mmap, DynasmError> { } IMM_REG.wrapping_sub(OFFSET) ); ; vmovd Rx(IMM_REG), eax ; mov eax, imm_u32 as i32 ; vpxor Rx(IMM_REG), Rx(IMM_REG), Rx(IMM_REG) dynasm!(self.0.ops let imm_u32 = imm.to_bits(); fn load_imm(&mut self, imm: f32) -> u8 { } self.0.ops.commit_local().unwrap(); ); ; O: // fallthrough to out ; vmovd Rx(reg(out_reg)), eax ; mov eax, 1f32.to_bits() as i32 ; R: ; jmp >O ; vaddss Rx(reg(out_reg)), Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) // TODO: this can't be the best way to make a NAN ; N: ; jmp >O ; vmovd Rx(reg(out_reg)), eax ; mov eax, (-1f32).to_bits() as i32 ; L: // Less than ; jmp >O ; vmovd Rx(reg(out_reg)), eax ; xor eax, eax // set eax to 0u32, which is also 0f32 // Fall-through for equal ; jb >L ; ja >R ; jp >N ; vcomiss Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) dynasm!(self.0.ops fn build_compare(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ad � ^ � � } Y > � � � � W < �
�
�
K
J
� � � � ` E � � � T S ! �
�
�
�
�
s
K
0
� � � v Y : � � � � s T . � � � | v u 0 � � � h A " � � � � W 8 � � � � l N 1 � � � � h I � T > = � � � � � � � � j T A , � � � ) { f: extern "sysv64" fn(Grad) -> Grad, arg_reg: u8, out_reg: u8, &mut self, fn call_fn_unary( impl GradSliceAssembler { #[expect(clippy::useless_conversion)] } } self.0.finalize() ); ; -> X: // Finalization code, which ); ; vmovq Rx(reg(out_reg)), r8 ; xor r8d, eax ; shr r8d, 22 ; mov r8d, eax ; imul eax, r9d, 277803737 ; xor r9d, eax ; shrx r9d, r9d, r8d ; mov r9d, eax ; add r8b, 4 ; shr r8d, 28 ; mov r8d, eax ; add eax, -1403630843 ; imul eax, r9d, 747796405 ; add r9d, eax ; xor r9d, r8d ; shr r9d, 22 ; mov r9d, r8d ; imul r8d, r10d, 277803737 ; xor r10d, r9d ; shrx r10d, r9d, r8d ; mov r10d, r9d ; add r8b, 4 ; shr r8d, 28 ; mov r8d, r9d ; add r9d, -1403630843 ; imul r9d, r8d, 747796405 ; movq r8, Rx(reg(rhs_reg)) ; movq rax, Rx(reg(lhs_reg)) dynasm!(self.0.ops // Ported straight from disassembly fn build_mix(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; addss Rx(reg(out_reg)), xmm0 ; vmovd xmm0, eax ; mov rax, 0xbf800000 // -1.0 ; vmovd Rx(reg(out_reg)), eax ; or eax, 0x3f800000 ; xor eax, r8d ; shr eax, 9 ; shr r8d, 31 ; mov r8d, eax ; imul eax, r9d, 277803737 ; xor r9d, eax ; shrx r9d, r9d, r8d ; mov r9d, eax ; add r8b, 4 ; shr r8d, 28 ; mov r8d, eax ; add eax, -1403630843 ; imul eax, eax, 747796405 ; movq rax, Rx(reg(lhs_reg)) dynasm!(self.0.ops // Ported straight from Godbolt fn build_rand(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; vorpd Rx(reg(out_reg)), xmm1, xmm2 ; vandpd xmm2, xmm2, Rx(reg(lhs_reg)) ; vandpd xmm1, xmm1, Rx(reg(rhs_reg)) ; vxorpd xmm2, xmm1, xmm2 // 1 ^ b = !b, so this inverts xmm1 ; vpcmpeqd xmm2, xmm2, xmm2 ; vbroadcastss xmm1, xmm1 ; vcmpeqss xmm1, Rx(reg(lhs_reg)), xmm1 ; vpxor xmm1, xmm1, xmm1 dynasm!(self.0.ops fn build_or(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vorpd Rx(reg(out_reg)), xmm1, xmm2 ; vandpd xmm2, xmm2, Rx(reg(rhs_reg)) ; vandpd xmm1, xmm1, Rx(reg(lhs_reg)) ; vxorpd xmm2, xmm1, xmm2 // 1 ^ b = !b, so this inverts xmm1 ; vpcmpeqd xmm2, xmm2, xmm2 ; vbroadcastss xmm1, xmm1 ; vcmpeqss xmm1, Rx(reg(lhs_reg)), xmm1 ; vpxor xmm1, xmm1, xmm1 dynasm!(self.0.ops fn build_and(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vandps Rx(reg(out_reg)), xmm1, Rx(reg(i)) ; vcmpeqss xmm1, Rx(reg(arg_reg)), xmm1 ; vpxor xmm1, xmm1, xmm1 dynasm!(self.0.ops let i = self.load_imm(1.0); fn build_not(&mut self, out_reg: u8, arg_reg: u8) { } self.call_fn_binary(out_reg, lhs_reg, rhs_reg, grad_modulo); ad [ � � K B � � � � e < �
�
�
m
C
� � � | Q � � � � � h ? �
�
�
p
F
� � � � ~ Z 6 � � � � � v c N 9 $ � � � � � E ; ' � � � n J I � � � c : � � � i ? � � Q � � � S , � � � � ; vmovups xmm4, [rsp] // Restore gradient registers ; add rsp, 0x20 ; call rdx ; mov rdx, QWORD addr as _ ; sub rsp, 0x20 // make room for the shadow stack ; movsd xmm0, Rx(reg(lhs_reg)) ; vpshufd xmm3, Rx(reg(rhs_reg)), 0b1110 ; movsd xmm2, Rx(reg(rhs_reg)) ; vpshufd xmm1, Rx(reg(lhs_reg)), 0b1110 // arguments if we're using IMM_REG // Note that we load xmm0 last, because it could be one of our // Call the function, packing the gradient into xmm0 + xmm1 ; vmovups [rsp + 0xb0], xmm15 ; vmovups [rsp + 0xa0], xmm14 ; vmovups [rsp + 0x90], xmm13 ; vmovups [rsp + 0x80], xmm12 ; vmovups [rsp + 0x70], xmm11 ; vmovups [rsp + 0x60], xmm10 ; vmovups [rsp + 0x50], xmm9 ; vmovups [rsp + 0x40], xmm8 ; vmovups [rsp + 0x30], xmm7 ; vmovups [rsp + 0x20], xmm6 ; vmovups [rsp + 0x10], xmm5 ; vmovups [rsp], xmm4 // Back up register values to the stack, saving all 128 bits ; mov [rbp - 0x20], rcx ; mov [rbp - 0x18], rdx ; mov [rbp - 0x10], rsi ; mov [rbp - 0x8], rdi // Back up X/Y/Z pointers to the stack dynasm!(self.0.ops println!(); } print!("{:02x}", unsafe { *(addr as *const u8).add(i) }); for i in 0..0x100 { println!("function text"); let addr = f as usize; ) { f: extern "sysv64" fn(Grad, Grad) -> Grad, rhs_reg: u8, lhs_reg: u8, out_reg: u8, &mut self, fn call_fn_binary( } ); ; vpunpcklqdq Rx(reg(out_reg)), xmm0, xmm1 // Collect the 4x floats into the out register ; mov rcx, [rbp - 0x20] ; mov rdx, [rbp - 0x18] ; mov rsi, [rbp - 0x10] ; mov rdi, [rbp - 0x8] // Restore X/Y/Z pointers ; vmovups xmm15, [rsp + 0xb0] ; vmovups xmm14, [rsp + 0xa0] ; vmovups xmm13, [rsp + 0x90] ; vmovups xmm12, [rsp + 0x80] ; vmovups xmm11, [rsp + 0x70] ; vmovups xmm10, [rsp + 0x60] ; vmovups xmm9, [rsp + 0x50] ; vmovups xmm8, [rsp + 0x40] ; vmovups xmm7, [rsp + 0x30] ; vmovups xmm6, [rsp + 0x20] ; vmovups xmm5, [rsp + 0x10] ; vmovups xmm4, [rsp] // Restore gradient registers ; call rdx ; mov rdx, QWORD addr as _ ; vpshufd xmm1, Rx(reg(arg_reg)), 0b1110 ; movsd xmm0, Rx(reg(arg_reg)) // call the function, packing the gradient into xmm0 + xmm1 ; vmovups [rsp + 0xb0], xmm15 ; vmovups [rsp + 0xa0], xmm14 ; vmovups [rsp + 0x90], xmm13 ; vmovups [rsp + 0x80], xmm12 ; vmovups [rsp + 0x70], xmm11 ; vmovups [rsp + 0x60], xmm10 ; vmovups [rsp + 0x50], xmm9 ; vmovups [rsp + 0x40], xmm8 ; vmovups [rsp + 0x30], xmm7 ; vmovups [rsp + 0x20], xmm6 ; vmovups [rsp + 0x10], xmm5 ; vmovups [rsp], xmm4 // Back up register values to the stack, saving all 128 bits ; mov [rbp - 0x20], rcx ; mov [rbp - 0x18], rdx // Back up input / output / pointers to // Back up input / output / counter registers to the stack dynasm!(self.0.ops let addr = f as usize; ad � h � � W 0 � � z o i . �
�
�
J
?
9
8
� � � \ ) � � � � R L �
�
�
�
G
� � � � y @ + ! � � � o Z P � � � E ? � � � x r ; � � � � p U � � � X
� � � ` _ 2 � � � � y Q � � // Fallthrough to end ; vpxor Rx(reg(out_reg)), xmm0, Rx(reg(lhs_reg)) ; vpslld xmm0, xmm0, 31 // set the sign bit ; vpcmpeqw xmm0, xmm0, xmm0 ; N: // negative ; jmp >E ; vmovups Rx(reg(out_reg)), Rx(reg(lhs_reg)) // Fallthrough: non-negative (or NaN) input ; jb >N ; vcomiss Rx(reg(lhs_reg)), xmm0 ; vpxor xmm0, xmm0, xmm0 // Store 0.0 to xmm0, for comparisons dynasm!(self.0.ops fn build_abs(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; vpxor Rx(reg(out_reg)), xmm0, Rx(reg(lhs_reg)) ; vpslld xmm0, xmm0, 31 // set the sign bit ; vpcmpeqw xmm0, xmm0, xmm0 dynasm!(self.0.ops fn build_neg(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; vmovups Rx(reg(out_reg)), Rx(reg(lhs_reg)) dynasm!(self.0.ops fn build_copy(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_ln); } f.ln() extern "sysv64" fn float_ln(f: Grad) -> Grad { fn build_ln(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_exp); } f.exp() extern "sysv64" fn float_exp(f: Grad) -> Grad { fn build_exp(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_atan); } f.atan() extern "sysv64" fn float_atan(f: Grad) -> Grad { fn build_atan(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_acos); } f.acos() extern "sysv64" fn float_acos(f: Grad) -> Grad { fn build_acos(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_asin); } f.asin() extern "sysv64" fn float_asin(f: Grad) -> Grad { fn build_asin(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_tan); } f.tan() extern "sysv64" fn float_tan(f: Grad) -> Grad { fn build_tan(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, float_cos); } f.cos() extern "sysv64" fn float_cos(f: Grad) -> Grad { fn build_cos(&mut self, out_reg: u8, lhs_reg: u8) { } self.call_fn_unary(out_reg, lhs_reg, grad_sin); } v.sin() extern "sysv64" fn grad_sin(v: Grad) -> Grad { fn build_sin(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; vmovups [r8 + rcx], Rx(reg(arg_reg)) ; mov r8, [rsi + pos] // read the *mut float from the array dynasm!(self.0.ops let pos = 8 * i32::try_from(out_index).unwrap(); fn build_output(&mut self, arg_reg: u8, out_index: u32) { } ); ; vmovaps Rx(reg(out_reg)), [r8 + rcx] // offset by array ; mov r8, [rdi + pos] // read the *const float from the array dynasm!(self.0.ops let pos = 8 * i32::try_from(src_arg).unwrap(); fn build_input(&mut self, out_reg: u8, src_arg: u32) { } ); ; vmovups [rsp + sp_offset], Rx(reg(src_reg)) dynasm!(self.0.ops // XXX could we use vmovaps here instead? .unwrap(); .try_into() + STACK_SIZE_LOWER as u32) let sp_offset: i32 = (self.0.stack_pos(dst_mem) assert!((src_reg as usize) < REGISTER_LIMIT); fn build_store(&mut self, dst_mem: u32, src_reg: u8) { ad B � i � � � � � w M 2 � � � � w Z < ; �
�
�
p
o
8
� � � � h ] W � � � o n N �
�
�
�
i
C
B
� � � g f 9 � � � � � i : � � � � � P O � � � � � m R � � � � Q 6 � � t n 4 � � � m A # � � p e _ ^ � � dynasm!(self.0.ops fn build_add(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vroundss Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)), 3 ; vaddss Rx(reg(out_reg)), xmm1, Rx(reg(lhs_reg)) ; vorps xmm1, xmm1, xmm2 ; vmovd xmm2, eax ; mov eax, 0x3effffffu32 as i32 ; vandps xmm1, xmm1, Rx(reg(lhs_reg)) ; vmovd xmm1, eax ; mov eax, 0x80000000u32 as i32 dynasm!(self.0.ops // Shenanigans figured through Godbolt fn build_round(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; movss Rx(reg(out_reg)), xmm1 ; vpxor Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)) ; vroundss xmm1, Rx(reg(lhs_reg)), Rx(reg(lhs_reg)), 2 dynasm!(self.0.ops fn build_ceil(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; movss Rx(reg(out_reg)), xmm1 ; vpxor Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)) ; vroundss xmm1, Rx(reg(lhs_reg)), Rx(reg(lhs_reg)), 1 dynasm!(self.0.ops fn build_floor(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; vmulps Rx(reg(out_reg)), xmm0, Rx(reg(lhs_reg)) ; vmulps xmm0, xmm0, xmm1 ; vbroadcastss xmm0, Rx(reg(lhs_reg)) // At this point, xmm1 contains [1, 2, 2, 2] ; movss xmm1, xmm0 ; vmovd xmm0, eax ; mov eax, 1.0f32.to_bits() as i32 ; vbroadcastss xmm1, xmm1 ; vmovd xmm1, eax ; mov eax, 2.0f32.to_bits() as i32 dynasm!(self.0.ops // d/dx f(x)**2 = 2 * f(x) * f'(x) fn build_square(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; vmovss Rx(reg(out_reg)), Rx(reg(out_reg)), xmm1 ; vmovups Rx(reg(out_reg)), xmm0 ; vsqrtss xmm1, xmm1, Rx(reg(lhs_reg)) // Compute the actual square root into xmm1 ; vdivps xmm0, Rx(reg(lhs_reg)), xmm0 // Set every element in xmm0 to -f'(x) / f(x)**2 ; vbroadcastss xmm0, xmm0 // Set every element in xmm0 to 2 * sqrt(f(x)) ; vmulss xmm0, xmm0, xmm1 ; vmovd xmm1, eax ; mov eax, 2.0f32.to_bits() as i32 // TODO is this the best way to make 2.0? // Multiply it by 2 ; vsqrtss xmm0, xmm0, Rx(reg(lhs_reg)) // Calculate xmm0[0] = f(x)**2 dynasm!(self.0.ops // d/dx sqrt(f(x)) = f'(x) / (2 * sqrt(f(x))) fn build_sqrt(&mut self, out_reg: u8, lhs_reg: u8) { } ); ; movss Rx(reg(out_reg)), xmm1 ; vmovups Rx(reg(out_reg)), xmm0 ; divss xmm1, Rx(reg(lhs_reg)) ; movd xmm1, eax ; mov eax, 1.0f32.to_bits() as i32 // Compute the actual reciprocal into xmm1 ; vdivps xmm0, Rx(reg(lhs_reg)), xmm0 // Set every element in xmm0 to -f'(x) / f(x)**2 ; vbroadcastss xmm0, xmm0 // Set every element in xmm0 to -f(x)**2 ; pxor xmm0, xmm1 ; movd xmm1, eax ; mov eax, 0x80000000u32 as i32 // Negate it ; mulss xmm0, xmm0 ; movss xmm0, Rx(reg(lhs_reg)) // Calculate xmm0[0] = f(x)**2 dynasm!(self.0.ops // d/dx 1/f(x) = -f'(x) / f(x)**2 fn build_recip(&mut self, out_reg: u8, lhs_reg: u8) { } self.0.ops.commit_local().unwrap(); ); ; E: ad � k � � � ` E � � � � o T " �
�
�
f
e
'
� � � � f $ � � � � l : �
�
�
�
d
>
=
� � � � Y
� � i _ � � { @ , + � � � � g � � � � � � W Q � � } i h M � � � � X C B 1 � � � � � � � I � � � } lhs.rem_euclid(rhs) extern "sysv64" fn grad_modulo(lhs: Grad, rhs: Grad) -> Grad { fn build_mod(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } self.0.ops.commit_local().unwrap(); ); ; O: // Fallthrough ; vmovups Rx(reg(out_reg)), Rx(reg(rhs_reg)) ; R: ; jmp >O ; vcmpeqss Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)) ; vpxor Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)) ; N: ; jmp >O ; vmovups Rx(reg(out_reg)), Rx(reg(lhs_reg)) // Fallthrough ; ja >R ; jp >N // Parity flag is set if result is NAN ; vcomiss Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) dynasm!(self.0.ops fn build_min(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } self.0.ops.commit_local().unwrap(); ); ; E: // Fallthrough ; vmovups Rx(reg(out_reg)), Rx(reg(lhs_reg)) ; L: ; jmp >E ; vcmpeqss Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)) ; vpxor Rx(reg(out_reg)), Rx(reg(out_reg)), Rx(reg(out_reg)) ; N: ; jmp >E ; vmovups Rx(reg(out_reg)), Rx(reg(rhs_reg)) // Fallthrough ; ja >L ; jp >N // Parity flag is set if result is NAN ; vcomiss Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) dynasm!(self.0.ops fn build_max(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } self.call_fn_binary(out_reg, lhs_reg, rhs_reg, grad_atan2); } y.atan2(x) extern "sysv64" fn grad_atan2(y: Grad, x: Grad) -> Grad { fn build_atan2(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vmovss Rx(reg(out_reg)), Rx(reg(out_reg)), xmm2 ; vmovups Rx(reg(out_reg)), xmm1 ; vdivss xmm2, Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) // Patch in the actual division result ; vdivps xmm1, xmm1, xmm2 // Do the division ; vbroadcastss xmm2, xmm2 ; vmulss xmm2, Rx(reg(rhs_reg)), Rx(reg(rhs_reg)) // g(x)**2 ; vsubps xmm1, xmm2, xmm1 // f'(x)*g(x) - f(x)*g'(x) ; vmulps xmm2, xmm2, Rx(reg(lhs_reg)) ; vbroadcastss xmm2, Rx(reg(rhs_reg)) // g(x) * f'(x) ; vmulps xmm1, xmm1, Rx(reg(rhs_reg)) ; vbroadcastss xmm1, Rx(reg(lhs_reg)) // f(x) * g'(x) dynasm!(self.0.ops // d/dx f(x) * g(x) = (f'(x)*g(x) - f(x)*g'(x)) / g(x)**2 fn build_div(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vmovss Rx(reg(out_reg)), Rx(reg(out_reg)), xmm2 ; vmovups Rx(reg(out_reg)), xmm1 ; vmulss xmm2, Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) ; vaddps xmm1, xmm1, xmm2 ; vmulps xmm2, xmm2, Rx(reg(lhs_reg)) ; vbroadcastss xmm2, Rx(reg(rhs_reg)) ; vmulps xmm1, xmm1, Rx(reg(rhs_reg)) ; vbroadcastss xmm1, Rx(reg(lhs_reg)) dynasm!(self.0.ops // d/dx f(x) * g(x) = f'(x)*g(x) + f(x)*g'(x) fn build_mul(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vsubps Rx(reg(out_reg)), Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) dynasm!(self.0.ops fn build_sub(&mut self, out_reg: u8, lhs_reg: u8, rhs_reg: u8) { } ); ; vaddps Rx(reg(out_reg)), Rx(reg(lhs_reg)), Rx(reg(rhs_reg)) ad p b � � � q p ' � � � a 8 �
�
�
f
<
;
� � � l U T * � � � d ; �
�
�
i
?
>
� � � v u : � � � � � � � � U M . � � � � p = � � � � d B � � � u K ! � � � y x 0 � � � U � � � y ] \ 2 ; vmovups xmm4, [rsp] // Restore gradient registers ; add rsp, 0x20 ; call rdx ; mov rdx, QWORD addr as _ ; sub rsp, 0x20 // make room for the shadow stack ; movsd xmm0, Rx(reg(lhs_reg)) ; vpshufd xmm3, Rx(reg(rhs_reg)), 0b1110 ; movsd xmm2, Rx(reg(rhs_reg)) ; vpshufd xmm1, Rx(reg(lhs_reg)), 0b1110 // arguments if we're using IMM_REG // Note that we load xmm0 last, because it could be one of our // Call the function, packing the gradient into xmm0 + xmm1 ; vmovups [rsp + 0xb0], xmm15 ; vmovups [rsp + 0xa0], xmm14 ; vmovups [rsp + 0x90], xmm13 ; vmovups [rsp + 0x80], xmm12 ; vmovups [rsp + 0x70], xmm11 ; vmovups [rsp + 0x60], xmm10 ; vmovups [rsp + 0x50], xmm9 ; vmovups [rsp + 0x40], xmm8 ; vmovups [rsp + 0x30], xmm7 ; vmovups [rsp + 0x20], xmm6 ; vmovups [rsp + 0x10], xmm5 ; vmovups [rsp], xmm4 // Back up register values to the stack, saving all 128 bits ; mov [rbp - 0x20], rcx ; mov [rbp - 0x18], rdx ; mov [rbp - 0x10], rsi ; mov [rbp - 0x8], rdi // Back up X/Y/Z pointers to the stack dynasm!(self.0.ops println!(); } print!("{:02x}", unsafe { *(addr as *const u8).add(i) }); for i in 0..0x100 { println!("function text"); let addr = f as usize; ) { f: extern "sysv64" fn(Grad, Grad) -> Grad, rhs_reg: u8, lhs_reg: u8, out_reg: u8, &mut self, fn call_fn_binary( } ); ; vpunpcklqdq Rx(reg(out_reg)), xmm0, xmm1 // Collect the 4x floats into the out register ; mov rcx, [rbp - 0x20] ; mov rdx, [rbp - 0x18] ; mov rsi, [rbp - 0x10] ; mov rdi, [rbp - 0x8] // Restore input / output / counter pointers ; vmovups xmm15, [rsp + 0xb0] ; vmovups xmm14, [rsp + 0xa0] ; vmovups xmm13, [rsp + 0x90] ; vmovups xmm12, [rsp + 0x80] ; vmovups xmm11, [rsp + 0x70] ; vmovups xmm10, [rsp + 0x60] ; vmovups xmm9, [rsp + 0x50] ; vmovups xmm8, [rsp + 0x40] ; vmovups xmm7, [rsp + 0x30] ; vmovups xmm6, [rsp + 0x20] ; vmovups xmm5, [rsp + 0x10] ; vmovups xmm4, [rsp] // Restore gradient registers ; call rdx ; mov rdx, QWORD addr as _ ; vpshufd xmm1, Rx(reg(arg_reg)), 0b1110 ; movsd xmm0, Rx(reg(arg_reg)) // call the function, packing the gradient into xmm0 + xmm1 ; vmovups [rsp + 0xb0], xmm15 ; vmovups [rsp + 0xa0], xmm14 ; vmovups [rsp + 0x90], xmm13 ; vmovups [rsp + 0x80], xmm12 ; vmovups [rsp + 0x70], xmm11 ; vmovups [rsp + 0x60], xmm10 ; vmovups [rsp + 0x50], xmm9 ; vmovups [rsp + 0x40], xmm8 ; vmovups [rsp + 0x30], xmm7 ; vmovups [rsp + 0x20], xmm6 ; vmovups [rsp + 0x10], xmm5 ; vmovups [rsp], xmm4 // Back up register values to the stack, saving all 128 bits ; mov [rbp - 0x20], rcx ; mov [rbp - 0x18], rdx ; mov [rbp - 0x10], rsi ; mov [rbp - 0x8], rdi