# Copyright (c) 2018-2019, tevador <tevador@gmail.com>
# Copyright (c) 2019, SChernykh <https://github.com/SChernykh>
#
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of the copyright holder nor the
# names of its contributors may be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#if defined(__APPLE__)
#define DECL(x) _##x
#else
#define DECL(x) x
#endif
.arch armv8-a
.text
.global DECL(randomx_program_aarch64)
.global DECL(randomx_program_aarch64_main_loop)
.global DECL(randomx_program_aarch64_vm_instructions)
.global DECL(randomx_program_aarch64_imul_rcp_literals_end)
.global DECL(randomx_program_aarch64_vm_instructions_end)
.global DECL(randomx_program_aarch64_cacheline_align_mask1)
.global DECL(randomx_program_aarch64_cacheline_align_mask2)
.global DECL(randomx_program_aarch64_update_spMix1)
.global DECL(randomx_program_aarch64_vm_instructions_end_light)
.global DECL(randomx_program_aarch64_light_cacheline_align_mask)
.global DECL(randomx_program_aarch64_light_dataset_offset)
.global DECL(randomx_init_dataset_aarch64)
.global DECL(randomx_init_dataset_aarch64_end)
.global DECL(randomx_calc_dataset_item_aarch64)
.global DECL(randomx_calc_dataset_item_aarch64_prefetch)
.global DECL(randomx_calc_dataset_item_aarch64_mix)
.global DECL(randomx_calc_dataset_item_aarch64_store_result)
.global DECL(randomx_calc_dataset_item_aarch64_end)
#include "configuration.h"
# Register allocation
# x0 -> pointer to reg buffer and then literal for IMUL_RCP
# x1 -> pointer to mem buffer and then to dataset
# x2 -> pointer to scratchpad
# x3 -> loop counter
# x4 -> "r0"
# x5 -> "r1"
# x6 -> "r2"
# x7 -> "r3"
# x8 -> fpcr (reversed bits)
# x9 -> mx, ma
# x10 -> spMix1
# x11 -> literal for IMUL_RCP
# x12 -> "r4"
# x13 -> "r5"
# x14 -> "r6"
# x15 -> "r7"
# x16 -> spAddr0
# x17 -> spAddr1
# x18 -> temporary
# x19 -> temporary
# x20 -> literal for IMUL_RCP
# x21 -> literal for IMUL_RCP
# x22 -> literal for IMUL_RCP
# x23 -> literal for IMUL_RCP
# x24 -> literal for IMUL_RCP
# x25 -> literal for IMUL_RCP
# x26 -> literal for IMUL_RCP
# x27 -> literal for IMUL_RCP
# x28 -> literal for IMUL_RCP
# x29 -> literal for IMUL_RCP
# x30 -> literal for IMUL_RCP
# v0-v15 -> store 32-bit literals
# v16 -> "f0"
# v17 -> "f1"
# v18 -> "f2"
# v19 -> "f3"
# v20 -> "e0"
# v21 -> "e1"
# v22 -> "e2"
# v23 -> "e3"
# v24 -> "a0"
# v25 -> "a1"
# v26 -> "a2"
# v27 -> "a3"
# v28 -> temporary
# v29 -> E 'and' mask = 0x00ffffffffffffff00ffffffffffffff
# v30 -> E 'or' mask = 0x3*00000000******3*00000000******
# v31 -> scale mask = 0x81f000000000000081f0000000000000
.balign 4
DECL(randomx_program_aarch64):
# Save callee-saved registers
sub sp, sp, 192
stp x16, x17, [sp]
stp x18, x19, [sp, 16]
stp x20, x21, [sp, 32]
stp x22, x23, [sp, 48]
stp x24, x25, [sp, 64]
stp x26, x27, [sp, 80]
stp x28, x29, [sp, 96]
stp x8, x30, [sp, 112]
stp d8, d9, [sp, 128]
stp d10, d11, [sp, 144]
stp d12, d13, [sp, 160]
stp d14, d15, [sp, 176]
# Zero integer registers
mov x4, xzr
mov x5, xzr
mov x6, xzr
mov x7, xzr
mov x12, xzr
mov x13, xzr
mov x14, xzr
mov x15, xzr
# Load ma, mx and dataset pointer
ldp x9, x1, [x1]
# Load initial spMix value
mov x10, x9
# Load group A registers
ldp q24, q25, [x0, 192]
ldp q26, q27, [x0, 224]
# Load E 'and' mask
mov x16, 0x00FFFFFFFFFFFFFF
ins v29.d[0], x16
ins v29.d[1], x16
# Load E 'or' mask (stored in reg.f[0])
ldr q30, [x0, 64]
# Load scale mask
mov x16, 0x80f0000000000000
ins v31.d[0], x16
ins v31.d[1], x16
# Read fpcr
mrs x8, fpcr
rbit x8, x8
# Save x0
str x0, [sp, -16]!
# Read literals
ldr x0, literal_x0
ldr x11, literal_x11
ldr x20, literal_x20
ldr x21, literal_x21
ldr x22, literal_x22
ldr x23, literal_x23
ldr x24, literal_x24
ldr x25, literal_x25
ldr x26, literal_x26
ldr x27, literal_x27
ldr x28, literal_x28
ldr x29, literal_x29
ldr x30, literal_x30
ldr q0, literal_v0
ldr q1, literal_v1
ldr q2, literal_v2
ldr q3, literal_v3
ldr q4, literal_v4
ldr q5, literal_v5
ldr q6, literal_v6
ldr q7, literal_v7
ldr q8, literal_v8
ldr q9, literal_v9
ldr q10, literal_v10
ldr q11, literal_v11
ldr q12, literal_v12
ldr q13, literal_v13
ldr q14, literal_v14
ldr q15, literal_v15
DECL(randomx_program_aarch64_main_loop):
# spAddr0 = spMix1 & ScratchpadL3Mask64;
# spAddr1 = (spMix1 >> 32) & ScratchpadL3Mask64;
lsr x18, x10, 32
# Actual mask will be inserted by JIT compiler
and w16, w10, 1
and w17, w18, 1
# x16 = scratchpad + spAddr0
# x17 = scratchpad + spAddr1
add x16, x16, x2
add x17, x17, x2
# xor integer registers with scratchpad data (spAddr0)
ldp x18, x19, [x16]
eor x4, x4, x18
eor x5, x5, x19
ldp x18, x19, [x16, 16]
eor x6, x6, x18
eor x7, x7, x19
ldp x18, x19, [x16, 32]
eor x12, x12, x18
eor x13, x13, x19
ldp x18, x19, [x16, 48]
eor x14, x14, x18
eor x15, x15, x19
# Load group F registers (spAddr1)
ldpsw x18, x19, [x17]
ins v16.d[0], x18
ins v16.d[1], x19
ldpsw x18, x19, [x17, 8]
ins v17.d[0], x18
ins v17.d[1], x19
ldpsw x18, x19, [x17, 16]
ins v18.d[0], x18
ins v18.d[1], x19
ldpsw x18, x19, [x17, 24]
ins v19.d[0], x18
ins v19.d[1], x19
scvtf v16.2d, v16.2d
scvtf v17.2d, v17.2d
scvtf v18.2d, v18.2d
scvtf v19.2d, v19.2d
# Load group E registers (spAddr1)
ldpsw x18, x19, [x17, 32]
ins v20.d[0], x18
ins v20.d[1], x19
ldpsw x18, x19, [x17, 40]
ins v21.d[0], x18
ins v21.d[1], x19
ldpsw x18, x19, [x17, 48]
ins v22.d[0], x18
ins v22.d[1], x19
ldpsw x18, x19, [x17, 56]
ins v23.d[0], x18
ins v23.d[1], x19
scvtf v20.2d, v20.2d
scvtf v21.2d, v21.2d
scvtf v22.2d, v22.2d
scvtf v23.2d, v23.2d
and v20.16b, v20.16b, v29.16b
and v21.16b, v21.16b, v29.16b
and v22.16b, v22.16b, v29.16b
and v23.16b, v23.16b, v29.16b
orr v20.16b, v20.16b, v30.16b
orr v21.16b, v21.16b, v30.16b
orr v22.16b, v22.16b, v30.16b
orr v23.16b, v23.16b, v30.16b
# Execute VM instructions
DECL(randomx_program_aarch64_vm_instructions):
# buffer for generated instructions
# FDIV_M is the largest instruction taking up to 12 ARMv8 instructions
.fill RANDOMX_PROGRAM_SIZE*12,4,0
literal_x0: .fill 1,8,0
literal_x11: .fill 1,8,0
literal_x20: .fill 1,8,0
literal_x21: .fill 1,8,0
literal_x22: .fill 1,8,0
literal_x23: .fill 1,8,0
literal_x24: .fill 1,8,0
literal_x25: .fill 1,8,0
literal_x26: .fill 1,8,0
literal_x27: .fill 1,8,0
literal_x28: .fill 1,8,0
literal_x29: .fill 1,8,0
literal_x30: .fill 1,8,0
DECL(randomx_program_aarch64_imul_rcp_literals_end):
literal_v0: .fill 2,8,0
literal_v1: .fill 2,8,0
literal_v2: .fill 2,8,0
literal_v3: .fill 2,8,0
literal_v4: .fill 2,8,0
literal_v5: .fill 2,8,0
literal_v6: .fill 2,8,0
literal_v7: .fill 2,8,0
literal_v8: .fill 2,8,0
literal_v9: .fill 2,8,0
literal_v10: .fill 2,8,0
literal_v11: .fill 2,8,0
literal_v12: .fill 2,8,0
literal_v13: .fill 2,8,0
literal_v14: .fill 2,8,0
literal_v15: .fill 2,8,0
DECL(randomx_program_aarch64_vm_instructions_end):
# mx ^= r[readReg2] ^ r[readReg3];
eor x9, x9, x18
# Calculate dataset pointer for dataset prefetch
mov w18, w9
DECL(randomx_program_aarch64_cacheline_align_mask1):
# Actual mask will be inserted by JIT compiler
and x18, x18, 1
add x18, x18, x1
# Prefetch dataset data
prfm pldl2strm, [x18]
# mx <-> ma
ror x9, x9, 32
# Calculate dataset pointer for dataset read
mov w10, w9
DECL(randomx_program_aarch64_cacheline_align_mask2):
# Actual mask will be inserted by JIT compiler
and x10, x10, 1
add x10, x10, x1
DECL(randomx_program_aarch64_xor_with_dataset_line):
# xor integer registers with dataset data
ldp x18, x19, [x10]
eor x4, x4, x18
eor x5, x5, x19
ldp x18, x19, [x10, 16]
eor x6, x6, x18
eor x7, x7, x19
ldp x18, x19, [x10, 32]
eor x12, x12, x18
eor x13, x13, x19
ldp x18, x19, [x10, 48]
eor x14, x14, x18
eor x15, x15, x19
DECL(randomx_program_aarch64_update_spMix1):
# JIT compiler will replace it with "eor x10, config.readReg0, config.readReg1"
eor x10, x0, x0
# Store integer registers to scratchpad (spAddr1)
stp x4, x5, [x17, 0]
stp x6, x7, [x17, 16]
stp x12, x13, [x17, 32]
stp x14, x15, [x17, 48]
# xor group F and group E registers
eor v16.16b, v16.16b, v20.16b
eor v17.16b, v17.16b, v21.16b
eor v18.16b, v18.16b, v22.16b
eor v19.16b, v19.16b, v23.16b
# Store FP registers to scratchpad (spAddr0)
stp q16, q17, [x16, 0]
stp q18, q19, [x16, 32]
subs x3, x3, 1
bne DECL(randomx_program_aarch64_main_loop)
# Restore x0
ldr x0, [sp], 16
# Store integer registers
stp x4, x5, [x0, 0]
stp x6, x7, [x0, 16]
stp x12, x13, [x0, 32]
stp x14, x15, [x0, 48]
# Store FP registers
stp q16, q17, [x0, 64]
stp q18, q19, [x0, 96]
stp q20, q21, [x0, 128]
stp q22, q23, [x0, 160]
# Restore callee-saved registers
ldp x16, x17, [sp]
ldp x18, x19, [sp, 16]
ldp x20, x21, [sp, 32]
ldp x22, x23, [sp, 48]
ldp x24, x25, [sp, 64]
ldp x26, x27, [sp, 80]
ldp x28, x29, [sp, 96]
ldp x8, x30, [sp, 112]
ldp d8, d9, [sp, 128]
ldp d10, d11, [sp, 144]
ldp d12, d13, [sp, 160]
ldp d14, d15, [sp, 176]
add sp, sp, 192
ret
DECL(randomx_program_aarch64_vm_instructions_end_light):
sub sp, sp, 96
stp x0, x1, [sp, 64]
stp x2, x30, [sp, 80]
# mx ^= r[readReg2] ^ r[readReg3];
eor x9, x9, x18
# mx <-> ma
ror x9, x9, 32
# x0 -> pointer to cache memory
mov x0, x1
# x1 -> pointer to output
mov x1, sp
DECL(randomx_program_aarch64_light_cacheline_align_mask):
# Actual mask will be inserted by JIT compiler
and w2, w9, 1
# x2 -> item number
lsr x2, x2, 6
DECL(randomx_program_aarch64_light_dataset_offset):
# Apply dataset offset (filled in by JIT compiler)
add x2, x2, 0
add x2, x2, 0
bl DECL(randomx_calc_dataset_item_aarch64)
mov x10, sp
ldp x0, x1, [sp, 64]
ldp x2, x30, [sp, 80]
add sp, sp, 96
b DECL(randomx_program_aarch64_xor_with_dataset_line)
# Input parameters
#
# x0 -> pointer to cache
# x1 -> pointer to dataset memory at startItem
# x2 -> start item
# x3 -> end item
DECL(randomx_init_dataset_aarch64):
# Save x30 (return address)
str x30, [sp, -16]!
# Load pointer to cache memory
ldr x0, [x0]
DECL(randomx_init_dataset_aarch64_main_loop):
bl DECL(randomx_calc_dataset_item_aarch64)
add x1, x1, 64
add x2, x2, 1
cmp x2, x3
bne DECL(randomx_init_dataset_aarch64_main_loop)
# Restore x30 (return address)
ldr x30, [sp], 16
ret
DECL(randomx_init_dataset_aarch64_end):
# Input parameters
#
# x0 -> pointer to cache memory
# x1 -> pointer to output
# x2 -> item number
#
# Register allocation
#
# x0-x7 -> output value (calculated dataset item)
# x8 -> pointer to cache memory
# x9 -> pointer to output
# x10 -> registerValue
# x11 -> mixBlock
# x12 -> temporary
# x13 -> temporary
DECL(randomx_calc_dataset_item_aarch64):
sub sp, sp, 112
stp x0, x1, [sp]
stp x2, x3, [sp, 16]
stp x4, x5, [sp, 32]
stp x6, x7, [sp, 48]
stp x8, x9, [sp, 64]
stp x10, x11, [sp, 80]
stp x12, x13, [sp, 96]
ldr x12, superscalarMul0
mov x8, x0
mov x9, x1
mov x10, x2
# rl[0] = (itemNumber + 1) * superscalarMul0;
madd x0, x2, x12, x12
# rl[1] = rl[0] ^ superscalarAdd1;
ldr x12, superscalarAdd1
eor x1, x0, x12
# rl[2] = rl[0] ^ superscalarAdd2;
ldr x12, superscalarAdd2
eor x2, x0, x12
# rl[3] = rl[0] ^ superscalarAdd3;
ldr x12, superscalarAdd3
eor x3, x0, x12
# rl[4] = rl[0] ^ superscalarAdd4;
ldr x12, superscalarAdd4
eor x4, x0, x12
# rl[5] = rl[0] ^ superscalarAdd5;
ldr x12, superscalarAdd5
eor x5, x0, x12
# rl[6] = rl[0] ^ superscalarAdd6;
ldr x12, superscalarAdd6
eor x6, x0, x12
# rl[7] = rl[0] ^ superscalarAdd7;
ldr x12, superscalarAdd7
eor x7, x0, x12
b DECL(randomx_calc_dataset_item_aarch64_prefetch)
superscalarMul0: .quad 6364136223846793005
superscalarAdd1: .quad 9298411001130361340
superscalarAdd2: .quad 12065312585734608966
superscalarAdd3: .quad 9306329213124626780
superscalarAdd4: .quad 5281919268842080866
superscalarAdd5: .quad 10536153434571861004
superscalarAdd6: .quad 3398623926847679864
superscalarAdd7: .quad 9549104520008361294
# Prefetch -> SuperScalar hash -> Mix will be repeated N times
DECL(randomx_calc_dataset_item_aarch64_prefetch):
# Actual mask will be inserted by JIT compiler
and x11, x10, 1
add x11, x8, x11, lsl 6
prfm pldl2strm, [x11]
# Generated SuperScalar hash program goes here
DECL(randomx_calc_dataset_item_aarch64_mix):
ldp x12, x13, [x11]
eor x0, x0, x12
eor x1, x1, x13
ldp x12, x13, [x11, 16]
eor x2, x2, x12
eor x3, x3, x13
ldp x12, x13, [x11, 32]
eor x4, x4, x12
eor x5, x5, x13
ldp x12, x13, [x11, 48]
eor x6, x6, x12
eor x7, x7, x13
DECL(randomx_calc_dataset_item_aarch64_store_result):
stp x0, x1, [x9]
stp x2, x3, [x9, 16]
stp x4, x5, [x9, 32]
stp x6, x7, [x9, 48]
ldp x0, x1, [sp]
ldp x2, x3, [sp, 16]
ldp x4, x5, [sp, 32]
ldp x6, x7, [sp, 48]
ldp x8, x9, [sp, 64]
ldp x10, x11, [sp, 80]
ldp x12, x13, [sp, 96]
add sp, sp, 112
ret
DECL(randomx_calc_dataset_item_aarch64_end):