;; Instruction formats.
(type MInst
(enum
;; A no-op of zero size.
(Nop0)
;; A no-op that is one instruction large.
(Nop4)
;; An ALU operation with two register sources and a register destination.
(AluRRR
(alu_op ALUOp)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(rm Reg))
;; An ALU operation with three register sources and a register destination.
(AluRRRR
(alu_op ALUOp3)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(rm Reg)
(ra Reg))
;; An ALU operation with a register source and an immediate-12 source, and a register
;; destination.
(AluRRImm12
(alu_op ALUOp)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(imm12 Imm12))
;; An ALU operation with a register source and an immediate-logic source, and a register destination.
(AluRRImmLogic
(alu_op ALUOp)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(imml ImmLogic))
;; An ALU operation with a register source and an immediate-shiftamt source, and a register destination.
(AluRRImmShift
(alu_op ALUOp)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(immshift ImmShift))
;; An ALU operation with two register sources, one of which can be shifted, and a register
;; destination.
(AluRRRShift
(alu_op ALUOp)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(rm Reg)
(shiftop ShiftOpAndAmt))
;; An ALU operation with two register sources, one of which can be {zero,sign}-extended and
;; shifted, and a register destination.
(AluRRRExtend
(alu_op ALUOp)
(size OperandSize)
(rd WritableReg)
(rn Reg)
(rm Reg)
(extendop ExtendOp))
;; A bit op instruction with a single register source.
(BitRR
(op BitOp)
(size OperandSize)
(rd WritableReg)
(rn Reg))
;; An unsigned (zero-extending) 8-bit load.
(ULoad8
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; A signed (sign-extending) 8-bit load.
(SLoad8
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; An unsigned (zero-extending) 16-bit load.
(ULoad16
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; A signed (sign-extending) 16-bit load.
(SLoad16
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; An unsigned (zero-extending) 32-bit load.
(ULoad32
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; A signed (sign-extending) 32-bit load.
(SLoad32
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; A 64-bit load.
(ULoad64
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; An 8-bit store.
(Store8
(rd Reg)
(mem AMode)
(flags MemFlags))
;; A 16-bit store.
(Store16
(rd Reg)
(mem AMode)
(flags MemFlags))
;; A 32-bit store.
(Store32
(rd Reg)
(mem AMode)
(flags MemFlags))
;; A 64-bit store.
(Store64
(rd Reg)
(mem AMode)
(flags MemFlags))
;; A store of a pair of registers.
(StoreP64
(rt Reg)
(rt2 Reg)
(mem PairAMode)
(flags MemFlags))
;; A load of a pair of registers.
(LoadP64
(rt WritableReg)
(rt2 WritableReg)
(mem PairAMode)
(flags MemFlags))
;; A MOV instruction. These are encoded as ORR's (AluRRR form).
;; The 32-bit version zeroes the top 32 bits of the
;; destination, which is effectively an alias for an unsigned
;; 32-to-64-bit extension.
(Mov
(size OperandSize)
(rd WritableReg)
(rm Reg))
;; Like `Move` but with a particular `PReg` source (for implementing CLIF
;; instructions like `get_stack_pointer`).
(MovPReg
(rd WritableReg)
(rm PReg))
;; A MOV[Z,N] with a 16-bit immediate.
(MovWide
(op MoveWideOp)
(rd WritableReg)
(imm MoveWideConst)
(size OperandSize))
;; A MOVK with a 16-bit immediate. Modifies its register; we
;; model this with a seprate input `rn` and output `rd` virtual
;; register, with a regalloc constraint to tie them together.
(MovK
(rd WritableReg)
(rn Reg)
(imm MoveWideConst)
(size OperandSize))
;; A sign- or zero-extend operation.
(Extend
(rd WritableReg)
(rn Reg)
(signed bool)
(from_bits u8)
(to_bits u8))
;; A conditional-select operation.
(CSel
(rd WritableReg)
(cond Cond)
(rn Reg)
(rm Reg))
;; A conditional-select negation operation.
(CSNeg
(rd WritableReg)
(cond Cond)
(rn Reg)
(rm Reg))
;; A conditional-set operation.
(CSet
(rd WritableReg)
(cond Cond))
;; A conditional-set-mask operation.
(CSetm
(rd WritableReg)
(cond Cond))
;; A conditional comparison with a second register.
(CCmp
(size OperandSize)
(rn Reg)
(rm Reg)
(nzcv NZCV)
(cond Cond))
;; A conditional comparison with an immediate.
(CCmpImm
(size OperandSize)
(rn Reg)
(imm UImm5)
(nzcv NZCV)
(cond Cond))
;; A synthetic insn, which is a load-linked store-conditional loop, that has the overall
;; effect of atomically modifying a memory location in a particular way. Because we have
;; no way to explain to the regalloc about earlyclobber registers, this instruction has
;; completely fixed operand registers, and we rely on the RA's coalescing to remove copies
;; in the surrounding code to the extent it can. Load- and store-exclusive instructions,
;; with acquire-release semantics, are used to access memory. The operand conventions are:
;;
;; x25 (rd) address
;; x26 (rd) second operand for `op`
;; x27 (wr) old value
;; x24 (wr) scratch reg; value afterwards has no meaning
;; x28 (wr) scratch reg; value afterwards has no meaning
(AtomicRMWLoop
(ty Type) ;; I8, I16, I32 or I64
(op AtomicRMWLoopOp)
(flags MemFlags)
(addr Reg)
(operand Reg)
(oldval WritableReg)
(scratch1 WritableReg)
(scratch2 WritableReg))
;; Similar to AtomicRMWLoop, a compare-and-swap operation implemented using a load-linked
;; store-conditional loop, with acquire-release semantics.
;; Note that the operand conventions, although very similar to AtomicRMWLoop, are different:
;;
;; x25 (rd) address
;; x26 (rd) expected value
;; x28 (rd) replacement value
;; x27 (wr) old value
;; x24 (wr) scratch reg; value afterwards has no meaning
(AtomicCASLoop
(ty Type) ;; I8, I16, I32 or I64
(flags MemFlags)
(addr Reg)
(expected Reg)
(replacement Reg)
(oldval WritableReg)
(scratch WritableReg))
;; An atomic read-modify-write operation. These instructions require the
;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
;; acquire-release semantics.
(AtomicRMW
(op AtomicRMWOp)
(rs Reg)
(rt WritableReg)
(rn Reg)
(ty Type)
(flags MemFlags))
;; An atomic compare-and-swap operation. These instructions require the
;; Large System Extension (LSE) ISA support (FEAT_LSE). The instructions have
;; acquire-release semantics.
(AtomicCAS
;; `rd` is really `rs` in the encoded instruction (so `rd` == `rs`); we separate
;; them here to have separate use and def vregs for regalloc.
(rd WritableReg)
(rs Reg)
(rt Reg)
(rn Reg)
(ty Type)
(flags MemFlags))
;; Read `access_ty` bits from address `rt`, either 8, 16, 32 or 64-bits, and put
;; it in `rn`, optionally zero-extending to fill a word or double word result.
;; This instruction is sequentially consistent.
(LoadAcquire
(access_ty Type) ;; I8, I16, I32 or I64
(rt WritableReg)
(rn Reg)
(flags MemFlags))
;; Write the lowest `ty` bits of `rt` to address `rn`.
;; This instruction is sequentially consistent.
(StoreRelease
(access_ty Type) ;; I8, I16, I32 or I64
(rt Reg)
(rn Reg)
(flags MemFlags))
;; A memory fence. This must provide ordering to ensure that, at a minimum, neither loads
;; nor stores may move forwards or backwards across the fence. Currently emitted as "dmb
;; ish". This instruction is sequentially consistent.
(Fence)
;; Consumption of speculative data barrier.
(Csdb)
;; FPU move. Note that this is distinct from a vector-register
;; move; moving just 64 bits seems to be significantly faster.
(FpuMove64
(rd WritableReg)
(rn Reg))
;; Vector register move.
(FpuMove128
(rd WritableReg)
(rn Reg))
;; Move to scalar from a vector element.
(FpuMoveFromVec
(rd WritableReg)
(rn Reg)
(idx u8)
(size VectorSize))
;; Zero-extend a SIMD & FP scalar to the full width of a vector register.
;; 16-bit scalars require half-precision floating-point support (FEAT_FP16).
(FpuExtend
(rd WritableReg)
(rn Reg)
(size ScalarSize))
;; 1-op FPU instruction.
(FpuRR
(fpu_op FPUOp1)
(size ScalarSize)
(rd WritableReg)
(rn Reg))
;; 2-op FPU instruction.
(FpuRRR
(fpu_op FPUOp2)
(size ScalarSize)
(rd WritableReg)
(rn Reg)
(rm Reg))
(FpuRRI
(fpu_op FPUOpRI)
(rd WritableReg)
(rn Reg))
;; Variant of FpuRRI that modifies its `rd`, and so we name the
;; input state `ri` (for "input") and constrain the two
;; together.
(FpuRRIMod
(fpu_op FPUOpRIMod)
(rd WritableReg)
(ri Reg)
(rn Reg))
;; 3-op FPU instruction.
;; 16-bit scalars require half-precision floating-point support (FEAT_FP16).
(FpuRRRR
(fpu_op FPUOp3)
(size ScalarSize)
(rd WritableReg)
(rn Reg)
(rm Reg)
(ra Reg))
;; FPU comparison.
(FpuCmp
(size ScalarSize)
(rn Reg)
(rm Reg))
;; Floating-point load, single-precision (32 bit).
(FpuLoad32
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; Floating-point store, single-precision (32 bit).
(FpuStore32
(rd Reg)
(mem AMode)
(flags MemFlags))
;; Floating-point load, double-precision (64 bit).
(FpuLoad64
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; Floating-point store, double-precision (64 bit).
(FpuStore64
(rd Reg)
(mem AMode)
(flags MemFlags))
;; Floating-point/vector load, 128 bit.
(FpuLoad128
(rd WritableReg)
(mem AMode)
(flags MemFlags))
;; Floating-point/vector store, 128 bit.
(FpuStore128
(rd Reg)
(mem AMode)
(flags MemFlags))
;; A load of a pair of floating-point registers, double precision (64-bit).
(FpuLoadP64
(rt WritableReg)
(rt2 WritableReg)
(mem PairAMode)
(flags MemFlags))
;; A store of a pair of floating-point registers, double precision (64-bit).
(FpuStoreP64
(rt Reg)
(rt2 Reg)
(mem PairAMode)
(flags MemFlags))
;; A load of a pair of floating-point registers, 128-bit.
(FpuLoadP128
(rt WritableReg)
(rt2 WritableReg)
(mem PairAMode)
(flags MemFlags))
;; A store of a pair of floating-point registers, 128-bit.
(FpuStoreP128
(rt Reg)
(rt2 Reg)
(mem PairAMode)
(flags MemFlags))
(LoadFpuConst64
(rd WritableReg)
(const_data u64))
(LoadFpuConst128
(rd WritableReg)
(const_data u128))
;; Conversion: FP -> integer.
(FpuToInt
(op FpuToIntOp)
(rd WritableReg)
(rn Reg))
;; Conversion: integer -> FP.
(IntToFpu
(op IntToFpuOp)
(rd WritableReg)
(rn Reg))
;; FP conditional select, 32 bit.
(FpuCSel32
(rd WritableReg)
(rn Reg)
(rm Reg)
(cond Cond))
;; FP conditional select, 64 bit.
(FpuCSel64
(rd WritableReg)
(rn Reg)
(rm Reg)
(cond Cond))
;; Round to integer.
(FpuRound
(op FpuRoundMode)
(rd WritableReg)
(rn Reg))
;; Move from a GPR to a vector register. The scalar value is parked in the lowest lane
;; of the destination, and all other lanes are zeroed out. Currently only 32- and 64-bit
;; transactions are supported.
(MovToFpu
(rd WritableReg)
(rn Reg)
(size ScalarSize))
;; Loads a floating-point immediate.
(FpuMoveFPImm
(rd WritableReg)
(imm ASIMDFPModImm)
(size ScalarSize))
;; Move to a vector element from a GPR.
(MovToVec
(rd WritableReg)
(ri Reg)
(rn Reg)
(idx u8)
(size VectorSize))
;; Unsigned move from a vector element to a GPR.
(MovFromVec
(rd WritableReg)
(rn Reg)
(idx u8)
(size ScalarSize))
;; Signed move from a vector element to a GPR.
(MovFromVecSigned
(rd WritableReg)
(rn Reg)
(idx u8)
(size VectorSize)
(scalar_size OperandSize))
;; Duplicate general-purpose register to vector.
(VecDup
(rd WritableReg)
(rn Reg)
(size VectorSize))
;; Duplicate scalar to vector.
(VecDupFromFpu
(rd WritableReg)
(rn Reg)
(size VectorSize))
;; Duplicate FP immediate to vector.
(VecDupFPImm
(rd WritableReg)
(imm ASIMDFPModImm)
(size VectorSize))
;; Duplicate immediate to vector.
(VecDupImm
(rd WritableReg)
(imm ASIMDMovModImm)
(invert bool)
(size VectorSize))
;; Vector extend.
(VecExtend
(t VecExtendOp)
(rd WritableReg)
(rn Reg)
(high_half bool)
(lane_size ScalarSize))
;; Move vector element to another vector element.
(VecMovElement
(rd WritableReg)
(ri Reg)
(rn Reg)
(dest_idx u8)
(src_idx u8)
(size VectorSize))
;; Vector widening operation.
(VecRRLong
(op VecRRLongOp)
(rd WritableReg)
(rn Reg)
(high_half bool))
;; Vector narrowing operation -- low half.
(VecRRNarrowLow
(op VecRRNarrowOp)
(rd WritableReg)
(rn Reg)
(lane_size ScalarSize))
;; Vector narrowing operation -- high half.
(VecRRNarrowHigh
(op VecRRNarrowOp)
(rd WritableReg)
(ri Reg)
(rn Reg)
(lane_size ScalarSize))
;; 1-operand vector instruction that operates on a pair of elements.
(VecRRPair
(op VecPairOp)
(rd WritableReg)
(rn Reg))
;; 2-operand vector instruction that produces a result with twice the
;; lane width and half the number of lanes.
(VecRRRLong
(alu_op VecRRRLongOp)
(rd WritableReg)
(rn Reg)
(rm Reg)
(high_half bool))
;; 2-operand vector instruction that produces a result with
;; twice the lane width and half the number of lanes. Variant
;; that modifies `rd` (so takes its initial state as `ri`).
(VecRRRLongMod
(alu_op VecRRRLongModOp)
(rd WritableReg)
(ri Reg)
(rn Reg)
(rm Reg)
(high_half bool))
;; 1-operand vector instruction that extends elements of the input
;; register and operates on a pair of elements. The output lane width
;; is double that of the input.
(VecRRPairLong
(op VecRRPairLongOp)
(rd WritableReg)
(rn Reg))
;; A vector ALU op.
(VecRRR
(alu_op VecALUOp)
(rd WritableReg)
(rn Reg)
(rm Reg)
(size VectorSize))
;; A vector ALU op modifying a source register.
(VecRRRMod
(alu_op VecALUModOp)
(rd WritableReg)
(ri Reg)
(rn Reg)
(rm Reg)
(size VectorSize))
;; Vector two register miscellaneous instruction.
(VecMisc
(op VecMisc2)
(rd WritableReg)
(rn Reg)
(size VectorSize))
;; Vector instruction across lanes.
(VecLanes
(op VecLanesOp)
(rd WritableReg)
(rn Reg)
(size VectorSize))
;; Vector shift by immediate Shift Left (immediate), Unsigned Shift Right (immediate)
;; Signed Shift Right (immediate). These are somewhat unusual in that, for right shifts,
;; the allowed range of `imm` values is 1 to lane-size-in-bits, inclusive. A zero
;; right-shift cannot be encoded. Left shifts are "normal", though, having valid `imm`
;; values from 0 to lane-size-in-bits - 1 inclusive.
(VecShiftImm
(op VecShiftImmOp)
(rd WritableReg)
(rn Reg)
(size VectorSize)
(imm u8))
;; Destructive vector shift by immediate.
(VecShiftImmMod
(op VecShiftImmModOp)
(rd WritableReg)
(ri Reg)
(rn Reg)
(size VectorSize)
(imm u8))
;; Vector extract - create a new vector, being the concatenation of the lowest `imm4` bytes
;; of `rm` followed by the uppermost `16 - imm4` bytes of `rn`.
(VecExtract
(rd WritableReg)
(rn Reg)
(rm Reg)
(imm4 u8))
;; Table vector lookup - single register table. The table
;; consists of 8-bit elements and is stored in `rn`, while `rm`
;; contains 8-bit element indices. This variant emits `TBL`,
;; which sets elements that correspond to out-of-range indices
;; (greater than 15) to 0.
(VecTbl
(rd WritableReg)
(rn Reg)
(rm Reg))
;; Table vector lookup - single register table. The table
;; consists of 8-bit elements and is stored in `rn`, while `rm`
;; contains 8-bit element indices. This variant emits `TBX`,
;; which leaves elements that correspond to out-of-range indices
;; (greater than 15) unmodified. Hence, it takes an input vreg in
;; `ri` that is constrained to the same allocation as `rd`.
(VecTblExt
(rd WritableReg)
(ri Reg)
(rn Reg)
(rm Reg))
;; Table vector lookup - two register table. The table consists
;; of 8-bit elements and is stored in `rn` and `rn2`, while
;; `rm` contains 8-bit element indices. The table registers
;; `rn` and `rn2` must have consecutive numbers modulo 32, that
;; is v31 and v0 (in that order) are consecutive registers.
;; This variant emits `TBL`, which sets out-of-range results to
;; 0.
(VecTbl2
(rd WritableReg)
(rn Reg)
(rn2 Reg)
(rm Reg))
;; Table vector lookup - two register table. The table consists
;; of 8-bit elements and is stored in `rn` and `rn2`, while
;; `rm` contains 8-bit element indices. The table registers
;; `rn` and `rn2` must have consecutive numbers modulo 32, that
;; is v31 and v0 (in that order) are consecutive registers.
;; This variant emits `TBX`, which leaves out-of-range results
;; unmodified, hence takes the initial state of the result
;; register in vreg `ri`.
(VecTbl2Ext
(rd WritableReg)
(ri Reg)
(rn Reg)
(rn2 Reg)
(rm Reg))
;; Load an element and replicate to all lanes of a vector.
(VecLoadReplicate
(rd WritableReg)
(rn Reg)
(size VectorSize)
(flags MemFlags))
;; Vector conditional select, 128 bit. A synthetic instruction, which generates a 4-insn
;; control-flow diamond.
(VecCSel
(rd WritableReg)
(rn Reg)
(rm Reg)
(cond Cond))
;; Move to the NZCV flags (actually a `MSR NZCV, Xn` insn).
(MovToNZCV
(rn Reg))
;; Move from the NZCV flags (actually a `MRS Xn, NZCV` insn).
(MovFromNZCV
(rd WritableReg))
;; A machine call instruction. N.B.: this allows only a +/- 128MB offset (it uses a relocation
;; of type `Reloc::Arm64Call`); if the destination distance is not `RelocDistance::Near`, the
;; code should use a `LoadExtName` / `CallInd` sequence instead, allowing an arbitrary 64-bit
;; target.
(Call
(info BoxCallInfo))
;; A machine indirect-call instruction.
(CallInd
(info BoxCallIndInfo))
;; A pseudo-instruction that captures register arguments in vregs.
(Args
(args VecArgPair))
;; ---- branches (exactly one must appear at end of BB) ----
;; A machine return instruction.
(Ret
(rets VecReg))
;; A machine return instruction with pointer authentication using SP as the
;; modifier. This instruction requires pointer authentication support
;; (FEAT_PAuth) unless `is_hint` is true, in which case it is equivalent to
;; the combination of a no-op and a return instruction on platforms without
;; the relevant support.
(AuthenticatedRet
(key APIKey)
(is_hint bool)
(rets VecReg))
;; An unconditional branch.
(Jump
(dest BranchTarget))
;; A conditional branch. Contains two targets; at emission time, both are emitted, but
;; the MachBuffer knows to truncate the trailing branch if fallthrough. We optimize the
;; choice of taken/not_taken (inverting the branch polarity as needed) based on the
;; fallthrough at the time of lowering.
(CondBr
(taken BranchTarget)
(not_taken BranchTarget)
(kind CondBrKind))
;; A conditional trap: execute a `udf` if the condition is true. This is
;; one VCode instruction because it uses embedded control flow; it is
;; logically a single-in, single-out region, but needs to appear as one
;; unit to the register allocator.
;;
;; The `CondBrKind` gives the conditional-branch condition that will
;; *execute* the embedded `Inst`. (In the emitted code, we use the inverse
;; of this condition in a branch that skips the trap instruction.)
(TrapIf
(kind CondBrKind)
(trap_code TrapCode))
;; An indirect branch through a register, augmented with set of all
;; possible successors.
(IndirectBr
(rn Reg)
(targets VecMachLabel))
;; A "break" instruction, used for e.g. traps and debug breakpoints.
(Brk)
;; An instruction guaranteed to always be undefined and to trigger an illegal instruction at
;; runtime.
(Udf
(trap_code TrapCode))
;; Compute the address (using a PC-relative offset) of a memory location, using the `ADR`
;; instruction. Note that we take a simple offset, not a `MemLabel`, here, because `Adr` is
;; only used for now in fixed lowering sequences with hardcoded offsets. In the future we may
;; need full `MemLabel` support.
(Adr
(rd WritableReg)
;; Offset in range -2^20 .. 2^20.
(off i32))
;; Raw 32-bit word, used for inline constants and jump-table entries.
(Word4
(data u32))
;; Raw 64-bit word, used for inline constants.
(Word8
(data u64))
;; Jump-table sequence, as one compound instruction (see note in lower_inst.rs for rationale).
(JTSequence
(info BoxJTSequenceInfo)
(ridx Reg)
(rtmp1 WritableReg)
(rtmp2 WritableReg))
;; Load an inline symbol reference.
(LoadExtName
(rd WritableReg)
(name BoxExternalName)
(offset i64))
;; Load address referenced by `mem` into `rd`.
(LoadAddr
(rd WritableReg)
(mem AMode))
;; Pointer authentication code for instruction address with modifier in SP;
;; equivalent to a no-op if Pointer authentication (FEAT_PAuth) is not
;; supported.
(Pacisp
(key APIKey))
;; Strip pointer authentication code from instruction address in LR;
;; equivalent to a no-op if Pointer authentication (FEAT_PAuth) is not
;; supported.
(Xpaclri)
;; Branch target identification; equivalent to a no-op if Branch Target
;; Identification (FEAT_BTI) is not supported.
(Bti
(targets BranchTargetType))
;; Marker, no-op in generated code: SP "virtual offset" is adjusted. This
;; controls how AMode::NominalSPOffset args are lowered.
(VirtualSPOffsetAdj
(offset i64))
;; Meta-insn, no-op in generated code: emit constant/branch veneer island
;; at this point (with a guard jump around it) if less than the needed
;; space is available before the next branch deadline. See the `MachBuffer`
;; implementation in `machinst/buffer.rs` for the overall algorithm. In
;; brief, we retain a set of "pending/unresolved label references" from
;; branches as we scan forward through instructions to emit machine code;
;; if we notice we're about to go out of range on an unresolved reference,
;; we stop, emit a bunch of "veneers" (branches in a form that has a longer
;; range, e.g. a 26-bit-offset unconditional jump), and point the original
;; label references to those. This is an "island" because it comes in the
;; middle of the code.
;;
;; This meta-instruction is a necessary part of the logic that determines
;; where to place islands. Ordinarily, we want to place them between basic
;; blocks, so we compute the worst-case size of each block, and emit the
;; island before starting a block if we would exceed a deadline before the
;; end of the block. However, some sequences (such as an inline jumptable)
;; are variable-length and not accounted for by this logic; so these
;; lowered sequences include an `EmitIsland` to trigger island generation
;; where necessary.
(EmitIsland
;; The needed space before the next deadline.
(needed_space CodeOffset))
;; A call to the `ElfTlsGetAddr` libcall. Returns address of TLS symbol in x0.
(ElfTlsGetAddr
(symbol ExternalName)
(rd WritableReg))
;; An unwind pseudo-instruction.
(Unwind
(inst UnwindInst))
;; A dummy use, useful to keep a value alive.
(DummyUse
(reg Reg))))
;; An ALU operation. This can be paired with several instruction formats
;; below (see `Inst`) in any combination.
(type ALUOp
(enum
(Add)
(Sub)
(Orr)
(OrrNot)
(And)
(AndS)
(AndNot)
;; XOR (AArch64 calls this "EOR")
(Eor)
;; XNOR (AArch64 calls this "EOR-NOT")
(EorNot)
;; Add, setting flags
(AddS)
;; Sub, setting flags
(SubS)
;; Signed multiply, high-word result
(SMulH)
;; Unsigned multiply, high-word result
(UMulH)
(SDiv)
(UDiv)
(RotR)
(Lsr)
(Asr)
(Lsl)
;; Add with carry
(Adc)
;; Add with carry, settings flags
(AdcS)
;; Subtract with carry
(Sbc)
;; Subtract with carry, settings flags
(SbcS)
))
;; An ALU operation with three arguments.
(type ALUOp3
(enum
;; Multiply-add
(MAdd)
;; Multiply-sub
(MSub)
))
(type MoveWideOp
(enum
(MovZ)
(MovN)
))
(type UImm5 (primitive UImm5))
(type Imm12 (primitive Imm12))
(type ImmLogic (primitive ImmLogic))
(type ImmShift (primitive ImmShift))
(type ShiftOpAndAmt (primitive ShiftOpAndAmt))
(type MoveWideConst (primitive MoveWideConst))
(type NZCV (primitive NZCV))
(type ASIMDFPModImm (primitive ASIMDFPModImm))
(type ASIMDMovModImm (primitive ASIMDMovModImm))
(type BoxCallInfo (primitive BoxCallInfo))
(type BoxCallIndInfo (primitive BoxCallIndInfo))
(type CondBrKind (primitive CondBrKind))
(type BranchTarget (primitive BranchTarget))
(type BoxJTSequenceInfo (primitive BoxJTSequenceInfo))
(type CodeOffset (primitive CodeOffset))
(type VecMachLabel extern (enum))
(type ExtendOp extern
(enum
(UXTB)
(UXTH)
(UXTW)
(UXTX)
(SXTB)
(SXTH)
(SXTW)
(SXTX)
))
;; An operation on the bits of a register. This can be paired with several instruction formats
;; below (see `Inst`) in any combination.
(type BitOp
(enum
;; Bit reverse
(RBit)
(Clz)
(Cls)
))
(type MemLabel extern (enum))
(type SImm9 extern (enum))
(type UImm12Scaled extern (enum))
;; An addressing mode specified for a load/store operation.
(type AMode
(enum
;;
;; Real ARM64 addressing modes:
;;
;; "post-indexed" mode as per AArch64 docs: postincrement reg after
;; address computation.
;; Specialized here to SP so we don't have to emit regalloc metadata.
(SPPostIndexed
(simm9 SImm9))
;; "pre-indexed" mode as per AArch64 docs: preincrement reg before
;; address computation.
;; Specialized here to SP so we don't have to emit regalloc metadata.
(SPPreIndexed
(simm9 SImm9))
;; N.B.: RegReg, RegScaled, and RegScaledExtended all correspond to
;; what the ISA calls the "register offset" addressing mode. We split
;; out several options here for more ergonomic codegen.
;;
;; Register plus register offset.
(RegReg
(rn Reg)
(rm Reg))
;; Register plus register offset, scaled by type's size.
(RegScaled
(rn Reg)
(rm Reg)
(ty Type))
;; Register plus register offset, scaled by type's size, with index
;; sign- or zero-extended first.
(RegScaledExtended
(rn Reg)
(rm Reg)
(ty Type)
(extendop ExtendOp))
;; Register plus register offset, with index sign- or zero-extended
;; first.
(RegExtended
(rn Reg)
(rm Reg)
(extendop ExtendOp))
;; Unscaled signed 9-bit immediate offset from reg.
(Unscaled
(rn Reg)
(simm9 SImm9))
;; Scaled (by size of a type) unsigned 12-bit immediate offset from reg.
(UnsignedOffset
(rn Reg)
(uimm12 UImm12Scaled))
;; virtual addressing modes that are lowered at emission time:
;;
;; Reference to a "label": e.g., a symbol.
(Label
(label MemLabel))
;; Arbitrary offset from a register. Converted to generation of large
;; offsets with multiple instructions as necessary during code emission.
(RegOffset
(rn Reg)
(off i64)
(ty Type))
;; Offset from the stack pointer.
(SPOffset
(off i64)
(ty Type))
;; Offset from the frame pointer.
(FPOffset
(off i64)
(ty Type))
;; Offset from the "nominal stack pointer", which is where the real SP is
;; just after stack and spill slots are allocated in the function prologue.
;; At emission time, this is converted to `SPOffset` with a fixup added to
;; the offset constant. The fixup is a running value that is tracked as
;; emission iterates through instructions in linear order, and can be
;; adjusted up and down with [Inst::VirtualSPOffsetAdj].
;;
;; The standard ABI is in charge of handling this (by emitting the
;; adjustment meta-instructions). It maintains the invariant that "nominal
;; SP" is where the actual SP is after the function prologue and before
;; clobber pushes. See the diagram in the documentation for
;; [crate::isa::aarch64::abi](the ABI module) for more details.
(NominalSPOffset
(off i64)
(ty Type))
))
(type PairAMode extern (enum))
(type FPUOpRI extern (enum))
(type FPUOpRIMod extern (enum))
(type OperandSize extern
(enum Size32
Size64))
;; Helper for calculating the `OperandSize` corresponding to a type
(decl operand_size (Type) OperandSize)
(rule 1 (operand_size (fits_in_32 _ty)) (OperandSize.Size32))
(rule (operand_size (fits_in_64 _ty)) (OperandSize.Size64))
(type ScalarSize extern
(enum Size8
Size16
Size32
Size64
Size128))
;; Helper for calculating the `ScalarSize` corresponding to a type
(decl scalar_size (Type) ScalarSize)
(rule (scalar_size $I8) (ScalarSize.Size8))
(rule (scalar_size $I16) (ScalarSize.Size16))
(rule (scalar_size $I32) (ScalarSize.Size32))
(rule (scalar_size $I64) (ScalarSize.Size64))
(rule (scalar_size $I128) (ScalarSize.Size128))
(rule (scalar_size $B8) (ScalarSize.Size8))
(rule (scalar_size $B16) (ScalarSize.Size16))
(rule (scalar_size $B32) (ScalarSize.Size32))
(rule (scalar_size $B64) (ScalarSize.Size64))
(rule (scalar_size $B128) (ScalarSize.Size128))
(rule (scalar_size $F32) (ScalarSize.Size32))
(rule (scalar_size $F64) (ScalarSize.Size64))
;; Helper for calculating the `ScalarSize` lane type from vector type
(decl lane_size (Type) ScalarSize)
(rule 1 (lane_size (multi_lane 8 _)) (ScalarSize.Size8))
(rule 1 (lane_size (multi_lane 16 _)) (ScalarSize.Size16))
(rule 1 (lane_size (multi_lane 32 _)) (ScalarSize.Size32))
(rule 1 (lane_size (multi_lane 64 _)) (ScalarSize.Size64))
(rule (lane_size (dynamic_lane 8 _)) (ScalarSize.Size8))
(rule (lane_size (dynamic_lane 16 _)) (ScalarSize.Size16))
(rule (lane_size (dynamic_lane 32 _)) (ScalarSize.Size32))
(rule (lane_size (dynamic_lane 64 _)) (ScalarSize.Size64))
(type Cond extern
(enum
(Eq)
(Ne)
(Hs)
(Lo)
(Mi)
(Pl)
(Vs)
(Vc)
(Hi)
(Ls)
(Ge)
(Lt)
(Gt)
(Le)
(Al)
(Nv)
))
(type VectorSize extern
(enum
(Size8x8)
(Size8x16)
(Size16x4)
(Size16x8)
(Size32x2)
(Size32x4)
(Size64x2)
))
;; Helper for calculating the `VectorSize` corresponding to a type
(decl vector_size (Type) VectorSize)
(rule 1 (vector_size (multi_lane 8 8)) (VectorSize.Size8x8))
(rule 1 (vector_size (multi_lane 8 16)) (VectorSize.Size8x16))
(rule 1 (vector_size (multi_lane 16 4)) (VectorSize.Size16x4))
(rule 1 (vector_size (multi_lane 16 8)) (VectorSize.Size16x8))
(rule 1 (vector_size (multi_lane 32 2)) (VectorSize.Size32x2))
(rule 1 (vector_size (multi_lane 32 4)) (VectorSize.Size32x4))
(rule 1 (vector_size (multi_lane 64 2)) (VectorSize.Size64x2))
(rule (vector_size (dynamic_lane 8 8)) (VectorSize.Size8x8))
(rule (vector_size (dynamic_lane 8 16)) (VectorSize.Size8x16))
(rule (vector_size (dynamic_lane 16 4)) (VectorSize.Size16x4))
(rule (vector_size (dynamic_lane 16 8)) (VectorSize.Size16x8))
(rule (vector_size (dynamic_lane 32 2)) (VectorSize.Size32x2))
(rule (vector_size (dynamic_lane 32 4)) (VectorSize.Size32x4))
(rule (vector_size (dynamic_lane 64 2)) (VectorSize.Size64x2))
;; A floating-point unit (FPU) operation with one arg.
(type FPUOp1
(enum
(Abs)
(Neg)
(Sqrt)
(Cvt32To64)
(Cvt64To32)
))
;; A floating-point unit (FPU) operation with two args.
(type FPUOp2
(enum
(Add)
(Sub)
(Mul)
(Div)
(Max)
(Min)
))
;; A floating-point unit (FPU) operation with three args.
(type FPUOp3
(enum
(MAdd)
))
;; A conversion from an FP to an integer value.
(type FpuToIntOp
(enum
(F32ToU32)
(F32ToI32)
(F32ToU64)
(F32ToI64)
(F64ToU32)
(F64ToI32)
(F64ToU64)
(F64ToI64)
))
;; A conversion from an integer to an FP value.
(type IntToFpuOp
(enum
(U32ToF32)
(I32ToF32)
(U32ToF64)
(I32ToF64)
(U64ToF32)
(I64ToF32)
(U64ToF64)
(I64ToF64)
))
;; Modes for FP rounding ops: round down (floor) or up (ceil), or toward zero (trunc), or to
;; nearest, and for 32- or 64-bit FP values.
(type FpuRoundMode
(enum
(Minus32)
(Minus64)
(Plus32)
(Plus64)
(Zero32)
(Zero64)
(Nearest32)
(Nearest64)
))
;; Type of vector element extensions.
(type VecExtendOp
(enum
;; Signed extension
(Sxtl)
;; Unsigned extension
(Uxtl)
))
;; A vector ALU operation.
(type VecALUOp
(enum
;; Signed saturating add
(Sqadd)
;; Unsigned saturating add
(Uqadd)
;; Signed saturating subtract
(Sqsub)
;; Unsigned saturating subtract
(Uqsub)
;; Compare bitwise equal
(Cmeq)
;; Compare signed greater than or equal
(Cmge)
;; Compare signed greater than
(Cmgt)
;; Compare unsigned higher
(Cmhs)
;; Compare unsigned higher or same
(Cmhi)
;; Floating-point compare equal
(Fcmeq)
;; Floating-point compare greater than
(Fcmgt)
;; Floating-point compare greater than or equal
(Fcmge)
;; Bitwise and
(And)
;; Bitwise bit clear
(Bic)
;; Bitwise inclusive or
(Orr)
;; Bitwise exclusive or
(Eor)
;; Unsigned maximum pairwise
(Umaxp)
;; Add
(Add)
;; Subtract
(Sub)
;; Multiply
(Mul)
;; Signed shift left
(Sshl)
;; Unsigned shift left
(Ushl)
;; Unsigned minimum
(Umin)
;; Signed minimum
(Smin)
;; Unsigned maximum
(Umax)
;; Signed maximum
(Smax)
;; Unsigned rounding halving add
(Urhadd)
;; Floating-point add
(Fadd)
;; Floating-point subtract
(Fsub)
;; Floating-point divide
(Fdiv)
;; Floating-point maximum
(Fmax)
;; Floating-point minimum
(Fmin)
;; Floating-point multiply
(Fmul)
;; Add pairwise
(Addp)
;; Zip vectors (primary) [meaning, high halves]
(Zip1)
;; Signed saturating rounding doubling multiply returning high half
(Sqrdmulh)
))
;; A Vector ALU operation which modifies a source register.
(type VecALUModOp
(enum
;; Bitwise select
(Bsl)
;; Floating-point fused multiply-add vectors
(Fmla)
))
;; A Vector miscellaneous operation with two registers.
(type VecMisc2
(enum
;; Bitwise NOT
(Not)
;; Negate
(Neg)
;; Absolute value
(Abs)
;; Floating-point absolute value
(Fabs)
;; Floating-point negate
(Fneg)
;; Floating-point square root
(Fsqrt)
;; Reverse elements in 64-bit doublewords
(Rev64)
;; Floating-point convert to signed integer, rounding toward zero
(Fcvtzs)
;; Floating-point convert to unsigned integer, rounding toward zero
(Fcvtzu)
;; Signed integer convert to floating-point
(Scvtf)
;; Unsigned integer convert to floating-point
(Ucvtf)
;; Floating point round to integral, rounding towards nearest
(Frintn)
;; Floating point round to integral, rounding towards zero
(Frintz)
;; Floating point round to integral, rounding towards minus infinity
(Frintm)
;; Floating point round to integral, rounding towards plus infinity
(Frintp)
;; Population count per byte
(Cnt)
;; Compare bitwise equal to 0
(Cmeq0)
;; Compare signed greater than or equal to 0
(Cmge0)
;; Compare signed greater than 0
(Cmgt0)
;; Compare signed less than or equal to 0
(Cmle0)
;; Compare signed less than 0
(Cmlt0)
;; Floating point compare equal to 0
(Fcmeq0)
;; Floating point compare greater than or equal to 0
(Fcmge0)
;; Floating point compare greater than 0
(Fcmgt0)
;; Floating point compare less than or equal to 0
(Fcmle0)
;; Floating point compare less than 0
(Fcmlt0)
))
;; A vector widening operation with one argument.
(type VecRRLongOp
(enum
;; Floating-point convert to higher precision long, 16-bit elements
(Fcvtl16)
;; Floating-point convert to higher precision long, 32-bit elements
(Fcvtl32)
;; Shift left long (by element size), 8-bit elements
(Shll8)
;; Shift left long (by element size), 16-bit elements
(Shll16)
;; Shift left long (by element size), 32-bit elements
(Shll32)
))
;; A vector narrowing operation with one argument.
(type VecRRNarrowOp
(enum
;; Extract narrow.
(Xtn)
;; Signed saturating extract narrow.
(Sqxtn)
;; Signed saturating extract unsigned narrow.
(Sqxtun)
;; Unsigned saturating extract narrow.
(Uqxtn)
;; Floating-point convert to lower precision narrow.
(Fcvtn)
))
(type VecRRRLongOp
(enum
;; Signed multiply long.
(Smull8)
(Smull16)
(Smull32)
;; Unsigned multiply long.
(Umull8)
(Umull16)
(Umull32)
))
(type VecRRRLongModOp
(enum
;; Unsigned multiply add long
(Umlal8)
(Umlal16)
(Umlal32)
))
;; A vector operation on a pair of elements with one register.
(type VecPairOp
(enum
;; Add pair of elements
(Addp)
))
;; 1-operand vector instruction that extends elements of the input register
;; and operates on a pair of elements.
(type VecRRPairLongOp
(enum
;; Sign extend and add pair of elements
(Saddlp8)
(Saddlp16)
;; Unsigned extend and add pair of elements
(Uaddlp8)
(Uaddlp16)
))
;; An operation across the lanes of vectors.
(type VecLanesOp
(enum
;; Integer addition across a vector
(Addv)
;; Unsigned minimum across a vector
(Uminv)
))
;; A shift-by-immediate operation on each lane of a vector.
(type VecShiftImmOp
(enum
;; Unsigned shift left
(Shl)
;; Unsigned shift right
(Ushr)
;; Signed shift right
(Sshr)
))
;; Destructive shift-by-immediate operation on each lane of a vector.
(type VecShiftImmModOp
(enum
;; Shift left and insert
(Sli)
))
;; Atomic read-modify-write operations with acquire-release semantics
(type AtomicRMWOp
(enum
(Add)
(Clr)
(Eor)
(Set)
(Smax)
(Smin)
(Umax)
(Umin)
(Swp)
))
;; Atomic read-modify-write operations, with acquire-release semantics,
;; implemented with a loop.
(type AtomicRMWLoopOp
(enum
(Add)
(Sub)
(And)
(Nand)
(Eor)
(Orr)
(Smax)
(Smin)
(Umax)
(Umin)
(Xchg)
))
;; Keys for instruction address PACs
(type APIKey
(enum
(A)
(B)
))
;; Branch target types
(type BranchTargetType
(enum
(None)
(C)
(J)
(JC)
))
;; Extractors for target features ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl pure sign_return_address_disabled () Unit)
(extern constructor sign_return_address_disabled sign_return_address_disabled)
(decl use_lse () Inst)
(extern extractor use_lse use_lse)
;; Extractor helpers for various immmediate constants ;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl pure imm_logic_from_u64 (Type u64) ImmLogic)
(extern constructor imm_logic_from_u64 imm_logic_from_u64)
(decl pure imm_logic_from_imm64 (Type Imm64) ImmLogic)
(extern constructor imm_logic_from_imm64 imm_logic_from_imm64)
(decl pure imm_shift_from_imm64 (Type Imm64) ImmShift)
(extern constructor imm_shift_from_imm64 imm_shift_from_imm64)
(decl imm_shift_from_u8 (u8) ImmShift)
(extern constructor imm_shift_from_u8 imm_shift_from_u8)
(decl imm12_from_u64 (Imm12) u64)
(extern extractor imm12_from_u64 imm12_from_u64)
(decl u8_into_uimm5 (u8) UImm5)
(extern constructor u8_into_uimm5 u8_into_uimm5)
(decl u8_into_imm12 (u8) Imm12)
(extern constructor u8_into_imm12 u8_into_imm12)
(decl u64_into_imm_logic (Type u64) ImmLogic)
(extern constructor u64_into_imm_logic u64_into_imm_logic)
(decl branch_target (VecMachLabel u8) BranchTarget)
(extern constructor branch_target branch_target)
(decl targets_jt_size (VecMachLabel) u32)
(extern constructor targets_jt_size targets_jt_size)
(decl targets_jt_space (VecMachLabel) CodeOffset)
(extern constructor targets_jt_space targets_jt_space)
(decl targets_jt_info (VecMachLabel) BoxJTSequenceInfo)
(extern constructor targets_jt_info targets_jt_info)
;; Calculate the minimum floating-point bound for a conversion to floating
;; point from an integer type.
;; Accepts whether the output is signed, the size of the input
;; floating point type in bits, and the size of the output integer type
;; in bits.
(decl min_fp_value (bool u8 u8) Reg)
(extern constructor min_fp_value min_fp_value)
;; Calculate the maximum floating-point bound for a conversion to floating
;; point from an integer type.
;; Accepts whether the output is signed, the size of the input
;; floating point type in bits, and the size of the output integer type
;; in bits.
(decl max_fp_value (bool u8 u8) Reg)
(extern constructor max_fp_value max_fp_value)
;; Constructs an FPUOpRI.Ushr* given the size in bits of the value (or lane)
;; and the amount to shift by.
(decl fpu_op_ri_ushr (u8 u8) FPUOpRI)
(extern constructor fpu_op_ri_ushr fpu_op_ri_ushr)
;; Constructs an FPUOpRIMod.Sli* given the size in bits of the value (or lane)
;; and the amount to shift by.
(decl fpu_op_ri_sli (u8 u8) FPUOpRIMod)
(extern constructor fpu_op_ri_sli fpu_op_ri_sli)
(decl imm12_from_negated_u64 (Imm12) u64)
(extern extractor imm12_from_negated_u64 imm12_from_negated_u64)
(decl pure lshr_from_u64 (Type u64) ShiftOpAndAmt)
(extern constructor lshr_from_u64 lshr_from_u64)
(decl pure lshl_from_imm64 (Type Imm64) ShiftOpAndAmt)
(extern constructor lshl_from_imm64 lshl_from_imm64)
(decl pure lshl_from_u64 (Type u64) ShiftOpAndAmt)
(extern constructor lshl_from_u64 lshl_from_u64)
(decl integral_ty (Type) Type)
(extern extractor integral_ty integral_ty)
(decl valid_atomic_transaction (Type) Type)
(extern extractor valid_atomic_transaction valid_atomic_transaction)
(decl pure is_zero_simm9 (SImm9) Unit)
(extern constructor is_zero_simm9 is_zero_simm9)
(decl pure is_zero_uimm12 (UImm12Scaled) Unit)
(extern constructor is_zero_uimm12 is_zero_uimm12)
;; Helper to go directly from a `Value`, when it's an `iconst`, to an `Imm12`.
(decl imm12_from_value (Imm12) Value)
(extractor
(imm12_from_value n)
(iconst (u64_from_imm64 (imm12_from_u64 n))))
;; Same as `imm12_from_value`, but tries negating the constant value.
(decl imm12_from_negated_value (Imm12) Value)
(extractor
(imm12_from_negated_value n)
(iconst (u64_from_imm64 (imm12_from_negated_u64 n))))
;; Helper type to represent a value and an extend operation fused together.
(type ExtendedValue extern (enum))
(decl extended_value_from_value (ExtendedValue) Value)
(extern extractor extended_value_from_value extended_value_from_value)
;; Constructors used to poke at the fields of an `ExtendedValue`.
(decl put_extended_in_reg (ExtendedValue) Reg)
(extern constructor put_extended_in_reg put_extended_in_reg)
(decl get_extended_op (ExtendedValue) ExtendOp)
(extern constructor get_extended_op get_extended_op)
(decl nzcv (bool bool bool bool) NZCV)
(extern constructor nzcv nzcv)
(decl cond_br_zero (Reg) CondBrKind)
(extern constructor cond_br_zero cond_br_zero)
(decl cond_br_not_zero (Reg) CondBrKind)
(extern constructor cond_br_not_zero cond_br_not_zero)
(decl cond_br_cond (Cond) CondBrKind)
(extern constructor cond_br_cond cond_br_cond)
(decl pair_amode (Value u32) PairAMode)
(extern constructor pair_amode pair_amode)
;; Instruction creation helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Helper for creating the zero register.
(decl zero_reg () Reg)
(extern constructor zero_reg zero_reg)
(decl fp_reg () Reg)
(extern constructor fp_reg fp_reg)
(decl stack_reg () Reg)
(extern constructor stack_reg stack_reg)
(decl writable_link_reg () WritableReg)
(extern constructor writable_link_reg writable_link_reg)
(decl writable_zero_reg () WritableReg)
(extern constructor writable_zero_reg writable_zero_reg)
;; Helper for emitting `MInst.Mov` instructions.
(decl mov (Reg Type) Reg)
(rule (mov src ty)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.Mov (operand_size ty) dst src))))
dst))
;; Helper for emitting `MInst.MovZ` instructions.
(decl movz (MoveWideConst OperandSize) Reg)
(rule (movz imm size)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.MovWide (MoveWideOp.MovZ) dst imm size))))
dst))
;; Helper for emitting `MInst.MovN` instructions.
(decl movn (MoveWideConst OperandSize) Reg)
(rule (movn imm size)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.MovWide (MoveWideOp.MovN) dst imm size))))
dst))
;; Helper for emitting `MInst.AluRRImmLogic` instructions.
(decl alu_rr_imm_logic (ALUOp Type Reg ImmLogic) Reg)
(rule (alu_rr_imm_logic op ty src imm)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRImmLogic op (operand_size ty) dst src imm))))
dst))
;; Helper for emitting `MInst.AluRRImmShift` instructions.
(decl alu_rr_imm_shift (ALUOp Type Reg ImmShift) Reg)
(rule (alu_rr_imm_shift op ty src imm)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRImmShift op (operand_size ty) dst src imm))))
dst))
;; Helper for emitting `MInst.AluRRR` instructions.
(decl alu_rrr (ALUOp Type Reg Reg) Reg)
(rule (alu_rrr op ty src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRR op (operand_size ty) dst src1 src2))))
dst))
;; Helper for emitting `MInst.VecRRR` instructions.
(decl vec_rrr (VecALUOp Reg Reg VectorSize) Reg)
(rule (vec_rrr op src1 src2 size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRR op dst src1 src2 size))))
dst))
;; Helper for emitting `MInst.FpuRR` instructions.
(decl fpu_rr (FPUOp1 Reg ScalarSize) Reg)
(rule (fpu_rr op src size)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuRR op size dst src))))
dst))
;; Helper for emitting `MInst.VecRRR` instructions which use three registers,
;; one of which is both source and output.
(decl vec_rrr_mod (VecALUModOp Reg Reg Reg VectorSize) Reg)
(rule (vec_rrr_mod op src1 src2 src3 size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_1 Unit (emit (MInst.VecRRRMod op dst src1 src2 src3 size))))
dst))
(decl fpu_rri (FPUOpRI Reg) Reg)
(rule (fpu_rri op src)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuRRI op dst src))))
dst))
(decl fpu_rri_mod (FPUOpRIMod Reg Reg) Reg)
(rule (fpu_rri_mod op dst_src src)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuRRIMod op dst dst_src src))))
dst))
;; Helper for emitting `MInst.FpuRRR` instructions.
(decl fpu_rrr (FPUOp2 Reg Reg ScalarSize) Reg)
(rule (fpu_rrr op src1 src2 size)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuRRR op size dst src1 src2))))
dst))
;; Helper for emitting `MInst.FpuRRRR` instructions.
(decl fpu_rrrr (FPUOp3 ScalarSize Reg Reg Reg) Reg)
(rule (fpu_rrrr size op src1 src2 src3)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuRRRR size op dst src1 src2 src3))))
dst))
;; Helper for emitting `MInst.FpuCmp` instructions.
(decl fpu_cmp (ScalarSize Reg Reg) ProducesFlags)
(rule (fpu_cmp size rn rm)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.FpuCmp size rn rm)))
;; Helper for emitting `MInst.VecLanes` instructions.
(decl vec_lanes (VecLanesOp Reg VectorSize) Reg)
(rule (vec_lanes op src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecLanes op dst src size))))
dst))
;; Helper for emitting `MInst.VecShiftImm` instructions.
(decl vec_shift_imm (VecShiftImmOp u8 Reg VectorSize) Reg)
(rule (vec_shift_imm op imm src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecShiftImm op dst src size imm))))
dst))
;; Helper for emitting `MInst.VecDup` instructions.
(decl vec_dup (Reg VectorSize) Reg)
(rule (vec_dup src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecDup dst src size))))
dst))
;; Helper for emitting `MInst.VecDupFromFpu` instructions.
(decl vec_dup_from_fpu (Reg VectorSize) Reg)
(rule (vec_dup_from_fpu src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecDupFromFpu dst src size))))
dst))
;; Helper for emitting `MInst.AluRRImm12` instructions.
(decl alu_rr_imm12 (ALUOp Type Reg Imm12) Reg)
(rule (alu_rr_imm12 op ty src imm)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRImm12 op (operand_size ty) dst src imm))))
dst))
;; Helper for emitting `MInst.AluRRRShift` instructions.
(decl alu_rrr_shift (ALUOp Type Reg Reg ShiftOpAndAmt) Reg)
(rule (alu_rrr_shift op ty src1 src2 shift)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRRShift op (operand_size ty) dst src1 src2 shift))))
dst))
;; Helper for emitting `cmp` instructions, setting flags, with a right-shifted
;; second operand register.
(decl cmp_rr_shift (OperandSize Reg Reg u64) ProducesFlags)
(rule (cmp_rr_shift size src1 src2 shift_amount)
(if-let shift (lshr_from_u64 $I64 shift_amount))
(ProducesFlags.ProducesFlagsSideEffect
(MInst.AluRRRShift (ALUOp.SubS) size (writable_zero_reg)
src1 src2 shift)))
;; Helper for emitting `MInst.AluRRRExtend` instructions.
(decl alu_rrr_extend (ALUOp Type Reg Reg ExtendOp) Reg)
(rule (alu_rrr_extend op ty src1 src2 extend)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRRExtend op (operand_size ty) dst src1 src2 extend))))
dst))
;; Same as `alu_rrr_extend`, but takes an `ExtendedValue` packed "pair" instead
;; of a `Reg` and an `ExtendOp`.
(decl alu_rr_extend_reg (ALUOp Type Reg ExtendedValue) Reg)
(rule (alu_rr_extend_reg op ty src1 extended_reg)
(let ((src2 Reg (put_extended_in_reg extended_reg))
(extend ExtendOp (get_extended_op extended_reg)))
(alu_rrr_extend op ty src1 src2 extend)))
;; Helper for emitting `MInst.AluRRRR` instructions.
(decl alu_rrrr (ALUOp3 Type Reg Reg Reg) Reg)
(rule (alu_rrrr op ty src1 src2 src3)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRRR op (operand_size ty) dst src1 src2 src3))))
dst))
;; Helper for emitting `MInst.BitRR` instructions.
(decl bit_rr (BitOp Type Reg) Reg)
(rule (bit_rr op ty src)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.BitRR op (operand_size ty) dst src))))
dst))
;; Helper for emitting `adds` instructions.
(decl add_with_flags_paired (Type Reg Reg) ProducesFlags)
(rule (add_with_flags_paired ty src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ProducesFlags.ProducesFlagsReturnsResultWithConsumer
(MInst.AluRRR (ALUOp.AddS) (operand_size ty) dst src1 src2)
dst)))
;; Helper for emitting `adds` instructions, setting flags in ambient
;; state. Used only for `iadd_ifcout`.
(decl add_with_flags (Type Reg Reg) Reg)
(rule (add_with_flags ty src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AluRRR (ALUOp.AddS) (operand_size ty) dst src1 src2))))
dst))
;; Helper for emitting `adc` instructions.
(decl adc_paired (Type Reg Reg) ConsumesFlags)
(rule (adc_paired ty src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
(MInst.AluRRR (ALUOp.Adc) (operand_size ty) dst src1 src2)
dst)))
;; Helper for emitting `subs` instructions.
(decl sub_with_flags_paired (Type Reg Reg) ProducesFlags)
(rule (sub_with_flags_paired ty src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ProducesFlags.ProducesFlagsReturnsResultWithConsumer
(MInst.AluRRR (ALUOp.SubS) (operand_size ty) dst src1 src2)
dst)))
;; Helper for materializing a boolean value into a register from
;; flags.
(decl materialize_bool_result (u8 Cond) ConsumesFlags)
(rule (materialize_bool_result 1 cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.CSet dst cond)
dst)))
(rule -1 (materialize_bool_result _ty_bits cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.CSetm dst cond)
dst)))
(decl cmn_imm (OperandSize Reg Imm12) ProducesFlags)
(rule (cmn_imm size src1 src2)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.AluRRImm12 (ALUOp.AddS) size (writable_zero_reg)
src1 src2)))
(decl cmp (OperandSize Reg Reg) ProducesFlags)
(rule (cmp size src1 src2)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.AluRRR (ALUOp.SubS) size (writable_zero_reg)
src1 src2)))
(decl cmp_imm (OperandSize Reg Imm12) ProducesFlags)
(rule (cmp_imm size src1 src2)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.AluRRImm12 (ALUOp.SubS) size (writable_zero_reg)
src1 src2)))
(decl cmp64_imm (Reg Imm12) ProducesFlags)
(rule (cmp64_imm src1 src2)
(cmp_imm (OperandSize.Size64) src1 src2))
(decl cmp_extend (OperandSize Reg Reg ExtendOp) ProducesFlags)
(rule (cmp_extend size src1 src2 extend)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.AluRRRExtend (ALUOp.SubS) size (writable_zero_reg)
src1 src2 extend)))
;; Helper for emitting `sbc` instructions.
(decl sbc_paired (Type Reg Reg) ConsumesFlags)
(rule (sbc_paired ty src1 src2)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsResultWithProducer
(MInst.AluRRR (ALUOp.Sbc) (operand_size ty) dst src1 src2)
dst)))
;; Helper for emitting `MInst.VecMisc` instructions.
(decl vec_misc (VecMisc2 Reg VectorSize) Reg)
(rule (vec_misc op src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecMisc op dst src size))))
dst))
;; Helper for emitting `MInst.VecTbl` instructions.
(decl vec_tbl (Reg Reg) Reg)
(rule (vec_tbl rn rm)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecTbl dst rn rm))))
dst))
(decl vec_tbl_ext (Reg Reg Reg) Reg)
(rule (vec_tbl_ext ri rn rm)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecTblExt dst ri rn rm))))
dst))
;; Helper for emitting `MInst.VecTbl2` instructions.
(decl vec_tbl2 (Reg Reg Reg Type) Reg)
(rule (vec_tbl2 rn rn2 rm ty)
(let (
(dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecTbl2 dst rn rn2 rm)))
)
dst))
;; Helper for emitting `MInst.VecTbl2Ext` instructions.
(decl vec_tbl2_ext (Reg Reg Reg Reg Type) Reg)
(rule (vec_tbl2_ext ri rn rn2 rm ty)
(let (
(dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecTbl2Ext dst ri rn rn2 rm)))
)
dst))
;; Helper for emitting `MInst.VecRRRLong` instructions.
(decl vec_rrr_long (VecRRRLongOp Reg Reg bool) Reg)
(rule (vec_rrr_long op src1 src2 high_half)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRRLong op dst src1 src2 high_half))))
dst))
;; Helper for emitting `MInst.VecRRPairLong` instructions.
(decl vec_rr_pair_long (VecRRPairLongOp Reg) Reg)
(rule (vec_rr_pair_long op src)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRPairLong op dst src))))
dst))
;; Helper for emitting `MInst.VecRRRLongMod` instructions.
(decl vec_rrrr_long (VecRRRLongModOp Reg Reg Reg bool) Reg)
(rule (vec_rrrr_long op src1 src2 src3 high_half)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRRLongMod op dst src1 src2 src3 high_half))))
dst))
;; Helper for emitting `MInst.VecRRNarrow` instructions.
(decl vec_rr_narrow_low (VecRRNarrowOp Reg ScalarSize) Reg)
(rule (vec_rr_narrow_low op src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRNarrowLow op dst src size))))
dst))
;; Helper for emitting `MInst.VecRRNarrow` instructions which update the
;; high half of the destination register.
(decl vec_rr_narrow_high (VecRRNarrowOp Reg Reg ScalarSize) Reg)
(rule (vec_rr_narrow_high op mod src size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRNarrowHigh op dst mod src size))))
dst))
;; Helper for emitting `MInst.VecRRLong` instructions.
(decl vec_rr_long (VecRRLongOp Reg bool) Reg)
(rule (vec_rr_long op src high_half)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecRRLong op dst src high_half))))
dst))
;; Helper for emitting `MInst.FpuCSel32` / `MInst.FpuCSel64`
;; instructions.
(decl fpu_csel (Type Cond Reg Reg) ConsumesFlags)
(rule (fpu_csel $F32 cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $F32)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.FpuCSel32 dst if_true if_false cond)
dst)))
(rule (fpu_csel $F64 cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $F64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.FpuCSel64 dst if_true if_false cond)
dst)))
;; Helper for emitting `MInst.VecCSel` instructions.
(decl vec_csel (Cond Reg Reg) ConsumesFlags)
(rule (vec_csel cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $I8X16)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.VecCSel dst if_true if_false cond)
dst)))
;; Helper for emitting `MInst.FpuRound` instructions.
(decl fpu_round (FpuRoundMode Reg) Reg)
(rule (fpu_round op rn)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuRound op dst rn))))
dst))
;; Helper for emitting `MInst.FpuMove64` and `MInst.FpuMove128` instructions.
(decl fpu_move (Type Reg) Reg)
(rule (fpu_move _ src)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.FpuMove128 dst src))))
dst))
(rule 1 (fpu_move (fits_in_64 _) src)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuMove64 dst src))))
dst))
;; Helper for emitting `MInst.MovToFpu` instructions.
(decl mov_to_fpu (Reg ScalarSize) Reg)
(rule (mov_to_fpu x size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.MovToFpu dst x size))))
dst))
;; Helper for emitting `MInst.MovToVec` instructions.
(decl mov_to_vec (Reg Reg u8 VectorSize) Reg)
(rule (mov_to_vec src1 src2 lane size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.MovToVec dst src1 src2 lane size))))
dst))
;; Helper for emitting `MInst.VecMovElement` instructions.
(decl mov_vec_elem (Reg Reg u8 u8 VectorSize) Reg)
(rule (mov_vec_elem src1 src2 dst_idx src_idx size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecMovElement dst src1 src2 dst_idx src_idx size))))
dst))
;; Helper for emitting `MInst.MovFromVec` instructions.
(decl mov_from_vec (Reg u8 ScalarSize) Reg)
(rule (mov_from_vec rn idx size)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.MovFromVec dst rn idx size))))
dst))
;; Helper for emitting `MInst.MovFromVecSigned` instructions.
(decl mov_from_vec_signed (Reg u8 VectorSize OperandSize) Reg)
(rule (mov_from_vec_signed rn idx size scalar_size)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.MovFromVecSigned dst rn idx size scalar_size))))
dst))
(decl fpu_move_from_vec (Reg u8 VectorSize) Reg)
(rule (fpu_move_from_vec rn idx size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.FpuMoveFromVec dst rn idx size))))
dst))
;; Helper for emitting `MInst.Extend` instructions.
(decl extend (Reg bool u8 u8) Reg)
(rule (extend rn signed from_bits to_bits)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.Extend dst rn signed from_bits to_bits))))
dst))
;; Helper for emitting `MInst.FpuExtend` instructions.
(decl fpu_extend (Reg ScalarSize) Reg)
(rule (fpu_extend src size)
(let ((dst WritableReg (temp_writable_reg $F32X4))
(_ Unit (emit (MInst.FpuExtend dst src size))))
dst))
;; Helper for emitting `MInst.VecExtend` instructions.
(decl vec_extend (VecExtendOp Reg bool ScalarSize) Reg)
(rule (vec_extend op src high_half size)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecExtend op dst src high_half size))))
dst))
;; Helper for emitting `MInst.VecExtract` instructions.
(decl vec_extract (Reg Reg u8) Reg)
(rule (vec_extract src1 src2 idx)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecExtract dst src1 src2 idx))))
dst))
;; Helper for emitting `MInst.LoadAcquire` instructions.
(decl load_acquire (Type MemFlags Reg) Reg)
(rule (load_acquire ty flags addr)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadAcquire ty dst addr flags))))
dst))
;; Helper for emitting `MInst.StoreRelease` instructions.
(decl store_release (Type MemFlags Reg Reg) SideEffectNoResult)
(rule (store_release ty flags src addr)
(SideEffectNoResult.Inst (MInst.StoreRelease ty src addr flags)))
;; Helper for generating a `tst` instruction.
;;
;; Produces a `ProducesFlags` rather than a register or emitted instruction
;; which must be paired with `with_flags*` helpers.
(decl tst_imm (Type Reg ImmLogic) ProducesFlags)
(rule (tst_imm ty reg imm)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.AluRRImmLogic (ALUOp.AndS)
(operand_size ty)
(writable_zero_reg)
reg
imm)))
;; Helper for generating a `CSel` instruction.
;;
;; Note that this doesn't actually emit anything, instead it produces a
;; `ConsumesFlags` instruction which must be consumed with `with_flags*`
;; helpers.
(decl csel (Cond Reg Reg) ConsumesFlags)
(rule (csel cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.CSel dst cond if_true if_false)
dst)))
;; Helper for generating a `CSNeg` instruction.
;;
;; Note that this doesn't actually emit anything, instead it produces a
;; `ConsumesFlags` instruction which must be consumed with `with_flags*`
;; helpers.
(decl csneg (Cond Reg Reg) ConsumesFlags)
(rule (csneg cond if_true if_false)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.CSNeg dst cond if_true if_false)
dst)))
;; Helper for generating `MInst.CCmp` instructions.
;; Creates a new `ProducesFlags` from the supplied `ProducesFlags` followed
;; immediately by the `MInst.CCmp` instruction.
(decl ccmp (OperandSize Reg Reg NZCV Cond ProducesFlags) ProducesFlags)
(rule (ccmp size rn rm nzcv cond inst_input)
(produces_flags_append inst_input (MInst.CCmp size rn rm nzcv cond)))
;; Helper for generating `MInst.CCmpImm` instructions.
(decl ccmp_imm (OperandSize u8 Reg UImm5 NZCV Cond) ConsumesFlags)
(rule 1 (ccmp_imm size 1 rn imm nzcv cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CCmpImm size rn imm nzcv cond)
(MInst.CSet dst cond)
(value_reg dst))))
(rule (ccmp_imm size _ty_bits rn imm nzcv cond)
(let ((dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CCmpImm size rn imm nzcv cond)
(MInst.CSetm dst cond)
(value_reg dst))))
;; Helpers for generating `add` instructions.
(decl add (Type Reg Reg) Reg)
(rule (add ty x y) (alu_rrr (ALUOp.Add) ty x y))
(decl add_imm (Type Reg Imm12) Reg)
(rule (add_imm ty x y) (alu_rr_imm12 (ALUOp.Add) ty x y))
(decl add_extend (Type Reg ExtendedValue) Reg)
(rule (add_extend ty x y) (alu_rr_extend_reg (ALUOp.Add) ty x y))
(decl add_shift (Type Reg Reg ShiftOpAndAmt) Reg)
(rule (add_shift ty x y z) (alu_rrr_shift (ALUOp.Add) ty x y z))
(decl add_vec (Reg Reg VectorSize) Reg)
(rule (add_vec x y size) (vec_rrr (VecALUOp.Add) x y size))
;; Helpers for generating `sub` instructions.
(decl sub (Type Reg Reg) Reg)
(rule (sub ty x y) (alu_rrr (ALUOp.Sub) ty x y))
(decl sub_imm (Type Reg Imm12) Reg)
(rule (sub_imm ty x y) (alu_rr_imm12 (ALUOp.Sub) ty x y))
(decl sub_extend (Type Reg ExtendedValue) Reg)
(rule (sub_extend ty x y) (alu_rr_extend_reg (ALUOp.Sub) ty x y))
(decl sub_shift (Type Reg Reg ShiftOpAndAmt) Reg)
(rule (sub_shift ty x y z) (alu_rrr_shift (ALUOp.Sub) ty x y z))
(decl sub_vec (Reg Reg VectorSize) Reg)
(rule (sub_vec x y size) (vec_rrr (VecALUOp.Sub) x y size))
;; Helpers for generating `madd` instructions.
(decl madd (Type Reg Reg Reg) Reg)
(rule (madd ty x y z) (alu_rrrr (ALUOp3.MAdd) ty x y z))
;; Helpers for generating `msub` instructions.
(decl msub (Type Reg Reg Reg) Reg)
(rule (msub ty x y z) (alu_rrrr (ALUOp3.MSub) ty x y z))
;; Helper for generating `uqadd` instructions.
(decl uqadd (Reg Reg VectorSize) Reg)
(rule (uqadd x y size) (vec_rrr (VecALUOp.Uqadd) x y size))
;; Helper for generating `sqadd` instructions.
(decl sqadd (Reg Reg VectorSize) Reg)
(rule (sqadd x y size) (vec_rrr (VecALUOp.Sqadd) x y size))
;; Helper for generating `uqsub` instructions.
(decl uqsub (Reg Reg VectorSize) Reg)
(rule (uqsub x y size) (vec_rrr (VecALUOp.Uqsub) x y size))
;; Helper for generating `sqsub` instructions.
(decl sqsub (Reg Reg VectorSize) Reg)
(rule (sqsub x y size) (vec_rrr (VecALUOp.Sqsub) x y size))
;; Helper for generating `umulh` instructions.
(decl umulh (Type Reg Reg) Reg)
(rule (umulh ty x y) (alu_rrr (ALUOp.UMulH) ty x y))
;; Helper for generating `smulh` instructions.
(decl smulh (Type Reg Reg) Reg)
(rule (smulh ty x y) (alu_rrr (ALUOp.SMulH) ty x y))
;; Helper for generating `mul` instructions.
(decl mul (Reg Reg VectorSize) Reg)
(rule (mul x y size) (vec_rrr (VecALUOp.Mul) x y size))
;; Helper for generating `neg` instructions.
(decl neg (Reg VectorSize) Reg)
(rule (neg x size) (vec_misc (VecMisc2.Neg) x size))
;; Helper for generating `rev64` instructions.
(decl rev64 (Reg VectorSize) Reg)
(rule (rev64 x size) (vec_misc (VecMisc2.Rev64) x size))
;; Helper for generating `xtn` instructions.
(decl xtn (Reg ScalarSize) Reg)
(rule (xtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Xtn) x size))
;; Helper for generating `fcvtn` instructions.
(decl fcvtn (Reg ScalarSize) Reg)
(rule (fcvtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Fcvtn) x size))
;; Helper for generating `sqxtn` instructions.
(decl sqxtn (Reg ScalarSize) Reg)
(rule (sqxtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Sqxtn) x size))
;; Helper for generating `sqxtn2` instructions.
(decl sqxtn2 (Reg Reg ScalarSize) Reg)
(rule (sqxtn2 x y size) (vec_rr_narrow_high (VecRRNarrowOp.Sqxtn) x y size))
;; Helper for generating `sqxtun` instructions.
(decl sqxtun (Reg ScalarSize) Reg)
(rule (sqxtun x size) (vec_rr_narrow_low (VecRRNarrowOp.Sqxtun) x size))
;; Helper for generating `sqxtun2` instructions.
(decl sqxtun2 (Reg Reg ScalarSize) Reg)
(rule (sqxtun2 x y size) (vec_rr_narrow_high (VecRRNarrowOp.Sqxtun) x y size))
;; Helper for generating `uqxtn` instructions.
(decl uqxtn (Reg ScalarSize) Reg)
(rule (uqxtn x size) (vec_rr_narrow_low (VecRRNarrowOp.Uqxtn) x size))
;; Helper for generating `uqxtn2` instructions.
(decl uqxtn2 (Reg Reg ScalarSize) Reg)
(rule (uqxtn2 x y size) (vec_rr_narrow_high (VecRRNarrowOp.Uqxtn) x y size))
;; Helper for generating `fence` instructions.
(decl aarch64_fence () SideEffectNoResult)
(rule (aarch64_fence)
(SideEffectNoResult.Inst (MInst.Fence)))
;; Helper for generating `csdb` instructions.
(decl csdb () SideEffectNoResult)
(rule (csdb)
(SideEffectNoResult.Inst (MInst.Csdb)))
;; Helper for generating `brk` instructions.
(decl brk () SideEffectNoResult)
(rule (brk)
(SideEffectNoResult.Inst (MInst.Brk)))
;; Helper for generating `addp` instructions.
(decl addp (Reg Reg VectorSize) Reg)
(rule (addp x y size) (vec_rrr (VecALUOp.Addp) x y size))
;; Helper for generating `zip1` instructions.
(decl zip1 (Reg Reg VectorSize) Reg)
(rule (zip1 x y size) (vec_rrr (VecALUOp.Zip1) x y size))
;; Helper for generating vector `abs` instructions.
(decl vec_abs (Reg VectorSize) Reg)
(rule (vec_abs x size) (vec_misc (VecMisc2.Abs) x size))
;; Helper for generating instruction sequences to calculate a scalar absolute
;; value.
(decl abs (OperandSize Reg) Reg)
(rule (abs size x)
(value_regs_get (with_flags (cmp_imm size x (u8_into_imm12 0))
(csneg (Cond.Gt) x x)) 0))
;; Helper for generating `addv` instructions.
(decl addv (Reg VectorSize) Reg)
(rule (addv x size) (vec_lanes (VecLanesOp.Addv) x size))
;; Helper for generating `shll32` instructions.
(decl shll32 (Reg bool) Reg)
(rule (shll32 x high_half) (vec_rr_long (VecRRLongOp.Shll32) x high_half))
;; Helpers for generating `addlp` instructions.
(decl saddlp8 (Reg) Reg)
(rule (saddlp8 x) (vec_rr_pair_long (VecRRPairLongOp.Saddlp8) x))
(decl saddlp16 (Reg) Reg)
(rule (saddlp16 x) (vec_rr_pair_long (VecRRPairLongOp.Saddlp16) x))
(decl uaddlp8 (Reg) Reg)
(rule (uaddlp8 x) (vec_rr_pair_long (VecRRPairLongOp.Uaddlp8) x))
(decl uaddlp16 (Reg) Reg)
(rule (uaddlp16 x) (vec_rr_pair_long (VecRRPairLongOp.Uaddlp16) x))
;; Helper for generating `umlal32` instructions.
(decl umlal32 (Reg Reg Reg bool) Reg)
(rule (umlal32 x y z high_half) (vec_rrrr_long (VecRRRLongModOp.Umlal32) x y z high_half))
;; Helper for generating `smull8` instructions.
(decl smull8 (Reg Reg bool) Reg)
(rule (smull8 x y high_half) (vec_rrr_long (VecRRRLongOp.Smull8) x y high_half))
;; Helper for generating `umull8` instructions.
(decl umull8 (Reg Reg bool) Reg)
(rule (umull8 x y high_half) (vec_rrr_long (VecRRRLongOp.Umull8) x y high_half))
;; Helper for generating `smull16` instructions.
(decl smull16 (Reg Reg bool) Reg)
(rule (smull16 x y high_half) (vec_rrr_long (VecRRRLongOp.Smull16) x y high_half))
;; Helper for generating `umull16` instructions.
(decl umull16 (Reg Reg bool) Reg)
(rule (umull16 x y high_half) (vec_rrr_long (VecRRRLongOp.Umull16) x y high_half))
;; Helper for generating `smull32` instructions.
(decl smull32 (Reg Reg bool) Reg)
(rule (smull32 x y high_half) (vec_rrr_long (VecRRRLongOp.Smull32) x y high_half))
;; Helper for generating `umull32` instructions.
(decl umull32 (Reg Reg bool) Reg)
(rule (umull32 x y high_half) (vec_rrr_long (VecRRRLongOp.Umull32) x y high_half))
;; Helper for generating `asr` instructions.
(decl asr (Type Reg Reg) Reg)
(rule (asr ty x y) (alu_rrr (ALUOp.Asr) ty x y))
(decl asr_imm (Type Reg ImmShift) Reg)
(rule (asr_imm ty x imm) (alu_rr_imm_shift (ALUOp.Asr) ty x imm))
;; Helper for generating `lsr` instructions.
(decl lsr (Type Reg Reg) Reg)
(rule (lsr ty x y) (alu_rrr (ALUOp.Lsr) ty x y))
(decl lsr_imm (Type Reg ImmShift) Reg)
(rule (lsr_imm ty x imm) (alu_rr_imm_shift (ALUOp.Lsr) ty x imm))
;; Helper for generating `lsl` instructions.
(decl lsl (Type Reg Reg) Reg)
(rule (lsl ty x y) (alu_rrr (ALUOp.Lsl) ty x y))
(decl lsl_imm (Type Reg ImmShift) Reg)
(rule (lsl_imm ty x imm) (alu_rr_imm_shift (ALUOp.Lsl) ty x imm))
;; Helper for generating `udiv` instructions.
(decl a64_udiv (Type Reg Reg) Reg)
(rule (a64_udiv ty x y) (alu_rrr (ALUOp.UDiv) ty x y))
;; Helper for generating `sdiv` instructions.
(decl a64_sdiv (Type Reg Reg) Reg)
(rule (a64_sdiv ty x y) (alu_rrr (ALUOp.SDiv) ty x y))
;; Helper for generating `not` instructions.
(decl not (Reg VectorSize) Reg)
(rule (not x size) (vec_misc (VecMisc2.Not) x size))
;; Helpers for generating `orr_not` instructions.
(decl orr_not (Type Reg Reg) Reg)
(rule (orr_not ty x y) (alu_rrr (ALUOp.OrrNot) ty x y))
(decl orr_not_shift (Type Reg Reg ShiftOpAndAmt) Reg)
(rule (orr_not_shift ty x y shift) (alu_rrr_shift (ALUOp.OrrNot) ty x y shift))
;; Helpers for generating `orr` instructions.
(decl orr (Type Reg Reg) Reg)
(rule (orr ty x y) (alu_rrr (ALUOp.Orr) ty x y))
(decl orr_imm (Type Reg ImmLogic) Reg)
(rule (orr_imm ty x y) (alu_rr_imm_logic (ALUOp.Orr) ty x y))
(decl orr_vec (Reg Reg VectorSize) Reg)
(rule (orr_vec x y size) (vec_rrr (VecALUOp.Orr) x y size))
;; Helpers for generating `and` instructions.
(decl and_reg (Type Reg Reg) Reg)
(rule (and_reg ty x y) (alu_rrr (ALUOp.And) ty x y))
(decl and_imm (Type Reg ImmLogic) Reg)
(rule (and_imm ty x y) (alu_rr_imm_logic (ALUOp.And) ty x y))
(decl and_vec (Reg Reg VectorSize) Reg)
(rule (and_vec x y size) (vec_rrr (VecALUOp.And) x y size))
;; Helpers for generating `eor` instructions.
(decl eor_vec (Reg Reg VectorSize) Reg)
(rule (eor_vec x y size) (vec_rrr (VecALUOp.Eor) x y size))
;; Helpers for generating `bic` instructions.
(decl bic (Type Reg Reg) Reg)
(rule (bic ty x y) (alu_rrr (ALUOp.AndNot) ty x y))
(decl bic_vec (Reg Reg VectorSize) Reg)
(rule (bic_vec x y size) (vec_rrr (VecALUOp.Bic) x y size))
;; Helpers for generating `sshl` instructions.
(decl sshl (Reg Reg VectorSize) Reg)
(rule (sshl x y size) (vec_rrr (VecALUOp.Sshl) x y size))
;; Helpers for generating `ushl` instructions.
(decl ushl (Reg Reg VectorSize) Reg)
(rule (ushl x y size) (vec_rrr (VecALUOp.Ushl) x y size))
;; Helpers for generating `rotr` instructions.
(decl a64_rotr (Type Reg Reg) Reg)
(rule (a64_rotr ty x y) (alu_rrr (ALUOp.RotR) ty x y))
(decl a64_rotr_imm (Type Reg ImmShift) Reg)
(rule (a64_rotr_imm ty x y) (alu_rr_imm_shift (ALUOp.RotR) ty x y))
;; Helpers for generating `rbit` instructions.
(decl rbit (Type Reg) Reg)
(rule (rbit ty x) (bit_rr (BitOp.RBit) ty x))
;; Helpers for generating `clz` instructions.
(decl a64_clz (Type Reg) Reg)
(rule (a64_clz ty x) (bit_rr (BitOp.Clz) ty x))
;; Helpers for generating `cls` instructions.
(decl a64_cls (Type Reg) Reg)
(rule (a64_cls ty x) (bit_rr (BitOp.Cls) ty x))
;; Helpers for generating `eon` instructions.
(decl eon (Type Reg Reg) Reg)
(rule (eon ty x y) (alu_rrr (ALUOp.EorNot) ty x y))
;; Helpers for generating `cnt` instructions.
(decl vec_cnt (Reg VectorSize) Reg)
(rule (vec_cnt x size) (vec_misc (VecMisc2.Cnt) x size))
;; Helpers for generating a `bsl` instruction.
(decl bsl (Type Reg Reg Reg) Reg)
(rule (bsl ty c x y)
(vec_rrr_mod (VecALUModOp.Bsl) c x y (vector_size ty)))
;; Helper for generating a `udf` instruction.
(decl udf (TrapCode) SideEffectNoResult)
(rule (udf trap_code)
(SideEffectNoResult.Inst (MInst.Udf trap_code)))
;; Helpers for generating various load instructions, with varying
;; widths and sign/zero-extending properties.
(decl aarch64_uload8 (AMode MemFlags) Reg)
(rule (aarch64_uload8 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.ULoad8 dst amode flags))))
dst))
(decl aarch64_sload8 (AMode MemFlags) Reg)
(rule (aarch64_sload8 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.SLoad8 dst amode flags))))
dst))
(decl aarch64_uload16 (AMode MemFlags) Reg)
(rule (aarch64_uload16 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.ULoad16 dst amode flags))))
dst))
(decl aarch64_sload16 (AMode MemFlags) Reg)
(rule (aarch64_sload16 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.SLoad16 dst amode flags))))
dst))
(decl aarch64_uload32 (AMode MemFlags) Reg)
(rule (aarch64_uload32 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.ULoad32 dst amode flags))))
dst))
(decl aarch64_sload32 (AMode MemFlags) Reg)
(rule (aarch64_sload32 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.SLoad32 dst amode flags))))
dst))
(decl aarch64_uload64 (AMode MemFlags) Reg)
(rule (aarch64_uload64 amode flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.ULoad64 dst amode flags))))
dst))
(decl aarch64_fpuload32 (AMode MemFlags) Reg)
(rule (aarch64_fpuload32 amode flags)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuLoad32 dst amode flags))))
dst))
(decl aarch64_fpuload64 (AMode MemFlags) Reg)
(rule (aarch64_fpuload64 amode flags)
(let ((dst WritableReg (temp_writable_reg $F64))
(_ Unit (emit (MInst.FpuLoad64 dst amode flags))))
dst))
(decl aarch64_fpuload128 (AMode MemFlags) Reg)
(rule (aarch64_fpuload128 amode flags)
(let ((dst WritableReg (temp_writable_reg $F64X2))
(_ Unit (emit (MInst.FpuLoad128 dst amode flags))))
dst))
(decl aarch64_loadp64 (PairAMode MemFlags) ValueRegs)
(rule (aarch64_loadp64 amode flags)
(let ((dst1 WritableReg (temp_writable_reg $I64))
(dst2 WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadP64 dst1 dst2 amode flags))))
(value_regs dst1 dst2)))
;; Helpers for generating various store instructions with varying
;; widths.
(decl aarch64_store8 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_store8 amode flags val)
(SideEffectNoResult.Inst (MInst.Store8 val amode flags)))
(decl aarch64_store16 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_store16 amode flags val)
(SideEffectNoResult.Inst (MInst.Store16 val amode flags)))
(decl aarch64_store32 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_store32 amode flags val)
(SideEffectNoResult.Inst (MInst.Store32 val amode flags)))
(decl aarch64_store64 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_store64 amode flags val)
(SideEffectNoResult.Inst (MInst.Store64 val amode flags)))
(decl aarch64_fpustore32 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_fpustore32 amode flags val)
(SideEffectNoResult.Inst (MInst.FpuStore32 val amode flags)))
(decl aarch64_fpustore64 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_fpustore64 amode flags val)
(SideEffectNoResult.Inst (MInst.FpuStore64 val amode flags)))
(decl aarch64_fpustore128 (AMode MemFlags Reg) SideEffectNoResult)
(rule (aarch64_fpustore128 amode flags val)
(SideEffectNoResult.Inst (MInst.FpuStore128 val amode flags)))
(decl aarch64_storep64 (PairAMode MemFlags Reg Reg) SideEffectNoResult)
(rule (aarch64_storep64 amode flags val1 val2)
(SideEffectNoResult.Inst (MInst.StoreP64 val1 val2 amode flags)))
;; Helper for generating a `trapif` instruction.
(decl trap_if (ProducesFlags TrapCode Cond) InstOutput)
(rule (trap_if flags trap_code cond)
(side_effect
(with_flags_side_effect flags
(ConsumesFlags.ConsumesFlagsSideEffect
(MInst.TrapIf (cond_br_cond cond) trap_code)))))
;; Immediate value helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Type of extension performed by an immediate helper
(type ImmExtend
(enum
(Sign)
(Zero)))
;; Arguments:
;; * Immediate type
;; * Way to extend the immediate value to the full width of the destination
;; register
;; * Immediate value - only the bits that fit within the type are used and
;; extended, while the rest are ignored
;;
;; Note that, unlike the convention in the AArch64 backend, this helper leaves
;; all bits in the destination register in a defined state, i.e. smaller types
;; such as `I8` are either sign- or zero-extended.
(decl imm (Type ImmExtend u64) Reg)
;; Weird logical-instruction immediate in ORI using zero register; to simplify,
;; we only match when we are zero-extending the value.
(rule 1 (imm (integral_ty ty) (ImmExtend.Zero) k)
(if-let n (imm_logic_from_u64 ty k))
(orr_imm ty (zero_reg) n))
(decl load_constant64_full (Type ImmExtend u64) Reg)
(extern constructor load_constant64_full load_constant64_full)
;; Fallback for integral 64-bit constants
(rule (imm (integral_ty ty) extend n)
(load_constant64_full ty extend n))
;; Sign extension helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Place a `Value` into a register, sign extending it to 32-bits
(decl put_in_reg_sext32 (Value) Reg)
(rule -1 (put_in_reg_sext32 val @ (value_type (fits_in_32 ty)))
(extend val $true (ty_bits ty) 32))
;; 32/64-bit passthrough.
(rule (put_in_reg_sext32 val @ (value_type $I32)) val)
(rule (put_in_reg_sext32 val @ (value_type $I64)) val)
;; Place a `Value` into a register, zero extending it to 32-bits
(decl put_in_reg_zext32 (Value) Reg)
(rule -1 (put_in_reg_zext32 val @ (value_type (fits_in_32 ty)))
(extend val $false (ty_bits ty) 32))
;; 32/64-bit passthrough.
(rule (put_in_reg_zext32 val @ (value_type $I32)) val)
(rule (put_in_reg_zext32 val @ (value_type $I64)) val)
;; Place a `Value` into a register, sign extending it to 64-bits
(decl put_in_reg_sext64 (Value) Reg)
(rule 1 (put_in_reg_sext64 val @ (value_type (fits_in_32 ty)))
(extend val $true (ty_bits ty) 64))
;; 64-bit passthrough.
(rule (put_in_reg_sext64 val @ (value_type $I64)) val)
;; Place a `Value` into a register, zero extending it to 64-bits
(decl put_in_reg_zext64 (Value) Reg)
(rule 1 (put_in_reg_zext64 val @ (value_type (fits_in_32 ty)))
(extend val $false (ty_bits ty) 64))
;; 64-bit passthrough.
(rule (put_in_reg_zext64 val @ (value_type $I64)) val)
;; Misc instruction helpers ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl trap_if_zero_divisor (Reg) Reg)
(rule (trap_if_zero_divisor reg)
(let ((_ Unit (emit (MInst.TrapIf (cond_br_zero reg) (trap_code_division_by_zero)))))
reg))
(decl size_from_ty (Type) OperandSize)
(rule 1 (size_from_ty (fits_in_32 _ty)) (OperandSize.Size32))
(rule (size_from_ty $I64) (OperandSize.Size64))
;; Check for signed overflow. The only case is min_value / -1.
;; The following checks must be done in 32-bit or 64-bit, depending
;; on the input type.
(decl trap_if_div_overflow (Type Reg Reg) Reg)
(rule (trap_if_div_overflow ty x y)
(let (
;; Check RHS is -1.
(_ Unit (emit (MInst.AluRRImm12 (ALUOp.AddS) (operand_size ty) (writable_zero_reg) y (u8_into_imm12 1))))
;; Check LHS is min_value, by subtracting 1 and branching if
;; there is overflow.
(_ Unit (emit (MInst.CCmpImm (size_from_ty ty)
x
(u8_into_uimm5 1)
(nzcv $false $false $false $false)
(Cond.Eq))))
(_ Unit (emit (MInst.TrapIf (cond_br_cond (Cond.Vs))
(trap_code_integer_overflow))))
)
x))
(decl sink_atomic_load (Inst) Reg)
(rule (sink_atomic_load x @ (atomic_load _ addr))
(let ((_ Unit (sink_inst x)))
(put_in_reg addr)))
;; Helper for generating either an `AluRRR`, `AluRRRShift`, or `AluRRImmLogic`
;; instruction depending on the input. Note that this requires that the `ALUOp`
;; specified is commutative.
(decl alu_rs_imm_logic_commutative (ALUOp Type Value Value) Reg)
;; Base case of operating on registers.
(rule -1 (alu_rs_imm_logic_commutative op ty x y)
(alu_rrr op ty x y))
;; Special cases for when one operand is a constant.
(rule (alu_rs_imm_logic_commutative op ty x (iconst k))
(if-let imm (imm_logic_from_imm64 ty k))
(alu_rr_imm_logic op ty x imm))
(rule 1 (alu_rs_imm_logic_commutative op ty (iconst k) x)
(if-let imm (imm_logic_from_imm64 ty k))
(alu_rr_imm_logic op ty x imm))
;; Special cases for when one operand is shifted left by a constant.
(rule (alu_rs_imm_logic_commutative op ty x (ishl y (iconst k)))
(if-let amt (lshl_from_imm64 ty k))
(alu_rrr_shift op ty x y amt))
(rule 1 (alu_rs_imm_logic_commutative op ty (ishl x (iconst k)) y)
(if-let amt (lshl_from_imm64 ty k))
(alu_rrr_shift op ty y x amt))
;; Same as `alu_rs_imm_logic_commutative` above, except that it doesn't require
;; that the operation is commutative.
(decl alu_rs_imm_logic (ALUOp Type Value Value) Reg)
(rule -1 (alu_rs_imm_logic op ty x y)
(alu_rrr op ty x y))
(rule (alu_rs_imm_logic op ty x (iconst k))
(if-let imm (imm_logic_from_imm64 ty k))
(alu_rr_imm_logic op ty x imm))
(rule (alu_rs_imm_logic op ty x (ishl y (iconst k)))
(if-let amt (lshl_from_imm64 ty k))
(alu_rrr_shift op ty x y amt))
;; Helper for generating i128 bitops which simply do the same operation to the
;; hi/lo registers.
;;
;; TODO: Support immlogic here
(decl i128_alu_bitop (ALUOp Type Value Value) ValueRegs)
(rule (i128_alu_bitop op ty x y)
(let (
(x_regs ValueRegs (put_in_regs x))
(x_lo Reg (value_regs_get x_regs 0))
(x_hi Reg (value_regs_get x_regs 1))
(y_regs ValueRegs (put_in_regs y))
(y_lo Reg (value_regs_get y_regs 0))
(y_hi Reg (value_regs_get y_regs 1))
)
(value_regs
(alu_rrr op ty x_lo y_lo)
(alu_rrr op ty x_hi y_hi))))
;; Helper for emitting `MInst.VecLoadReplicate` instructions.
(decl ld1r (Reg VectorSize MemFlags) Reg)
(rule (ld1r src size flags)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.VecLoadReplicate dst src size flags))))
dst))
;; Helper for emitting `MInst.LoadExtName` instructions.
(decl load_ext_name (BoxExternalName i64) Reg)
(rule (load_ext_name extname offset)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadExtName dst extname offset))))
dst))
;; Helper for emitting `MInst.LoadAddr` instructions.
(decl load_addr (AMode) Reg)
(rule (load_addr (AMode.UnsignedOffset r imm))
(if (is_zero_uimm12 imm))
r)
(rule (load_addr (AMode.Unscaled r imm))
(if (is_zero_simm9 imm))
r)
(rule (load_addr (AMode.RegOffset r 0 _)) r)
(rule (load_addr (AMode.FPOffset 0 _)) (fp_reg))
(rule (load_addr (AMode.SPOffset 0 _)) (stack_reg))
(rule -1 (load_addr addr)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.LoadAddr dst addr))))
dst))
;; Lower the address of a load or a store.
(decl amode (Type Value u32) AMode)
;; TODO: Port lower_address() to ISLE.
(extern constructor amode amode)
(decl sink_load_into_amode (Type Inst) AMode)
(rule (sink_load_into_amode ty x @ (load _ addr offset))
(let ((_ Unit (sink_inst x)))
(amode ty addr offset)))
;; Lower a constant f64.
(decl constant_f64 (u64) Reg)
;; TODO: Port lower_constant_f64() to ISLE.
(extern constructor constant_f64 constant_f64)
;; Lower a constant f128.
(decl constant_f128 (u128) Reg)
;; TODO: Port lower_constant_f128() to ISLE.
(extern constructor constant_f128 constant_f128)
;; Lower a vector splat with a constant parameter.
(decl splat_const (u64 VectorSize) Reg)
;; TODO: Port lower_splat_const() to ISLE.
(extern constructor splat_const splat_const)
;; Lower a FloatCC to a Cond.
(decl fp_cond_code (FloatCC) Cond)
;; TODO: Port lower_fp_condcode() to ISLE.
(extern constructor fp_cond_code fp_cond_code)
;; Lower an integer cond code.
(decl cond_code (IntCC) Cond)
;; TODO: Port lower_condcode() to ISLE.
(extern constructor cond_code cond_code)
;; Invert a condition code.
(decl invert_cond (Cond) Cond)
;; TODO: Port cond.invert() to ISLE.
(extern constructor invert_cond invert_cond)
;; Generate comparison to zero operator from input condition code
(decl float_cc_cmp_zero_to_vec_misc_op (FloatCC) VecMisc2)
(extern constructor float_cc_cmp_zero_to_vec_misc_op float_cc_cmp_zero_to_vec_misc_op)
(decl float_cc_cmp_zero_to_vec_misc_op_swap (FloatCC) VecMisc2)
(extern constructor float_cc_cmp_zero_to_vec_misc_op_swap float_cc_cmp_zero_to_vec_misc_op_swap)
;; Match valid generic compare to zero cases
(decl fcmp_zero_cond (FloatCC) FloatCC)
(extern extractor fcmp_zero_cond fcmp_zero_cond)
;; Match not equal compare to zero separately as it requires two output instructions
(decl fcmp_zero_cond_not_eq (FloatCC) FloatCC)
(extern extractor fcmp_zero_cond_not_eq fcmp_zero_cond_not_eq)
;; Helper for generating float compare to zero instructions where 2nd argument is zero
(decl float_cmp_zero (FloatCC Reg VectorSize) Reg)
(rule (float_cmp_zero cond rn size)
(vec_misc (float_cc_cmp_zero_to_vec_misc_op cond) rn size))
;; Helper for generating float compare to zero instructions in case where 1st argument is zero
(decl float_cmp_zero_swap (FloatCC Reg VectorSize) Reg)
(rule (float_cmp_zero_swap cond rn size)
(vec_misc (float_cc_cmp_zero_to_vec_misc_op_swap cond) rn size))
;; Helper for generating float compare equal to zero instruction
(decl fcmeq0 (Reg VectorSize) Reg)
(rule (fcmeq0 rn size)
(vec_misc (VecMisc2.Fcmeq0) rn size))
;; Generate comparison to zero operator from input condition code
(decl int_cc_cmp_zero_to_vec_misc_op (IntCC) VecMisc2)
(extern constructor int_cc_cmp_zero_to_vec_misc_op int_cc_cmp_zero_to_vec_misc_op)
(decl int_cc_cmp_zero_to_vec_misc_op_swap (IntCC) VecMisc2)
(extern constructor int_cc_cmp_zero_to_vec_misc_op_swap int_cc_cmp_zero_to_vec_misc_op_swap)
;; Match valid generic compare to zero cases
(decl icmp_zero_cond (IntCC) IntCC)
(extern extractor icmp_zero_cond icmp_zero_cond)
;; Match not equal compare to zero separately as it requires two output instructions
(decl icmp_zero_cond_not_eq (IntCC) IntCC)
(extern extractor icmp_zero_cond_not_eq icmp_zero_cond_not_eq)
;; Helper for generating int compare to zero instructions where 2nd argument is zero
(decl int_cmp_zero (IntCC Reg VectorSize) Reg)
(rule (int_cmp_zero cond rn size)
(vec_misc (int_cc_cmp_zero_to_vec_misc_op cond) rn size))
;; Helper for generating int compare to zero instructions in case where 1st argument is zero
(decl int_cmp_zero_swap (IntCC Reg VectorSize) Reg)
(rule (int_cmp_zero_swap cond rn size)
(vec_misc (int_cc_cmp_zero_to_vec_misc_op_swap cond) rn size))
;; Helper for generating int compare equal to zero instruction
(decl cmeq0 (Reg VectorSize) Reg)
(rule (cmeq0 rn size)
(vec_misc (VecMisc2.Cmeq0) rn size))
;; Helper for emitting `MInst.AtomicRMW` instructions.
(decl lse_atomic_rmw (AtomicRMWOp Value Reg Type MemFlags) Reg)
(rule (lse_atomic_rmw op p r_arg2 ty flags)
(let (
(r_addr Reg p)
(dst WritableReg (temp_writable_reg ty))
(_ Unit (emit (MInst.AtomicRMW op r_arg2 dst r_addr ty flags)))
)
dst))
;; Helper for emitting `MInst.AtomicCAS` instructions.
(decl lse_atomic_cas (Reg Reg Reg Type MemFlags) Reg)
(rule (lse_atomic_cas addr expect replace ty flags)
(let (
(dst WritableReg (temp_writable_reg ty))
(_ Unit (emit (MInst.AtomicCAS dst expect replace addr ty flags)))
)
dst))
;; Helper for emitting `MInst.AtomicRMWLoop` instructions.
;; - Make sure that both args are in virtual regs, since in effect
;; we have to do a parallel copy to get them safely to the AtomicRMW input
;; regs, and that's not guaranteed safe if either is in a real reg.
;; - Move the args to the preordained AtomicRMW input regs
;; - And finally, copy the preordained AtomicRMW output reg to its destination.
(decl atomic_rmw_loop (AtomicRMWLoopOp Reg Reg Type MemFlags) Reg)
(rule (atomic_rmw_loop op addr operand ty flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(scratch1 WritableReg (temp_writable_reg $I64))
(scratch2 WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AtomicRMWLoop ty op flags addr operand dst scratch1 scratch2))))
dst))
;; Helper for emitting `MInst.AtomicCASLoop` instructions.
;; This is very similar to, but not identical to, the AtomicRmw case. Note
;; that the AtomicCASLoop sequence does its own masking, so we don't need to worry
;; about zero-extending narrow (I8/I16/I32) values here.
;; Make sure that all three args are in virtual regs. See corresponding comment
;; for `atomic_rmw_loop` above.
(decl atomic_cas_loop (Reg Reg Reg Type MemFlags) Reg)
(rule (atomic_cas_loop addr expect replace ty flags)
(let ((dst WritableReg (temp_writable_reg $I64))
(scratch WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.AtomicCASLoop ty flags addr expect replace dst scratch))))
dst))
;; Helper for emitting `MInst.MovPReg` instructions.
(decl mov_preg (PReg) Reg)
(rule (mov_preg src)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.MovPReg dst src))))
dst))
(decl preg_sp () PReg)
(extern constructor preg_sp preg_sp)
(decl preg_fp () PReg)
(extern constructor preg_fp preg_fp)
(decl preg_link () PReg)
(extern constructor preg_link preg_link)
(decl aarch64_sp () Reg)
(rule (aarch64_sp)
(mov_preg (preg_sp)))
(decl aarch64_fp () Reg)
(rule (aarch64_fp)
(mov_preg (preg_fp)))
(decl aarch64_link () Reg)
(rule 1 (aarch64_link)
(if (preserve_frame_pointers))
(if (sign_return_address_disabled))
(let ((dst WritableReg (temp_writable_reg $I64))
;; Even though LR is not an allocatable register, whether it
;; contains the return address for the current function is
;; unknown at this point. For example, this operation may come
;; immediately after a call, in which case LR would not have a
;; valid value. That's why we must obtain the return address from
;; the frame record that corresponds to the current subroutine on
;; the stack; the presence of the record is guaranteed by the
;; `preserve_frame_pointers` setting.
(addr AMode (AMode.FPOffset 8 $I64))
(_ Unit (emit (MInst.ULoad64 dst addr (mem_flags_trusted)))))
dst))
(rule (aarch64_link)
(if (preserve_frame_pointers))
;; Similarly to the rule above, we must load the return address from the
;; the frame record. Furthermore, we can use LR as a scratch register
;; because the function will set it to the return address immediately
;; before returning.
(let ((addr AMode (AMode.FPOffset 8 $I64))
(lr WritableReg (writable_link_reg))
(_ Unit (emit (MInst.ULoad64 lr addr (mem_flags_trusted))))
(_ Unit (emit (MInst.Xpaclri))))
(mov_preg (preg_link))))
;; Helper for getting the maximum shift amount for a type.
(decl max_shift (Type) u8)
(rule (max_shift $F64) 63)
(rule (max_shift $F32) 31)
;; Helper for generating `fcopysign` instruction sequences.
(decl fcopy_sign (Reg Reg Type) Reg)
(rule 1 (fcopy_sign x y (ty_scalar_float ty))
(let ((dst WritableReg (temp_writable_reg $F64))
(tmp Reg (fpu_rri (fpu_op_ri_ushr (ty_bits ty) (max_shift ty)) y))
(_ Unit (emit (MInst.FpuRRIMod (fpu_op_ri_sli (ty_bits ty) (max_shift ty)) dst x tmp))))
dst))
(rule (fcopy_sign x y ty @ (multi_lane _ _))
(let ((dst WritableReg (temp_writable_reg $I8X16))
(tmp Reg (vec_shift_imm (VecShiftImmOp.Ushr) (max_shift (lane_type ty)) y (vector_size ty)))
(_ Unit (emit (MInst.VecShiftImmMod (VecShiftImmModOp.Sli) dst x tmp (vector_size ty) (max_shift (lane_type ty))))))
dst))
;; Helpers for generating `MInst.FpuToInt` instructions.
(decl fpu_to_int_nan_check (ScalarSize Reg) Reg)
(rule (fpu_to_int_nan_check size src)
(let ((r ValueRegs
(with_flags (fpu_cmp size src src)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Vs))
(trap_code_bad_conversion_to_integer))
src))))
(value_regs_get r 0)))
;; Checks that the value is not less than the minimum bound,
;; accepting a boolean (whether the type is signed), input type,
;; output type, and registers containing the source and minimum bound.
(decl fpu_to_int_underflow_check (bool Type Type Reg Reg) Reg)
(rule (fpu_to_int_underflow_check $true $F32 (fits_in_16 out_ty) src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (ScalarSize.Size32) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Le))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(rule (fpu_to_int_underflow_check $true $F64 (fits_in_32 out_ty) src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (ScalarSize.Size64) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Le))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(rule -1 (fpu_to_int_underflow_check $true in_ty _out_ty src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (scalar_size in_ty) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Lt))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(rule (fpu_to_int_underflow_check $false in_ty _out_ty src min)
(let ((r ValueRegs
(with_flags (fpu_cmp (scalar_size in_ty) src min)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Le))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
(decl fpu_to_int_overflow_check (ScalarSize Reg Reg) Reg)
(rule (fpu_to_int_overflow_check size src max)
(let ((r ValueRegs
(with_flags (fpu_cmp size src max)
(ConsumesFlags.ConsumesFlagsReturnsReg
(MInst.TrapIf (cond_br_cond (Cond.Ge))
(trap_code_integer_overflow))
src))))
(value_regs_get r 0)))
;; Emits the appropriate instruction sequence to convert a
;; floating-point value to an integer, trapping if the value
;; is a NaN or does not fit in the target type.
;; Accepts the specific conversion op, the source register,
;; whether the input is signed, and finally the input and output
;; types.
(decl fpu_to_int_cvt (FpuToIntOp Reg bool Type Type) Reg)
(rule (fpu_to_int_cvt op src signed in_ty out_ty)
(let ((size ScalarSize (scalar_size in_ty))
(in_bits u8 (ty_bits in_ty))
(out_bits u8 (ty_bits out_ty))
(src Reg (fpu_to_int_nan_check size src))
(min Reg (min_fp_value signed in_bits out_bits))
(src Reg (fpu_to_int_underflow_check signed in_ty out_ty src min))
(max Reg (max_fp_value signed in_bits out_bits))
(src Reg (fpu_to_int_overflow_check size src max)))
(fpu_to_int op src)))
;; Emits the appropriate instruction sequence to convert a
;; floating-point value to an integer, saturating if the value
;; does not fit in the target type.
;; Accepts the specific conversion op, the source register,
;; whether the input is signed, and finally the output type.
(decl fpu_to_int_cvt_sat (FpuToIntOp Reg bool Type) Reg)
(rule 1 (fpu_to_int_cvt_sat op src _ $I64)
(fpu_to_int op src))
(rule 1 (fpu_to_int_cvt_sat op src _ $I32)
(fpu_to_int op src))
(rule (fpu_to_int_cvt_sat op src $false (fits_in_16 out_ty))
(let ((result Reg (fpu_to_int op src))
(max Reg (imm out_ty (ImmExtend.Zero) -1)))
(with_flags_reg
(cmp (OperandSize.Size32) result max)
(csel (Cond.Hi) max result))))
(rule (fpu_to_int_cvt_sat op src $true (fits_in_16 out_ty))
(let ((result Reg (fpu_to_int op src))
(max Reg (imm $I32 (ImmExtend.Sign) (signed_max out_ty)))
(min Reg (imm $I32 (ImmExtend.Sign) (signed_min out_ty)))
(result Reg (with_flags_reg
(cmp (operand_size out_ty) result max)
(csel (Cond.Gt) max result)))
(result Reg (with_flags_reg
(cmp (operand_size out_ty) result min)
(csel (Cond.Lt) min result))))
result))
(decl signed_min (Type) u64)
(rule (signed_min $I8) -128)
(rule (signed_min $I16) -32768)
(decl signed_max (Type) u64)
(rule (signed_max $I8) 127)
(rule (signed_max $I16) 32767)
(decl fpu_to_int (FpuToIntOp Reg) Reg)
(rule (fpu_to_int op src)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.FpuToInt op dst src))))
dst))
;; Helper for generating `MInst.IntToFpu` instructions.
(decl int_to_fpu (IntToFpuOp Reg) Reg)
(rule (int_to_fpu op src)
(let ((dst WritableReg (temp_writable_reg $I8X16))
(_ Unit (emit (MInst.IntToFpu op dst src))))
dst))
;;;; Helpers for Emitting Calls ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
(decl gen_call (SigRef ExternalName RelocDistance ValueSlice) InstOutput)
(extern constructor gen_call gen_call)
(decl gen_call_indirect (SigRef Value ValueSlice) InstOutput)
(extern constructor gen_call_indirect gen_call_indirect)
;; Helpers for pinned register manipulation.
(decl writable_pinned_reg () WritableReg)
(extern constructor writable_pinned_reg writable_pinned_reg)
(decl pinned_reg () Reg)
(rule (pinned_reg) (writable_pinned_reg))
(decl write_pinned_reg (Reg) SideEffectNoResult)
(rule (write_pinned_reg val)
(let ((dst WritableReg (writable_pinned_reg)))
(SideEffectNoResult.Inst (gen_move $I64 dst val))))
;; Helpers for stackslot effective address generation.
(decl compute_stack_addr (StackSlot Offset32) Reg)
(rule (compute_stack_addr stack_slot offset)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (abi_stackslot_addr dst stack_slot offset))))
dst))
;; Helper for emitting instruction sequences to perform a vector comparison.
(decl vec_cmp_vc (Reg Reg VectorSize) Reg)
(rule (vec_cmp_vc rn rm size)
(let ((dst Reg (vec_rrr (VecALUOp.Fcmeq) rn rn size))
(tmp Reg (vec_rrr (VecALUOp.Fcmeq) rm rm size))
(dst Reg (vec_rrr (VecALUOp.And) dst tmp size)))
dst))
(decl vec_cmp (Reg Reg Type Cond) Reg)
;; Floating point Vs / Vc
(rule (vec_cmp rn rm ty (Cond.Vc))
(if (ty_vector_float ty))
(vec_cmp_vc rn rm (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Vs))
(if (ty_vector_float ty))
(let ((tmp Reg (vec_cmp_vc rn rm (vector_size ty))))
(vec_misc (VecMisc2.Not) tmp (vector_size ty))))
;; 'Less than' operations are implemented by swapping the order of
;; operands and using the 'greater than' instructions.
;; 'Not equal' is implemented with 'equal' and inverting the result.
;; Floating-point
(rule (vec_cmp rn rm ty (Cond.Eq))
(if (ty_vector_float ty))
(vec_rrr (VecALUOp.Fcmeq) rn rm (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Ne))
(if (ty_vector_float ty))
(let ((tmp Reg (vec_rrr (VecALUOp.Fcmeq) rn rm (vector_size ty))))
(vec_misc (VecMisc2.Not) tmp (vector_size ty))))
(rule (vec_cmp rn rm ty (Cond.Ge))
(if (ty_vector_float ty))
(vec_rrr (VecALUOp.Fcmge) rn rm (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Gt))
(if (ty_vector_float ty))
(vec_rrr (VecALUOp.Fcmgt) rn rm (vector_size ty)))
;; Floating-point swapped-operands
(rule (vec_cmp rn rm ty (Cond.Mi))
(if (ty_vector_float ty))
(vec_rrr (VecALUOp.Fcmgt) rm rn (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Ls))
(if (ty_vector_float ty))
(vec_rrr (VecALUOp.Fcmge) rm rn (vector_size ty)))
;; Integer
(rule 1 (vec_cmp rn rm ty (Cond.Eq))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmeq) rn rm (vector_size ty)))
(rule 1 (vec_cmp rn rm ty (Cond.Ne))
(if (ty_vector_not_float ty))
(let ((tmp Reg (vec_rrr (VecALUOp.Cmeq) rn rm (vector_size ty))))
(vec_misc (VecMisc2.Not) tmp (vector_size ty))))
(rule 1 (vec_cmp rn rm ty (Cond.Ge))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmge) rn rm (vector_size ty)))
(rule 1 (vec_cmp rn rm ty (Cond.Gt))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmgt) rn rm (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Hs))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmhs) rn rm (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Hi))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmhi) rn rm (vector_size ty)))
;; Integer swapped-operands
(rule (vec_cmp rn rm ty (Cond.Le))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmge) rm rn (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Lt))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmgt) rm rn (vector_size ty)))
(rule 1 (vec_cmp rn rm ty (Cond.Ls))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmhs) rm rn (vector_size ty)))
(rule (vec_cmp rn rm ty (Cond.Lo))
(if (ty_vector_not_float ty))
(vec_rrr (VecALUOp.Cmhi) rm rn (vector_size ty)))
;; Helper for determining if any value in a vector is true.
;; This operation is implemented by using umaxp to create a scalar value, which
;; is then compared against zero.
;;
;; umaxp vn.4s, vm.4s, vm.4s
;; mov xm, vn.d[0]
;; cmp xm, #0
(decl vanytrue (Reg Type) ProducesFlags)
(rule 1 (vanytrue src (ty_vec128 ty))
(let ((src Reg (vec_rrr (VecALUOp.Umaxp) src src (VectorSize.Size32x4)))
(src Reg (mov_from_vec src 0 (ScalarSize.Size64))))
(cmp_imm (OperandSize.Size64) src (u8_into_imm12 0))))
(rule (vanytrue src ty)
(if (ty_vec64 ty))
(let ((src Reg (mov_from_vec src 0 (ScalarSize.Size64))))
(cmp_imm (OperandSize.Size64) src (u8_into_imm12 0))))
;;;; TLS Values ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
;; Helper for emitting ElfTlsGetAddr.
(decl elf_tls_get_addr (ExternalName) Reg)
(rule (elf_tls_get_addr name)
(let ((dst WritableReg (temp_writable_reg $I64))
(_ Unit (emit (MInst.ElfTlsGetAddr name dst))))
dst))
;; Helpers for lowering `icmp` sequences.
;; `lower_icmp` contains shared functionality for lowering `icmp`
;; sequences, which `lower_icmp_into_{reg,flags}` extend from.
(decl lower_icmp (IntCC Value Value Type) ProducesFlags)
(decl lower_icmp_into_reg (IntCC Value Value Type Type) ValueRegs)
(decl lower_icmp_into_flags (IntCC Value Value Type) ProducesFlags)
;; For most cases, `lower_icmp_into_flags` is the same as `lower_icmp`,
;; except for some I128 cases (see below).
(rule -1 (lower_icmp_into_flags cond x y ty) (lower_icmp cond x y ty))
;; Vectors.
;; `icmp` into flags for vectors is invalid.
(rule 1 (lower_icmp_into_reg cond x y in_ty @ (multi_lane _ _) _out_ty)
(let ((cond Cond (cond_code cond))
(rn Reg (put_in_reg x))
(rm Reg (put_in_reg y)))
(vec_cmp rn rm in_ty cond)))
;; Determines the appropriate extend op given the value type and whether it is signed.
(decl lower_icmp_extend (Type bool) ExtendOp)
(rule (lower_icmp_extend $I8 $true) (ExtendOp.SXTB))
(rule (lower_icmp_extend $I16 $true) (ExtendOp.SXTH))
(rule (lower_icmp_extend $I8 $false) (ExtendOp.UXTB))
(rule (lower_icmp_extend $I16 $false) (ExtendOp.UXTH))
;; Integers <= 64-bits.
(rule -2 (lower_icmp_into_reg cond rn rm in_ty out_ty)
(if (ty_int_bool_ref_scalar_64 in_ty))
(let ((cc Cond (cond_code cond)))
(with_flags
(lower_icmp cond rn rm in_ty)
(materialize_bool_result (ty_bits out_ty) cc))))
(rule 1 (lower_icmp cond rn rm (fits_in_16 ty))
(if (signed_cond_code cond))
(let ((rn Reg (put_in_reg_sext32 rn)))
(cmp_extend (operand_size ty) rn rm (lower_icmp_extend ty $true))))
(rule -1 (lower_icmp cond rn (imm12_from_value rm) (fits_in_16 ty))
(let ((rn Reg (put_in_reg_zext32 rn)))
(cmp_imm (operand_size ty) rn rm)))
(rule -2 (lower_icmp cond rn rm (fits_in_16 ty))
(let ((rn Reg (put_in_reg_zext32 rn)))
(cmp_extend (operand_size ty) rn rm (lower_icmp_extend ty $false))))
(rule -3 (lower_icmp cond rn (imm12_from_value rm) ty)
(if (ty_int_bool_ref_scalar_64 ty))
(cmp_imm (operand_size ty) rn rm))
(rule -4 (lower_icmp cond rn rm ty)
(if (ty_int_bool_ref_scalar_64 ty))
(cmp (operand_size ty) rn rm))
;; 128-bit integers.
(rule (lower_icmp_into_reg cond @ (IntCC.Equal) rn rm $I128 out_ty)
(let ((cc Cond (cond_code cond)))
(with_flags
(lower_icmp cond rn rm $I128)
(materialize_bool_result (ty_bits out_ty) cc))))
(rule (lower_icmp_into_reg cond @ (IntCC.NotEqual) rn rm $I128 out_ty)
(let ((cc Cond (cond_code cond)))
(with_flags
(lower_icmp cond rn rm $I128)
(materialize_bool_result (ty_bits out_ty) cc))))
;; cmp lhs_lo, rhs_lo
;; ccmp lhs_hi, rhs_hi, #0, eq
(decl lower_icmp_i128_eq_ne (Value Value) ProducesFlags)
(rule (lower_icmp_i128_eq_ne lhs rhs)
(let ((lhs ValueRegs (put_in_regs lhs))
(rhs ValueRegs (put_in_regs rhs))
(lhs_lo Reg (value_regs_get lhs 0))
(lhs_hi Reg (value_regs_get lhs 1))
(rhs_lo Reg (value_regs_get rhs 0))
(rhs_hi Reg (value_regs_get rhs 1))
(cmp_inst ProducesFlags (cmp (OperandSize.Size64) lhs_lo rhs_lo)))
(ccmp (OperandSize.Size64) lhs_hi rhs_hi
(nzcv $false $false $false $false) (Cond.Eq) cmp_inst)))
(rule (lower_icmp (IntCC.Equal) lhs rhs $I128)
(lower_icmp_i128_eq_ne lhs rhs))
(rule (lower_icmp (IntCC.NotEqual) lhs rhs $I128)
(lower_icmp_i128_eq_ne lhs rhs))
;; cmp lhs_lo, rhs_lo
;; cset tmp1, unsigned_cond
;; cmp lhs_hi, rhs_hi
;; cset tmp2, cond
;; csel dst, tmp1, tmp2, eq
(rule -1 (lower_icmp_into_reg cond lhs rhs $I128 out_ty)
(let ((unsigned_cond Cond (cond_code (intcc_unsigned cond)))
(cond Cond (cond_code cond))
(lhs ValueRegs (put_in_regs lhs))
(rhs ValueRegs (put_in_regs rhs))
(lhs_lo Reg (value_regs_get lhs 0))
(lhs_hi Reg (value_regs_get lhs 1))
(rhs_lo Reg (value_regs_get rhs 0))
(rhs_hi Reg (value_regs_get rhs 1))
(tmp1 ValueRegs
(with_flags (cmp (OperandSize.Size64) lhs_lo rhs_lo)
(materialize_bool_result
(ty_bits out_ty) unsigned_cond)))
(tmp1 Reg (value_regs_get tmp1 0))
(dst ValueRegs
(with_flags (cmp (OperandSize.Size64) lhs_hi rhs_hi)
(lower_icmp_i128_consumer cond (ty_bits out_ty)
tmp1 lhs_hi rhs_hi))))
dst))
(decl lower_icmp_i128_consumer (Cond u8 Reg Reg Reg) ConsumesFlags)
(rule (lower_icmp_i128_consumer cond 1 tmp1 lhs_hi rhs_hi)
(let ((tmp2 WritableReg (temp_writable_reg $I64))
(dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSet tmp2 cond)
(MInst.CSel dst (Cond.Eq) tmp1 tmp2)
(value_reg dst))))
(rule (lower_icmp_i128_consumer cond 128 tmp1 lhs_hi rhs_hi)
(let ((tmp2 WritableReg (temp_writable_reg $I64))
(dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSetm tmp2 cond)
(MInst.CSel dst (Cond.Eq) tmp1 tmp2)
(value_regs dst dst))))
(rule -1 (lower_icmp_i128_consumer cond _out_ty_bits tmp1 lhs_hi rhs_hi)
(let ((tmp2 WritableReg (temp_writable_reg $I64))
(dst WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSetm tmp2 cond)
(MInst.CSel dst (Cond.Eq) tmp1 tmp2)
(value_reg dst))))
;; Exceptional `lower_icmp_into_flags` rules.
;; We need to guarantee that the flags for `cond` are correct, so we
;; compare `dst` with 1.
(rule (lower_icmp_into_flags cond @ (IntCC.SignedGreaterThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Sign) 1))) ;; mov tmp, #1
(cmp (OperandSize.Size64) dst tmp)))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedGreaterThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Zero) 1)))
(cmp (OperandSize.Size64) dst tmp)))
(rule (lower_icmp_into_flags cond @ (IntCC.SignedLessThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Sign) 1)))
(cmp (OperandSize.Size64) tmp dst)))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedLessThanOrEqual) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0))
(tmp Reg (imm $I64 (ImmExtend.Zero) 1)))
(cmp (OperandSize.Size64) tmp dst)))
;; For strict comparisons, we compare with 0.
(rule (lower_icmp_into_flags cond @ (IntCC.SignedGreaterThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) dst (zero_reg))))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedGreaterThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) dst (zero_reg))))
(rule (lower_icmp_into_flags cond @ (IntCC.SignedLessThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) (zero_reg) dst)))
(rule (lower_icmp_into_flags cond @ (IntCC.UnsignedLessThan) lhs rhs $I128)
(let ((dst ValueRegs (lower_icmp_into_reg cond lhs rhs $I128 $B1))
(dst Reg (value_regs_get dst 0)))
(cmp (OperandSize.Size64) (zero_reg) dst)))
;; Helpers for generating select instruction sequences.
(decl lower_select (ProducesFlags Cond Type Value Value) ValueRegs)
(rule 2 (lower_select flags cond (ty_scalar_float ty) rn rm)
(with_flags flags (fpu_csel ty cond rn rm)))
(rule 3 (lower_select flags cond (ty_vec128 ty) rn rm)
(with_flags flags (vec_csel cond rn rm)))
(rule (lower_select flags cond ty rn rm)
(if (ty_vec64 ty))
(with_flags flags (fpu_csel $F64 cond rn rm)))
(rule 4 (lower_select flags cond $I128 rn rm)
(let ((dst_lo WritableReg (temp_writable_reg $I64))
(dst_hi WritableReg (temp_writable_reg $I64))
(rn ValueRegs (put_in_regs rn))
(rm ValueRegs (put_in_regs rm))
(rn_lo Reg (value_regs_get rn 0))
(rn_hi Reg (value_regs_get rn 1))
(rm_lo Reg (value_regs_get rm 0))
(rm_hi Reg (value_regs_get rm 1)))
(with_flags flags
(ConsumesFlags.ConsumesFlagsTwiceReturnsValueRegs
(MInst.CSel dst_lo cond rn_lo rm_lo)
(MInst.CSel dst_hi cond rn_hi rm_hi)
(value_regs dst_lo dst_hi)))))
(rule 1 (lower_select flags cond ty rn rm)
(if (ty_int_bool_ref_scalar_64 ty))
(with_flags flags (csel cond rn rm)))
;; Helper for emitting `MInst.Jump` instructions.
(decl aarch64_jump (BranchTarget) SideEffectNoResult)
(rule (aarch64_jump target)
(SideEffectNoResult.Inst (MInst.Jump target)))
;; Helper for emitting `MInst.JTSequence` instructions.
;; Emit the compound instruction that does:
;;
;; b.hs default
;; csel rB, xzr, rIndex, hs
;; csdb
;; adr rA, jt
;; ldrsw rB, [rA, rB, uxtw #2]
;; add rA, rA, rB
;; br rA
;; [jt entries]
;;
;; This must be *one* instruction in the vcode because
;; we cannot allow regalloc to insert any spills/fills
;; in the middle of the sequence; otherwise, the ADR's
;; PC-rel offset to the jumptable would be incorrect.
;; (The alternative is to introduce a relocation pass
;; for inlined jumptables, which is much worse, IMHO.)
(decl jt_sequence (Reg BoxJTSequenceInfo) ConsumesFlags)
(rule (jt_sequence ridx info)
(let ((rtmp1 WritableReg (temp_writable_reg $I64))
(rtmp2 WritableReg (temp_writable_reg $I64)))
(ConsumesFlags.ConsumesFlagsSideEffect
(MInst.JTSequence info ridx rtmp1 rtmp2))))
;; Helper for emitting `MInst.CondBr` instructions.
(decl cond_br (BranchTarget BranchTarget CondBrKind) ConsumesFlags)
(rule (cond_br taken not_taken kind)
(ConsumesFlags.ConsumesFlagsSideEffect
(MInst.CondBr taken not_taken kind)))
;; Helper for emitting `MInst.MovToNZCV` instructions.
(decl mov_to_nzcv (Reg) ProducesFlags)
(rule (mov_to_nzcv rn)
(ProducesFlags.ProducesFlagsSideEffect
(MInst.MovToNZCV rn)))
;; Helper for emitting `MInst.EmitIsland` instructions.
(decl emit_island (CodeOffset) SideEffectNoResult)
(rule (emit_island needed_space)
(SideEffectNoResult.Inst
(MInst.EmitIsland needed_space)))
;; Helper for emitting `br_table` sequences.
(decl br_table_impl (u64 Reg VecMachLabel) InstOutput)
(rule (br_table_impl (imm12_from_u64 jt_size) ridx targets)
(let ((jt_info BoxJTSequenceInfo (targets_jt_info targets)))
(side_effect (with_flags_side_effect
(cmp_imm (OperandSize.Size32) ridx jt_size)
(jt_sequence ridx jt_info)))))
(rule -1 (br_table_impl jt_size ridx targets)
(let ((jt_size Reg (imm $I64 (ImmExtend.Zero) jt_size))
(jt_info BoxJTSequenceInfo (targets_jt_info targets)))
(side_effect (with_flags_side_effect
(cmp (OperandSize.Size32) ridx jt_size)
(jt_sequence ridx jt_info)))))