cranelift_codegen/isa/x64/inst/

emit.rs

use crate::ir::KnownSymbol;
use crate::ir::immediates::{Ieee32, Ieee64};
use crate::isa::x64::external::{AsmInst, CraneliftRegisters, PairedGpr};
use crate::isa::x64::inst::args::*;
use crate::isa::x64::inst::*;
use crate::isa::x64::lower::isle::generated_code::{Atomic128RmwSeqOp, AtomicRmwSeqOp};
use cranelift_assembler_x64 as asm;

/// A small helper to generate a signed conversion instruction.
fn emit_signed_cvt(
    sink: &mut MachBuffer<Inst>,
    info: &EmitInfo,
    state: &mut EmitState,
    src: Reg,
    dst: Writable<Reg>,
    to_f64: bool,
) {
    assert!(src.is_real());
    assert!(dst.to_reg().is_real());

    // Handle an unsigned int, which is the "easy" case: a signed conversion
    // will do the right thing.
    let dst = WritableXmm::from_writable_reg(dst).unwrap();
    if to_f64 {
        asm::inst::cvtsi2sdq_a::new(dst, src).emit(sink, info, state);
    } else {
        asm::inst::cvtsi2ssq_a::new(dst, src).emit(sink, info, state);
    }
}

/// Emits a one way conditional jump if CC is set (true).
fn one_way_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
    let cond_start = sink.cur_offset();
    let cond_disp_off = cond_start + 2;
    sink.use_label_at_offset(cond_disp_off, label, LabelUse::JmpRel32);
    emit_jcc_no_offset(sink, cc);
    debug_assert_eq!(sink.cur_offset(), cond_disp_off + 4);
}

/// Like `one_way_jmp` above emitting a conditional jump, but also using
/// `MachBuffer::add_cond_branch`.
fn cond_jmp(sink: &mut MachBuffer<Inst>, cc: CC, label: MachLabel) {
    let cond_start = sink.cur_offset();
    let cond_disp_off = cond_start + 2;
    let cond_end = cond_start + 6;

    sink.use_label_at_offset(cond_disp_off, label, LabelUse::JmpRel32);
    // FIXME: ideally this `inverted` calculation would go through the external
    // assembler, but for now it's left done manually.
    let inverted: [u8; 6] = [0x0F, 0x80 + (cc.invert().get_enc()), 0x00, 0x00, 0x00, 0x00];
    sink.add_cond_branch(cond_start, cond_end, label, &inverted[..]);

    emit_jcc_no_offset(sink, cc);

    debug_assert_eq!(sink.cur_offset(), cond_disp_off + 4);
    debug_assert_eq!(sink.cur_offset(), cond_end);
}

fn emit_jcc_no_offset(sink: &mut MachBuffer<Inst>, cc: CC) {
    // Note that the disassembler matches Capstone which doesn't match the `CC`
    // enum directly as Intel has multiple mnemonics use the same encoding.
    let inst: AsmInst = match cc {
        CC::Z => asm::inst::je_d32::new(0).into(),   // jz == je
        CC::NZ => asm::inst::jne_d32::new(0).into(), // jnz == jne
        CC::B => asm::inst::jb_d32::new(0).into(),
        CC::NB => asm::inst::jae_d32::new(0).into(), // jnb == jae
        CC::BE => asm::inst::jbe_d32::new(0).into(),
        CC::NBE => asm::inst::ja_d32::new(0).into(), // jnbe == ja
        CC::L => asm::inst::jl_d32::new(0).into(),
        CC::LE => asm::inst::jle_d32::new(0).into(),
        CC::NL => asm::inst::jge_d32::new(0).into(), // jnl == jge
        CC::NLE => asm::inst::jg_d32::new(0).into(), // jnle == jg
        CC::O => asm::inst::jo_d32::new(0).into(),
        CC::NO => asm::inst::jno_d32::new(0).into(),
        CC::P => asm::inst::jp_d32::new(0).into(),
        CC::NP => asm::inst::jnp_d32::new(0).into(),
        CC::S => asm::inst::js_d32::new(0).into(),
        CC::NS => asm::inst::jns_d32::new(0).into(),
    };
    inst.encode(&mut external::AsmCodeSink {
        sink,
        incoming_arg_offset: 0,
        slot_offset: 0,
    });
}

/// Emits an unconditional branch.
fn uncond_jmp(sink: &mut MachBuffer<Inst>, label: MachLabel) {
    let uncond_start = sink.cur_offset();
    let uncond_disp_off = uncond_start + 1;
    let uncond_end = uncond_start + 5;

    sink.use_label_at_offset(uncond_disp_off, label, LabelUse::JmpRel32);
    sink.add_uncond_branch(uncond_start, uncond_end, label);

    asm::inst::jmp_d32::new(0).encode(&mut external::AsmCodeSink {
        sink,
        incoming_arg_offset: 0,
        slot_offset: 0,
    });
    debug_assert_eq!(sink.cur_offset(), uncond_disp_off + 4);
    debug_assert_eq!(sink.cur_offset(), uncond_end);
}

/// Emits a relocation, attaching the current source location as well.
fn emit_reloc(sink: &mut MachBuffer<Inst>, kind: Reloc, name: &ExternalName, addend: Addend) {
    sink.add_reloc(kind, name, addend);
}

/// The top-level emit function.
///
/// Important!  Do not add improved (shortened) encoding cases to existing
/// instructions without also adding tests for those improved encodings.  That
/// is a dangerous game that leads to hard-to-track-down errors in the emitted
/// code.
///
/// For all instructions, make sure to have test coverage for all of the
/// following situations.  Do this by creating the cross product resulting from
/// applying the following rules to each operand:
///
/// (1) for any insn that mentions a register: one test using a register from
///     the group [rax, rcx, rdx, rbx, rsp, rbp, rsi, rdi] and a second one
///     using a register from the group [r8, r9, r10, r11, r12, r13, r14, r15].
///     This helps detect incorrect REX prefix construction.
///
/// (2) for any insn that mentions a byte register: one test for each of the
///     four encoding groups [al, cl, dl, bl], [spl, bpl, sil, dil],
///     [r8b .. r11b] and [r12b .. r15b].  This checks that
///     apparently-redundant REX prefixes are retained when required.
///
/// (3) for any insn that contains an immediate field, check the following
///     cases: field is zero, field is in simm8 range (-128 .. 127), field is
///     in simm32 range (-0x8000_0000 .. 0x7FFF_FFFF).  This is because some
///     instructions that require a 32-bit immediate have a short-form encoding
///     when the imm is in simm8 range.
///
/// Rules (1), (2) and (3) don't apply for registers within address expressions
/// (`Addr`s).  Those are already pretty well tested, and the registers in them
/// don't have any effect on the containing instruction (apart from possibly
/// require REX prefix bits).
///
/// When choosing registers for a test, avoid using registers with the same
/// offset within a given group.  For example, don't use rax and r8, since they
/// both have the lowest 3 bits as 000, and so the test won't detect errors
/// where those 3-bit register sub-fields are confused by the emitter.  Instead
/// use (eg) rax (lo3 = 000) and r9 (lo3 = 001).  Similarly, don't use (eg) cl
/// and bpl since they have the same offset in their group; use instead (eg) cl
/// and sil.
///
/// For all instructions, also add a test that uses only low-half registers
/// (rax .. rdi, xmm0 .. xmm7) etc, so as to check that any redundant REX
/// prefixes are correctly omitted.  This low-half restriction must apply to
/// _all_ registers in the insn, even those in address expressions.
///
/// Following these rules creates large numbers of test cases, but it's the
/// only way to make the emitter reliable.
///
/// Known possible improvements:
///
/// * there's a shorter encoding for shl/shr/sar by a 1-bit immediate.  (Do we
///   care?)
pub(crate) fn emit(
    inst: &Inst,
    sink: &mut MachBuffer<Inst>,
    info: &EmitInfo,
    state: &mut EmitState,
) {
    if !inst.is_available(&info) {
        let features = if let Inst::External { inst } = inst {
            inst.features().to_string()
        } else {
            "see `is_available` source for feature term".to_string()
        };
        panic!(
            "Cannot emit inst '{inst:?}' for target; failed to match ISA requirements: {features}"
        );
    }

    match inst {
        Inst::CheckedSRemSeq { divisor, .. } | Inst::CheckedSRemSeq8 { divisor, .. } => {
            // Validate that the register constraints of the dividend and the
            // destination are all as expected.
            let (dst, size) = match inst {
                Inst::CheckedSRemSeq {
                    dividend_lo,
                    dividend_hi,
                    dst_quotient,
                    dst_remainder,
                    size,
                    ..
                } => {
                    let dividend_lo = dividend_lo.to_reg();
                    let dividend_hi = dividend_hi.to_reg();
                    let dst_quotient = dst_quotient.to_reg().to_reg();
                    let dst_remainder = dst_remainder.to_reg().to_reg();
                    debug_assert_eq!(dividend_lo, regs::rax());
                    debug_assert_eq!(dividend_hi, regs::rdx());
                    debug_assert_eq!(dst_quotient, regs::rax());
                    debug_assert_eq!(dst_remainder, regs::rdx());
                    (regs::rdx(), *size)
                }
                Inst::CheckedSRemSeq8 { dividend, dst, .. } => {
                    let dividend = dividend.to_reg();
                    let dst = dst.to_reg().to_reg();
                    debug_assert_eq!(dividend, regs::rax());
                    debug_assert_eq!(dst, regs::rax());
                    (regs::rax(), OperandSize::Size8)
                }
                _ => unreachable!(),
            };

            // Generates the following code sequence:
            //
            // cmp -1 %divisor
            // jnz $do_op
            //
            // ;; for srem, result is 0
            // mov #0, %dst
            // j $done
            //
            // $do_op:
            // idiv %divisor
            //
            // $done:

            let do_op = sink.get_label();
            let done_label = sink.get_label();

            // Check if the divisor is -1, and if it isn't then immediately
            // go to the `idiv`.
            let inst = Inst::cmp_mi_sxb(size, *divisor, -1);
            inst.emit(sink, info, state);
            one_way_jmp(sink, CC::NZ, do_op);

            // ... otherwise the divisor is -1 and the result is always 0. This
            // is written to the destination register which will be %rax for
            // 8-bit srem and %rdx otherwise.
            //
            // Note that for 16-to-64-bit srem operations this leaves the
            // second destination, %rax, unchanged. This isn't semantically
            // correct if a lowering actually tries to use the `dst_quotient`
            // output but for srem only the `dst_remainder` output is used for
            // now.
            let inst = Inst::imm(OperandSize::Size64, 0, Writable::from_reg(dst));
            inst.emit(sink, info, state);
            let inst = Inst::jmp_known(done_label);
            inst.emit(sink, info, state);

            // Here the `idiv` is executed, which is different depending on the
            // size
            sink.bind_label(do_op, state.ctrl_plane_mut());
            let rax = Gpr::RAX;
            let rdx = Gpr::RDX;
            let writable_rax = Writable::from_reg(rax);
            let writable_rdx = Writable::from_reg(rdx);
            let inst: AsmInst = match size {
                OperandSize::Size8 => asm::inst::idivb_m::new(
                    PairedGpr::from(writable_rax),
                    *divisor,
                    TrapCode::INTEGER_DIVISION_BY_ZERO,
                )
                .into(),

                OperandSize::Size16 => asm::inst::idivw_m::new(
                    PairedGpr::from(writable_rax),
                    PairedGpr::from(writable_rdx),
                    *divisor,
                    TrapCode::INTEGER_DIVISION_BY_ZERO,
                )
                .into(),

                OperandSize::Size32 => asm::inst::idivl_m::new(
                    PairedGpr::from(writable_rax),
                    PairedGpr::from(writable_rdx),
                    *divisor,
                    TrapCode::INTEGER_DIVISION_BY_ZERO,
                )
                .into(),

                OperandSize::Size64 => asm::inst::idivq_m::new(
                    PairedGpr::from(writable_rax),
                    PairedGpr::from(writable_rdx),
                    *divisor,
                    TrapCode::INTEGER_DIVISION_BY_ZERO,
                )
                .into(),
            };
            inst.emit(sink, info, state);

            sink.bind_label(done_label, state.ctrl_plane_mut());
        }

        Inst::MovFromPReg { src, dst } => {
            let src: Reg = (*src).into();
            debug_assert!([regs::rsp(), regs::rbp(), regs::pinned_reg()].contains(&src));
            asm::inst::movq_mr::new(*dst, Gpr::unwrap_new(src)).emit(sink, info, state);
        }

        Inst::MovToPReg { src, dst } => {
            let dst: Reg = (*dst).into();
            debug_assert!([regs::rsp(), regs::rbp(), regs::pinned_reg()].contains(&dst));
            let dst = WritableGpr::from_writable_reg(Writable::from_reg(dst)).unwrap();
            asm::inst::movq_mr::new(dst, *src).emit(sink, info, state);
        }

        Inst::XmmCmove {
            ty,
            cc,
            consequent,
            alternative,
            dst,
        } => {
            let alternative = *alternative;
            let dst = *dst;
            debug_assert_eq!(alternative, dst.to_reg());
            let consequent = *consequent;

            // Lowering of the Select IR opcode when the input is an fcmp relies on the fact that
            // this doesn't clobber flags. Make sure to not do so here.
            let next = sink.get_label();

            // Jump if cc is *not* set.
            one_way_jmp(sink, cc.invert(), next);
            Inst::gen_move(dst.map(|r| r.to_reg()), consequent.to_reg(), *ty)
                .emit(sink, info, state);

            sink.bind_label(next, state.ctrl_plane_mut());
        }

        Inst::StackProbeLoop {
            tmp,
            frame_size,
            guard_size,
        } => {
            assert!(info.flags.enable_probestack());
            assert!(guard_size.is_power_of_two());

            let tmp = *tmp;

            // Number of probes that we need to perform
            let probe_count = align_to(*frame_size, *guard_size) / guard_size;

            // The inline stack probe loop has 3 phases:
            //
            // We generate the "guard area" register which is essentially the frame_size aligned to
            // guard_size. We copy the stack pointer and subtract the guard area from it. This
            // gets us a register that we can use to compare when looping.
            //
            // After that we emit the loop. Essentially we just adjust the stack pointer one guard_size'd
            // distance at a time and then touch the stack by writing anything to it. We use the previously
            // created "guard area" register to know when to stop looping.
            //
            // When we have touched all the pages that we need, we have to restore the stack pointer
            // to where it was before.
            //
            // Generate the following code:
            //         mov  tmp_reg, rsp
            //         sub  tmp_reg, guard_size * probe_count
            // .loop_start:
            //         sub  rsp, guard_size
            //         mov  [rsp], rsp
            //         cmp  rsp, tmp_reg
            //         jne  .loop_start
            //         add  rsp, guard_size * probe_count

            // Create the guard bound register
            // mov  tmp_reg, rsp
            let inst = Inst::gen_move(tmp, regs::rsp(), types::I64);
            inst.emit(sink, info, state);

            // sub  tmp_reg, GUARD_SIZE * probe_count
            let guard_plus_count = i32::try_from(guard_size * probe_count)
                .expect("`guard_size * probe_count` is too large to fit in a 32-bit immediate");
            Inst::subq_mi(tmp, guard_plus_count).emit(sink, info, state);

            // Emit the main loop!
            let loop_start = sink.get_label();
            sink.bind_label(loop_start, state.ctrl_plane_mut());

            // sub  rsp, GUARD_SIZE
            let rsp = Writable::from_reg(regs::rsp());
            let guard_size_ = i32::try_from(*guard_size)
                .expect("`guard_size` is too large to fit in a 32-bit immediate");
            Inst::subq_mi(rsp, guard_size_).emit(sink, info, state);

            // TODO: `mov [rsp], 0` would be better, but we don't have that instruction
            // Probe the stack! We don't use Inst::gen_store_stack here because we need a predictable
            // instruction size.
            // mov  [rsp], rsp
            asm::inst::movl_mr::new(Amode::imm_reg(0, regs::rsp()), Gpr::RSP)
                .emit(sink, info, state);

            // Compare and jump if we are not done yet
            // cmp  rsp, tmp_reg
            let tmp = Gpr::unwrap_new(tmp.to_reg());
            asm::inst::cmpq_rm::new(tmp, Gpr::RSP).emit(sink, info, state);

            // jne  .loop_start
            // TODO: Encoding the conditional jump as a short jump
            // could save us us 4 bytes here.
            one_way_jmp(sink, CC::NZ, loop_start);

            // The regular prologue code is going to emit a `sub` after this, so we need to
            // reset the stack pointer
            //
            // TODO: It would be better if we could avoid the `add` + `sub` that is generated here
            // and in the stack adj portion of the prologue
            //
            // add rsp, GUARD_SIZE * probe_count
            Inst::addq_mi(rsp, guard_plus_count).emit(sink, info, state);
        }

        Inst::CallKnown { info: call_info } => {
            let stack_map = state.take_stack_map();

            asm::inst::callq_d::new(0).emit(sink, info, state);

            // The last 4 bytes of `callq` is the relative displacement to where
            // we're calling, so that's where the reloc is registered.
            //
            // The addend adjusts for the difference between the end of the
            // instruction and the beginning of the immediate field.
            let len = sink.cur_offset();
            sink.add_reloc_at_offset(len - 4, Reloc::X86CallPCRel4, &call_info.dest, -4);

            if let Some(s) = stack_map {
                sink.push_user_stack_map(state, len, s);
            }

            if let Some(try_call) = call_info.try_call_info.as_ref() {
                sink.add_try_call_site(
                    Some(state.frame_layout().sp_to_fp()),
                    try_call.exception_handlers(&state.frame_layout()),
                );
            } else {
                sink.add_call_site();
            }

            // Reclaim the outgoing argument area that was released by the
            // callee, to ensure that StackAMode values are always computed from
            // a consistent SP.
            if call_info.callee_pop_size > 0 {
                let rsp = Writable::from_reg(regs::rsp());
                let callee_pop_size = i32::try_from(call_info.callee_pop_size)
                    .expect("`callee_pop_size` is too large to fit in a 32-bit immediate");
                Inst::subq_mi(rsp, callee_pop_size).emit(sink, info, state);
            }

            // Load any stack-carried return values.
            call_info.emit_retval_loads::<X64ABIMachineSpec, _, _>(
                state.frame_layout().stackslots_size,
                |inst| inst.emit(sink, info, state),
                |_space_needed| None,
            );

            // If this is a try-call, jump to the continuation
            // (normal-return) block.
            if let Some(try_call) = call_info.try_call_info.as_ref() {
                let jmp = Inst::JmpKnown {
                    dst: try_call.continuation,
                };
                jmp.emit(sink, info, state);
            }
        }

        Inst::ReturnCallKnown { info: call_info } => {
            emit_return_call_common_sequence(sink, info, state, &call_info);

            // Finally, jump to the callee!
            //
            // Note: this is not `Inst::Jmp { .. }.emit(..)` because we have
            // different metadata in this case: we don't have a label for the
            // target, but rather a function relocation.
            asm::inst::jmp_d32::new(0).emit(sink, info, state);
            let offset = sink.cur_offset();
            // The addend adjusts for the difference between the end of the instruction and the
            // beginning of the immediate field.
            sink.add_reloc_at_offset(offset - 4, Reloc::X86CallPCRel4, &call_info.dest, -4);
            sink.add_call_site();
        }

        Inst::ReturnCallUnknown { info: call_info } => {
            let callee = call_info.dest;

            emit_return_call_common_sequence(sink, info, state, &call_info);

            asm::inst::jmpq_m::new(callee).emit(sink, info, state);
            sink.add_call_site();
        }

        Inst::CallUnknown {
            info: call_info, ..
        } => {
            let stack_map = state.take_stack_map();

            let dest = match call_info.dest.clone() {
                RegMem::Reg { reg } => asm::GprMem::Gpr(Gpr::unwrap_new(reg)),
                RegMem::Mem { addr } => asm::GprMem::Mem(addr.into()),
            };

            asm::inst::callq_m::new(dest).emit(sink, info, state);

            if let Some(s) = stack_map {
                let offset = sink.cur_offset();
                sink.push_user_stack_map(state, offset, s);
            }

            if let Some(try_call) = call_info.try_call_info.as_ref() {
                sink.add_try_call_site(
                    Some(state.frame_layout().sp_to_fp()),
                    try_call.exception_handlers(&state.frame_layout()),
                );
            } else {
                sink.add_call_site();
            }

            // Reclaim the outgoing argument area that was released by the callee, to ensure that
            // StackAMode values are always computed from a consistent SP.
            if call_info.callee_pop_size > 0 {
                let rsp = Writable::from_reg(regs::rsp());
                let callee_pop_size = i32::try_from(call_info.callee_pop_size)
                    .expect("`callee_pop_size` is too large to fit in a 32-bit immediate");
                Inst::subq_mi(rsp, callee_pop_size).emit(sink, info, state);
            }

            // Load any stack-carried return values.
            call_info.emit_retval_loads::<X64ABIMachineSpec, _, _>(
                state.frame_layout().stackslots_size,
                |inst| inst.emit(sink, info, state),
                |_space_needed| None,
            );

            if let Some(try_call) = call_info.try_call_info.as_ref() {
                let jmp = Inst::JmpKnown {
                    dst: try_call.continuation,
                };
                jmp.emit(sink, info, state);
            }
        }

        Inst::Args { .. } => {}
        Inst::Rets { .. } => {}

        Inst::StackSwitchBasic {
            store_context_ptr,
            load_context_ptr,
            in_payload0,
            out_payload0,
        } => {
            // Note that we do not emit anything for preserving and restoring
            // ordinary registers here: That's taken care of by regalloc for us,
            // since we marked this instruction as clobbering all registers.
            //
            // Also note that we do nothing about passing the single payload
            // value: We've informed regalloc that it is sent and received via
            // the fixed register given by [stack_switch::payload_register]

            let (tmp1, tmp2) = {
                // Ideally we would just ask regalloc for two temporary registers.
                // However, adding any early defs to the constraints on StackSwitch
                // causes TooManyLiveRegs. Fortunately, we can manually find tmp
                // registers without regalloc: Since our instruction clobbers all
                // registers, we can simply pick any register that is not assigned
                // to the operands.

                let all = crate::isa::x64::abi::ALL_CLOBBERS;

                let used_regs = [
                    **load_context_ptr,
                    **store_context_ptr,
                    **in_payload0,
                    *out_payload0.to_reg(),
                ];

                let mut tmps = all.into_iter().filter_map(|preg| {
                    let reg: Reg = preg.into();
                    if !used_regs.contains(&reg) {
                        WritableGpr::from_writable_reg(isle::WritableReg::from_reg(reg))
                    } else {
                        None
                    }
                });
                (tmps.next().unwrap(), tmps.next().unwrap())
            };

            let layout = stack_switch::control_context_layout();
            let rsp_offset = layout.stack_pointer_offset as i32;
            let pc_offset = layout.ip_offset as i32;
            let rbp_offset = layout.frame_pointer_offset as i32;

            // Location to which someone switch-ing back to this stack will jump
            // to: Right behind the `StackSwitch` instruction
            let resume = sink.get_label();

            //
            // For RBP and RSP we do the following:
            // - Load new value for register from `load_context_ptr` +
            // corresponding offset.
            // - Store previous (!) value of register at `store_context_ptr` +
            // corresponding offset.
            //
            // Since `load_context_ptr` and `store_context_ptr` are allowed to be
            // equal, we need to use a temporary register here.
            //

            let mut exchange = |offset, reg| {
                let addr = SyntheticAmode::real(Amode::imm_reg(offset, **load_context_ptr));
                asm::inst::movq_rm::new(tmp1, addr).emit(sink, info, state);

                asm::inst::movq_mr::new(
                    Amode::imm_reg(offset, **store_context_ptr),
                    Gpr::new(reg).unwrap(),
                )
                .emit(sink, info, state);

                let dst = Writable::from_reg(reg);
                asm::inst::movq_mr::new(dst.map(Gpr::unwrap_new), tmp1.to_reg())
                    .emit(sink, info, state);
            };

            exchange(rsp_offset, regs::rsp());
            exchange(rbp_offset, regs::rbp());

            //
            // Load target PC, store resume PC, jump to target PC
            //

            let addr = SyntheticAmode::real(Amode::imm_reg(pc_offset, **load_context_ptr));
            asm::inst::movq_rm::new(tmp1, addr).emit(sink, info, state);

            let amode = Amode::RipRelative { target: resume };
            asm::inst::leaq_rm::new(tmp2, amode).emit(sink, info, state);

            asm::inst::movq_mr::new(
                Amode::imm_reg(pc_offset, **store_context_ptr),
                tmp2.to_reg(),
            )
            .emit(sink, info, state);

            asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state);

            sink.bind_label(resume, state.ctrl_plane_mut());
        }

        Inst::JmpKnown { dst } => uncond_jmp(sink, *dst),

        Inst::WinchJmpIf { cc, taken } => one_way_jmp(sink, *cc, *taken),

        Inst::JmpCond {
            cc,
            taken,
            not_taken,
        } => {
            cond_jmp(sink, *cc, *taken);
            uncond_jmp(sink, *not_taken);
        }

        Inst::JmpCondOr {
            cc1,
            cc2,
            taken,
            not_taken,
        } => {
            // Emit:
            //   jcc1 taken
            //   jcc2 taken
            //   jmp not_taken
            //
            // Note that we enroll both conditionals in the
            // branch-chomping mechanism because MachBuffer
            // simplification can continue upward as long as it keeps
            // chomping branches. In the best case, if taken ==
            // not_taken and that one block is the fallthrough block,
            // all three branches can disappear.

            cond_jmp(sink, *cc1, *taken);
            cond_jmp(sink, *cc2, *taken);
            uncond_jmp(sink, *not_taken);
        }

        &Inst::JmpTableSeq {
            idx,
            tmp1,
            tmp2,
            ref targets,
            ref default_target,
            ..
        } => {
            // This sequence is *one* instruction in the vcode, and is expanded only here at
            // emission time, because we cannot allow the regalloc to insert spills/reloads in
            // the middle; we depend on hardcoded PC-rel addressing below.
            //
            // We don't have to worry about emitting islands, because the only label-use type has a
            // maximum range of 2 GB. If we later consider using shorter-range label references,
            // this will need to be revisited.

            // We generate the following sequence. Note that the only read of %idx is before the
            // write to %tmp2, so regalloc may use the same register for both; fix x64/inst/mod.rs
            // if you change this.
            // lea start_of_jump_table_offset(%rip), %tmp1
            // movslq [%tmp1, %idx, 4], %tmp2 ;; shift of 2, viz. multiply index by 4
            // addq %tmp2, %tmp1
            // j *%tmp1
            // $start_of_jump_table:
            // -- jump table entries

            // Load base address of jump table.
            let start_of_jumptable = sink.get_label();
            asm::inst::leaq_rm::new(tmp1, Amode::rip_relative(start_of_jumptable))
                .emit(sink, info, state);

            // Load value out of the jump table. It's a relative offset to the target block, so it
            // might be negative; use a sign-extension.
            let inst = Inst::movsx_rm_r(
                ExtMode::LQ,
                RegMem::mem(Amode::imm_reg_reg_shift(
                    0,
                    Gpr::unwrap_new(tmp1.to_reg()),
                    Gpr::unwrap_new(idx),
                    2,
                )),
                tmp2,
            );
            inst.emit(sink, info, state);

            // Add base of jump table to jump-table-sourced block offset.
            asm::inst::addq_rm::new(tmp1, tmp2).emit(sink, info, state);

            // Branch to computed address.
            asm::inst::jmpq_m::new(tmp1.to_reg()).emit(sink, info, state);

            // Emit jump table (table of 32-bit offsets).
            sink.bind_label(start_of_jumptable, state.ctrl_plane_mut());
            let jt_off = sink.cur_offset();
            for &target in targets.iter().chain(std::iter::once(default_target)) {
                let word_off = sink.cur_offset();
                // off_into_table is an addend here embedded in the label to be later patched at
                // the end of codegen. The offset is initially relative to this jump table entry;
                // with the extra addend, it'll be relative to the jump table's start, after
                // patching.
                let off_into_table = word_off - jt_off;
                sink.use_label_at_offset(word_off, target, LabelUse::PCRel32);
                sink.put4(off_into_table);
            }
        }

        Inst::TrapIf { cc, trap_code } => {
            let trap_label = sink.defer_trap(*trap_code);
            one_way_jmp(sink, *cc, trap_label);
        }

        Inst::TrapIfAnd {
            cc1,
            cc2,
            trap_code,
        } => {
            let trap_label = sink.defer_trap(*trap_code);
            let else_label = sink.get_label();

            // Jump to the end if the first condition isn't true, and then if
            // the second condition is true go to the trap.
            one_way_jmp(sink, cc1.invert(), else_label);
            one_way_jmp(sink, *cc2, trap_label);

            sink.bind_label(else_label, state.ctrl_plane_mut());
        }

        Inst::TrapIfOr {
            cc1,
            cc2,
            trap_code,
        } => {
            let trap_label = sink.defer_trap(*trap_code);

            // Emit two jumps to the same trap if either condition code is true.
            one_way_jmp(sink, *cc1, trap_label);
            one_way_jmp(sink, *cc2, trap_label);
        }

        Inst::XmmMinMaxSeq {
            size,
            is_min,
            lhs,
            rhs,
            dst,
        } => {
            let rhs = rhs.to_reg();
            let lhs = lhs.to_reg();
            let dst = dst.to_writable_reg();
            debug_assert_eq!(rhs, dst.to_reg());

            // Generates the following sequence:
            // cmpss/cmpsd %lhs, %rhs_dst
            // jnz do_min_max
            // jp propagate_nan
            //
            // ;; ordered and equal: propagate the sign bit (for -0 vs 0):
            // {and,or}{ss,sd} %lhs, %rhs_dst
            // j done
            //
            // ;; to get the desired NaN behavior (signalling NaN transformed into a quiet NaN, the
            // ;; NaN value is returned), we add both inputs.
            // propagate_nan:
            // add{ss,sd} %lhs, %rhs_dst
            // j done
            //
            // do_min_max:
            // {min,max}{ss,sd} %lhs, %rhs_dst
            //
            // done:
            let done = sink.get_label();
            let propagate_nan = sink.get_label();
            let do_min_max = sink.get_label();

            let (add_op, cmp_op, and_op, or_op, min_max_op) = match size {
                OperandSize::Size32 => (
                    asm::inst::addss_a::new(dst, lhs).into(),
                    asm::inst::ucomiss_a::new(dst.to_reg(), lhs).into(),
                    asm::inst::andps_a::new(dst, lhs).into(),
                    asm::inst::orps_a::new(dst, lhs).into(),
                    if *is_min {
                        asm::inst::minss_a::new(dst, lhs).into()
                    } else {
                        asm::inst::maxss_a::new(dst, lhs).into()
                    },
                ),
                OperandSize::Size64 => (
                    asm::inst::addsd_a::new(dst, lhs).into(),
                    asm::inst::ucomisd_a::new(dst.to_reg(), lhs).into(),
                    asm::inst::andpd_a::new(dst, lhs).into(),
                    asm::inst::orpd_a::new(dst, lhs).into(),
                    if *is_min {
                        asm::inst::minsd_a::new(dst, lhs).into()
                    } else {
                        asm::inst::maxsd_a::new(dst, lhs).into()
                    },
                ),
                _ => unreachable!(),
            };
            let add_op: AsmInst = add_op;
            let or_op: AsmInst = or_op;
            let min_max_op: AsmInst = min_max_op;
            let cmp_op: AsmInst = cmp_op;

            cmp_op.emit(sink, info, state);

            one_way_jmp(sink, CC::NZ, do_min_max);
            one_way_jmp(sink, CC::P, propagate_nan);

            // Ordered and equal. The operands are bit-identical unless they are zero
            // and negative zero. These instructions merge the sign bits in that
            // case, and are no-ops otherwise.
            let inst: AsmInst = if *is_min { or_op } else { and_op };
            inst.emit(sink, info, state);

            let inst = Inst::jmp_known(done);
            inst.emit(sink, info, state);

            // x86's min/max are not symmetric; if either operand is a NaN, they return the
            // read-only operand: perform an addition between the two operands, which has the
            // desired NaN propagation effects.
            sink.bind_label(propagate_nan, state.ctrl_plane_mut());
            add_op.emit(sink, info, state);

            one_way_jmp(sink, CC::P, done);

            sink.bind_label(do_min_max, state.ctrl_plane_mut());
            min_max_op.emit(sink, info, state);

            sink.bind_label(done, state.ctrl_plane_mut());
        }

        Inst::XmmUninitializedValue { .. } | Inst::GprUninitializedValue { .. } => {
            // These instruction formats only exist to declare a register as a
            // `def`; no code is emitted. This is always immediately followed by
            // an instruction, such as `xor <tmp>, <tmp>`, that semantically
            // reads this undefined value but arithmetically produces the same
            // result regardless of its value.
        }

        Inst::CvtUint64ToFloatSeq {
            dst_size,
            src,
            dst,
            tmp_gpr1,
            tmp_gpr2,
        } => {
            let src = src.to_reg();
            let dst = dst.to_writable_reg();
            let tmp_gpr1 = tmp_gpr1.to_writable_reg();
            let tmp_gpr2 = tmp_gpr2.to_writable_reg();

            // Note: this sequence is specific to 64-bit mode; a 32-bit mode would require a
            // different sequence.
            //
            // Emit the following sequence:
            //
            //  cmp 0, %src
            //  jl handle_negative
            //
            //  ;; handle positive, which can't overflow
            //  cvtsi2sd/cvtsi2ss %src, %dst
            //  j done
            //
            //  ;; handle negative: see below for an explanation of what it's doing.
            //  handle_negative:
            //  mov %src, %tmp_gpr1
            //  shr $1, %tmp_gpr1
            //  mov %src, %tmp_gpr2
            //  and $1, %tmp_gpr2
            //  or %tmp_gpr1, %tmp_gpr2
            //  cvtsi2sd/cvtsi2ss %tmp_gpr2, %dst
            //  addsd/addss %dst, %dst
            //
            //  done:

            assert_ne!(src, tmp_gpr1.to_reg());
            assert_ne!(src, tmp_gpr2.to_reg());

            let handle_negative = sink.get_label();
            let done = sink.get_label();

            // If x seen as a signed int64 is not negative, a signed-conversion will do the right
            // thing.
            // TODO use tst src, src here.
            asm::inst::cmpq_mi_sxb::new(src, 0).emit(sink, info, state);

            one_way_jmp(sink, CC::L, handle_negative);

            // Handle a positive int64, which is the "easy" case: a signed conversion will do the
            // right thing.
            emit_signed_cvt(
                sink,
                info,
                state,
                src,
                dst,
                *dst_size == OperandSize::Size64,
            );

            let inst = Inst::jmp_known(done);
            inst.emit(sink, info, state);

            sink.bind_label(handle_negative, state.ctrl_plane_mut());

            // Divide x by two to get it in range for the signed conversion, keep the LSB, and
            // scale it back up on the FP side.
            let inst = Inst::gen_move(tmp_gpr1, src, types::I64);
            inst.emit(sink, info, state);

            // tmp_gpr1 := src >> 1
            asm::inst::shrq_mi::new(tmp_gpr1, 1).emit(sink, info, state);

            let inst = Inst::gen_move(tmp_gpr2, src, types::I64);
            inst.emit(sink, info, state);

            asm::inst::andq_mi_sxb::new(tmp_gpr2, 1).emit(sink, info, state);

            asm::inst::orq_rm::new(tmp_gpr2, tmp_gpr1).emit(sink, info, state);

            emit_signed_cvt(
                sink,
                info,
                state,
                tmp_gpr2.to_reg(),
                dst,
                *dst_size == OperandSize::Size64,
            );

            let inst: AsmInst = match *dst_size {
                OperandSize::Size64 => asm::inst::addsd_a::new(dst, dst.to_reg()).into(),
                OperandSize::Size32 => asm::inst::addss_a::new(dst, dst.to_reg()).into(),
                _ => unreachable!(),
            };
            inst.emit(sink, info, state);

            sink.bind_label(done, state.ctrl_plane_mut());
        }

        Inst::CvtFloatToSintSeq {
            src_size,
            dst_size,
            is_saturating,
            src,
            dst,
            tmp_gpr,
            tmp_xmm,
        } => {
            use OperandSize::*;

            let src = src.to_reg();
            let dst = dst.to_writable_reg();
            let tmp_gpr = tmp_gpr.to_writable_reg();
            let tmp_xmm = tmp_xmm.to_writable_reg();

            // Emits the following common sequence:
            //
            // cvttss2si/cvttsd2si %src, %dst
            // cmp %dst, 1
            // jno done
            //
            // Then, for saturating conversions:
            //
            // ;; check for NaN
            // cmpss/cmpsd %src, %src
            // jnp not_nan
            // xor %dst, %dst
            //
            // ;; positive inputs get saturated to INT_MAX; negative ones to INT_MIN, which is
            // ;; already in %dst.
            // xorpd %tmp_xmm, %tmp_xmm
            // cmpss/cmpsd %src, %tmp_xmm
            // jnb done
            // mov/movaps $INT_MAX, %dst
            //
            // done:
            //
            // Then, for non-saturating conversions:
            //
            // ;; check for NaN
            // cmpss/cmpsd %src, %src
            // jnp not_nan
            // ud2 trap BadConversionToInteger
            //
            // ;; check if INT_MIN was the correct result, against a magic constant:
            // not_nan:
            // movaps/mov $magic, %tmp_gpr
            // movq/movd %tmp_gpr, %tmp_xmm
            // cmpss/cmpsd %tmp_xmm, %src
            // jnb/jnbe $check_positive
            // ud2 trap IntegerOverflow
            //
            // ;; if positive, it was a real overflow
            // check_positive:
            // xorpd %tmp_xmm, %tmp_xmm
            // cmpss/cmpsd %src, %tmp_xmm
            // jnb done
            // ud2 trap IntegerOverflow
            //
            // done:

            let cmp_op: AsmInst = match src_size {
                Size64 => asm::inst::ucomisd_a::new(src, src).into(),
                Size32 => asm::inst::ucomiss_a::new(src, src).into(),
                _ => unreachable!(),
            };

            let cvtt_op = |dst, src| Inst::External {
                inst: match (*src_size, *dst_size) {
                    (Size32, Size32) => asm::inst::cvttss2si_a::new(dst, src).into(),
                    (Size32, Size64) => asm::inst::cvttss2si_aq::new(dst, src).into(),
                    (Size64, Size32) => asm::inst::cvttsd2si_a::new(dst, src).into(),
                    (Size64, Size64) => asm::inst::cvttsd2si_aq::new(dst, src).into(),
                    _ => unreachable!(),
                },
            };

            let done = sink.get_label();

            // The truncation.
            cvtt_op(dst, src).emit(sink, info, state);

            // Compare against 1, in case of overflow the dst operand was INT_MIN.
            let inst = Inst::cmp_mi_sxb(*dst_size, Gpr::unwrap_new(dst.to_reg()), 1);
            inst.emit(sink, info, state);

            one_way_jmp(sink, CC::NO, done); // no overflow => done

            // Check for NaN.
            cmp_op.emit(sink, info, state);

            if *is_saturating {
                let not_nan = sink.get_label();
                one_way_jmp(sink, CC::NP, not_nan); // go to not_nan if not a NaN

                // For NaN, emit 0.
                let inst: AsmInst = match *dst_size {
                    OperandSize::Size32 => asm::inst::xorl_rm::new(dst, dst).into(),
                    OperandSize::Size64 => asm::inst::xorq_rm::new(dst, dst).into(),
                    _ => unreachable!(),
                };
                inst.emit(sink, info, state);

                let inst = Inst::jmp_known(done);
                inst.emit(sink, info, state);

                sink.bind_label(not_nan, state.ctrl_plane_mut());

                // If the input was positive, saturate to INT_MAX.

                // Zero out tmp_xmm.
                asm::inst::xorpd_a::new(tmp_xmm, tmp_xmm.to_reg()).emit(sink, info, state);

                let inst: AsmInst = match src_size {
                    Size64 => asm::inst::ucomisd_a::new(tmp_xmm.to_reg(), src).into(),
                    Size32 => asm::inst::ucomiss_a::new(tmp_xmm.to_reg(), src).into(),
                    _ => unreachable!(),
                };
                inst.emit(sink, info, state);

                // Jump if >= to done.
                one_way_jmp(sink, CC::NB, done);

                // Otherwise, put INT_MAX.
                if *dst_size == OperandSize::Size64 {
                    let inst = Inst::imm(OperandSize::Size64, 0x7fffffffffffffff, dst);
                    inst.emit(sink, info, state);
                } else {
                    let inst = Inst::imm(OperandSize::Size32, 0x7fffffff, dst);
                    inst.emit(sink, info, state);
                }
            } else {
                let inst = Inst::trap_if(CC::P, TrapCode::BAD_CONVERSION_TO_INTEGER);
                inst.emit(sink, info, state);

                // Check if INT_MIN was the correct result: determine the smallest floating point
                // number that would convert to INT_MIN, put it in a temporary register, and compare
                // against the src register.
                // If the src register is less (or in some cases, less-or-equal) than the threshold,
                // trap!

                let mut no_overflow_cc = CC::NB; // >=
                let output_bits = dst_size.to_bits();
                match *src_size {
                    OperandSize::Size32 => {
                        let cst = (-Ieee32::pow2(output_bits - 1)).bits();
                        let inst = Inst::imm(OperandSize::Size32, cst as u64, tmp_gpr);
                        inst.emit(sink, info, state);
                    }
                    OperandSize::Size64 => {
                        // An f64 can represent `i32::min_value() - 1` exactly with precision to spare,
                        // so there are values less than -2^(N-1) that convert correctly to INT_MIN.
                        let cst = if output_bits < 64 {
                            no_overflow_cc = CC::NBE; // >
                            Ieee64::fcvt_to_sint_negative_overflow(output_bits)
                        } else {
                            -Ieee64::pow2(output_bits - 1)
                        };
                        let inst = Inst::imm(OperandSize::Size64, cst.bits(), tmp_gpr);
                        inst.emit(sink, info, state);
                    }
                    _ => unreachable!(),
                }

                let inst: AsmInst = {
                    let tmp_xmm: WritableXmm = tmp_xmm.map(|r| Xmm::new(r).unwrap());
                    match src_size {
                        Size32 => asm::inst::movd_a::new(tmp_xmm, tmp_gpr).into(),
                        Size64 => asm::inst::movq_a::new(tmp_xmm, tmp_gpr).into(),
                        _ => unreachable!(),
                    }
                };
                inst.emit(sink, info, state);

                let inst: AsmInst = match src_size {
                    Size64 => asm::inst::ucomisd_a::new(src, tmp_xmm.to_reg()).into(),
                    Size32 => asm::inst::ucomiss_a::new(src, tmp_xmm.to_reg()).into(),
                    _ => unreachable!(),
                };
                inst.emit(sink, info, state);

                // no trap if src >= or > threshold
                let inst = Inst::trap_if(no_overflow_cc.invert(), TrapCode::INTEGER_OVERFLOW);
                inst.emit(sink, info, state);

                // If positive, it was a real overflow.

                // Zero out the tmp_xmm register.
                asm::inst::xorpd_a::new(tmp_xmm, tmp_xmm.to_reg()).emit(sink, info, state);

                let inst: AsmInst = match src_size {
                    Size64 => asm::inst::ucomisd_a::new(tmp_xmm.to_reg(), src).into(),
                    Size32 => asm::inst::ucomiss_a::new(tmp_xmm.to_reg(), src).into(),
                    _ => unreachable!(),
                };
                inst.emit(sink, info, state);

                // no trap if 0 >= src
                let inst = Inst::trap_if(CC::B, TrapCode::INTEGER_OVERFLOW);
                inst.emit(sink, info, state);
            }

            sink.bind_label(done, state.ctrl_plane_mut());
        }

        Inst::CvtFloatToUintSeq {
            src_size,
            dst_size,
            is_saturating,
            src,
            dst,
            tmp_gpr,
            tmp_xmm,
            tmp_xmm2,
        } => {
            use OperandSize::*;

            let src = src.to_reg();
            let dst = dst.to_writable_reg();
            let tmp_gpr = tmp_gpr.to_writable_reg();
            let tmp_xmm = tmp_xmm.to_writable_reg();
            let tmp_xmm2 = tmp_xmm2.to_writable_reg();

            // The only difference in behavior between saturating and non-saturating is how we
            // handle errors. Emits the following sequence:
            //
            // movaps/mov 2**(int_width - 1), %tmp_gpr
            // movq/movd %tmp_gpr, %tmp_xmm
            // cmpss/cmpsd %tmp_xmm, %src
            // jnb is_large
            //
            // ;; check for NaN inputs
            // jnp not_nan
            // -- non-saturating: ud2 trap BadConversionToInteger
            // -- saturating: xor %dst, %dst; j done
            //
            // not_nan:
            // cvttss2si/cvttsd2si %src, %dst
            // cmp 0, %dst
            // jnl done
            // -- non-saturating: ud2 trap IntegerOverflow
            // -- saturating: xor %dst, %dst; j done
            //
            // is_large:
            // mov %src, %tmp_xmm2
            // subss/subsd %tmp_xmm, %tmp_xmm2
            // cvttss2si/cvttss2sd %tmp_x, %dst
            // cmp 0, %dst
            // jnl next_is_large
            // -- non-saturating: ud2 trap IntegerOverflow
            // -- saturating: movaps $UINT_MAX, %dst; j done
            //
            // next_is_large:
            // add 2**(int_width -1), %dst ;; 2 instructions for 64-bits integers
            //
            // done:

            assert_ne!(tmp_xmm.to_reg(), src, "tmp_xmm clobbers src!");

            let xor_op = |dst, src| Inst::External {
                inst: match *dst_size {
                    Size32 => asm::inst::xorl_rm::new(dst, src).into(),
                    Size64 => asm::inst::xorq_rm::new(dst, src).into(),
                    _ => unreachable!(),
                },
            };

            let subs_op = |dst, src| Inst::External {
                inst: match *src_size {
                    Size32 => asm::inst::subss_a::new(dst, src).into(),
                    Size64 => asm::inst::subsd_a::new(dst, src).into(),
                    _ => unreachable!(),
                },
            };

            let cvtt_op = |dst, src| Inst::External {
                inst: match (*src_size, *dst_size) {
                    (Size32, Size32) => asm::inst::cvttss2si_a::new(dst, src).into(),
                    (Size32, Size64) => asm::inst::cvttss2si_aq::new(dst, src).into(),
                    (Size64, Size32) => asm::inst::cvttsd2si_a::new(dst, src).into(),
                    (Size64, Size64) => asm::inst::cvttsd2si_aq::new(dst, src).into(),
                    _ => unreachable!(),
                },
            };

            let done = sink.get_label();

            let cst = match src_size {
                OperandSize::Size32 => Ieee32::pow2(dst_size.to_bits() - 1).bits() as u64,
                OperandSize::Size64 => Ieee64::pow2(dst_size.to_bits() - 1).bits(),
                _ => unreachable!(),
            };

            let inst = Inst::imm(*src_size, cst, tmp_gpr);
            inst.emit(sink, info, state);

            let inst: AsmInst = {
                let tmp_xmm: WritableXmm = tmp_xmm.map(|r| Xmm::new(r).unwrap());
                match src_size {
                    Size32 => asm::inst::movd_a::new(tmp_xmm, tmp_gpr).into(),
                    Size64 => asm::inst::movq_a::new(tmp_xmm, tmp_gpr).into(),
                    _ => unreachable!(),
                }
            };
            inst.emit(sink, info, state);

            let inst: AsmInst = match src_size {
                Size64 => asm::inst::ucomisd_a::new(src, tmp_xmm.to_reg()).into(),
                Size32 => asm::inst::ucomiss_a::new(src, tmp_xmm.to_reg()).into(),
                _ => unreachable!(),
            };
            inst.emit(sink, info, state);

            let handle_large = sink.get_label();
            one_way_jmp(sink, CC::NB, handle_large); // jump to handle_large if src >= large_threshold

            if *is_saturating {
                // If not NaN jump over this 0-return, otherwise return 0
                let not_nan = sink.get_label();
                one_way_jmp(sink, CC::NP, not_nan);

                xor_op(dst, dst).emit(sink, info, state);

                let inst = Inst::jmp_known(done);
                inst.emit(sink, info, state);
                sink.bind_label(not_nan, state.ctrl_plane_mut());
            } else {
                // Trap.
                let inst = Inst::trap_if(CC::P, TrapCode::BAD_CONVERSION_TO_INTEGER);
                inst.emit(sink, info, state);
            }

            // Actual truncation for small inputs: if the result is not positive, then we had an
            // overflow.

            cvtt_op(dst, src).emit(sink, info, state);

            let inst = Inst::cmp_mi_sxb(*dst_size, Gpr::unwrap_new(dst.to_reg()), 0);
            inst.emit(sink, info, state);

            one_way_jmp(sink, CC::NL, done); // if dst >= 0, jump to done

            if *is_saturating {
                // The input was "small" (< 2**(width -1)), so the only way to get an integer
                // overflow is because the input was too small: saturate to the min value, i.e. 0.
                let inst: AsmInst = match *dst_size {
                    OperandSize::Size32 => asm::inst::xorl_rm::new(dst, dst).into(),
                    OperandSize::Size64 => asm::inst::xorq_rm::new(dst, dst).into(),
                    _ => unreachable!(),
                };
                inst.emit(sink, info, state);

                let inst = Inst::jmp_known(done);
                inst.emit(sink, info, state);
            } else {
                // Trap.
                asm::inst::ud2_zo::new(TrapCode::INTEGER_OVERFLOW).emit(sink, info, state);
            }

            // Now handle large inputs.

            sink.bind_label(handle_large, state.ctrl_plane_mut());

            let inst = Inst::gen_move(tmp_xmm2, src, types::F64);
            inst.emit(sink, info, state);

            subs_op(tmp_xmm2, tmp_xmm.to_reg()).emit(sink, info, state);

            cvtt_op(dst, tmp_xmm2.to_reg()).emit(sink, info, state);

            let inst = Inst::cmp_mi_sxb(*dst_size, Gpr::unwrap_new(dst.to_reg()), 0);
            inst.emit(sink, info, state);

            if *is_saturating {
                let next_is_large = sink.get_label();
                one_way_jmp(sink, CC::NL, next_is_large); // if dst >= 0, jump to next_is_large

                // The input was "large" (>= 2**(width -1)), so the only way to get an integer
                // overflow is because the input was too large: saturate to the max value.
                let inst = Inst::imm(
                    OperandSize::Size64,
                    if *dst_size == OperandSize::Size64 {
                        u64::max_value()
                    } else {
                        u32::max_value() as u64
                    },
                    dst,
                );
                inst.emit(sink, info, state);

                let inst = Inst::jmp_known(done);
                inst.emit(sink, info, state);
                sink.bind_label(next_is_large, state.ctrl_plane_mut());
            } else {
                let inst = Inst::trap_if(CC::L, TrapCode::INTEGER_OVERFLOW);
                inst.emit(sink, info, state);
            }

            if *dst_size == OperandSize::Size64 {
                let inst = Inst::imm(OperandSize::Size64, 1 << 63, tmp_gpr);
                inst.emit(sink, info, state);

                asm::inst::addq_rm::new(dst, tmp_gpr).emit(sink, info, state);
            } else {
                asm::inst::addl_mi::new(dst, asm::Imm32::new(1 << 31)).emit(sink, info, state);
            }

            sink.bind_label(done, state.ctrl_plane_mut());
        }

        Inst::LoadExtName {
            dst,
            name,
            offset,
            distance,
        } => {
            let name = &**name;
            let riprel = asm::Amode::RipRelative {
                target: asm::DeferredTarget::None,
            };
            if info.flags.is_pic() {
                // Generates: movq symbol@GOTPCREL(%rip), %dst
                asm::inst::movq_rm::new(*dst, riprel).emit(sink, info, state);
                let cur = sink.cur_offset();
                sink.add_reloc_at_offset(cur - 4, Reloc::X86GOTPCRel4, name, -4);

                // Offset in the relocation above applies to the address of the
                // *GOT entry*, not the loaded address; so we emit a separate
                // add instruction if needed.
                let offset = i32::try_from(*offset).unwrap();
                if offset != 0 {
                    asm::inst::addq_mi_sxl::new(PairedGpr::from(*dst), offset)
                        .emit(sink, info, state);
                }
            } else if distance == &RelocDistance::Near {
                // If we know the distance to the name is within 2GB (e.g., a
                // module-local function), we can generate a RIP-relative
                // address, with a relocation.
                asm::inst::leaq_rm::new(*dst, riprel).emit(sink, info, state);
                let cur = sink.cur_offset();
                sink.add_reloc_at_offset(cur - 4, Reloc::X86CallPCRel4, name, *offset - 4);
            } else {
                // The full address can be encoded in the register, with a
                // relocation.
                asm::inst::movabsq_oi::new(*dst, 0).emit(sink, info, state);
                let cur = sink.cur_offset();
                sink.add_reloc_at_offset(cur - 8, Reloc::Abs8, name, *offset);
            }
        }

        Inst::AtomicRmwSeq {
            ty,
            op,
            mem,
            operand,
            temp,
            dst_old,
        } => {
            let operand = *operand;
            let temp = *temp;
            let temp_r = temp.map(|r| *r);
            let dst_old = *dst_old;
            let dst_old_r = dst_old.map(|r| *r);
            debug_assert_eq!(dst_old.to_reg(), regs::rax());
            let mem = mem.finalize(state.frame_layout(), sink).clone();

            // Emit this:
            //    mov{zbq,zwq,zlq,q}     (%r_address), %rax    // rax = old value
            //  again:
            //    movq                   %rax, %r_temp         // rax = old value, r_temp = old value
            //    `op`q                  %r_operand, %r_temp   // rax = old value, r_temp = new value
            //    lock cmpxchg{b,w,l,q}  %r_temp, (%r_address) // try to store new value
            //    jnz again // If this is taken, rax will have a "revised" old value
            //
            // Operand conventions: IN:  %r_address, %r_operand OUT: %rax (old
            //    value), %r_temp (trashed), %rflags (trashed)
            let again_label = sink.get_label();

            // mov{zbq,zwq,zlq,q} (%r_address), %rax
            // No need to call `add_trap` here, since the `i1` emit will do that.
            let i1 = Inst::load(*ty, mem.clone(), dst_old_r, ExtKind::ZeroExtend);
            i1.emit(sink, info, state);

            // again:
            sink.bind_label(again_label, state.ctrl_plane_mut());

            // movq %rax, %r_temp
            asm::inst::movq_mr::new(temp, dst_old.to_reg()).emit(sink, info, state);

            use AtomicRmwSeqOp as RmwOp;
            match op {
                RmwOp::Nand => {
                    // andq %r_operand, %r_temp
                    asm::inst::andq_rm::new(temp, operand).emit(sink, info, state);

                    // notq %r_temp
                    asm::inst::notq_m::new(PairedGpr::from(temp)).emit(sink, info, state);
                }
                RmwOp::Umin | RmwOp::Umax | RmwOp::Smin | RmwOp::Smax => {
                    // cmp %r_temp, %r_operand
                    let temp = temp.to_reg();
                    match *ty {
                        types::I8 => asm::inst::cmpb_mr::new(operand, temp).emit(sink, info, state),
                        types::I16 => {
                            asm::inst::cmpw_mr::new(operand, temp).emit(sink, info, state)
                        }
                        types::I32 => {
                            asm::inst::cmpl_mr::new(operand, temp).emit(sink, info, state)
                        }
                        types::I64 => {
                            asm::inst::cmpq_mr::new(operand, temp).emit(sink, info, state)
                        }
                        _ => unreachable!(),
                    }

                    // cmovcc %r_operand, %r_temp
                    match op {
                        RmwOp::Umin => {
                            asm::inst::cmovbeq_rm::new(temp_r, *operand).emit(sink, info, state)
                        }
                        RmwOp::Umax => {
                            asm::inst::cmovaeq_rm::new(temp_r, *operand).emit(sink, info, state)
                        }
                        RmwOp::Smin => {
                            asm::inst::cmovleq_rm::new(temp_r, *operand).emit(sink, info, state)
                        }
                        RmwOp::Smax => {
                            asm::inst::cmovgeq_rm::new(temp_r, *operand).emit(sink, info, state)
                        }
                        _ => unreachable!(),
                    }
                }
                RmwOp::And => {
                    // andq %r_operand, %r_temp
                    asm::inst::andq_rm::new(temp, operand).emit(sink, info, state);
                }
                RmwOp::Or => {
                    // orq %r_operand, %r_temp
                    asm::inst::orq_rm::new(temp, operand).emit(sink, info, state);
                }
                RmwOp::Xor => {
                    // xorq %r_operand, %r_temp
                    asm::inst::xorq_rm::new(temp, operand).emit(sink, info, state);
                }
            }

            // lock cmpxchg{b,w,l,q} %r_temp, (%r_address)
            // No need to call `add_trap` here, since the `i4` emit will do that.
            let temp = temp.to_reg();
            let dst_old = PairedGpr::from(dst_old);
            let inst: AsmInst = match *ty {
                types::I8 => asm::inst::lock_cmpxchgb_mr::new(mem, temp, dst_old).into(),
                types::I16 => asm::inst::lock_cmpxchgw_mr::new(mem, temp, dst_old).into(),
                types::I32 => asm::inst::lock_cmpxchgl_mr::new(mem, temp, dst_old).into(),
                types::I64 => asm::inst::lock_cmpxchgq_mr::new(mem, temp, dst_old).into(),
                _ => unreachable!(),
            };
            inst.emit(sink, info, state);

            // jnz again
            one_way_jmp(sink, CC::NZ, again_label);
        }

        Inst::Atomic128RmwSeq {
            op,
            mem,
            operand_low,
            operand_high,
            temp_low,
            temp_high,
            dst_old_low,
            dst_old_high,
        } => {
            let operand_low = *operand_low;
            let operand_high = *operand_high;
            let temp_low = *temp_low;
            let temp_high = *temp_high;
            let dst_old_low = *dst_old_low;
            let dst_old_high = *dst_old_high;
            debug_assert_eq!(temp_low.to_reg(), regs::rbx());
            debug_assert_eq!(temp_high.to_reg(), regs::rcx());
            debug_assert_eq!(dst_old_low.to_reg(), regs::rax());
            debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());
            let mem = mem.finalize(state.frame_layout(), sink).clone();

            let again_label = sink.get_label();

            // Load the initial value.
            asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);
            asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);

            // again:
            sink.bind_label(again_label, state.ctrl_plane_mut());

            // Move old value to temp registers.
            asm::inst::movq_mr::new(temp_low, dst_old_low.to_reg()).emit(sink, info, state);
            asm::inst::movq_mr::new(temp_high, dst_old_high.to_reg()).emit(sink, info, state);

            // Perform the operation.
            use Atomic128RmwSeqOp as RmwOp;
            match op {
                RmwOp::Nand => {
                    // temp &= operand
                    asm::inst::andq_rm::new(temp_low, operand_low).emit(sink, info, state);
                    asm::inst::andq_rm::new(temp_high, operand_high).emit(sink, info, state);

                    // temp = !temp
                    asm::inst::notq_m::new(PairedGpr::from(temp_low)).emit(sink, info, state);
                    asm::inst::notq_m::new(PairedGpr::from(temp_high)).emit(sink, info, state);
                }
                RmwOp::Umin | RmwOp::Umax | RmwOp::Smin | RmwOp::Smax => {
                    // Do a comparison with LHS temp and RHS operand.
                    // Note the opposite argument orders.
                    asm::inst::cmpq_mr::new(temp_low.to_reg(), operand_low).emit(sink, info, state);
                    // This will clobber `temp_high`
                    asm::inst::sbbq_rm::new(temp_high, operand_high).emit(sink, info, state);
                    // Restore the clobbered value
                    asm::inst::movq_mr::new(temp_high, dst_old_high.to_reg())
                        .emit(sink, info, state);
                    match op {
                        RmwOp::Umin => {
                            asm::inst::cmovaeq_rm::new(temp_low, operand_low)
                                .emit(sink, info, state);
                            asm::inst::cmovaeq_rm::new(temp_high, operand_high)
                                .emit(sink, info, state);
                        }
                        RmwOp::Umax => {
                            asm::inst::cmovbq_rm::new(temp_low, operand_low)
                                .emit(sink, info, state);
                            asm::inst::cmovbq_rm::new(temp_high, operand_high)
                                .emit(sink, info, state);
                        }
                        RmwOp::Smin => {
                            asm::inst::cmovgeq_rm::new(temp_low, operand_low)
                                .emit(sink, info, state);
                            asm::inst::cmovgeq_rm::new(temp_high, operand_high)
                                .emit(sink, info, state);
                        }
                        RmwOp::Smax => {
                            asm::inst::cmovlq_rm::new(temp_low, operand_low)
                                .emit(sink, info, state);
                            asm::inst::cmovlq_rm::new(temp_high, operand_high)
                                .emit(sink, info, state);
                        }
                        _ => unreachable!(),
                    }
                }
                RmwOp::Add => {
                    asm::inst::addq_rm::new(temp_low, operand_low).emit(sink, info, state);
                    asm::inst::adcq_rm::new(temp_high, operand_high).emit(sink, info, state);
                }
                RmwOp::Sub => {
                    asm::inst::subq_rm::new(temp_low, operand_low).emit(sink, info, state);
                    asm::inst::sbbq_rm::new(temp_high, operand_high).emit(sink, info, state);
                }
                RmwOp::And => {
                    asm::inst::andq_rm::new(temp_low, operand_low).emit(sink, info, state);
                    asm::inst::andq_rm::new(temp_high, operand_high).emit(sink, info, state);
                }
                RmwOp::Or => {
                    asm::inst::orq_rm::new(temp_low, operand_low).emit(sink, info, state);
                    asm::inst::orq_rm::new(temp_high, operand_high).emit(sink, info, state);
                }
                RmwOp::Xor => {
                    asm::inst::xorq_rm::new(temp_low, operand_low).emit(sink, info, state);
                    asm::inst::xorq_rm::new(temp_high, operand_high).emit(sink, info, state);
                }
            }

            // cmpxchg16b (mem)
            asm::inst::lock_cmpxchg16b_m::new(
                PairedGpr::from(dst_old_low),
                PairedGpr::from(dst_old_high),
                temp_low.to_reg(),
                temp_high.to_reg(),
                mem,
            )
            .emit(sink, info, state);

            // jnz again
            one_way_jmp(sink, CC::NZ, again_label);
        }

        Inst::Atomic128XchgSeq {
            mem,
            operand_low,
            operand_high,
            dst_old_low,
            dst_old_high,
        } => {
            let operand_low = *operand_low;
            let operand_high = *operand_high;
            let dst_old_low = *dst_old_low;
            let dst_old_high = *dst_old_high;
            debug_assert_eq!(operand_low, regs::rbx());
            debug_assert_eq!(operand_high, regs::rcx());
            debug_assert_eq!(dst_old_low.to_reg(), regs::rax());
            debug_assert_eq!(dst_old_high.to_reg(), regs::rdx());
            let mem = mem.finalize(state.frame_layout(), sink).clone();

            let again_label = sink.get_label();

            // Load the initial value.
            asm::inst::movq_rm::new(dst_old_low, mem.clone()).emit(sink, info, state);
            asm::inst::movq_rm::new(dst_old_high, mem.offset(8)).emit(sink, info, state);

            // again:
            sink.bind_label(again_label, state.ctrl_plane_mut());

            // cmpxchg16b (mem)
            asm::inst::lock_cmpxchg16b_m::new(
                PairedGpr::from(dst_old_low),
                PairedGpr::from(dst_old_high),
                operand_low,
                operand_high,
                mem,
            )
            .emit(sink, info, state);

            // jnz again
            one_way_jmp(sink, CC::NZ, again_label);
        }

        Inst::ElfTlsGetAddr { symbol, dst } => {
            let dst = dst.to_reg().to_reg();
            debug_assert_eq!(dst, regs::rax());

            // N.B.: Must be exactly this byte sequence; the linker requires it,
            // because it must know how to rewrite the bytes.

            // data16 lea gv@tlsgd(%rip),%rdi
            sink.put1(0x66); // data16
            sink.put1(0b01001000); // REX.W
            sink.put1(0x8d); // LEA
            sink.put1(0x3d); // ModRM byte
            emit_reloc(sink, Reloc::ElfX86_64TlsGd, symbol, -4);
            sink.put4(0); // offset

            // data16 data16 callq __tls_get_addr-4
            sink.put1(0x66); // data16
            sink.put1(0x66); // data16
            sink.put1(0b01001000); // REX.W
            sink.put1(0xe8); // CALL
            emit_reloc(
                sink,
                Reloc::X86CallPLTRel4,
                &ExternalName::LibCall(LibCall::ElfTlsGetAddr),
                -4,
            );
            sink.put4(0); // offset
        }

        Inst::MachOTlsGetAddr { symbol, dst } => {
            let dst = dst.to_reg().to_reg();
            debug_assert_eq!(dst, regs::rax());

            // movq gv@tlv(%rip), %rdi
            sink.put1(0x48); // REX.w
            sink.put1(0x8b); // MOV
            sink.put1(0x3d); // ModRM byte
            emit_reloc(sink, Reloc::MachOX86_64Tlv, symbol, -4);
            sink.put4(0); // offset

            asm::inst::callq_m::new(asm::Amode::ImmReg {
                base: Gpr::RDI,
                simm32: asm::AmodeOffsetPlusKnownOffset::ZERO,
                trap: None,
            })
            .emit(sink, info, state);
        }

        Inst::CoffTlsGetAddr { symbol, dst, tmp } => {
            let dst = dst.to_reg().to_reg();
            debug_assert_eq!(dst, regs::rax());

            // tmp is used below directly as %rcx
            let tmp = tmp.to_reg().to_reg();
            debug_assert_eq!(tmp, regs::rcx());

            // See: https://gcc.godbolt.org/z/M8or9x6ss
            // And: https://github.com/bjorn3/rustc_codegen_cranelift/issues/388#issuecomment-532930282

            // Emit the following sequence
            // movl	(%rip), %eax          ; IMAGE_REL_AMD64_REL32	_tls_index
            // movq	%gs:88, %rcx
            // movq	(%rcx,%rax,8), %rax
            // leaq	(%rax), %rax          ; Reloc: IMAGE_REL_AMD64_SECREL	symbol

            // Load TLS index for current thread
            // movl	(%rip), %eax
            sink.put1(0x8b); // mov
            sink.put1(0x05);
            emit_reloc(
                sink,
                Reloc::X86PCRel4,
                &ExternalName::KnownSymbol(KnownSymbol::CoffTlsIndex),
                -4,
            );
            sink.put4(0); // offset

            // movq	%gs:88, %rcx
            // Load the TLS Storage Array pointer
            // The gs segment register refers to the base address of the TEB on x64.
            // 0x58 is the offset in the TEB for the ThreadLocalStoragePointer member on x64:
            sink.put_data(&[
                0x65, 0x48, // REX.W
                0x8b, // MOV
                0x0c, 0x25, 0x58, // 0x58 - ThreadLocalStoragePointer offset
                0x00, 0x00, 0x00,
            ]);

            // movq	(%rcx,%rax,8), %rax
            // Load the actual TLS entry for this thread.
            // Computes ThreadLocalStoragePointer + _tls_index*8
            sink.put_data(&[0x48, 0x8b, 0x04, 0xc1]);

            // leaq	(%rax), %rax
            sink.put1(0x48);
            sink.put1(0x8d);
            sink.put1(0x80);
            emit_reloc(sink, Reloc::X86SecRel, symbol, 0);
            sink.put4(0); // offset
        }

        Inst::Unwind { inst } => {
            sink.add_unwind(inst.clone());
        }

        Inst::DummyUse { .. } => {
            // Nothing.
        }

        Inst::LabelAddress { dst, label } => {
            // Emit an LEA with a LabelUse given this label.
            asm::inst::leaq_rm::new(*dst, Amode::rip_relative(*label)).emit(sink, info, state);
        }

        Inst::External { inst } => {
            let frame = state.frame_layout();
            emit_maybe_shrink(
                inst,
                &mut external::AsmCodeSink {
                    sink,

                    // These values are transcribed from what is happening in
                    // `SyntheticAmode::finalize`. This, plus the `Into` logic
                    // converting a `SyntheticAmode` to its external counterpart, are
                    // necessary to communicate Cranelift's internal offsets to the
                    // assembler; due to when Cranelift determines these offsets, this
                    // happens quite late (i.e., here during emission).
                    incoming_arg_offset: i32::try_from(
                        frame.tail_args_size + frame.setup_area_size,
                    )
                    .unwrap(),
                    slot_offset: i32::try_from(frame.outgoing_args_size).unwrap(),
                },
            );
        }
    }

    state.clear_post_insn();
}

/// Emit the common sequence used for both direct and indirect tail calls:
///
/// * Copy the new frame's stack arguments over the top of our current frame.
///
/// * Restore the old frame pointer.
///
/// * Initialize the tail callee's stack pointer (simultaneously deallocating
///   the temporary stack space we allocated when creating the new frame's stack
///   arguments).
///
/// * Move the return address into its stack slot.
fn emit_return_call_common_sequence<T>(
    sink: &mut MachBuffer<Inst>,
    info: &EmitInfo,
    state: &mut EmitState,
    call_info: &ReturnCallInfo<T>,
) {
    assert!(
        info.flags.preserve_frame_pointers(),
        "frame pointers aren't fundamentally required for tail calls, \
                 but the current implementation relies on them being present"
    );

    let tmp = call_info.tmp.to_writable_reg();

    for inst in
        X64ABIMachineSpec::gen_clobber_restore(CallConv::Tail, &info.flags, state.frame_layout())
    {
        inst.emit(sink, info, state);
    }

    for inst in X64ABIMachineSpec::gen_epilogue_frame_restore(
        CallConv::Tail,
        &info.flags,
        &info.isa_flags,
        state.frame_layout(),
    ) {
        inst.emit(sink, info, state);
    }

    let incoming_args_diff = state.frame_layout().tail_args_size - call_info.new_stack_arg_size;
    if incoming_args_diff > 0 {
        // Move the saved return address up by `incoming_args_diff`.
        let addr = Amode::imm_reg(0, regs::rsp());
        asm::inst::movq_rm::new(tmp, addr).emit(sink, info, state);
        asm::inst::movq_mr::new(
            Amode::imm_reg(i32::try_from(incoming_args_diff).unwrap(), regs::rsp()),
            Gpr::unwrap_new(tmp.to_reg()),
        )
        .emit(sink, info, state);

        // Increment the stack pointer to shrink the argument area for the new
        // call.
        let rsp = Writable::from_reg(regs::rsp());
        let incoming_args_diff = i32::try_from(incoming_args_diff)
            .expect("`incoming_args_diff` is too large to fit in a 32-bit signed immediate");
        Inst::addq_mi(rsp, incoming_args_diff).emit(sink, info, state);
    }
}

/// Conveniene trait to have an `emit` method on all `asm::inst::*` variants.
trait ExternalEmit {
    fn emit(self, sink: &mut MachBuffer<Inst>, info: &EmitInfo, state: &mut EmitState);
}

impl<I> ExternalEmit for I
where
    I: Into<asm::inst::Inst<CraneliftRegisters>>,
{
    fn emit(self, sink: &mut MachBuffer<Inst>, info: &EmitInfo, state: &mut EmitState) {
        Inst::External { inst: self.into() }.emit(sink, info, state)
    }
}

/// Attempt to "shrink" the provided `inst`.
///
/// This function will inspect `inst` and attempt to return a new instruction
/// which is equivalent semantically but will encode to a smaller binary
/// representation. This is only done for instructions which require register
/// allocation to have already happened, for example shrinking immediates should
/// be done during instruction selection not at this point.
///
/// An example of this optimization is the `AND` instruction. The Intel manual
/// has a smaller encoding for `AND AL, imm8` than it does for `AND r/m8, imm8`.
/// Here the instructions are matched against and if regalloc state indicates
/// that a smaller variant is available then that's swapped to instead.
fn emit_maybe_shrink(inst: &AsmInst, sink: &mut impl asm::CodeSink) {
    use cranelift_assembler_x64::GprMem;
    use cranelift_assembler_x64::inst::*;

    type R = CraneliftRegisters;
    const RAX: PairedGpr = PairedGpr {
        read: Gpr::RAX,
        write: Writable::from_reg(Gpr::RAX),
    };
    const RAX_RM: GprMem<PairedGpr, Gpr> = GprMem::Gpr(RAX);

    match *inst {
        // and
        Inst::andb_mi(andb_mi { rm8: RAX_RM, imm8 }) => andb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::andw_mi(andw_mi {
            rm16: RAX_RM,
            imm16,
        }) => andw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::andl_mi(andl_mi {
            rm32: RAX_RM,
            imm32,
        }) => andl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::andq_mi_sxl(andq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => andq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // or
        Inst::orb_mi(orb_mi { rm8: RAX_RM, imm8 }) => orb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::orw_mi(orw_mi {
            rm16: RAX_RM,
            imm16,
        }) => orw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::orl_mi(orl_mi {
            rm32: RAX_RM,
            imm32,
        }) => orl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::orq_mi_sxl(orq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => orq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // xor
        Inst::xorb_mi(xorb_mi { rm8: RAX_RM, imm8 }) => xorb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::xorw_mi(xorw_mi {
            rm16: RAX_RM,
            imm16,
        }) => xorw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::xorl_mi(xorl_mi {
            rm32: RAX_RM,
            imm32,
        }) => xorl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::xorq_mi_sxl(xorq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => xorq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // add
        Inst::addb_mi(addb_mi { rm8: RAX_RM, imm8 }) => addb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::addw_mi(addw_mi {
            rm16: RAX_RM,
            imm16,
        }) => addw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::addl_mi(addl_mi {
            rm32: RAX_RM,
            imm32,
        }) => addl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::addq_mi_sxl(addq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => addq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // adc
        Inst::adcb_mi(adcb_mi { rm8: RAX_RM, imm8 }) => adcb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::adcw_mi(adcw_mi {
            rm16: RAX_RM,
            imm16,
        }) => adcw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::adcl_mi(adcl_mi {
            rm32: RAX_RM,
            imm32,
        }) => adcl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::adcq_mi_sxl(adcq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => adcq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // sub
        Inst::subb_mi(subb_mi { rm8: RAX_RM, imm8 }) => subb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::subw_mi(subw_mi {
            rm16: RAX_RM,
            imm16,
        }) => subw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::subl_mi(subl_mi {
            rm32: RAX_RM,
            imm32,
        }) => subl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::subq_mi_sxl(subq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => subq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // sbb
        Inst::sbbb_mi(sbbb_mi { rm8: RAX_RM, imm8 }) => sbbb_i::<R>::new(RAX, imm8).encode(sink),
        Inst::sbbw_mi(sbbw_mi {
            rm16: RAX_RM,
            imm16,
        }) => sbbw_i::<R>::new(RAX, imm16).encode(sink),
        Inst::sbbl_mi(sbbl_mi {
            rm32: RAX_RM,
            imm32,
        }) => sbbl_i::<R>::new(RAX, imm32).encode(sink),
        Inst::sbbq_mi_sxl(sbbq_mi_sxl {
            rm64: RAX_RM,
            imm32,
        }) => sbbq_i_sxl::<R>::new(RAX, imm32).encode(sink),

        // cmp
        Inst::cmpb_mi(cmpb_mi {
            rm8: GprMem::Gpr(Gpr::RAX),
            imm8,
        }) => cmpb_i::<R>::new(Gpr::RAX, imm8).encode(sink),
        Inst::cmpw_mi(cmpw_mi {
            rm16: GprMem::Gpr(Gpr::RAX),
            imm16,
        }) => cmpw_i::<R>::new(Gpr::RAX, imm16).encode(sink),
        Inst::cmpl_mi(cmpl_mi {
            rm32: GprMem::Gpr(Gpr::RAX),
            imm32,
        }) => cmpl_i::<R>::new(Gpr::RAX, imm32).encode(sink),
        Inst::cmpq_mi(cmpq_mi {
            rm64: GprMem::Gpr(Gpr::RAX),
            imm32,
        }) => cmpq_i::<R>::new(Gpr::RAX, imm32).encode(sink),

        // test
        Inst::testb_mi(testb_mi {
            rm8: GprMem::Gpr(Gpr::RAX),
            imm8,
        }) => testb_i::<R>::new(Gpr::RAX, imm8).encode(sink),
        Inst::testw_mi(testw_mi {
            rm16: GprMem::Gpr(Gpr::RAX),
            imm16,
        }) => testw_i::<R>::new(Gpr::RAX, imm16).encode(sink),
        Inst::testl_mi(testl_mi {
            rm32: GprMem::Gpr(Gpr::RAX),
            imm32,
        }) => testl_i::<R>::new(Gpr::RAX, imm32).encode(sink),
        Inst::testq_mi(testq_mi {
            rm64: GprMem::Gpr(Gpr::RAX),
            imm32,
        }) => testq_i::<R>::new(Gpr::RAX, imm32).encode(sink),

        // lea
        Inst::leal_rm(leal_rm { r32, m32 }) => emit_lea(
            r32,
            m32,
            sink,
            |dst, amode, s| leal_rm::<R>::new(dst, amode).encode(s),
            |dst, simm32, s| addl_mi::<R>::new(dst, simm32.cast_unsigned()).encode(s),
            |dst, reg, s| addl_rm::<R>::new(dst, reg).encode(s),
        ),
        Inst::leaq_rm(leaq_rm { r64, m64 }) => emit_lea(
            r64,
            m64,
            sink,
            |dst, amode, s| leaq_rm::<R>::new(dst, amode).encode(s),
            |dst, simm32, s| addq_mi_sxl::<R>::new(dst, simm32).encode(s),
            |dst, reg, s| addq_rm::<R>::new(dst, reg).encode(s),
        ),

        // All other instructions fall through to here and cannot be shrunk, so
        // return `false` to emit them as usual.
        _ => inst.encode(sink),
    }
}

/// If `lea` can actually get encoded as an `add` then do that instead.
/// Currently all candidate `iadd`s become an `lea` pseudo-instruction here but
/// maximizing the use of `lea` is not necessarily optimal. The `lea`
/// instruction goes through dedicated address units on cores which are finite
/// and disjoint from the general ALU, so if everything uses `lea` then those
/// units can get saturated while leaving the ALU idle.
///
/// To help make use of more parts of a CPU, this attempts to use `add` when
/// it's semantically equivalent to `lea`, or otherwise when the `dst` register
/// is the same as the `base` or `index` register.
///
/// FIXME: ideally regalloc is informed of this constraint. Register allocation
/// of `lea` should "attempt" to put the `base` in the same register as `dst`
/// but not at the expense of generating a `mov` instruction. Currently that's
/// not possible but perhaps one day it may be worth it.
fn emit_lea<S>(
    dst: asm::Gpr<WritableGpr>,
    addr: asm::Amode<Gpr>,
    sink: &mut S,
    lea: fn(WritableGpr, asm::Amode<Gpr>, &mut S),
    add_mi: fn(PairedGpr, i32, &mut S),
    add_rm: fn(PairedGpr, Gpr, &mut S),
) where
    S: asm::CodeSink,
{
    match addr {
        // If `base == dst` then this is `add dst, $imm`, so encode that
        // instead.
        asm::Amode::ImmReg {
            base,
            simm32:
                asm::AmodeOffsetPlusKnownOffset {
                    simm32,
                    offset: None,
                },
            trap: None,
        } if dst.as_ref().to_reg() == base => add_mi(
            PairedGpr {
                read: base,
                write: *dst.as_ref(),
            },
            simm32.value(),
            sink,
        ),

        // If the offset is 0 and the shift is a scale of 1, then:
        //
        // * If `base == dst`, then this is `addq dst, index`
        // * If `index == dst`, then this is `addq dst, base`
        asm::Amode::ImmRegRegShift {
            base,
            index,
            scale: asm::Scale::One,
            simm32: asm::AmodeOffset::ZERO,
            trap: None,
        } => {
            if dst.as_ref().to_reg() == base {
                add_rm(
                    PairedGpr {
                        read: base,
                        write: *dst.as_ref(),
                    },
                    *index.as_ref(),
                    sink,
                )
            } else if dst.as_ref().to_reg() == *index.as_ref() {
                add_rm(
                    PairedGpr {
                        read: *index.as_ref(),
                        write: *dst.as_ref(),
                    },
                    base,
                    sink,
                )
            } else {
                lea(*dst.as_ref(), addr, sink)
            }
        }

        _ => lea(*dst.as_ref(), addr, sink),
    }
}