Skip to main content

offlineasm_ir/
expand.rs

1//! Expands mnemonic invocations into [`Inst`] sequences.
2
3use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6use smallvec::{smallvec, SmallVec};
7
8/// Inline capacity of an [`Expansion`].
9///
10/// ~97% of mnemonic expansions hold 2 or fewer nodes
11pub const EXPANSION_INLINE: usize = 2;
12
13/// The lowered instruction sequence for one mnemonic invocation.
14pub type Expansion = SmallVec<[Inst; EXPANSION_INLINE]>;
15
16/// Lowers one mnemonic invocation to the [`Inst`] sequence that implements it.
17///
18/// `sc` supplies the scratch registers that multi-instruction expansions
19/// and the destination-aliasing fixup may need. It must not alias any
20/// operand in `ops`.
21/// When `patch` is set, the mnemonic must be a [`patch_kind`] site, and the
22/// returned sequence is shaped so a backend can patch it after emission
23/// (a fixed instruction count, no operand folding).
24///
25/// # Errors
26///
27/// - [`ExpandError::NotPatchable`] if `patch` is set and `mn` has no
28///   [`patch_kind`].
29/// - [`ExpandError::BadOperand`] if `ops` has the wrong count or kind for
30///   `mn`.
31pub fn expand(
32    mn: Mnemonic,
33    ops: &[Op],
34    sc: &Scratch,
35    patch: bool,
36) -> Result<Expansion, ExpandError> {
37    if patch && patch_kind(mn).is_none() {
38        return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
39    }
40    check_extern_operands(mn, ops, patch)?;
41    let mut insts = expand_inner(mn, ops, sc, patch)?;
42    stage_aliasing_mem(&mut insts, sc);
43    Ok(insts)
44}
45
46/// Rejects external symbols outside the shapes that take one: the target of
47/// `jmp`/`call`, the source of `move`, and the source of a 64-bit `lea`
48/// (`leap`, `globaladdr`, `la`). None of them is a patch site.
49fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
50    let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
51        return Ok(());
52    };
53    if patch {
54        return Err(bad(mn, "an external symbol cannot be a patch site"));
55    }
56    let ok = match mn {
57        Mnemonic::Jmp | Mnemonic::Call => at == 0,
58        Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
59        Mnemonic::Lea {
60            w: Width::B64,
61            form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
62        } => at == 1,
63        _ => false,
64    };
65    if ok {
66        Ok(())
67    } else {
68        Err(bad(
69            mn,
70            "an external symbol is only a `jmp`/`call` target or the source of \
71             `move`/`leap`/`globaladdr`/`la`",
72        ))
73    }
74}
75
76/// Stages aliased memory operands through the scratch registers.
77///
78/// `addis t1, t0, 8[t1]` reads `8[t1]` with the *old* `t1`: all operands are
79/// evaluated before the destination is written. A backend that first lowers
80/// `dst = lhs` into `t1` and then reuses `t1` as the memory base (the
81/// natural 2-operand sequence on every target) reads `8[t1]` after the move
82/// instead and dereferences the wrong address. The fused `Alu3`/op-branch
83/// nodes hand that ordering choice to each backend, so the fix is pinned
84/// down in the front half both assemblers share: the load happens before
85/// the operation, through the expansion scratch register.
86///
87/// The oracle interprets this same sequence, and scratch is clobberable by
88/// design (it is never in the portable register pool, so it cannot alias
89/// `dst` or the operand bases).
90fn stage_aliasing_mem(insts: &mut Expansion, sc: &Scratch) {
91    // Nearly every expansion is alias-free; checking first avoids moving
92    // every instruction into a fresh `Expansion` on the hot path.
93    if !insts.iter().any(needs_staging) {
94        return;
95    }
96    let old = std::mem::take(insts);
97    let mut out = Expansion::new();
98    for mut inst in old {
99        if let Some(loads) = stage_one(&mut inst, sc) {
100            out.extend(loads);
101        }
102        out.push(inst);
103    }
104    *insts = out;
105}
106
107/// Reports whether `m` uses `g` as its base or index.
108fn aliases(m: &Mem, g: Gpr) -> bool {
109    m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g)
110}
111
112/// Reports whether `dst` is a GPR and any of `operands` is a memory operand
113/// aliasing it.
114fn aliasing_mem<'a>(dst: &Op, operands: impl IntoIterator<Item = &'a Op>) -> bool {
115    let Op::Gpr(g) = *dst else {
116        return false;
117    };
118    operands
119        .into_iter()
120        .any(|op| matches!(op, Op::Mem(m) if aliases(m, g)))
121}
122
123/// Checks whether `inst` has an aliasing memory operand to stage.
124///
125/// This mirrors the patterns of [`stage_one`] without borrowing mutably, so
126/// the common no-staging case can bail out before any rewrite.
127fn needs_staging(inst: &Inst) -> bool {
128    match inst {
129        Inst::Add { op, .. }
130        | Inst::Sub { op, .. }
131        | Inst::Mul { op, .. }
132        | Inst::And { op, .. }
133        | Inst::Or { op, .. }
134        | Inst::Xor { op, .. } => aliasing_mem(&op.dst, [&op.lhs, &op.rhs]),
135        Inst::AluS { alu, .. } => aliasing_mem(&alu.dst, [&alu.lhs, &alu.rhs]),
136        Inst::OpBranch { args, .. } => {
137            let ops = args.as_slice();
138            let Some((dst, srcs)) = ops.split_last() else {
139                return false;
140            };
141            aliasing_mem(dst, srcs.iter())
142        }
143        _ => false,
144    }
145}
146
147/// Collects the staging loads for `inst` and rewrites its aliased operands.
148///
149/// Returns `None` when `inst` needs no staging.
150fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Expansion> {
151    match inst {
152        Inst::Add { w, op }
153        | Inst::Sub { w, op }
154        | Inst::Mul { w, op }
155        | Inst::And { w, op }
156        | Inst::Or { w, op }
157        | Inst::Xor { w, op } => stage_ops(*w, &mut op.dst, [&mut op.lhs, &mut op.rhs], sc),
158        Inst::AluS { w, alu, .. } => stage_ops(*w, &mut alu.dst, [&mut alu.lhs, &mut alu.rhs], sc),
159        Inst::OpBranch { w, args, .. } => {
160            let (dst, srcs) = args.as_mut_slice().split_last_mut()?;
161            stage_ops(*w, dst, srcs.iter_mut(), sc)
162        }
163        _ => None,
164    }
165}
166
167/// Stages every operand of `inst` that aliases `dst` through `sc`.
168fn stage_ops<'a>(
169    w: Width,
170    dst: &mut Op,
171    operands: impl IntoIterator<Item = &'a mut Op>,
172    sc: &Scratch,
173) -> Option<Expansion> {
174    let Op::Gpr(g) = *dst else {
175        return None;
176    };
177    let slots = [sc.gpr, sc.gpr2];
178    let mut loads = Expansion::new();
179    let mut slot = 0usize;
180    for operand in operands {
181        let Op::Mem(m) = operand else {
182            continue;
183        };
184        if !aliases(m, g) {
185            continue;
186        }
187        // At most two memory operands can alias `dst` (op-branch takes
188        // `[lhs, rhs, dst]`), and both scratch GPRs are available.
189        let &s = slots.get(slot)?;
190        slot += 1;
191        loads.push(Inst::Load {
192            w,
193            sx: Sx::Zero,
194            dst: s,
195            addr: *m,
196            acquire: false,
197        });
198        *operand = Op::Gpr(s);
199    }
200    (!loads.is_empty()).then_some(loads)
201}
202
203/// Returns the patch shape for `mn`.
204pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
205    Some(match mn {
206        Mnemonic::Jmp => PatchKind::JumpSite,
207        Mnemonic::Call => PatchKind::CallSite,
208        Mnemonic::Move => PatchKind::MovBlock,
209        Mnemonic::FlagBranch(_)
210        | Mnemonic::OpBranch { .. }
211        | Mnemonic::RiscAluS { .. }
212        | Mnemonic::TestBranch { .. }
213        | Mnemonic::IntBranch { .. }
214        | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
215        _ => return None,
216    })
217}
218
219/// Describes what a `#[patch]` site patches.
220///
221/// Jump shapes become jump handles (in onlineasm, retargeted after
222/// emission); [`MovBlock`](PatchKind::MovBlock) becomes a 64-bit immediate
223/// slot (rewritten after emission).
224#[derive(Clone, Copy, Debug, PartialEq, Eq)]
225pub enum PatchKind {
226    /// Patches an unconditional jump target (`jmp` to a label).
227    JumpSite,
228    /// Patches a conditional branch target (a single-branch expansion).
229    CondJumpSite,
230    /// Patches a call target (`call` to a label).
231    CallSite,
232    /// Patches a 64-bit move slot (`move` of an immediate into a register).
233    MovBlock,
234}
235
236fn float_cc(cc: FCc) -> (FCc, bool) {
237    match cc {
238        FCc::Lt => (FCc::Gt, true),
239        FCc::Le => (FCc::Ge, true),
240        FCc::GtUn => (FCc::GtUn, false),
241        FCc::GeUn => (FCc::GeUn, false),
242        FCc::LtUn => (FCc::GtUn, true),
243        FCc::LeUn => (FCc::GeUn, true),
244        other => (other, false),
245    }
246}
247
248fn test_set_node(
249    mn: Mnemonic,
250    ops: &[Op],
251    w: Width,
252    test: TestKind,
253) -> Result<Expansion, ExpandError> {
254    if ops.len() != 2 && ops.len() != 3 {
255        return Err(bad(mn, "test-set takes 2 or 3 operands"));
256    }
257    let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
258    Ok(smallvec![Inst::TestSet {
259        w,
260        value: ops[1],
261        mask,
262        test,
263        dst: ops[0],
264    }])
265}
266
267fn test_branch_node(
268    mn: Mnemonic,
269    ops: &[Op],
270    w: Width,
271    test: TestKind,
272    patch: bool,
273) -> Result<Expansion, ExpandError> {
274    if ops.len() != 2 && ops.len() != 3 {
275        return Err(bad(mn, "test-branch takes 2 or 3 operands"));
276    }
277    let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
278    let target = label_op(
279        mn,
280        ops.last()
281            .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
282    )?;
283    Ok(smallvec![Inst::TestBranch {
284        w,
285        value: ops[0],
286        mask,
287        test,
288        target,
289        patch,
290    }])
291}
292
293fn div_rem_node(
294    mn: Mnemonic,
295    ops: &[Op],
296    signed: bool,
297    w: Width,
298    rem: bool,
299) -> Result<Expansion, ExpandError> {
300    let (dvd, dvs, dst) = match ops {
301        [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
302        [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
303        _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
304    };
305    Ok(smallvec![if rem {
306        Inst::Rem3 {
307            signed,
308            w,
309            dst,
310            dvd,
311            dvs,
312        }
313    } else {
314        Inst::Div3 {
315            signed,
316            w,
317            dst,
318            dvd,
319            dvs,
320        }
321    }])
322}
323
324fn shift_mem_canon(
325    mn: Mnemonic,
326    w: Width,
327    op: ShiftOp,
328    cnt: &Op,
329    dst: &Mem,
330    sc: &Scratch,
331) -> Result<Expansion, ExpandError> {
332    let shift_cnt = match cnt {
333        Op::Imm(v) => ShiftCnt::Imm(*v),
334        Op::Gpr(g) => ShiftCnt::Reg(*g),
335        _ => return Err(bad(mn, "shift count must be an immediate or register")),
336    };
337    Ok(smallvec![
338        Inst::Load {
339            w,
340            sx: Sx::Zero,
341            dst: sc.gpr,
342            addr: *dst,
343            acquire: false,
344        },
345        Inst::Shift {
346            w,
347            op,
348            dst: sc.gpr,
349            cnt: shift_cnt,
350        },
351        Inst::Store {
352            w,
353            src: Op::Gpr(sc.gpr),
354            addr: *dst,
355        },
356    ])
357}
358
359fn shift_node(
360    mn: Mnemonic,
361    w: Width,
362    op: ShiftOp,
363    ops: &[Op],
364    sc: &Scratch,
365) -> Result<Expansion, ExpandError> {
366    if ops.len() != 2 {
367        return Err(bad(mn, "shift takes 2 operands"));
368    }
369    if let Op::Mem(a) = &ops[0] {
370        return shift_mem_canon(mn, w, op, &ops[1], a, sc);
371    }
372    let dst = gpr_op(mn, &ops[0])?;
373    let cnt = match &ops[1] {
374        Op::Imm(v) => ShiftCnt::Imm(*v),
375        Op::Gpr(g) => ShiftCnt::Reg(*g),
376        _ => return Err(bad(mn, "shift count must be an immediate or register")),
377    };
378    Ok(smallvec![Inst::Shift { w, op, dst, cnt }])
379}
380
381fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
382    Alu3::from_operands(mn, ops)
383}
384
385fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<BranchArgs, ExpandError> {
386    match ops {
387        [dst, rhs] => Ok(BranchArgs::two(*rhs, *dst)),
388        [dst, lhs, rhs] => Ok(BranchArgs::three(*lhs, *rhs, *dst)),
389        _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
390    }
391}
392
393fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
394    match op {
395        Op::Label(l) => Ok(Target::Label(*l)),
396        Op::Gpr(g) => Ok(Target::Reg(*g)),
397        Op::Mem(a) => Ok(Target::Mem(*a)),
398        Op::Imm(v) => Ok(Target::Abs(*v)),
399        Op::Extern(e) => Ok(Target::Extern(*e)),
400        _ => Err(bad(
401            mn,
402            "jump/call needs a label, register, memory, or immediate",
403        )),
404    }
405}
406
407#[allow(clippy::too_many_lines)]
408fn expand_inner(
409    mn: Mnemonic,
410    ops: &[Op],
411    sc: &Scratch,
412    patch: bool,
413) -> Result<Expansion, ExpandError> {
414    let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
415        expect_len(mn, ops, 1)?;
416        Ok(ops[0])
417    };
418    match mn {
419        Mnemonic::Move => Ok(smallvec![mov_node(mn, ops, Width::B64, patch)?]),
420        Mnemonic::Load(w) => Ok(smallvec![mov_node(mn, ops, w, patch)?]),
421        Mnemonic::AtomicLoad(w) => Ok(smallvec![atomic_load_node(mn, ops, w)?]),
422        Mnemonic::Store(w) => Ok(smallvec![store_node(mn, ops, w)?]),
423        Mnemonic::Extend { from, to, sx } => Ok(smallvec![extend_node(mn, ops, from, to, sx)?]),
424        Mnemonic::MovZero32 => Ok(smallvec![movzero_node(mn, ops)?]),
425        Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
426        Mnemonic::FMove(k) => Ok(smallvec![fmove_node(mn, ops, k)?]),
427        Mnemonic::VMove { store } => Ok(smallvec![vmove_node(mn, ops, store)?]),
428        Mnemonic::Lea { w, .. } => Ok(smallvec![lea_node(mn, ops, w)?]),
429        Mnemonic::LoadQInc => loadqinc_node(mn, ops),
430        Mnemonic::LoadPair(w) => Ok(smallvec![pair_node(mn, ops, w, true)?]),
431        Mnemonic::StorePair(w) => Ok(smallvec![pair_node(mn, ops, w, false)?]),
432        Mnemonic::Push => stack_nodes(mn, ops, true, false),
433        Mnemonic::Pop => stack_nodes(mn, ops, false, false),
434        Mnemonic::PushV => stack_nodes(mn, ops, true, true),
435        Mnemonic::PopV => stack_nodes(mn, ops, false, true),
436        Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
437        Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
438        Mnemonic::Peek => Ok(smallvec![peek_node(mn, ops, true)?]),
439        Mnemonic::Poke => Ok(smallvec![peek_node(mn, ops, false)?]),
440        Mnemonic::TlsLoad(w) => Ok(smallvec![tls_node(mn, ops, w, true)?]),
441        Mnemonic::TlsStore(w) => Ok(smallvec![tls_node(mn, ops, w, false)?]),
442        Mnemonic::TlsLea => Ok(smallvec![tls_lea_node(mn, ops)?]),
443
444        Mnemonic::Add(w) => Ok(smallvec![Inst::Add {
445            w,
446            op: alu3(mn, ops)?,
447        }]),
448        Mnemonic::Sub(w) => Ok(smallvec![Inst::Sub {
449            w,
450            op: alu3(mn, ops)?,
451        }]),
452        Mnemonic::Mul(w) => Ok(smallvec![mul_node(mn, w, ops)?]),
453        Mnemonic::And(w) => Ok(smallvec![Inst::And {
454            w,
455            op: alu3(mn, ops)?,
456        }]),
457        Mnemonic::Or(w) => Ok(smallvec![Inst::Or {
458            w,
459            op: alu3(mn, ops)?,
460        }]),
461        Mnemonic::Xor(w) => Ok(smallvec![Inst::Xor {
462            w,
463            op: alu3(mn, ops)?,
464        }]),
465        Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
466        Mnemonic::Not(w) => Ok(smallvec![Inst::Not {
467            w,
468            dst: one(mn, ops)?,
469        }]),
470        Mnemonic::Neg(w) => Ok(smallvec![Inst::Neg {
471            w,
472            dst: one(mn, ops)?,
473        }]),
474        Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
475        Mnemonic::Bswap(w) => Ok(smallvec![bswap_node(mn, ops, w)?]),
476        Mnemonic::Adc => Ok(smallvec![adc_node(mn, ops, false)?]),
477        Mnemonic::Sbb => Ok(smallvec![adc_node(mn, ops, true)?]),
478        Mnemonic::MulHigh { signed } => Ok(smallvec![Inst::MulHigh {
479            signed,
480            rhs: one(mn, ops)?,
481        }]),
482        Mnemonic::MulWide { signed } => Ok(smallvec![mulwide_node(mn, ops, signed)?]),
483        Mnemonic::AluS { op, w } => Ok(smallvec![Inst::AluS {
484            op,
485            w,
486            alu: alu3(mn, ops)?,
487        }]),
488        Mnemonic::RiscAluS { op, w } => {
489            let has_label =
490                matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
491            if has_label {
492                let flag_op = match op {
493                    AluOp::Add => FlagAlu::Add,
494                    AluOp::Sub => FlagAlu::Sub,
495                    AluOp::Or => FlagAlu::Or,
496                };
497                return Ok(smallvec![Inst::OpBranch {
498                    op: flag_op,
499                    w,
500                    flag: PredFlag::S,
501                    args: op_branch_args(mn, &ops[..ops.len() - 1])?,
502                    target: label_op(
503                        mn,
504                        ops.last()
505                            .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
506                    )?,
507                    patch,
508                }]);
509            }
510            Ok(smallvec![Inst::AluS {
511                op,
512                w,
513                alu: alu3(mn, ops)?,
514            }])
515        }
516        Mnemonic::AddShifted => Ok(smallvec![addshifted_node(mn, ops)?]),
517        Mnemonic::Cqo { quad } => Ok(smallvec![Inst::Cqo { quad }]),
518        Mnemonic::DivMod1 { signed, w } => Ok(smallvec![Inst::DivMod1 {
519            signed,
520            w,
521            divisor: one(mn, ops)?,
522        }]),
523        Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
524        Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
525        Mnemonic::IntSet { w, cc } => {
526            expect_len(mn, ops, 3)?;
527            Ok(smallvec![Inst::IntSet {
528                cc,
529                w,
530                dst: ops[0],
531                lhs: ops[1],
532                rhs: ops[2],
533            }])
534        }
535        Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
536        Mnemonic::Cmov { zero } => Ok(smallvec![cmov_node(mn, ops, zero)?]),
537        Mnemonic::BitCount { w, trailing } => Ok(smallvec![cz_node(mn, ops, w, trailing)?]),
538        Mnemonic::Popcnt(w) => Ok(smallvec![popcnt_node(mn, ops, w)?]),
539        Mnemonic::Bfi => bfi_node(mn, ops, sc),
540        Mnemonic::IntBranch { w, cc } => {
541            expect_len(mn, ops, 3)?;
542            Ok(smallvec![Inst::Branch {
543                cc,
544                w,
545                lhs: ops[0],
546                rhs: ops[1],
547                target: label_op(mn, &ops[2])?,
548                patch,
549            }])
550        }
551        Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
552        Mnemonic::OpBranch { op, w, flag } => {
553            if ops.len() != 3 && ops.len() != 4 {
554                return Err(bad(mn, "op-branch takes 3 or 4 operands"));
555            }
556            Ok(smallvec![Inst::OpBranch {
557                op,
558                w,
559                flag,
560                args: op_branch_args(mn, &ops[..ops.len() - 1])?,
561                target: label_op(
562                    mn,
563                    ops.last()
564                        .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
565                )?,
566                patch,
567            }])
568        }
569        Mnemonic::FlagBranch(flag) => Ok(smallvec![flag_branch_node(mn, ops, flag, patch)?]),
570        Mnemonic::BranchF { cc, k } => {
571            expect_len(mn, ops, 3)?;
572            let (cc, reverse) = float_cc(cc);
573            let (lhs, rhs) = if reverse {
574                (ops[1], ops[0])
575            } else {
576                (ops[0], ops[1])
577            };
578            Ok(smallvec![Inst::BranchF {
579                cc,
580                k,
581                lhs,
582                rhs,
583                target: label_op(mn, &ops[2])?,
584                patch,
585            }])
586        }
587        Mnemonic::Jmp => {
588            expect_len(mn, ops, 1)?;
589            Ok(smallvec![Inst::Jump {
590                target: jump_target(mn, &ops[0])?,
591                patch,
592            }])
593        }
594        Mnemonic::Call => {
595            expect_len(mn, ops, 1)?;
596            Ok(smallvec![Inst::Call {
597                target: jump_target(mn, &ops[0])?,
598                patch,
599            }])
600        }
601        Mnemonic::Ret => {
602            expect_len(mn, ops, 0)?;
603            Ok(smallvec![Inst::Ret])
604        }
605        Mnemonic::Break => {
606            expect_len(mn, ops, 0)?;
607            Ok(smallvec![Inst::Break])
608        }
609        Mnemonic::Nop => {
610            expect_len(mn, ops, 0)?;
611            Ok(smallvec![Inst::Nop])
612        }
613        Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
614        Mnemonic::Fma { op, k } => Ok(smallvec![fma_node(mn, ops, k, op)?]),
615        Mnemonic::FSqrt(k) => Ok(smallvec![fsqrt_node(mn, ops, k)?]),
616        Mnemonic::FRound { k, mode } => Ok(smallvec![fround_node(mn, ops, k, mode)?]),
617        Mnemonic::FAbs { k, negate } => Ok(smallvec![fabs_node(mn, ops, k, negate)?]),
618        Mnemonic::FSet { cc, k } => {
619            expect_len(mn, ops, 3)?;
620            let (cc, reverse) = float_cc(cc);
621            let (lhs, rhs) = if reverse {
622                (ops[2], ops[1])
623            } else {
624                (ops[1], ops[2])
625            };
626            Ok(smallvec![Inst::FSet {
627                cc,
628                k,
629                dst: ops[0],
630                lhs,
631                rhs,
632            }])
633        }
634        Mnemonic::CvtFI { k, w } => Ok(smallvec![cvtfi_node(mn, ops, k, w)?]),
635        Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
636        Mnemonic::CvtIF { signed, from, k } => Ok(smallvec![cvtif_node(mn, ops, signed, from, k)?]),
637        Mnemonic::CvtFF { from, to } => Ok(smallvec![cvtff_node(mn, ops, from, to)?]),
638        Mnemonic::Bitcast(dir) => Ok(smallvec![bitcast_node(mn, ops, dir)?]),
639        Mnemonic::MovDZ => Ok(smallvec![movdz_node(mn, ops)?]),
640        Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
641        Mnemonic::Btd2i => btd2i_node(mn, ops),
642        Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
643        Mnemonic::VecExtract(w) => Ok(smallvec![umov_node(mn, ops, w)?]),
644        Mnemonic::AtomicRmw { op, w } => Ok(smallvec![rmw_node(mn, ops, w, op)?]),
645        Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
646        Mnemonic::AtomicCas { w, branch } => Ok(smallvec![cas_node(mn, ops, w, branch)?]),
647        Mnemonic::LoadLinked(w) => Ok(smallvec![ll_node(mn, ops, w)?]),
648        Mnemonic::StoreCond(w) => Ok(smallvec![sc_node(mn, ops, w)?]),
649        Mnemonic::Fence { full } => Ok(smallvec![Inst::Fence { full }]),
650        Mnemonic::Pause => Ok(smallvec![Inst::Pause]),
651        Mnemonic::TagCodePtr
652        | Mnemonic::TagReturnAddress
653        | Mnemonic::UntagReturnAddress
654        | Mnemonic::RemoveCodePtrTag
655        | Mnemonic::UntagArrayPtr
656        | Mnemonic::RemoveArrayPtrTag => Ok(smallvec![]),
657    }
658}
659
660fn flag_branch_node(
661    mn: Mnemonic,
662    ops: &[Op],
663    flag: PredFlag,
664    patch: bool,
665) -> Result<Inst, ExpandError> {
666    expect_len(mn, ops, 1)?;
667    Ok(Inst::FlagBranch {
668        flag,
669        target: label_op(mn, &ops[0])?,
670        patch,
671    })
672}
673
674fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
675    expect_len(mn, ops, 2)?;
676    Ok(Inst::Popcnt {
677        w,
678        dst: gpr_op(mn, &ops[0])?,
679        src: ops[1],
680    })
681}
682
683fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
684    expect_len(mn, ops, 2)?;
685    let dst = gpr_op(mn, &ops[0])?;
686    let inst = if trailing {
687        Inst::Ctz {
688            w,
689            dst,
690            src: ops[1],
691        }
692    } else {
693        Inst::Clz {
694            w,
695            dst,
696            src: ops[1],
697        }
698    };
699    Ok(inst)
700}
701
702fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
703    expect_len(mn, ops, 2)?;
704    if load {
705        let idx = imm_op(mn, &ops[1])?;
706        Ok(Inst::TlsLoad {
707            w,
708            dst: gpr_op(mn, &ops[0])?,
709            idx,
710        })
711    } else {
712        let idx = imm_op(mn, &ops[0])?;
713        Ok(Inst::TlsStore {
714            w,
715            src: gpr_op(mn, &ops[1])?,
716            idx,
717        })
718    }
719}
720
721fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
722    expect_len(mn, ops, 2)?;
723    Ok(Inst::TlsLea {
724        dst: gpr_op(mn, &ops[0])?,
725        idx: imm_op(mn, &ops[1])?,
726    })
727}
728
729fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
730    expect_len(mn, ops, 2)?;
731
732    Ok(Inst::AtomicRmw {
733        op,
734        w,
735        a: ops[1],
736        b: ops[0],
737    })
738}
739
740fn rmw_not_node(
741    mn: Mnemonic,
742    ops: &[Op],
743    sc: &Scratch,
744    w: Width,
745    op: AtomicOp,
746) -> Result<Expansion, ExpandError> {
747    expect_len(mn, ops, 2)?;
748    let mut out = Expansion::new();
749    // `mem` is ops[0] and `src` is ops[1].
750    let a = match &ops[1] {
751        Op::Imm(v) => Op::Imm(!v),
752        Op::Gpr(g) => {
753            out.push(Inst::Mov {
754                w: Width::B64,
755                dst: Op::Gpr(sc.gpr),
756                src: Op::Gpr(*g),
757                patch: false,
758            });
759            out.push(Inst::Not {
760                w,
761                dst: Op::Gpr(sc.gpr),
762            });
763            Op::Gpr(sc.gpr)
764        }
765        Op::Mem(addr) => {
766            let lw = if w == Width::B64 {
767                Width::B64
768            } else {
769                Width::B32
770            };
771            out.push(Inst::Load {
772                w: lw,
773                sx: Sx::Zero,
774                dst: sc.gpr,
775                addr: *addr,
776                acquire: false,
777            });
778            out.push(Inst::Not {
779                w,
780                dst: Op::Gpr(sc.gpr),
781            });
782            Op::Gpr(sc.gpr)
783        }
784        _ => {
785            return Err(bad(
786                mn,
787                "atomic clear needs an immediate, register, or memory operand",
788            ))
789        }
790    };
791    out.push(Inst::AtomicRmw {
792        op,
793        w,
794        a,
795        b: ops[0],
796    });
797    Ok(out)
798}
799
800fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
801    let n = if branch { 4 } else { 3 };
802    expect_len(mn, ops, n)?;
803
804    Ok(Inst::AtomicCas {
805        w,
806        mem: mem_op(mn, &ops[0])?,
807        src: gpr_op(mn, &ops[2])?,
808        branch: if branch {
809            Some(label_op(mn, &ops[3])?)
810        } else {
811            None
812        },
813    })
814}
815
816fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
817    expect_len(mn, ops, 2)?;
818    Ok(Inst::LoadLinked {
819        w,
820        mem: mem_op(mn, &ops[1])?,
821        dst: gpr_op(mn, &ops[0])?,
822    })
823}
824
825fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
826    expect_len(mn, ops, 3)?;
827    Ok(Inst::StoreCond {
828        w,
829        status: gpr_op(mn, &ops[1])?,
830        src: gpr_op(mn, &ops[2])?,
831        mem: mem_op(mn, &ops[0])?,
832    })
833}
834
835/// Expands `bfiq` to a mask, shift, and OR sequence.
836///
837/// Computes `dst = (dst & ~mask) | ((src & wmask) << lsb)`.
838fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
839    expect_len(mn, ops, 4)?;
840    let lsb = imm_op(mn, &ops[2])?;
841    let width = imm_op(mn, &ops[3])?;
842    if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
843        return Err(bad(mn, "bitfield out of range"));
844    }
845    let wmask: u64 = if width == 64 {
846        u64::MAX
847    } else {
848        (1u64 << width) - 1
849    };
850    let mask: i64 = (wmask << lsb) as i64;
851    let s = sc.gpr;
852    let dst = gpr_op(mn, &ops[0])?;
853
854    Ok(smallvec![
855        Inst::Mov {
856            w: Width::B64,
857            dst: Op::Gpr(s),
858            src: ops[1],
859            patch: false,
860        },
861        Inst::And {
862            w: Width::B64,
863            op: Alu3 {
864                dst: Op::Gpr(s),
865                lhs: Op::Gpr(s),
866                rhs: Op::Imm(wmask as i64),
867            },
868        },
869        Inst::Shift {
870            w: Width::B64,
871            op: ShiftOp::Shl,
872            dst: s,
873            cnt: ShiftCnt::Imm(lsb),
874        },
875        Inst::And {
876            w: Width::B64,
877            op: Alu3 {
878                dst: Op::Gpr(dst),
879                lhs: Op::Gpr(dst),
880                rhs: Op::Imm(!mask),
881            },
882        },
883        Inst::Or {
884            w: Width::B64,
885            op: Alu3 {
886                dst: Op::Gpr(dst),
887                lhs: Op::Gpr(dst),
888                rhs: Op::Gpr(s),
889            },
890        },
891    ])
892}
893
894fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
895    expect_len(mn, ops, 3)?;
896    Ok(Inst::Cmov {
897        w: Width::B64,
898        zero,
899        dst: gpr_op(mn, &ops[0])?,
900        src: ops[1],
901        test: gpr_op(mn, &ops[2])?,
902    })
903}
904
905/// Expands `peek dst, off` and `poke off, src`, with the destination first
906/// in both. The offset counts quad slots and is scaled here.
907fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
908    expect_len(mn, ops, 2)?;
909    if peek {
910        let disp = slot_disp(mn, &ops[1])?;
911        return Ok(Inst::Peek {
912            disp,
913            dst: gpr_op(mn, &ops[0])?,
914        });
915    }
916    Ok(Inst::Poke {
917        disp: slot_disp(mn, &ops[0])?,
918        src: gpr_op(mn, &ops[1])?,
919    })
920}
921
922fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
923    let idx = imm_op(mn, op)?;
924    let off = idx
925        .checked_mul(8)
926        .ok_or_else(|| bad(mn, "stack offset overflow"))?;
927    if off < i32::MIN as i64 || off > i32::MAX as i64 {
928        return Err(bad(mn, "stack offset out of range"));
929    }
930    Ok(off as i32)
931}
932
933fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
934    expect_len(mn, ops, 3)?;
935    let dst = gpr_op(mn, &ops[0])?;
936    let base = gpr_op(mn, &ops[1])?;
937    let inc = imm_op(mn, &ops[2])?;
938    Ok(smallvec![
939        Inst::Load {
940            w: Width::B64,
941            sx: Sx::Zero,
942            dst,
943            addr: Mem::base(base, 0),
944            acquire: false,
945        },
946        Inst::Add {
947            w: Width::B64,
948            op: Alu3 {
949                dst: Op::Gpr(base),
950                lhs: Op::Gpr(base),
951                rhs: Op::Imm(inc),
952            },
953        },
954    ])
955}
956
957fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
958    expect_len(mn, ops, 3)?;
959    if load {
960        Ok(Inst::PairLoad {
961            w,
962            dst1: ops[0],
963            dst2: ops[1],
964            mem: mem_op(mn, &ops[2])?,
965        })
966    } else {
967        Ok(Inst::PairStore {
968            w,
969            src1: ops[1],
970            src2: ops[2],
971            mem: mem_op(mn, &ops[0])?,
972        })
973    }
974}
975
976fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
977    expect_len(mn, ops, 2)?;
978    let dst = gpr_op(mn, &ops[0])?;
979    let src = match &ops[1] {
980        Op::Mem(a) => LeaSrc::Mem(*a),
981        Op::Label(l) => LeaSrc::Label(*l),
982        Op::Extern(e) => LeaSrc::Extern(*e),
983        _ => return Err(bad(mn, "lea needs an address or label source")),
984    };
985    Ok(Inst::Lea { w, dst, src })
986}
987
988fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
989    expect_len(mn, ops, 2)?;
990    // `move reg, extern` takes the symbol's address, like `leap`.
991    if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
992        return Ok(Inst::Lea {
993            w: Width::B64,
994            dst: *dst,
995            src: LeaSrc::Extern(*e),
996        });
997    }
998    Ok(Inst::Mov {
999        w,
1000        dst: ops[0],
1001        src: ops[1],
1002        patch,
1003    })
1004}
1005
1006fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1007    expect_len(mn, ops, 2)?;
1008    Ok(Inst::MovZero32 {
1009        dst: ops[0],
1010        src: ops[1],
1011    })
1012}
1013
1014fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1015    expect_len(mn, ops, 2)?;
1016    Ok(Inst::Load {
1017        w,
1018        sx: Sx::Zero,
1019        dst: gpr_op(mn, &ops[0])?,
1020        addr: mem_op(mn, &ops[1])?,
1021        acquire: true,
1022    })
1023}
1024
1025fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1026    expect_len(mn, ops, 2)?;
1027    let addr = mem_op(mn, &ops[0])?;
1028    Ok(Inst::Store {
1029        w,
1030        src: ops[1],
1031        addr,
1032    })
1033}
1034
1035fn extend_node(
1036    mn: Mnemonic,
1037    ops: &[Op],
1038    from: Width,
1039    to: Width,
1040    sx: Sx,
1041) -> Result<Inst, ExpandError> {
1042    expect_len(mn, ops, 2)?;
1043    Ok(Inst::Extend {
1044        from,
1045        to,
1046        sx,
1047        dst: gpr_op(mn, &ops[0])?,
1048        src: ops[1],
1049    })
1050}
1051
1052fn transfer_node(
1053    mn: Mnemonic,
1054    ops: &[Op],
1055    sc: &Scratch,
1056    w: Width,
1057) -> Result<Expansion, ExpandError> {
1058    expect_len(mn, ops, 2)?;
1059    let mut out = Expansion::new();
1060    match &ops[1] {
1061        Op::Gpr(_) => {}
1062        Op::Mem(a) => {
1063            out.push(Inst::Load {
1064                w,
1065                sx: Sx::Zero,
1066                dst: sc.gpr,
1067                addr: *a,
1068                acquire: false,
1069            });
1070        }
1071        Op::Imm(v) => {
1072            out.push(Inst::Mov {
1073                w,
1074                dst: Op::Gpr(sc.gpr),
1075                src: Op::Imm(*v),
1076                patch: false,
1077            });
1078        }
1079        Op::Label(l) => {
1080            out.push(Inst::Lea {
1081                w: Width::B64,
1082                dst: sc.gpr,
1083                src: LeaSrc::Label(*l),
1084            });
1085            let addr = Mem::base(sc.gpr, 0);
1086            out.push(Inst::Load {
1087                w,
1088                sx: Sx::Zero,
1089                dst: sc.gpr,
1090                addr,
1091                acquire: false,
1092            });
1093        }
1094        Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1095        Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1096        Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1097    }
1098    let src = if out.is_empty() {
1099        ops[1]
1100    } else {
1101        Op::Gpr(sc.gpr)
1102    };
1103    out.push(Inst::Mov {
1104        w,
1105        dst: ops[0],
1106        src,
1107        patch: false,
1108    });
1109    Ok(out)
1110}
1111
1112fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1113    expect_len(mn, ops, 2)?;
1114    match (&ops[0], &ops[1]) {
1115        (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1116            k,
1117            dst: *d,
1118            src: *s,
1119        }),
1120        (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1121            k,
1122            dst: *d,
1123            addr: *a,
1124        }),
1125        (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1126            k,
1127            src: *s,
1128            addr: *a,
1129        }),
1130        _ => Err(bad(mn, "float move needs FPR/memory operands")),
1131    }
1132}
1133
1134fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1135    expect_len(mn, ops, 2)?;
1136    if !store {
1137        match (&ops[0], &ops[1]) {
1138            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1139            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1140            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1141                dst: *d,
1142                addr: Mem::base(*s, 0),
1143            }),
1144            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1145        }
1146    } else {
1147        match (&ops[0], &ops[1]) {
1148            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1149            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1150            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1151                dst: *d,
1152                addr: Mem::base(*s, 0),
1153            }),
1154            (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1155            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1156        }
1157    }
1158}
1159
1160/// Checks that every callee-save operand is a GPR in `saved`.
1161///
1162/// Does nothing for every other mnemonic.
1163///
1164/// # Errors
1165///
1166/// [`ExpandError::BadOperand`] if `mnemonic` is `PushCalleeSaves` or
1167/// `PopCalleeSaves` and `ops` contains a non-GPR operand or a GPR whose
1168/// number is not in `saved`.
1169#[inline]
1170pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1171    if !matches!(
1172        mnemonic,
1173        Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1174    ) {
1175        return Ok(());
1176    }
1177    for op in ops {
1178        match op {
1179            Op::Gpr(g) if saved.contains(&g.0) => {}
1180            _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1181        }
1182    }
1183    Ok(())
1184}
1185
1186/// Bytes one AArch64 `push`/`pop` of `n` quads moves `sp` by: `8 * n`
1187/// rounded up to 16, because `sp` must stay 16-byte aligned whenever it is
1188/// used as a base address.
1189///
1190/// The operands keep the layout they have on the targets with 8-byte slots
1191/// (see [`aligned_push_offset`]), and an odd count leaves one padding word
1192/// above them. A `pop` must name as many operands as the `push` it undoes.
1193pub fn aligned_push_bytes(n: usize) -> i32 {
1194    ((8 * n + 15) & !15) as i32
1195}
1196
1197/// Offset from the lowered `sp` of operand `i` in an `n`-operand AArch64
1198/// `push`/`pop` group. The last operand sits at `[sp]`, as it would after
1199/// `n` separate 8-byte pushes.
1200pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1201    8 * (n - 1 - i) as i32
1202}
1203
1204/// Lowers one `push`/`pop` operand to its [`Inst`].
1205fn push_inst(mn: Mnemonic, op: &Op, push: bool, vector: bool) -> Result<Inst, ExpandError> {
1206    if vector {
1207        let x = fpr_op(mn, op)?;
1208        Ok(if push {
1209            Inst::PushV { src: x }
1210        } else {
1211            Inst::PopV { dst: x }
1212        })
1213    } else {
1214        match op {
1215            Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => Ok(if push {
1216                Inst::Push { op: *op }
1217            } else {
1218                Inst::Pop { op: *op }
1219            }),
1220            _ => Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1221        }
1222    }
1223}
1224
1225fn stack_nodes(
1226    mn: Mnemonic,
1227    ops: &[Op],
1228    push: bool,
1229    vector: bool,
1230) -> Result<Expansion, ExpandError> {
1231    if ops.is_empty() {
1232        return Err(bad(mn, "needs at least one operand"));
1233    }
1234    let mut out = Expansion::new();
1235    // `push a, b` stores `a` then `b`, leaving `b` on top; `pop a, b` must
1236    // unwind LIFO, so it reads `b` before `a` - the reverse of the operand
1237    // list order that `push` uses.
1238    if push {
1239        for op in ops {
1240            out.push(push_inst(mn, op, true, vector)?);
1241        }
1242    } else {
1243        for op in ops.iter().rev() {
1244            out.push(push_inst(mn, op, false, vector)?);
1245        }
1246    }
1247    Ok(out)
1248}
1249
1250fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1251    if ops.len() == 3 {
1252        if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1253            return Ok(Inst::MulImm {
1254                w,
1255                dst: gpr_op(mn, &ops[0])?,
1256                lhs,
1257                imm: *v,
1258            });
1259        }
1260        if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1261            return Ok(Inst::MulImm {
1262                w,
1263                dst: gpr_op(mn, &ops[0])?,
1264                lhs,
1265                imm: *v,
1266            });
1267        }
1268    }
1269    Ok(Inst::Mul {
1270        w,
1271        op: Alu3::from_operands(mn, ops)?,
1272    })
1273}
1274
1275fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1276    expect_len(mn, ops, 3)?;
1277    Ok(Inst::MulWide {
1278        signed,
1279        dst: gpr_op(mn, &ops[0])?,
1280        lhs: gpr_op(mn, &ops[1])?,
1281        rhs: gpr_op(mn, &ops[2])?,
1282    })
1283}
1284
1285fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1286    expect_len(mn, ops, 4)?;
1287    let amount = imm_op(mn, &ops[3])?;
1288    if !(0..=63).contains(&amount) {
1289        return Err(bad(mn, "shift amount out of range"));
1290    }
1291    Ok(Inst::AddShifted {
1292        dst: gpr_op(mn, &ops[0])?,
1293        lhs: gpr_op(mn, &ops[1])?,
1294        rhs: gpr_op(mn, &ops[2])?,
1295        amount: amount as u8,
1296    })
1297}
1298
1299fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1300    expect_len(mn, ops, 2)?;
1301    let inst = if borrow {
1302        Inst::Sbb {
1303            dst: ops[0],
1304            src: ops[1],
1305        }
1306    } else {
1307        Inst::Adc {
1308            dst: ops[0],
1309            src: ops[1],
1310        }
1311    };
1312    Ok(inst)
1313}
1314
1315fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Expansion, ExpandError> {
1316    if ops.len() != 2 && ops.len() != 3 {
1317        return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1318    }
1319    let mut out = Expansion::new();
1320    let (src, dst) = if ops.len() == 3 {
1321        let d = fpr_op(mn, &ops[0])?;
1322        if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1323            // `O1 = O2 op O1`: and/or commute, so fold `O2` into `O1`.
1324            (ops[1], d)
1325        } else {
1326            if !op_eq(&ops[1], &ops[0]) {
1327                match &ops[1] {
1328                    Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1329                    Op::Mem(a) => out.push(Inst::LoadF {
1330                        k,
1331                        dst: d,
1332                        addr: *a,
1333                    }),
1334                    _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1335                }
1336            }
1337            (ops[2], d)
1338        }
1339    } else {
1340        (ops[1], fpr_op(mn, &ops[0])?)
1341    };
1342    out.push(Inst::FBit { or, k, dst, src });
1343    Ok(out)
1344}
1345
1346fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Expansion, ExpandError> {
1347    if ops.len() == 3 {
1348        return Ok(smallvec![Inst::FArith {
1349            op,
1350            k,
1351            arith: FArith3 {
1352                dst: fpr_op(mn, &ops[0])?,
1353                lhs: ops[1],
1354                rhs: Some(ops[2]),
1355            },
1356        }]);
1357    }
1358    Ok(smallvec![Inst::FArith {
1359        op,
1360        k,
1361        arith: FArith3::from_operands(mn, ops)?,
1362    }])
1363}
1364
1365fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1366    expect_len(mn, ops, 2)?;
1367    Ok(Inst::FSqrt {
1368        k,
1369        dst: fpr_op(mn, &ops[0])?,
1370        src: ops[1],
1371    })
1372}
1373
1374fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1375    expect_len(mn, ops, 4)?;
1376    Ok(Inst::Fma {
1377        op,
1378        k,
1379        a: ops[1],
1380        b: ops[2],
1381        c: ops[3],
1382        dst: fpr_op(mn, &ops[0])?,
1383    })
1384}
1385
1386/// Reverses the bytes of `src` into `dst`.
1387fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1388    expect_len(mn, ops, 2)?;
1389    Ok(Inst::Bswap {
1390        w,
1391        dst: ops[0],
1392        src: ops[1],
1393    })
1394}
1395
1396fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1397    expect_len(mn, ops, 2)?;
1398    Ok(Inst::FRound {
1399        k,
1400        mode,
1401        dst: fpr_op(mn, &ops[0])?,
1402        src: ops[1],
1403    })
1404}
1405
1406fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1407    expect_len(mn, ops, 2)?;
1408    Ok(Inst::CvtFI {
1409        k,
1410        w,
1411        dst: gpr_op(mn, &ops[0])?,
1412        src: ops[1],
1413    })
1414}
1415
1416fn cvtif_node(
1417    mn: Mnemonic,
1418    ops: &[Op],
1419    signed: bool,
1420    from: Width,
1421    k: FKind,
1422) -> Result<Inst, ExpandError> {
1423    expect_len(mn, ops, 2)?;
1424    Ok(Inst::CvtIF {
1425        signed,
1426        from,
1427        k,
1428        dst: fpr_op(mn, &ops[0])?,
1429        src: ops[1],
1430    })
1431}
1432
1433fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1434    expect_len(mn, ops, 2)?;
1435    Ok(Inst::CvtFF {
1436        from,
1437        to,
1438        dst: fpr_op(mn, &ops[0])?,
1439        src: ops[1],
1440    })
1441}
1442
1443fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1444    expect_len(mn, ops, 2)?;
1445    Ok(Inst::Bitcast {
1446        dir,
1447        dst: ops[0],
1448        src: ops[1],
1449    })
1450}
1451
1452fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1453    expect_len(mn, ops, 2)?;
1454    let dst = fpr_op(mn, &ops[0])?;
1455    let inst = if negate {
1456        Inst::FNeg {
1457            k,
1458            dst,
1459            src: ops[1],
1460        }
1461    } else {
1462        Inst::FAbs {
1463            k,
1464            dst,
1465            src: ops[1],
1466        }
1467    };
1468    Ok(inst)
1469}
1470
1471fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1472    expect_len(mn, ops, 3)?;
1473    let lane = imm_op(mn, &ops[2])?;
1474    let lanes: i64 = 128 / w.bytes() as i64;
1475    if lane < 0 || lane >= lanes {
1476        return Err(bad(mn, "lane out of range"));
1477    }
1478    Ok(Inst::VecExtract {
1479        w,
1480        dst: gpr_op(mn, &ops[0])?,
1481        src: fpr_op(mn, &ops[1])?,
1482        lane: lane as u8,
1483    })
1484}
1485
1486fn trunc_q_slow(
1487    mn: Mnemonic,
1488    ops: &[Op],
1489    sc: &Scratch,
1490    k: FKind,
1491) -> Result<Expansion, ExpandError> {
1492    expect_len(mn, ops, 2)?;
1493    let (dst, src) = (ops[0], ops[1]);
1494    let slow = Lbl::Fresh(0);
1495    let done = Lbl::Fresh(1);
1496    let gs = Op::Gpr(sc.gpr);
1497    let fs = Op::Fpr(sc.fpr);
1498    let (int64_min, neg_min) = match k {
1499        FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1500        FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1501    };
1502    let dst_gpr = gpr_op(mn, &dst)?;
1503    Ok(smallvec![
1504        Inst::Mov {
1505            w: Width::B64,
1506            dst: gs,
1507            src: Op::Imm(neg_min),
1508            patch: false,
1509        },
1510        Inst::Bitcast {
1511            dir: if k == FKind::F32 {
1512                BitcastDir::I2F
1513            } else {
1514                BitcastDir::Q2D
1515            },
1516            dst: fs,
1517            src: gs,
1518        },
1519        Inst::BranchF {
1520            cc: FCc::Ge,
1521            k,
1522            lhs: src,
1523            rhs: fs,
1524            target: slow,
1525            patch: false,
1526        },
1527        Inst::CvtFI {
1528            k,
1529            w: Width::B64,
1530            dst: dst_gpr,
1531            src,
1532        },
1533        Inst::Jump {
1534            target: Target::Label(done),
1535            patch: false,
1536        },
1537        Inst::Bind(slow),
1538        Inst::Mov {
1539            w: Width::B64,
1540            dst: gs,
1541            src: Op::Imm(int64_min),
1542            patch: false,
1543        },
1544        Inst::Bitcast {
1545            dir: if k == FKind::F32 {
1546                BitcastDir::I2F
1547            } else {
1548                BitcastDir::Q2D
1549            },
1550            dst: fs,
1551            src: gs,
1552        },
1553        Inst::FArith {
1554            op: FAlu::Add,
1555            k,
1556            arith: FArith3 {
1557                dst: sc.fpr,
1558                lhs: src,
1559                rhs: None,
1560            },
1561        },
1562        Inst::CvtFI {
1563            k,
1564            w: Width::B64,
1565            dst: dst_gpr,
1566            src: fs,
1567        },
1568        Inst::Mov {
1569            w: Width::B64,
1570            dst: gs,
1571            src: Op::Imm(0x8000000000000000u64 as i64),
1572            patch: false,
1573        },
1574        Inst::Or {
1575            w: Width::B64,
1576            op: Alu3 {
1577                dst,
1578                lhs: dst,
1579                rhs: gs,
1580            },
1581        },
1582        Inst::Bind(done),
1583    ])
1584}
1585
1586fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Expansion, ExpandError> {
1587    expect_len(mn, ops, 3)?;
1588    let (src, dst) = (ops[1], ops[0]);
1589    let slow = Lbl::Fresh(0);
1590    let done = Lbl::Fresh(1);
1591    let s1 = gpr_op(mn, &ops[2])?;
1592    let s2 = sc.gpr;
1593    let dst_fpr = fpr_op(mn, &dst)?;
1594    Ok(smallvec![
1595        Inst::TestBranch {
1596            w: Width::B64,
1597            value: src,
1598            mask: None,
1599            test: TestKind::Sign,
1600            target: slow,
1601            patch: false,
1602        },
1603        Inst::CvtIF {
1604            signed: true,
1605            from: Width::B64,
1606            k,
1607            dst: dst_fpr,
1608            src,
1609        },
1610        Inst::Jump {
1611            target: Target::Label(done),
1612            patch: false,
1613        },
1614        Inst::Bind(slow),
1615        Inst::Mov {
1616            w: Width::B64,
1617            dst: Op::Gpr(s1),
1618            src,
1619            patch: false,
1620        },
1621        Inst::Mov {
1622            w: Width::B64,
1623            dst: Op::Gpr(s2),
1624            src,
1625            patch: false,
1626        },
1627        Inst::Shift {
1628            w: Width::B64,
1629            op: ShiftOp::Shr,
1630            dst: s1,
1631            cnt: ShiftCnt::Imm(1),
1632        },
1633        Inst::And {
1634            w: Width::B64,
1635            op: Alu3 {
1636                dst: Op::Gpr(s2),
1637                lhs: Op::Gpr(s2),
1638                rhs: Op::Imm(1),
1639            },
1640        },
1641        Inst::Or {
1642            w: Width::B64,
1643            op: Alu3 {
1644                dst: Op::Gpr(s2),
1645                lhs: Op::Gpr(s2),
1646                rhs: Op::Gpr(s1),
1647            },
1648        },
1649        Inst::CvtIF {
1650            signed: true,
1651            from: Width::B64,
1652            k,
1653            dst: dst_fpr,
1654            src: Op::Gpr(s2),
1655        },
1656        Inst::FArith {
1657            op: FAlu::Add,
1658            k,
1659            arith: FArith3 {
1660                dst: dst_fpr,
1661                lhs: Op::Fpr(dst_fpr),
1662                rhs: None,
1663            },
1664        },
1665        Inst::Bind(done),
1666    ])
1667}
1668
1669/// Expands `btd2i` to a convert plus an overflow-sentinel branch.
1670///
1671/// Takes the branch when the low 32 bits equal `0x80000000`.
1672fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
1673    expect_len(mn, ops, 3)?;
1674    Ok(smallvec![
1675        Inst::CvtFI {
1676            k: FKind::F64,
1677            w: Width::B64,
1678            dst: gpr_op(mn, &ops[0])?,
1679            src: ops[1],
1680        },
1681        Inst::Branch {
1682            cc: IntCc::Eq,
1683            w: Width::B32,
1684            lhs: ops[0],
1685            rhs: Op::Imm(-0x8000_0000i64),
1686            target: label_op(mn, &ops[2])?,
1687            patch: false,
1688        },
1689    ])
1690}
1691
1692fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
1693    expect_len(mn, ops, 3)?;
1694    let target = label_op(mn, &ops[2])?;
1695    Ok(smallvec![
1696        Inst::CvtFI {
1697            k: FKind::F64,
1698            w: Width::B64,
1699            dst: gpr_op(mn, &ops[0])?,
1700            src: ops[1],
1701        },
1702        Inst::TestBranch {
1703            w: Width::B64,
1704            value: ops[0],
1705            mask: None,
1706            test: TestKind::Zero,
1707            target,
1708            patch: false,
1709        },
1710        Inst::CvtIF {
1711            signed: true,
1712            from: Width::B64,
1713            k: FKind::F64,
1714            dst: sc.fpr,
1715            src: ops[0],
1716        },
1717        Inst::BranchF {
1718            cc: FCc::NeUn,
1719            k: FKind::F64,
1720            lhs: ops[1],
1721            rhs: Op::Fpr(sc.fpr),
1722            target,
1723            patch: false,
1724        },
1725    ])
1726}
1727
1728/// Zeros a double by computing `dst -= dst`.
1729fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1730    expect_len(mn, ops, 1)?;
1731    let dst = fpr_op(mn, &ops[0])?;
1732    Ok(Inst::FArith {
1733        op: FAlu::Sub,
1734        k: FKind::F64,
1735        arith: FArith3 {
1736            dst,
1737            lhs: Op::Fpr(dst),
1738            rhs: None,
1739        },
1740    })
1741}