Skip to main content

offlineasm_ir/
expand.rs

1//! Expands mnemonic invocations into [`Inst`] sequences.
2
3use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6
7/// Lowers one mnemonic invocation to the [`Inst`] sequence that implements it.
8///
9/// `sc` supplies the scratch registers that multi-instruction expansions
10/// and the destination-aliasing fixup may need. It must not alias any
11/// operand in `ops`.
12/// When `patch` is set, the mnemonic must be a [`patch_kind`] site, and the
13/// returned sequence is shaped so a backend can patch it after emission
14/// (a fixed instruction count, no operand folding).
15///
16/// # Errors
17///
18/// - [`ExpandError::NotPatchable`] if `patch` is set and `mn` has no
19///   [`patch_kind`].
20/// - [`ExpandError::BadOperand`] if `ops` has the wrong count or kind for
21///   `mn`.
22pub fn expand(
23    mn: Mnemonic,
24    ops: &[Op],
25    sc: &Scratch,
26    patch: bool,
27) -> Result<Vec<Inst>, ExpandError> {
28    if patch && patch_kind(mn).is_none() {
29        return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
30    }
31    check_extern_operands(mn, ops, patch)?;
32    let mut insts = expand_inner(mn, ops, sc, patch)?;
33    stage_aliasing_mem(&mut insts, sc);
34    Ok(insts)
35}
36
37/// Rejects external symbols outside the shapes that take one: the target of
38/// `jmp`/`call`, the source of `move`, and the source of a 64-bit `lea`
39/// (`leap`, `globaladdr`, `la`). None of them is a patch site.
40fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
41    let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
42        return Ok(());
43    };
44    if patch {
45        return Err(bad(mn, "an external symbol cannot be a patch site"));
46    }
47    let ok = match mn {
48        Mnemonic::Jmp | Mnemonic::Call => at == 0,
49        Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
50        Mnemonic::Lea {
51            w: Width::B64,
52            form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
53        } => at == 1,
54        _ => false,
55    };
56    if ok {
57        Ok(())
58    } else {
59        Err(bad(
60            mn,
61            "an external symbol is only a `jmp`/`call` target or the source of \
62             `move`/`leap`/`globaladdr`/`la`",
63        ))
64    }
65}
66
67/// Stages aliased memory operands through the scratch registers.
68///
69/// `addis t1, t0, 8[t1]` reads `8[t1]` with the *old* `t1`: all operands are
70/// evaluated before the destination is written. A backend that first lowers
71/// `dst = lhs` into `t1` and then reuses `t1` as the memory base (the
72/// natural 2-operand sequence on every target) reads `8[t1]` after the move
73/// instead and dereferences the wrong address. The fused `Alu3`/op-branch
74/// nodes hand that ordering choice to each backend, so the fix is pinned
75/// down in the front half both assemblers share: the load happens before
76/// the operation, through the expansion scratch register.
77///
78/// The oracle interprets this same sequence, and scratch is clobberable by
79/// design (it is never in the portable register pool, so it cannot alias
80/// `dst` or the operand bases).
81fn stage_aliasing_mem(insts: &mut Vec<Inst>, sc: &Scratch) {
82    let old = std::mem::take(insts);
83    let mut out = Vec::with_capacity(old.len() + 2);
84    for mut inst in old {
85        let loads = stage_one(&mut inst, sc);
86        if let Some(loads) = loads {
87            out.extend(loads);
88        }
89        out.push(inst);
90    }
91    *insts = out;
92}
93
94/// Collects the staging loads for `inst` and rewrites its aliased operands.
95///
96/// Returns `None` when `inst` needs no staging.
97fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Vec<Inst>> {
98    let (w, operands, dst) = match inst {
99        Inst::Add { w, op }
100        | Inst::Sub { w, op }
101        | Inst::Mul { w, op }
102        | Inst::And { w, op }
103        | Inst::Or { w, op }
104        | Inst::Xor { w, op } => (*w, vec![&mut op.lhs, &mut op.rhs], &mut op.dst),
105        Inst::AluS { w, alu, .. } => (*w, vec![&mut alu.lhs, &mut alu.rhs], &mut alu.dst),
106        Inst::OpBranch { w, args, .. } => {
107            let (last, init) = args.split_last_mut()?;
108            (*w, init.iter_mut().collect(), last)
109        }
110        _ => return None,
111    };
112    let Op::Gpr(g) = *dst else {
113        return None;
114    };
115    let slots = [sc.gpr, sc.gpr2];
116    let mut loads = Vec::new();
117    let mut slot = 0usize;
118    for operand in operands {
119        let Op::Mem(m) = operand else {
120            continue;
121        };
122        let aliases = m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g);
123        if !aliases {
124            continue;
125        }
126        // At most two memory operands can alias `dst` (op-branch takes
127        // `[lhs, rhs, dst]`), and both scratch GPRs are available.
128        let &s = slots.get(slot)?;
129        slot += 1;
130        loads.push(Inst::Load {
131            w,
132            sx: Sx::Zero,
133            dst: s,
134            addr: *m,
135            acquire: false,
136        });
137        *operand = Op::Gpr(s);
138    }
139    (!loads.is_empty()).then_some(loads)
140}
141
142/// Returns the patch shape for `mn`.
143pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
144    Some(match mn {
145        Mnemonic::Jmp => PatchKind::JumpSite,
146        Mnemonic::Call => PatchKind::CallSite,
147        Mnemonic::Move => PatchKind::MovBlock,
148        Mnemonic::FlagBranch(_)
149        | Mnemonic::OpBranch { .. }
150        | Mnemonic::RiscAluS { .. }
151        | Mnemonic::TestBranch { .. }
152        | Mnemonic::IntBranch { .. }
153        | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
154        _ => return None,
155    })
156}
157
158/// Describes what a `#[patch]` site patches.
159///
160/// Jump shapes become jump handles (in onlineasm, retargeted after
161/// emission); [`MovBlock`](PatchKind::MovBlock) becomes a 64-bit immediate
162/// slot (rewritten after emission).
163#[derive(Clone, Copy, Debug, PartialEq, Eq)]
164pub enum PatchKind {
165    /// Patches an unconditional jump target (`jmp` to a label).
166    JumpSite,
167    /// Patches a conditional branch target (a single-branch expansion).
168    CondJumpSite,
169    /// Patches a call target (`call` to a label).
170    CallSite,
171    /// Patches a 64-bit move slot (`move` of an immediate into a register).
172    MovBlock,
173}
174
175fn float_cc(cc: FCc) -> (FCc, bool) {
176    match cc {
177        FCc::Lt => (FCc::Gt, true),
178        FCc::Le => (FCc::Ge, true),
179        FCc::GtUn => (FCc::GtUn, false),
180        FCc::GeUn => (FCc::GeUn, false),
181        FCc::LtUn => (FCc::GtUn, true),
182        FCc::LeUn => (FCc::GeUn, true),
183        other => (other, false),
184    }
185}
186
187fn test_set_node(
188    mn: Mnemonic,
189    ops: &[Op],
190    w: Width,
191    test: TestKind,
192) -> Result<Vec<Inst>, ExpandError> {
193    if ops.len() != 2 && ops.len() != 3 {
194        return Err(bad(mn, "test-set takes 2 or 3 operands"));
195    }
196    let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
197    Ok(vec![Inst::TestSet {
198        w,
199        value: ops[1],
200        mask,
201        test,
202        dst: ops[0],
203    }])
204}
205
206fn test_branch_node(
207    mn: Mnemonic,
208    ops: &[Op],
209    w: Width,
210    test: TestKind,
211    patch: bool,
212) -> Result<Vec<Inst>, ExpandError> {
213    if ops.len() != 2 && ops.len() != 3 {
214        return Err(bad(mn, "test-branch takes 2 or 3 operands"));
215    }
216    let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
217    let target = label_op(
218        mn,
219        ops.last()
220            .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
221    )?;
222    Ok(vec![Inst::TestBranch {
223        w,
224        value: ops[0],
225        mask,
226        test,
227        target,
228        patch,
229    }])
230}
231
232fn div_rem_node(
233    mn: Mnemonic,
234    ops: &[Op],
235    signed: bool,
236    w: Width,
237    rem: bool,
238) -> Result<Vec<Inst>, ExpandError> {
239    let (dvd, dvs, dst) = match ops {
240        [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
241        [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
242        _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
243    };
244    Ok(vec![if rem {
245        Inst::Rem3 {
246            signed,
247            w,
248            dst,
249            dvd,
250            dvs,
251        }
252    } else {
253        Inst::Div3 {
254            signed,
255            w,
256            dst,
257            dvd,
258            dvs,
259        }
260    }])
261}
262
263fn shift_mem_canon(
264    mn: Mnemonic,
265    w: Width,
266    op: ShiftOp,
267    cnt: &Op,
268    dst: &Mem,
269    sc: &Scratch,
270) -> Result<Vec<Inst>, ExpandError> {
271    let shift_cnt = match cnt {
272        Op::Imm(v) => ShiftCnt::Imm(*v),
273        Op::Gpr(g) => ShiftCnt::Reg(*g),
274        _ => return Err(bad(mn, "shift count must be an immediate or register")),
275    };
276    Ok(vec![
277        Inst::Load {
278            w,
279            sx: Sx::Zero,
280            dst: sc.gpr,
281            addr: *dst,
282            acquire: false,
283        },
284        Inst::Shift {
285            w,
286            op,
287            dst: sc.gpr,
288            cnt: shift_cnt,
289        },
290        Inst::Store {
291            w,
292            src: Op::Gpr(sc.gpr),
293            addr: *dst,
294        },
295    ])
296}
297
298fn shift_node(
299    mn: Mnemonic,
300    w: Width,
301    op: ShiftOp,
302    ops: &[Op],
303    sc: &Scratch,
304) -> Result<Vec<Inst>, ExpandError> {
305    if ops.len() != 2 {
306        return Err(bad(mn, "shift takes 2 operands"));
307    }
308    if let Op::Mem(a) = &ops[0] {
309        return shift_mem_canon(mn, w, op, &ops[1], a, sc);
310    }
311    let dst = gpr_op(mn, &ops[0])?;
312    let cnt = match &ops[1] {
313        Op::Imm(v) => ShiftCnt::Imm(*v),
314        Op::Gpr(g) => ShiftCnt::Reg(*g),
315        _ => return Err(bad(mn, "shift count must be an immediate or register")),
316    };
317    Ok(vec![Inst::Shift { w, op, dst, cnt }])
318}
319
320fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
321    Alu3::from_operands(mn, ops)
322}
323
324fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Op>, ExpandError> {
325    match ops {
326        [dst, rhs] => Ok(vec![*rhs, *dst]),
327        [dst, lhs, rhs] => Ok(vec![*lhs, *rhs, *dst]),
328        _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
329    }
330}
331
332fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
333    match op {
334        Op::Label(l) => Ok(Target::Label(*l)),
335        Op::Gpr(g) => Ok(Target::Reg(*g)),
336        Op::Mem(a) => Ok(Target::Mem(*a)),
337        Op::Imm(v) => Ok(Target::Abs(*v)),
338        Op::Extern(e) => Ok(Target::Extern(*e)),
339        _ => Err(bad(
340            mn,
341            "jump/call needs a label, register, memory, or immediate",
342        )),
343    }
344}
345
346#[allow(clippy::too_many_lines)]
347fn expand_inner(
348    mn: Mnemonic,
349    ops: &[Op],
350    sc: &Scratch,
351    patch: bool,
352) -> Result<Vec<Inst>, ExpandError> {
353    let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
354        expect_len(mn, ops, 1)?;
355        Ok(ops[0])
356    };
357    match mn {
358        Mnemonic::Move => Ok(vec![mov_node(mn, ops, Width::B64, patch)?]),
359        Mnemonic::Load(w) => Ok(vec![mov_node(mn, ops, w, patch)?]),
360        Mnemonic::AtomicLoad(w) => Ok(vec![atomic_load_node(mn, ops, w)?]),
361        Mnemonic::Store(w) => Ok(vec![store_node(mn, ops, w)?]),
362        Mnemonic::Extend { from, to, sx } => Ok(vec![extend_node(mn, ops, from, to, sx)?]),
363        Mnemonic::MovZero32 => Ok(vec![movzero_node(mn, ops)?]),
364        Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
365        Mnemonic::FMove(k) => Ok(vec![fmove_node(mn, ops, k)?]),
366        Mnemonic::VMove { store } => Ok(vec![vmove_node(mn, ops, store)?]),
367        Mnemonic::Lea { w, .. } => Ok(vec![lea_node(mn, ops, w)?]),
368        Mnemonic::LoadQInc => loadqinc_node(mn, ops),
369        Mnemonic::LoadPair(w) => Ok(vec![pair_node(mn, ops, w, true)?]),
370        Mnemonic::StorePair(w) => Ok(vec![pair_node(mn, ops, w, false)?]),
371        Mnemonic::Push => stack_nodes(mn, ops, true, false),
372        Mnemonic::Pop => stack_nodes(mn, ops, false, false),
373        Mnemonic::PushV => stack_nodes(mn, ops, true, true),
374        Mnemonic::PopV => stack_nodes(mn, ops, false, true),
375        Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
376        Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
377        Mnemonic::Peek => Ok(vec![peek_node(mn, ops, true)?]),
378        Mnemonic::Poke => Ok(vec![peek_node(mn, ops, false)?]),
379        Mnemonic::TlsLoad(w) => Ok(vec![tls_node(mn, ops, w, true)?]),
380        Mnemonic::TlsStore(w) => Ok(vec![tls_node(mn, ops, w, false)?]),
381        Mnemonic::TlsLea => Ok(vec![tls_lea_node(mn, ops)?]),
382
383        Mnemonic::Add(w) => Ok(vec![Inst::Add {
384            w,
385            op: alu3(mn, ops)?,
386        }]),
387        Mnemonic::Sub(w) => Ok(vec![Inst::Sub {
388            w,
389            op: alu3(mn, ops)?,
390        }]),
391        Mnemonic::Mul(w) => Ok(vec![mul_node(mn, w, ops)?]),
392        Mnemonic::And(w) => Ok(vec![Inst::And {
393            w,
394            op: alu3(mn, ops)?,
395        }]),
396        Mnemonic::Or(w) => Ok(vec![Inst::Or {
397            w,
398            op: alu3(mn, ops)?,
399        }]),
400        Mnemonic::Xor(w) => Ok(vec![Inst::Xor {
401            w,
402            op: alu3(mn, ops)?,
403        }]),
404        Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
405        Mnemonic::Not(w) => Ok(vec![Inst::Not {
406            w,
407            dst: one(mn, ops)?,
408        }]),
409        Mnemonic::Neg(w) => Ok(vec![Inst::Neg {
410            w,
411            dst: one(mn, ops)?,
412        }]),
413        Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
414        Mnemonic::Bswap(w) => Ok(vec![bswap_node(mn, ops, w)?]),
415        Mnemonic::Adc => Ok(vec![adc_node(mn, ops, false)?]),
416        Mnemonic::Sbb => Ok(vec![adc_node(mn, ops, true)?]),
417        Mnemonic::MulHigh { signed } => Ok(vec![Inst::MulHigh {
418            signed,
419            rhs: one(mn, ops)?,
420        }]),
421        Mnemonic::MulWide { signed } => Ok(vec![mulwide_node(mn, ops, signed)?]),
422        Mnemonic::AluS { op, w } => Ok(vec![Inst::AluS {
423            op,
424            w,
425            alu: alu3(mn, ops)?,
426        }]),
427        Mnemonic::RiscAluS { op, w } => {
428            let has_label =
429                matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
430            if has_label {
431                let flag_op = match op {
432                    AluOp::Add => FlagAlu::Add,
433                    AluOp::Sub => FlagAlu::Sub,
434                    AluOp::Or => FlagAlu::Or,
435                };
436                return Ok(vec![Inst::OpBranch {
437                    op: flag_op,
438                    w,
439                    flag: PredFlag::S,
440                    args: op_branch_args(mn, &ops[..ops.len() - 1])?,
441                    target: label_op(
442                        mn,
443                        ops.last()
444                            .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
445                    )?,
446                    patch,
447                }]);
448            }
449            Ok(vec![Inst::AluS {
450                op,
451                w,
452                alu: alu3(mn, ops)?,
453            }])
454        }
455        Mnemonic::AddShifted => Ok(vec![addshifted_node(mn, ops)?]),
456        Mnemonic::Cqo { quad } => Ok(vec![Inst::Cqo { quad }]),
457        Mnemonic::DivMod1 { signed, w } => Ok(vec![Inst::DivMod1 {
458            signed,
459            w,
460            divisor: one(mn, ops)?,
461        }]),
462        Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
463        Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
464        Mnemonic::IntSet { w, cc } => {
465            expect_len(mn, ops, 3)?;
466            Ok(vec![Inst::IntSet {
467                cc,
468                w,
469                dst: ops[0],
470                lhs: ops[1],
471                rhs: ops[2],
472            }])
473        }
474        Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
475        Mnemonic::Cmov { zero } => Ok(vec![cmov_node(mn, ops, zero)?]),
476        Mnemonic::BitCount { w, trailing } => Ok(vec![cz_node(mn, ops, w, trailing)?]),
477        Mnemonic::Popcnt(w) => Ok(vec![popcnt_node(mn, ops, w)?]),
478        Mnemonic::Bfi => bfi_node(mn, ops, sc),
479        Mnemonic::IntBranch { w, cc } => {
480            expect_len(mn, ops, 3)?;
481            Ok(vec![Inst::Branch {
482                cc,
483                w,
484                lhs: ops[0],
485                rhs: ops[1],
486                target: label_op(mn, &ops[2])?,
487                patch,
488            }])
489        }
490        Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
491        Mnemonic::OpBranch { op, w, flag } => {
492            if ops.len() != 3 && ops.len() != 4 {
493                return Err(bad(mn, "op-branch takes 3 or 4 operands"));
494            }
495            Ok(vec![Inst::OpBranch {
496                op,
497                w,
498                flag,
499                args: op_branch_args(mn, &ops[..ops.len() - 1])?,
500                target: label_op(
501                    mn,
502                    ops.last()
503                        .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
504                )?,
505                patch,
506            }])
507        }
508        Mnemonic::FlagBranch(flag) => Ok(vec![flag_branch_node(mn, ops, flag, patch)?]),
509        Mnemonic::BranchF { cc, k } => {
510            expect_len(mn, ops, 3)?;
511            let (cc, reverse) = float_cc(cc);
512            let (lhs, rhs) = if reverse {
513                (ops[1], ops[0])
514            } else {
515                (ops[0], ops[1])
516            };
517            Ok(vec![Inst::BranchF {
518                cc,
519                k,
520                lhs,
521                rhs,
522                target: label_op(mn, &ops[2])?,
523                patch,
524            }])
525        }
526        Mnemonic::Jmp => {
527            expect_len(mn, ops, 1)?;
528            Ok(vec![Inst::Jump {
529                target: jump_target(mn, &ops[0])?,
530                patch,
531            }])
532        }
533        Mnemonic::Call => {
534            expect_len(mn, ops, 1)?;
535            Ok(vec![Inst::Call {
536                target: jump_target(mn, &ops[0])?,
537                patch,
538            }])
539        }
540        Mnemonic::Ret => {
541            expect_len(mn, ops, 0)?;
542            Ok(vec![Inst::Ret])
543        }
544        Mnemonic::Break => {
545            expect_len(mn, ops, 0)?;
546            Ok(vec![Inst::Break])
547        }
548        Mnemonic::Nop => {
549            expect_len(mn, ops, 0)?;
550            Ok(vec![Inst::Nop])
551        }
552        Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
553        Mnemonic::Fma { op, k } => Ok(vec![fma_node(mn, ops, k, op)?]),
554        Mnemonic::FSqrt(k) => Ok(vec![fsqrt_node(mn, ops, k)?]),
555        Mnemonic::FRound { k, mode } => Ok(vec![fround_node(mn, ops, k, mode)?]),
556        Mnemonic::FAbs { k, negate } => Ok(vec![fabs_node(mn, ops, k, negate)?]),
557        Mnemonic::FSet { cc, k } => {
558            expect_len(mn, ops, 3)?;
559            let (cc, reverse) = float_cc(cc);
560            let (lhs, rhs) = if reverse {
561                (ops[2], ops[1])
562            } else {
563                (ops[1], ops[2])
564            };
565            Ok(vec![Inst::FSet {
566                cc,
567                k,
568                dst: ops[0],
569                lhs,
570                rhs,
571            }])
572        }
573        Mnemonic::CvtFI { k, w } => Ok(vec![cvtfi_node(mn, ops, k, w)?]),
574        Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
575        Mnemonic::CvtIF { signed, from, k } => Ok(vec![cvtif_node(mn, ops, signed, from, k)?]),
576        Mnemonic::CvtFF { from, to } => Ok(vec![cvtff_node(mn, ops, from, to)?]),
577        Mnemonic::Bitcast(dir) => Ok(vec![bitcast_node(mn, ops, dir)?]),
578        Mnemonic::MovDZ => Ok(vec![movdz_node(mn, ops)?]),
579        Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
580        Mnemonic::Btd2i => btd2i_node(mn, ops),
581        Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
582        Mnemonic::VecExtract(w) => Ok(vec![umov_node(mn, ops, w)?]),
583        Mnemonic::AtomicRmw { op, w } => Ok(vec![rmw_node(mn, ops, w, op)?]),
584        Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
585        Mnemonic::AtomicCas { w, branch } => Ok(vec![cas_node(mn, ops, w, branch)?]),
586        Mnemonic::LoadLinked(w) => Ok(vec![ll_node(mn, ops, w)?]),
587        Mnemonic::StoreCond(w) => Ok(vec![sc_node(mn, ops, w)?]),
588        Mnemonic::Fence { full } => Ok(vec![Inst::Fence { full }]),
589        Mnemonic::Pause => Ok(vec![Inst::Pause]),
590        Mnemonic::TagCodePtr
591        | Mnemonic::TagReturnAddress
592        | Mnemonic::UntagReturnAddress
593        | Mnemonic::RemoveCodePtrTag
594        | Mnemonic::UntagArrayPtr
595        | Mnemonic::RemoveArrayPtrTag => Ok(vec![]),
596    }
597}
598
599fn flag_branch_node(
600    mn: Mnemonic,
601    ops: &[Op],
602    flag: PredFlag,
603    patch: bool,
604) -> Result<Inst, ExpandError> {
605    expect_len(mn, ops, 1)?;
606    Ok(Inst::FlagBranch {
607        flag,
608        target: label_op(mn, &ops[0])?,
609        patch,
610    })
611}
612
613fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
614    expect_len(mn, ops, 2)?;
615    Ok(Inst::Popcnt {
616        w,
617        dst: gpr_op(mn, &ops[0])?,
618        src: ops[1],
619    })
620}
621
622fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
623    expect_len(mn, ops, 2)?;
624    let dst = gpr_op(mn, &ops[0])?;
625    let inst = if trailing {
626        Inst::Ctz {
627            w,
628            dst,
629            src: ops[1],
630        }
631    } else {
632        Inst::Clz {
633            w,
634            dst,
635            src: ops[1],
636        }
637    };
638    Ok(inst)
639}
640
641fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
642    expect_len(mn, ops, 2)?;
643    if load {
644        let idx = imm_op(mn, &ops[1])?;
645        Ok(Inst::TlsLoad {
646            w,
647            dst: gpr_op(mn, &ops[0])?,
648            idx,
649        })
650    } else {
651        let idx = imm_op(mn, &ops[0])?;
652        Ok(Inst::TlsStore {
653            w,
654            src: gpr_op(mn, &ops[1])?,
655            idx,
656        })
657    }
658}
659
660fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
661    expect_len(mn, ops, 2)?;
662    Ok(Inst::TlsLea {
663        dst: gpr_op(mn, &ops[0])?,
664        idx: imm_op(mn, &ops[1])?,
665    })
666}
667
668fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
669    expect_len(mn, ops, 2)?;
670
671    Ok(Inst::AtomicRmw {
672        op,
673        w,
674        a: ops[1],
675        b: ops[0],
676    })
677}
678
679fn rmw_not_node(
680    mn: Mnemonic,
681    ops: &[Op],
682    sc: &Scratch,
683    w: Width,
684    op: AtomicOp,
685) -> Result<Vec<Inst>, ExpandError> {
686    expect_len(mn, ops, 2)?;
687    let mut out = Vec::new();
688    // `mem` is ops[0] and `src` is ops[1].
689    let a = match &ops[1] {
690        Op::Imm(v) => Op::Imm(!v),
691        Op::Gpr(g) => {
692            out.push(Inst::Mov {
693                w: Width::B64,
694                dst: Op::Gpr(sc.gpr),
695                src: Op::Gpr(*g),
696                patch: false,
697            });
698            out.push(Inst::Not {
699                w,
700                dst: Op::Gpr(sc.gpr),
701            });
702            Op::Gpr(sc.gpr)
703        }
704        Op::Mem(addr) => {
705            let lw = if w == Width::B64 {
706                Width::B64
707            } else {
708                Width::B32
709            };
710            out.push(Inst::Load {
711                w: lw,
712                sx: Sx::Zero,
713                dst: sc.gpr,
714                addr: *addr,
715                acquire: false,
716            });
717            out.push(Inst::Not {
718                w,
719                dst: Op::Gpr(sc.gpr),
720            });
721            Op::Gpr(sc.gpr)
722        }
723        _ => {
724            return Err(bad(
725                mn,
726                "atomic clear needs an immediate, register, or memory operand",
727            ))
728        }
729    };
730    out.push(Inst::AtomicRmw {
731        op,
732        w,
733        a,
734        b: ops[0],
735    });
736    Ok(out)
737}
738
739fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
740    let n = if branch { 4 } else { 3 };
741    expect_len(mn, ops, n)?;
742
743    Ok(Inst::AtomicCas {
744        w,
745        mem: mem_op(mn, &ops[0])?,
746        src: gpr_op(mn, &ops[2])?,
747        branch: if branch {
748            Some(label_op(mn, &ops[3])?)
749        } else {
750            None
751        },
752    })
753}
754
755fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
756    expect_len(mn, ops, 2)?;
757    Ok(Inst::LoadLinked {
758        w,
759        mem: mem_op(mn, &ops[1])?,
760        dst: gpr_op(mn, &ops[0])?,
761    })
762}
763
764fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
765    expect_len(mn, ops, 3)?;
766    Ok(Inst::StoreCond {
767        w,
768        status: gpr_op(mn, &ops[1])?,
769        src: gpr_op(mn, &ops[2])?,
770        mem: mem_op(mn, &ops[0])?,
771    })
772}
773
774/// Expands `bfiq` to a mask, shift, and OR sequence.
775///
776/// Computes `dst = (dst & ~mask) | ((src & wmask) << lsb)`.
777fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
778    expect_len(mn, ops, 4)?;
779    let lsb = imm_op(mn, &ops[2])?;
780    let width = imm_op(mn, &ops[3])?;
781    if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
782        return Err(bad(mn, "bitfield out of range"));
783    }
784    let wmask: u64 = if width == 64 {
785        u64::MAX
786    } else {
787        (1u64 << width) - 1
788    };
789    let mask: i64 = (wmask << lsb) as i64;
790    let s = sc.gpr;
791    let dst = gpr_op(mn, &ops[0])?;
792
793    Ok(vec![
794        Inst::Mov {
795            w: Width::B64,
796            dst: Op::Gpr(s),
797            src: ops[1],
798            patch: false,
799        },
800        Inst::And {
801            w: Width::B64,
802            op: Alu3 {
803                dst: Op::Gpr(s),
804                lhs: Op::Gpr(s),
805                rhs: Op::Imm(wmask as i64),
806            },
807        },
808        Inst::Shift {
809            w: Width::B64,
810            op: ShiftOp::Shl,
811            dst: s,
812            cnt: ShiftCnt::Imm(lsb),
813        },
814        Inst::And {
815            w: Width::B64,
816            op: Alu3 {
817                dst: Op::Gpr(dst),
818                lhs: Op::Gpr(dst),
819                rhs: Op::Imm(!mask),
820            },
821        },
822        Inst::Or {
823            w: Width::B64,
824            op: Alu3 {
825                dst: Op::Gpr(dst),
826                lhs: Op::Gpr(dst),
827                rhs: Op::Gpr(s),
828            },
829        },
830    ])
831}
832
833fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
834    expect_len(mn, ops, 3)?;
835    Ok(Inst::Cmov {
836        w: Width::B64,
837        zero,
838        dst: gpr_op(mn, &ops[0])?,
839        src: ops[1],
840        test: gpr_op(mn, &ops[2])?,
841    })
842}
843
844/// Expands `peek dst, off` and `poke off, src`, with the destination first
845/// in both. The offset counts quad slots and is scaled here.
846fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
847    expect_len(mn, ops, 2)?;
848    if peek {
849        let disp = slot_disp(mn, &ops[1])?;
850        return Ok(Inst::Peek {
851            disp,
852            dst: gpr_op(mn, &ops[0])?,
853        });
854    }
855    Ok(Inst::Poke {
856        disp: slot_disp(mn, &ops[0])?,
857        src: gpr_op(mn, &ops[1])?,
858    })
859}
860
861fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
862    let idx = imm_op(mn, op)?;
863    let off = idx
864        .checked_mul(8)
865        .ok_or_else(|| bad(mn, "stack offset overflow"))?;
866    if off < i32::MIN as i64 || off > i32::MAX as i64 {
867        return Err(bad(mn, "stack offset out of range"));
868    }
869    Ok(off as i32)
870}
871
872fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
873    expect_len(mn, ops, 3)?;
874    let dst = gpr_op(mn, &ops[0])?;
875    let base = gpr_op(mn, &ops[1])?;
876    let inc = imm_op(mn, &ops[2])?;
877    Ok(vec![
878        Inst::Load {
879            w: Width::B64,
880            sx: Sx::Zero,
881            dst,
882            addr: Mem::base(base, 0),
883            acquire: false,
884        },
885        Inst::Add {
886            w: Width::B64,
887            op: Alu3 {
888                dst: Op::Gpr(base),
889                lhs: Op::Gpr(base),
890                rhs: Op::Imm(inc),
891            },
892        },
893    ])
894}
895
896fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
897    expect_len(mn, ops, 3)?;
898    if load {
899        Ok(Inst::PairLoad {
900            w,
901            dst1: ops[0],
902            dst2: ops[1],
903            mem: mem_op(mn, &ops[2])?,
904        })
905    } else {
906        Ok(Inst::PairStore {
907            w,
908            src1: ops[1],
909            src2: ops[2],
910            mem: mem_op(mn, &ops[0])?,
911        })
912    }
913}
914
915fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
916    expect_len(mn, ops, 2)?;
917    let dst = gpr_op(mn, &ops[0])?;
918    let src = match &ops[1] {
919        Op::Mem(a) => LeaSrc::Mem(*a),
920        Op::Label(l) => LeaSrc::Label(*l),
921        Op::Extern(e) => LeaSrc::Extern(*e),
922        _ => return Err(bad(mn, "lea needs an address or label source")),
923    };
924    Ok(Inst::Lea { w, dst, src })
925}
926
927fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
928    expect_len(mn, ops, 2)?;
929    // `move reg, extern` takes the symbol's address, like `leap`.
930    if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
931        return Ok(Inst::Lea {
932            w: Width::B64,
933            dst: *dst,
934            src: LeaSrc::Extern(*e),
935        });
936    }
937    Ok(Inst::Mov {
938        w,
939        dst: ops[0],
940        src: ops[1],
941        patch,
942    })
943}
944
945fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
946    expect_len(mn, ops, 2)?;
947    Ok(Inst::MovZero32 {
948        dst: ops[0],
949        src: ops[1],
950    })
951}
952
953fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
954    expect_len(mn, ops, 2)?;
955    Ok(Inst::Load {
956        w,
957        sx: Sx::Zero,
958        dst: gpr_op(mn, &ops[0])?,
959        addr: mem_op(mn, &ops[1])?,
960        acquire: true,
961    })
962}
963
964fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
965    expect_len(mn, ops, 2)?;
966    let addr = mem_op(mn, &ops[0])?;
967    Ok(Inst::Store {
968        w,
969        src: ops[1],
970        addr,
971    })
972}
973
974fn extend_node(
975    mn: Mnemonic,
976    ops: &[Op],
977    from: Width,
978    to: Width,
979    sx: Sx,
980) -> Result<Inst, ExpandError> {
981    expect_len(mn, ops, 2)?;
982    Ok(Inst::Extend {
983        from,
984        to,
985        sx,
986        dst: gpr_op(mn, &ops[0])?,
987        src: ops[1],
988    })
989}
990
991fn transfer_node(
992    mn: Mnemonic,
993    ops: &[Op],
994    sc: &Scratch,
995    w: Width,
996) -> Result<Vec<Inst>, ExpandError> {
997    expect_len(mn, ops, 2)?;
998    let mut out = Vec::new();
999    match &ops[1] {
1000        Op::Gpr(_) => {}
1001        Op::Mem(a) => {
1002            out.push(Inst::Load {
1003                w,
1004                sx: Sx::Zero,
1005                dst: sc.gpr,
1006                addr: *a,
1007                acquire: false,
1008            });
1009        }
1010        Op::Imm(v) => {
1011            out.push(Inst::Mov {
1012                w,
1013                dst: Op::Gpr(sc.gpr),
1014                src: Op::Imm(*v),
1015                patch: false,
1016            });
1017        }
1018        Op::Label(l) => {
1019            out.push(Inst::Lea {
1020                w: Width::B64,
1021                dst: sc.gpr,
1022                src: LeaSrc::Label(*l),
1023            });
1024            let addr = Mem::base(sc.gpr, 0);
1025            out.push(Inst::Load {
1026                w,
1027                sx: Sx::Zero,
1028                dst: sc.gpr,
1029                addr,
1030                acquire: false,
1031            });
1032        }
1033        Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1034        Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1035        Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1036    }
1037    let src = if out.is_empty() {
1038        ops[1]
1039    } else {
1040        Op::Gpr(sc.gpr)
1041    };
1042    out.push(Inst::Mov {
1043        w,
1044        dst: ops[0],
1045        src,
1046        patch: false,
1047    });
1048    Ok(out)
1049}
1050
1051fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1052    expect_len(mn, ops, 2)?;
1053    match (&ops[0], &ops[1]) {
1054        (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1055            k,
1056            dst: *d,
1057            src: *s,
1058        }),
1059        (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1060            k,
1061            dst: *d,
1062            addr: *a,
1063        }),
1064        (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1065            k,
1066            src: *s,
1067            addr: *a,
1068        }),
1069        _ => Err(bad(mn, "float move needs FPR/memory operands")),
1070    }
1071}
1072
1073fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1074    expect_len(mn, ops, 2)?;
1075    if !store {
1076        match (&ops[0], &ops[1]) {
1077            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1078            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1079            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1080                dst: *d,
1081                addr: Mem::base(*s, 0),
1082            }),
1083            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1084        }
1085    } else {
1086        match (&ops[0], &ops[1]) {
1087            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1088            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1089            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1090                dst: *d,
1091                addr: Mem::base(*s, 0),
1092            }),
1093            (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1094            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1095        }
1096    }
1097}
1098
1099/// Checks that every callee-save operand is a GPR in `saved`.
1100///
1101/// Does nothing for every other mnemonic.
1102///
1103/// # Errors
1104///
1105/// [`ExpandError::BadOperand`] if `mnemonic` is `PushCalleeSaves` or
1106/// `PopCalleeSaves` and `ops` contains a non-GPR operand or a GPR whose
1107/// number is not in `saved`.
1108pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1109    if !matches!(
1110        mnemonic,
1111        Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1112    ) {
1113        return Ok(());
1114    }
1115    for op in ops {
1116        match op {
1117            Op::Gpr(g) if saved.contains(&g.0) => {}
1118            _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1119        }
1120    }
1121    Ok(())
1122}
1123
1124/// Bytes one AArch64 `push`/`pop` of `n` quads moves `sp` by: `8 * n`
1125/// rounded up to 16, because `sp` must stay 16-byte aligned whenever it is
1126/// used as a base address.
1127///
1128/// The operands keep the layout they have on the targets with 8-byte slots
1129/// (see [`aligned_push_offset`]), and an odd count leaves one padding word
1130/// above them. A `pop` must name as many operands as the `push` it undoes.
1131pub fn aligned_push_bytes(n: usize) -> i32 {
1132    ((8 * n + 15) & !15) as i32
1133}
1134
1135/// Offset from the lowered `sp` of operand `i` in an `n`-operand AArch64
1136/// `push`/`pop` group. The last operand sits at `[sp]`, as it would after
1137/// `n` separate 8-byte pushes.
1138pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1139    8 * (n - 1 - i) as i32
1140}
1141
1142fn stack_nodes(
1143    mn: Mnemonic,
1144    ops: &[Op],
1145    push: bool,
1146    vector: bool,
1147) -> Result<Vec<Inst>, ExpandError> {
1148    if ops.is_empty() {
1149        return Err(bad(mn, "needs at least one operand"));
1150    }
1151    let mut out = Vec::with_capacity(ops.len());
1152    // `push a, b` stores `a` then `b`, leaving `b` on top; `pop a, b` must
1153    // unwind LIFO, so it reads `b` before `a` - the reverse of the operand
1154    // list order that `push` uses.
1155    let ordered: Box<dyn Iterator<Item = &Op>> = if push {
1156        Box::new(ops.iter())
1157    } else {
1158        Box::new(ops.iter().rev())
1159    };
1160    for op in ordered {
1161        if vector {
1162            let x = fpr_op(mn, op)?;
1163            out.push(if push {
1164                Inst::PushV { src: x }
1165            } else {
1166                Inst::PopV { dst: x }
1167            });
1168        } else {
1169            match op {
1170                Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => {
1171                    out.push(if push {
1172                        Inst::Push { op: *op }
1173                    } else {
1174                        Inst::Pop { op: *op }
1175                    });
1176                }
1177                _ => return Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1178            }
1179        }
1180    }
1181    Ok(out)
1182}
1183
1184fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1185    if ops.len() == 3 {
1186        if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1187            return Ok(Inst::MulImm {
1188                w,
1189                dst: gpr_op(mn, &ops[0])?,
1190                lhs,
1191                imm: *v,
1192            });
1193        }
1194        if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1195            return Ok(Inst::MulImm {
1196                w,
1197                dst: gpr_op(mn, &ops[0])?,
1198                lhs,
1199                imm: *v,
1200            });
1201        }
1202    }
1203    Ok(Inst::Mul {
1204        w,
1205        op: Alu3::from_operands(mn, ops)?,
1206    })
1207}
1208
1209fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1210    expect_len(mn, ops, 3)?;
1211    Ok(Inst::MulWide {
1212        signed,
1213        dst: gpr_op(mn, &ops[0])?,
1214        lhs: gpr_op(mn, &ops[1])?,
1215        rhs: gpr_op(mn, &ops[2])?,
1216    })
1217}
1218
1219fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1220    expect_len(mn, ops, 4)?;
1221    let amount = imm_op(mn, &ops[3])?;
1222    if !(0..=63).contains(&amount) {
1223        return Err(bad(mn, "shift amount out of range"));
1224    }
1225    Ok(Inst::AddShifted {
1226        dst: gpr_op(mn, &ops[0])?,
1227        lhs: gpr_op(mn, &ops[1])?,
1228        rhs: gpr_op(mn, &ops[2])?,
1229        amount: amount as u8,
1230    })
1231}
1232
1233fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1234    expect_len(mn, ops, 2)?;
1235    let inst = if borrow {
1236        Inst::Sbb {
1237            dst: ops[0],
1238            src: ops[1],
1239        }
1240    } else {
1241        Inst::Adc {
1242            dst: ops[0],
1243            src: ops[1],
1244        }
1245    };
1246    Ok(inst)
1247}
1248
1249fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Vec<Inst>, ExpandError> {
1250    if ops.len() != 2 && ops.len() != 3 {
1251        return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1252    }
1253    let mut out = Vec::new();
1254    let (src, dst) = if ops.len() == 3 {
1255        let d = fpr_op(mn, &ops[0])?;
1256        if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1257            // `O1 = O2 op O1`: and/or commute, so fold `O2` into `O1`.
1258            (ops[1], d)
1259        } else {
1260            if !op_eq(&ops[1], &ops[0]) {
1261                match &ops[1] {
1262                    Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1263                    Op::Mem(a) => out.push(Inst::LoadF {
1264                        k,
1265                        dst: d,
1266                        addr: *a,
1267                    }),
1268                    _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1269                }
1270            }
1271            (ops[2], d)
1272        }
1273    } else {
1274        (ops[1], fpr_op(mn, &ops[0])?)
1275    };
1276    out.push(Inst::FBit { or, k, dst, src });
1277    Ok(out)
1278}
1279
1280fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Vec<Inst>, ExpandError> {
1281    if ops.len() == 3 {
1282        return Ok(vec![Inst::FArith {
1283            op,
1284            k,
1285            arith: FArith3 {
1286                dst: fpr_op(mn, &ops[0])?,
1287                lhs: ops[1],
1288                rhs: Some(ops[2]),
1289            },
1290        }]);
1291    }
1292    Ok(vec![Inst::FArith {
1293        op,
1294        k,
1295        arith: FArith3::from_operands(mn, ops)?,
1296    }])
1297}
1298
1299fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1300    expect_len(mn, ops, 2)?;
1301    Ok(Inst::FSqrt {
1302        k,
1303        dst: fpr_op(mn, &ops[0])?,
1304        src: ops[1],
1305    })
1306}
1307
1308fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1309    expect_len(mn, ops, 4)?;
1310    Ok(Inst::Fma {
1311        op,
1312        k,
1313        a: ops[1],
1314        b: ops[2],
1315        c: ops[3],
1316        dst: fpr_op(mn, &ops[0])?,
1317    })
1318}
1319
1320/// Reverses the bytes of `src` into `dst`.
1321fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1322    expect_len(mn, ops, 2)?;
1323    Ok(Inst::Bswap {
1324        w,
1325        dst: ops[0],
1326        src: ops[1],
1327    })
1328}
1329
1330fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1331    expect_len(mn, ops, 2)?;
1332    Ok(Inst::FRound {
1333        k,
1334        mode,
1335        dst: fpr_op(mn, &ops[0])?,
1336        src: ops[1],
1337    })
1338}
1339
1340fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1341    expect_len(mn, ops, 2)?;
1342    Ok(Inst::CvtFI {
1343        k,
1344        w,
1345        dst: gpr_op(mn, &ops[0])?,
1346        src: ops[1],
1347    })
1348}
1349
1350fn cvtif_node(
1351    mn: Mnemonic,
1352    ops: &[Op],
1353    signed: bool,
1354    from: Width,
1355    k: FKind,
1356) -> Result<Inst, ExpandError> {
1357    expect_len(mn, ops, 2)?;
1358    Ok(Inst::CvtIF {
1359        signed,
1360        from,
1361        k,
1362        dst: fpr_op(mn, &ops[0])?,
1363        src: ops[1],
1364    })
1365}
1366
1367fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1368    expect_len(mn, ops, 2)?;
1369    Ok(Inst::CvtFF {
1370        from,
1371        to,
1372        dst: fpr_op(mn, &ops[0])?,
1373        src: ops[1],
1374    })
1375}
1376
1377fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1378    expect_len(mn, ops, 2)?;
1379    Ok(Inst::Bitcast {
1380        dir,
1381        dst: ops[0],
1382        src: ops[1],
1383    })
1384}
1385
1386fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1387    expect_len(mn, ops, 2)?;
1388    let dst = fpr_op(mn, &ops[0])?;
1389    let inst = if negate {
1390        Inst::FNeg {
1391            k,
1392            dst,
1393            src: ops[1],
1394        }
1395    } else {
1396        Inst::FAbs {
1397            k,
1398            dst,
1399            src: ops[1],
1400        }
1401    };
1402    Ok(inst)
1403}
1404
1405fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1406    expect_len(mn, ops, 3)?;
1407    let lane = imm_op(mn, &ops[2])?;
1408    let lanes: i64 = 128 / w.bytes() as i64;
1409    if lane < 0 || lane >= lanes {
1410        return Err(bad(mn, "lane out of range"));
1411    }
1412    Ok(Inst::VecExtract {
1413        w,
1414        dst: gpr_op(mn, &ops[0])?,
1415        src: fpr_op(mn, &ops[1])?,
1416        lane: lane as u8,
1417    })
1418}
1419
1420fn trunc_q_slow(
1421    mn: Mnemonic,
1422    ops: &[Op],
1423    sc: &Scratch,
1424    k: FKind,
1425) -> Result<Vec<Inst>, ExpandError> {
1426    expect_len(mn, ops, 2)?;
1427    let (dst, src) = (ops[0], ops[1]);
1428    let slow = Lbl::Fresh(0);
1429    let done = Lbl::Fresh(1);
1430    let gs = Op::Gpr(sc.gpr);
1431    let fs = Op::Fpr(sc.fpr);
1432    let (int64_min, neg_min) = match k {
1433        FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1434        FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1435    };
1436    let dst_gpr = gpr_op(mn, &dst)?;
1437    Ok(vec![
1438        Inst::Mov {
1439            w: Width::B64,
1440            dst: gs,
1441            src: Op::Imm(neg_min),
1442            patch: false,
1443        },
1444        Inst::Bitcast {
1445            dir: if k == FKind::F32 {
1446                BitcastDir::I2F
1447            } else {
1448                BitcastDir::Q2D
1449            },
1450            dst: fs,
1451            src: gs,
1452        },
1453        Inst::BranchF {
1454            cc: FCc::Ge,
1455            k,
1456            lhs: src,
1457            rhs: fs,
1458            target: slow,
1459            patch: false,
1460        },
1461        Inst::CvtFI {
1462            k,
1463            w: Width::B64,
1464            dst: dst_gpr,
1465            src,
1466        },
1467        Inst::Jump {
1468            target: Target::Label(done),
1469            patch: false,
1470        },
1471        Inst::Bind(slow),
1472        Inst::Mov {
1473            w: Width::B64,
1474            dst: gs,
1475            src: Op::Imm(int64_min),
1476            patch: false,
1477        },
1478        Inst::Bitcast {
1479            dir: if k == FKind::F32 {
1480                BitcastDir::I2F
1481            } else {
1482                BitcastDir::Q2D
1483            },
1484            dst: fs,
1485            src: gs,
1486        },
1487        Inst::FArith {
1488            op: FAlu::Add,
1489            k,
1490            arith: FArith3 {
1491                dst: sc.fpr,
1492                lhs: src,
1493                rhs: None,
1494            },
1495        },
1496        Inst::CvtFI {
1497            k,
1498            w: Width::B64,
1499            dst: dst_gpr,
1500            src: fs,
1501        },
1502        Inst::Mov {
1503            w: Width::B64,
1504            dst: gs,
1505            src: Op::Imm(0x8000000000000000u64 as i64),
1506            patch: false,
1507        },
1508        Inst::Or {
1509            w: Width::B64,
1510            op: Alu3 {
1511                dst,
1512                lhs: dst,
1513                rhs: gs,
1514            },
1515        },
1516        Inst::Bind(done),
1517    ])
1518}
1519
1520fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Vec<Inst>, ExpandError> {
1521    expect_len(mn, ops, 3)?;
1522    let (src, dst) = (ops[1], ops[0]);
1523    let slow = Lbl::Fresh(0);
1524    let done = Lbl::Fresh(1);
1525    let s1 = gpr_op(mn, &ops[2])?;
1526    let s2 = sc.gpr;
1527    let dst_fpr = fpr_op(mn, &dst)?;
1528    Ok(vec![
1529        Inst::TestBranch {
1530            w: Width::B64,
1531            value: src,
1532            mask: None,
1533            test: TestKind::Sign,
1534            target: slow,
1535            patch: false,
1536        },
1537        Inst::CvtIF {
1538            signed: true,
1539            from: Width::B64,
1540            k,
1541            dst: dst_fpr,
1542            src,
1543        },
1544        Inst::Jump {
1545            target: Target::Label(done),
1546            patch: false,
1547        },
1548        Inst::Bind(slow),
1549        Inst::Mov {
1550            w: Width::B64,
1551            dst: Op::Gpr(s1),
1552            src,
1553            patch: false,
1554        },
1555        Inst::Mov {
1556            w: Width::B64,
1557            dst: Op::Gpr(s2),
1558            src,
1559            patch: false,
1560        },
1561        Inst::Shift {
1562            w: Width::B64,
1563            op: ShiftOp::Shr,
1564            dst: s1,
1565            cnt: ShiftCnt::Imm(1),
1566        },
1567        Inst::And {
1568            w: Width::B64,
1569            op: Alu3 {
1570                dst: Op::Gpr(s2),
1571                lhs: Op::Gpr(s2),
1572                rhs: Op::Imm(1),
1573            },
1574        },
1575        Inst::Or {
1576            w: Width::B64,
1577            op: Alu3 {
1578                dst: Op::Gpr(s2),
1579                lhs: Op::Gpr(s2),
1580                rhs: Op::Gpr(s1),
1581            },
1582        },
1583        Inst::CvtIF {
1584            signed: true,
1585            from: Width::B64,
1586            k,
1587            dst: dst_fpr,
1588            src: Op::Gpr(s2),
1589        },
1590        Inst::FArith {
1591            op: FAlu::Add,
1592            k,
1593            arith: FArith3 {
1594                dst: dst_fpr,
1595                lhs: Op::Fpr(dst_fpr),
1596                rhs: None,
1597            },
1598        },
1599        Inst::Bind(done),
1600    ])
1601}
1602
1603/// Expands `btd2i` to a convert plus an overflow-sentinel branch.
1604///
1605/// Takes the branch when the low 32 bits equal `0x80000000`.
1606fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
1607    expect_len(mn, ops, 3)?;
1608    Ok(vec![
1609        Inst::CvtFI {
1610            k: FKind::F64,
1611            w: Width::B64,
1612            dst: gpr_op(mn, &ops[0])?,
1613            src: ops[1],
1614        },
1615        Inst::Branch {
1616            cc: IntCc::Eq,
1617            w: Width::B32,
1618            lhs: ops[0],
1619            rhs: Op::Imm(-0x8000_0000i64),
1620            target: label_op(mn, &ops[2])?,
1621            patch: false,
1622        },
1623    ])
1624}
1625
1626fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
1627    expect_len(mn, ops, 3)?;
1628    let target = label_op(mn, &ops[2])?;
1629    Ok(vec![
1630        Inst::CvtFI {
1631            k: FKind::F64,
1632            w: Width::B64,
1633            dst: gpr_op(mn, &ops[0])?,
1634            src: ops[1],
1635        },
1636        Inst::TestBranch {
1637            w: Width::B64,
1638            value: ops[0],
1639            mask: None,
1640            test: TestKind::Zero,
1641            target,
1642            patch: false,
1643        },
1644        Inst::CvtIF {
1645            signed: true,
1646            from: Width::B64,
1647            k: FKind::F64,
1648            dst: sc.fpr,
1649            src: ops[0],
1650        },
1651        Inst::BranchF {
1652            cc: FCc::NeUn,
1653            k: FKind::F64,
1654            lhs: ops[1],
1655            rhs: Op::Fpr(sc.fpr),
1656            target,
1657            patch: false,
1658        },
1659    ])
1660}
1661
1662/// Zeros a double by computing `dst -= dst`.
1663fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1664    expect_len(mn, ops, 1)?;
1665    let dst = fpr_op(mn, &ops[0])?;
1666    Ok(Inst::FArith {
1667        op: FAlu::Sub,
1668        k: FKind::F64,
1669        arith: FArith3 {
1670            dst,
1671            lhs: Op::Fpr(dst),
1672            rhs: None,
1673        },
1674    })
1675}