Skip to main content

offlineasm_ir/
expand.rs

1//! Expands mnemonic invocations into [`Inst`] sequences.
2
3use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6use smallvec::{smallvec, SmallVec};
7
8/// Inline capacity of an [`Expansion`].
9///
10/// ~97% of mnemonic expansions hold 2 or fewer nodes
11pub const EXPANSION_INLINE: usize = 2;
12
13/// The lowered instruction sequence for one mnemonic invocation.
14pub type Expansion = SmallVec<[Inst; EXPANSION_INLINE]>;
15
16/// Lowers one mnemonic invocation to the [`Inst`] sequence that implements it.
17///
18/// `sc` supplies the scratch registers that multi-instruction expansions
19/// and the destination-aliasing fixup may need. It must not alias any
20/// operand in `ops`.
21/// When `patch` is set, the mnemonic must be a [`patch_kind`] site, and the
22/// returned sequence is shaped so a backend can patch it after emission
23/// (a fixed instruction count, no operand folding).
24///
25/// # Errors
26///
27/// - [`ExpandError::NotPatchable`] if `patch` is set and `mn` has no
28///   [`patch_kind`].
29/// - [`ExpandError::BadOperand`] if `ops` has the wrong count or kind for
30///   `mn`.
31pub fn expand(
32    mn: Mnemonic,
33    ops: &[Op],
34    sc: &Scratch,
35    patch: bool,
36) -> Result<Expansion, ExpandError> {
37    if patch && patch_kind(mn).is_none() {
38        return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
39    }
40    check_extern_operands(mn, ops, patch)?;
41    let mut insts = expand_inner(mn, ops, sc, patch)?;
42    stage_aliasing_mem(&mut insts, sc);
43    Ok(insts)
44}
45
46/// Rejects external symbols outside the shapes that take one: the target of
47/// `jmp`/`call`, the source of `move`, and the source of a 64-bit `lea`
48/// (`leap`, `globaladdr`, `la`). None of them is a patch site.
49fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
50    let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
51        return Ok(());
52    };
53    if patch {
54        return Err(bad(mn, "an external symbol cannot be a patch site"));
55    }
56    let ok = match mn {
57        Mnemonic::Jmp | Mnemonic::Call => at == 0,
58        Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
59        Mnemonic::Lea {
60            w: Width::B64,
61            form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
62        } => at == 1,
63        _ => false,
64    };
65    if ok {
66        Ok(())
67    } else {
68        Err(bad(
69            mn,
70            "an external symbol is only a `jmp`/`call` target or the source of \
71             `move`/`leap`/`globaladdr`/`la`",
72        ))
73    }
74}
75
76/// Stages aliased memory operands through the scratch registers.
77///
78/// `addis t1, t0, 8[t1]` reads `8[t1]` with the *old* `t1`: all operands are
79/// evaluated before the destination is written. A backend that first lowers
80/// `dst = lhs` into `t1` and then reuses `t1` as the memory base (the
81/// natural 2-operand sequence on every target) reads `8[t1]` after the move
82/// instead and dereferences the wrong address. The fused `Alu3`/op-branch
83/// nodes hand that ordering choice to each backend, so the fix is pinned
84/// down in the front half both assemblers share: the load happens before
85/// the operation, through the expansion scratch register.
86///
87/// The oracle interprets this same sequence, and scratch is clobberable by
88/// design (it is never in the portable register pool, so it cannot alias
89/// `dst` or the operand bases).
90fn stage_aliasing_mem(insts: &mut Expansion, sc: &Scratch) {
91    let old = std::mem::take(insts);
92    let mut out = Expansion::new();
93    for mut inst in old {
94        let loads = stage_one(&mut inst, sc);
95        if let Some(loads) = loads {
96            out.extend(loads);
97        }
98        out.push(inst);
99    }
100    *insts = out;
101}
102
103/// Collects the staging loads for `inst` and rewrites its aliased operands.
104///
105/// Returns `None` when `inst` needs no staging.
106fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Expansion> {
107    let (w, operands, dst) = match inst {
108        Inst::Add { w, op }
109        | Inst::Sub { w, op }
110        | Inst::Mul { w, op }
111        | Inst::And { w, op }
112        | Inst::Or { w, op }
113        | Inst::Xor { w, op } => (*w, vec![&mut op.lhs, &mut op.rhs], &mut op.dst),
114        Inst::AluS { w, alu, .. } => (*w, vec![&mut alu.lhs, &mut alu.rhs], &mut alu.dst),
115        Inst::OpBranch { w, args, .. } => {
116            let (last, init) = args.as_mut_slice().split_last_mut()?;
117            (*w, init.iter_mut().collect(), last)
118        }
119        _ => return None,
120    };
121    let Op::Gpr(g) = *dst else {
122        return None;
123    };
124    let slots = [sc.gpr, sc.gpr2];
125    let mut loads = Expansion::new();
126    let mut slot = 0usize;
127    for operand in operands {
128        let Op::Mem(m) = operand else {
129            continue;
130        };
131        let aliases = m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g);
132        if !aliases {
133            continue;
134        }
135        // At most two memory operands can alias `dst` (op-branch takes
136        // `[lhs, rhs, dst]`), and both scratch GPRs are available.
137        let &s = slots.get(slot)?;
138        slot += 1;
139        loads.push(Inst::Load {
140            w,
141            sx: Sx::Zero,
142            dst: s,
143            addr: *m,
144            acquire: false,
145        });
146        *operand = Op::Gpr(s);
147    }
148    (!loads.is_empty()).then_some(loads)
149}
150
151/// Returns the patch shape for `mn`.
152pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
153    Some(match mn {
154        Mnemonic::Jmp => PatchKind::JumpSite,
155        Mnemonic::Call => PatchKind::CallSite,
156        Mnemonic::Move => PatchKind::MovBlock,
157        Mnemonic::FlagBranch(_)
158        | Mnemonic::OpBranch { .. }
159        | Mnemonic::RiscAluS { .. }
160        | Mnemonic::TestBranch { .. }
161        | Mnemonic::IntBranch { .. }
162        | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
163        _ => return None,
164    })
165}
166
167/// Describes what a `#[patch]` site patches.
168///
169/// Jump shapes become jump handles (in onlineasm, retargeted after
170/// emission); [`MovBlock`](PatchKind::MovBlock) becomes a 64-bit immediate
171/// slot (rewritten after emission).
172#[derive(Clone, Copy, Debug, PartialEq, Eq)]
173pub enum PatchKind {
174    /// Patches an unconditional jump target (`jmp` to a label).
175    JumpSite,
176    /// Patches a conditional branch target (a single-branch expansion).
177    CondJumpSite,
178    /// Patches a call target (`call` to a label).
179    CallSite,
180    /// Patches a 64-bit move slot (`move` of an immediate into a register).
181    MovBlock,
182}
183
184fn float_cc(cc: FCc) -> (FCc, bool) {
185    match cc {
186        FCc::Lt => (FCc::Gt, true),
187        FCc::Le => (FCc::Ge, true),
188        FCc::GtUn => (FCc::GtUn, false),
189        FCc::GeUn => (FCc::GeUn, false),
190        FCc::LtUn => (FCc::GtUn, true),
191        FCc::LeUn => (FCc::GeUn, true),
192        other => (other, false),
193    }
194}
195
196fn test_set_node(
197    mn: Mnemonic,
198    ops: &[Op],
199    w: Width,
200    test: TestKind,
201) -> Result<Expansion, ExpandError> {
202    if ops.len() != 2 && ops.len() != 3 {
203        return Err(bad(mn, "test-set takes 2 or 3 operands"));
204    }
205    let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
206    Ok(smallvec![Inst::TestSet {
207        w,
208        value: ops[1],
209        mask,
210        test,
211        dst: ops[0],
212    }])
213}
214
215fn test_branch_node(
216    mn: Mnemonic,
217    ops: &[Op],
218    w: Width,
219    test: TestKind,
220    patch: bool,
221) -> Result<Expansion, ExpandError> {
222    if ops.len() != 2 && ops.len() != 3 {
223        return Err(bad(mn, "test-branch takes 2 or 3 operands"));
224    }
225    let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
226    let target = label_op(
227        mn,
228        ops.last()
229            .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
230    )?;
231    Ok(smallvec![Inst::TestBranch {
232        w,
233        value: ops[0],
234        mask,
235        test,
236        target,
237        patch,
238    }])
239}
240
241fn div_rem_node(
242    mn: Mnemonic,
243    ops: &[Op],
244    signed: bool,
245    w: Width,
246    rem: bool,
247) -> Result<Expansion, ExpandError> {
248    let (dvd, dvs, dst) = match ops {
249        [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
250        [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
251        _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
252    };
253    Ok(smallvec![if rem {
254        Inst::Rem3 {
255            signed,
256            w,
257            dst,
258            dvd,
259            dvs,
260        }
261    } else {
262        Inst::Div3 {
263            signed,
264            w,
265            dst,
266            dvd,
267            dvs,
268        }
269    }])
270}
271
272fn shift_mem_canon(
273    mn: Mnemonic,
274    w: Width,
275    op: ShiftOp,
276    cnt: &Op,
277    dst: &Mem,
278    sc: &Scratch,
279) -> Result<Expansion, ExpandError> {
280    let shift_cnt = match cnt {
281        Op::Imm(v) => ShiftCnt::Imm(*v),
282        Op::Gpr(g) => ShiftCnt::Reg(*g),
283        _ => return Err(bad(mn, "shift count must be an immediate or register")),
284    };
285    Ok(smallvec![
286        Inst::Load {
287            w,
288            sx: Sx::Zero,
289            dst: sc.gpr,
290            addr: *dst,
291            acquire: false,
292        },
293        Inst::Shift {
294            w,
295            op,
296            dst: sc.gpr,
297            cnt: shift_cnt,
298        },
299        Inst::Store {
300            w,
301            src: Op::Gpr(sc.gpr),
302            addr: *dst,
303        },
304    ])
305}
306
307fn shift_node(
308    mn: Mnemonic,
309    w: Width,
310    op: ShiftOp,
311    ops: &[Op],
312    sc: &Scratch,
313) -> Result<Expansion, ExpandError> {
314    if ops.len() != 2 {
315        return Err(bad(mn, "shift takes 2 operands"));
316    }
317    if let Op::Mem(a) = &ops[0] {
318        return shift_mem_canon(mn, w, op, &ops[1], a, sc);
319    }
320    let dst = gpr_op(mn, &ops[0])?;
321    let cnt = match &ops[1] {
322        Op::Imm(v) => ShiftCnt::Imm(*v),
323        Op::Gpr(g) => ShiftCnt::Reg(*g),
324        _ => return Err(bad(mn, "shift count must be an immediate or register")),
325    };
326    Ok(smallvec![Inst::Shift { w, op, dst, cnt }])
327}
328
329fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
330    Alu3::from_operands(mn, ops)
331}
332
333fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<BranchArgs, ExpandError> {
334    match ops {
335        [dst, rhs] => Ok(BranchArgs::two(*rhs, *dst)),
336        [dst, lhs, rhs] => Ok(BranchArgs::three(*lhs, *rhs, *dst)),
337        _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
338    }
339}
340
341fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
342    match op {
343        Op::Label(l) => Ok(Target::Label(*l)),
344        Op::Gpr(g) => Ok(Target::Reg(*g)),
345        Op::Mem(a) => Ok(Target::Mem(*a)),
346        Op::Imm(v) => Ok(Target::Abs(*v)),
347        Op::Extern(e) => Ok(Target::Extern(*e)),
348        _ => Err(bad(
349            mn,
350            "jump/call needs a label, register, memory, or immediate",
351        )),
352    }
353}
354
355#[allow(clippy::too_many_lines)]
356fn expand_inner(
357    mn: Mnemonic,
358    ops: &[Op],
359    sc: &Scratch,
360    patch: bool,
361) -> Result<Expansion, ExpandError> {
362    let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
363        expect_len(mn, ops, 1)?;
364        Ok(ops[0])
365    };
366    match mn {
367        Mnemonic::Move => Ok(smallvec![mov_node(mn, ops, Width::B64, patch)?]),
368        Mnemonic::Load(w) => Ok(smallvec![mov_node(mn, ops, w, patch)?]),
369        Mnemonic::AtomicLoad(w) => Ok(smallvec![atomic_load_node(mn, ops, w)?]),
370        Mnemonic::Store(w) => Ok(smallvec![store_node(mn, ops, w)?]),
371        Mnemonic::Extend { from, to, sx } => Ok(smallvec![extend_node(mn, ops, from, to, sx)?]),
372        Mnemonic::MovZero32 => Ok(smallvec![movzero_node(mn, ops)?]),
373        Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
374        Mnemonic::FMove(k) => Ok(smallvec![fmove_node(mn, ops, k)?]),
375        Mnemonic::VMove { store } => Ok(smallvec![vmove_node(mn, ops, store)?]),
376        Mnemonic::Lea { w, .. } => Ok(smallvec![lea_node(mn, ops, w)?]),
377        Mnemonic::LoadQInc => loadqinc_node(mn, ops),
378        Mnemonic::LoadPair(w) => Ok(smallvec![pair_node(mn, ops, w, true)?]),
379        Mnemonic::StorePair(w) => Ok(smallvec![pair_node(mn, ops, w, false)?]),
380        Mnemonic::Push => stack_nodes(mn, ops, true, false),
381        Mnemonic::Pop => stack_nodes(mn, ops, false, false),
382        Mnemonic::PushV => stack_nodes(mn, ops, true, true),
383        Mnemonic::PopV => stack_nodes(mn, ops, false, true),
384        Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
385        Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
386        Mnemonic::Peek => Ok(smallvec![peek_node(mn, ops, true)?]),
387        Mnemonic::Poke => Ok(smallvec![peek_node(mn, ops, false)?]),
388        Mnemonic::TlsLoad(w) => Ok(smallvec![tls_node(mn, ops, w, true)?]),
389        Mnemonic::TlsStore(w) => Ok(smallvec![tls_node(mn, ops, w, false)?]),
390        Mnemonic::TlsLea => Ok(smallvec![tls_lea_node(mn, ops)?]),
391
392        Mnemonic::Add(w) => Ok(smallvec![Inst::Add {
393            w,
394            op: alu3(mn, ops)?,
395        }]),
396        Mnemonic::Sub(w) => Ok(smallvec![Inst::Sub {
397            w,
398            op: alu3(mn, ops)?,
399        }]),
400        Mnemonic::Mul(w) => Ok(smallvec![mul_node(mn, w, ops)?]),
401        Mnemonic::And(w) => Ok(smallvec![Inst::And {
402            w,
403            op: alu3(mn, ops)?,
404        }]),
405        Mnemonic::Or(w) => Ok(smallvec![Inst::Or {
406            w,
407            op: alu3(mn, ops)?,
408        }]),
409        Mnemonic::Xor(w) => Ok(smallvec![Inst::Xor {
410            w,
411            op: alu3(mn, ops)?,
412        }]),
413        Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
414        Mnemonic::Not(w) => Ok(smallvec![Inst::Not {
415            w,
416            dst: one(mn, ops)?,
417        }]),
418        Mnemonic::Neg(w) => Ok(smallvec![Inst::Neg {
419            w,
420            dst: one(mn, ops)?,
421        }]),
422        Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
423        Mnemonic::Bswap(w) => Ok(smallvec![bswap_node(mn, ops, w)?]),
424        Mnemonic::Adc => Ok(smallvec![adc_node(mn, ops, false)?]),
425        Mnemonic::Sbb => Ok(smallvec![adc_node(mn, ops, true)?]),
426        Mnemonic::MulHigh { signed } => Ok(smallvec![Inst::MulHigh {
427            signed,
428            rhs: one(mn, ops)?,
429        }]),
430        Mnemonic::MulWide { signed } => Ok(smallvec![mulwide_node(mn, ops, signed)?]),
431        Mnemonic::AluS { op, w } => Ok(smallvec![Inst::AluS {
432            op,
433            w,
434            alu: alu3(mn, ops)?,
435        }]),
436        Mnemonic::RiscAluS { op, w } => {
437            let has_label =
438                matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
439            if has_label {
440                let flag_op = match op {
441                    AluOp::Add => FlagAlu::Add,
442                    AluOp::Sub => FlagAlu::Sub,
443                    AluOp::Or => FlagAlu::Or,
444                };
445                return Ok(smallvec![Inst::OpBranch {
446                    op: flag_op,
447                    w,
448                    flag: PredFlag::S,
449                    args: op_branch_args(mn, &ops[..ops.len() - 1])?,
450                    target: label_op(
451                        mn,
452                        ops.last()
453                            .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
454                    )?,
455                    patch,
456                }]);
457            }
458            Ok(smallvec![Inst::AluS {
459                op,
460                w,
461                alu: alu3(mn, ops)?,
462            }])
463        }
464        Mnemonic::AddShifted => Ok(smallvec![addshifted_node(mn, ops)?]),
465        Mnemonic::Cqo { quad } => Ok(smallvec![Inst::Cqo { quad }]),
466        Mnemonic::DivMod1 { signed, w } => Ok(smallvec![Inst::DivMod1 {
467            signed,
468            w,
469            divisor: one(mn, ops)?,
470        }]),
471        Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
472        Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
473        Mnemonic::IntSet { w, cc } => {
474            expect_len(mn, ops, 3)?;
475            Ok(smallvec![Inst::IntSet {
476                cc,
477                w,
478                dst: ops[0],
479                lhs: ops[1],
480                rhs: ops[2],
481            }])
482        }
483        Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
484        Mnemonic::Cmov { zero } => Ok(smallvec![cmov_node(mn, ops, zero)?]),
485        Mnemonic::BitCount { w, trailing } => Ok(smallvec![cz_node(mn, ops, w, trailing)?]),
486        Mnemonic::Popcnt(w) => Ok(smallvec![popcnt_node(mn, ops, w)?]),
487        Mnemonic::Bfi => bfi_node(mn, ops, sc),
488        Mnemonic::IntBranch { w, cc } => {
489            expect_len(mn, ops, 3)?;
490            Ok(smallvec![Inst::Branch {
491                cc,
492                w,
493                lhs: ops[0],
494                rhs: ops[1],
495                target: label_op(mn, &ops[2])?,
496                patch,
497            }])
498        }
499        Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
500        Mnemonic::OpBranch { op, w, flag } => {
501            if ops.len() != 3 && ops.len() != 4 {
502                return Err(bad(mn, "op-branch takes 3 or 4 operands"));
503            }
504            Ok(smallvec![Inst::OpBranch {
505                op,
506                w,
507                flag,
508                args: op_branch_args(mn, &ops[..ops.len() - 1])?,
509                target: label_op(
510                    mn,
511                    ops.last()
512                        .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
513                )?,
514                patch,
515            }])
516        }
517        Mnemonic::FlagBranch(flag) => Ok(smallvec![flag_branch_node(mn, ops, flag, patch)?]),
518        Mnemonic::BranchF { cc, k } => {
519            expect_len(mn, ops, 3)?;
520            let (cc, reverse) = float_cc(cc);
521            let (lhs, rhs) = if reverse {
522                (ops[1], ops[0])
523            } else {
524                (ops[0], ops[1])
525            };
526            Ok(smallvec![Inst::BranchF {
527                cc,
528                k,
529                lhs,
530                rhs,
531                target: label_op(mn, &ops[2])?,
532                patch,
533            }])
534        }
535        Mnemonic::Jmp => {
536            expect_len(mn, ops, 1)?;
537            Ok(smallvec![Inst::Jump {
538                target: jump_target(mn, &ops[0])?,
539                patch,
540            }])
541        }
542        Mnemonic::Call => {
543            expect_len(mn, ops, 1)?;
544            Ok(smallvec![Inst::Call {
545                target: jump_target(mn, &ops[0])?,
546                patch,
547            }])
548        }
549        Mnemonic::Ret => {
550            expect_len(mn, ops, 0)?;
551            Ok(smallvec![Inst::Ret])
552        }
553        Mnemonic::Break => {
554            expect_len(mn, ops, 0)?;
555            Ok(smallvec![Inst::Break])
556        }
557        Mnemonic::Nop => {
558            expect_len(mn, ops, 0)?;
559            Ok(smallvec![Inst::Nop])
560        }
561        Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
562        Mnemonic::Fma { op, k } => Ok(smallvec![fma_node(mn, ops, k, op)?]),
563        Mnemonic::FSqrt(k) => Ok(smallvec![fsqrt_node(mn, ops, k)?]),
564        Mnemonic::FRound { k, mode } => Ok(smallvec![fround_node(mn, ops, k, mode)?]),
565        Mnemonic::FAbs { k, negate } => Ok(smallvec![fabs_node(mn, ops, k, negate)?]),
566        Mnemonic::FSet { cc, k } => {
567            expect_len(mn, ops, 3)?;
568            let (cc, reverse) = float_cc(cc);
569            let (lhs, rhs) = if reverse {
570                (ops[2], ops[1])
571            } else {
572                (ops[1], ops[2])
573            };
574            Ok(smallvec![Inst::FSet {
575                cc,
576                k,
577                dst: ops[0],
578                lhs,
579                rhs,
580            }])
581        }
582        Mnemonic::CvtFI { k, w } => Ok(smallvec![cvtfi_node(mn, ops, k, w)?]),
583        Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
584        Mnemonic::CvtIF { signed, from, k } => Ok(smallvec![cvtif_node(mn, ops, signed, from, k)?]),
585        Mnemonic::CvtFF { from, to } => Ok(smallvec![cvtff_node(mn, ops, from, to)?]),
586        Mnemonic::Bitcast(dir) => Ok(smallvec![bitcast_node(mn, ops, dir)?]),
587        Mnemonic::MovDZ => Ok(smallvec![movdz_node(mn, ops)?]),
588        Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
589        Mnemonic::Btd2i => btd2i_node(mn, ops),
590        Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
591        Mnemonic::VecExtract(w) => Ok(smallvec![umov_node(mn, ops, w)?]),
592        Mnemonic::AtomicRmw { op, w } => Ok(smallvec![rmw_node(mn, ops, w, op)?]),
593        Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
594        Mnemonic::AtomicCas { w, branch } => Ok(smallvec![cas_node(mn, ops, w, branch)?]),
595        Mnemonic::LoadLinked(w) => Ok(smallvec![ll_node(mn, ops, w)?]),
596        Mnemonic::StoreCond(w) => Ok(smallvec![sc_node(mn, ops, w)?]),
597        Mnemonic::Fence { full } => Ok(smallvec![Inst::Fence { full }]),
598        Mnemonic::Pause => Ok(smallvec![Inst::Pause]),
599        Mnemonic::TagCodePtr
600        | Mnemonic::TagReturnAddress
601        | Mnemonic::UntagReturnAddress
602        | Mnemonic::RemoveCodePtrTag
603        | Mnemonic::UntagArrayPtr
604        | Mnemonic::RemoveArrayPtrTag => Ok(smallvec![]),
605    }
606}
607
608fn flag_branch_node(
609    mn: Mnemonic,
610    ops: &[Op],
611    flag: PredFlag,
612    patch: bool,
613) -> Result<Inst, ExpandError> {
614    expect_len(mn, ops, 1)?;
615    Ok(Inst::FlagBranch {
616        flag,
617        target: label_op(mn, &ops[0])?,
618        patch,
619    })
620}
621
622fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
623    expect_len(mn, ops, 2)?;
624    Ok(Inst::Popcnt {
625        w,
626        dst: gpr_op(mn, &ops[0])?,
627        src: ops[1],
628    })
629}
630
631fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
632    expect_len(mn, ops, 2)?;
633    let dst = gpr_op(mn, &ops[0])?;
634    let inst = if trailing {
635        Inst::Ctz {
636            w,
637            dst,
638            src: ops[1],
639        }
640    } else {
641        Inst::Clz {
642            w,
643            dst,
644            src: ops[1],
645        }
646    };
647    Ok(inst)
648}
649
650fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
651    expect_len(mn, ops, 2)?;
652    if load {
653        let idx = imm_op(mn, &ops[1])?;
654        Ok(Inst::TlsLoad {
655            w,
656            dst: gpr_op(mn, &ops[0])?,
657            idx,
658        })
659    } else {
660        let idx = imm_op(mn, &ops[0])?;
661        Ok(Inst::TlsStore {
662            w,
663            src: gpr_op(mn, &ops[1])?,
664            idx,
665        })
666    }
667}
668
669fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
670    expect_len(mn, ops, 2)?;
671    Ok(Inst::TlsLea {
672        dst: gpr_op(mn, &ops[0])?,
673        idx: imm_op(mn, &ops[1])?,
674    })
675}
676
677fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
678    expect_len(mn, ops, 2)?;
679
680    Ok(Inst::AtomicRmw {
681        op,
682        w,
683        a: ops[1],
684        b: ops[0],
685    })
686}
687
688fn rmw_not_node(
689    mn: Mnemonic,
690    ops: &[Op],
691    sc: &Scratch,
692    w: Width,
693    op: AtomicOp,
694) -> Result<Expansion, ExpandError> {
695    expect_len(mn, ops, 2)?;
696    let mut out = Expansion::new();
697    // `mem` is ops[0] and `src` is ops[1].
698    let a = match &ops[1] {
699        Op::Imm(v) => Op::Imm(!v),
700        Op::Gpr(g) => {
701            out.push(Inst::Mov {
702                w: Width::B64,
703                dst: Op::Gpr(sc.gpr),
704                src: Op::Gpr(*g),
705                patch: false,
706            });
707            out.push(Inst::Not {
708                w,
709                dst: Op::Gpr(sc.gpr),
710            });
711            Op::Gpr(sc.gpr)
712        }
713        Op::Mem(addr) => {
714            let lw = if w == Width::B64 {
715                Width::B64
716            } else {
717                Width::B32
718            };
719            out.push(Inst::Load {
720                w: lw,
721                sx: Sx::Zero,
722                dst: sc.gpr,
723                addr: *addr,
724                acquire: false,
725            });
726            out.push(Inst::Not {
727                w,
728                dst: Op::Gpr(sc.gpr),
729            });
730            Op::Gpr(sc.gpr)
731        }
732        _ => {
733            return Err(bad(
734                mn,
735                "atomic clear needs an immediate, register, or memory operand",
736            ))
737        }
738    };
739    out.push(Inst::AtomicRmw {
740        op,
741        w,
742        a,
743        b: ops[0],
744    });
745    Ok(out)
746}
747
748fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
749    let n = if branch { 4 } else { 3 };
750    expect_len(mn, ops, n)?;
751
752    Ok(Inst::AtomicCas {
753        w,
754        mem: mem_op(mn, &ops[0])?,
755        src: gpr_op(mn, &ops[2])?,
756        branch: if branch {
757            Some(label_op(mn, &ops[3])?)
758        } else {
759            None
760        },
761    })
762}
763
764fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
765    expect_len(mn, ops, 2)?;
766    Ok(Inst::LoadLinked {
767        w,
768        mem: mem_op(mn, &ops[1])?,
769        dst: gpr_op(mn, &ops[0])?,
770    })
771}
772
773fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
774    expect_len(mn, ops, 3)?;
775    Ok(Inst::StoreCond {
776        w,
777        status: gpr_op(mn, &ops[1])?,
778        src: gpr_op(mn, &ops[2])?,
779        mem: mem_op(mn, &ops[0])?,
780    })
781}
782
783/// Expands `bfiq` to a mask, shift, and OR sequence.
784///
785/// Computes `dst = (dst & ~mask) | ((src & wmask) << lsb)`.
786fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
787    expect_len(mn, ops, 4)?;
788    let lsb = imm_op(mn, &ops[2])?;
789    let width = imm_op(mn, &ops[3])?;
790    if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
791        return Err(bad(mn, "bitfield out of range"));
792    }
793    let wmask: u64 = if width == 64 {
794        u64::MAX
795    } else {
796        (1u64 << width) - 1
797    };
798    let mask: i64 = (wmask << lsb) as i64;
799    let s = sc.gpr;
800    let dst = gpr_op(mn, &ops[0])?;
801
802    Ok(smallvec![
803        Inst::Mov {
804            w: Width::B64,
805            dst: Op::Gpr(s),
806            src: ops[1],
807            patch: false,
808        },
809        Inst::And {
810            w: Width::B64,
811            op: Alu3 {
812                dst: Op::Gpr(s),
813                lhs: Op::Gpr(s),
814                rhs: Op::Imm(wmask as i64),
815            },
816        },
817        Inst::Shift {
818            w: Width::B64,
819            op: ShiftOp::Shl,
820            dst: s,
821            cnt: ShiftCnt::Imm(lsb),
822        },
823        Inst::And {
824            w: Width::B64,
825            op: Alu3 {
826                dst: Op::Gpr(dst),
827                lhs: Op::Gpr(dst),
828                rhs: Op::Imm(!mask),
829            },
830        },
831        Inst::Or {
832            w: Width::B64,
833            op: Alu3 {
834                dst: Op::Gpr(dst),
835                lhs: Op::Gpr(dst),
836                rhs: Op::Gpr(s),
837            },
838        },
839    ])
840}
841
842fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
843    expect_len(mn, ops, 3)?;
844    Ok(Inst::Cmov {
845        w: Width::B64,
846        zero,
847        dst: gpr_op(mn, &ops[0])?,
848        src: ops[1],
849        test: gpr_op(mn, &ops[2])?,
850    })
851}
852
853/// Expands `peek dst, off` and `poke off, src`, with the destination first
854/// in both. The offset counts quad slots and is scaled here.
855fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
856    expect_len(mn, ops, 2)?;
857    if peek {
858        let disp = slot_disp(mn, &ops[1])?;
859        return Ok(Inst::Peek {
860            disp,
861            dst: gpr_op(mn, &ops[0])?,
862        });
863    }
864    Ok(Inst::Poke {
865        disp: slot_disp(mn, &ops[0])?,
866        src: gpr_op(mn, &ops[1])?,
867    })
868}
869
870fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
871    let idx = imm_op(mn, op)?;
872    let off = idx
873        .checked_mul(8)
874        .ok_or_else(|| bad(mn, "stack offset overflow"))?;
875    if off < i32::MIN as i64 || off > i32::MAX as i64 {
876        return Err(bad(mn, "stack offset out of range"));
877    }
878    Ok(off as i32)
879}
880
881fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
882    expect_len(mn, ops, 3)?;
883    let dst = gpr_op(mn, &ops[0])?;
884    let base = gpr_op(mn, &ops[1])?;
885    let inc = imm_op(mn, &ops[2])?;
886    Ok(smallvec![
887        Inst::Load {
888            w: Width::B64,
889            sx: Sx::Zero,
890            dst,
891            addr: Mem::base(base, 0),
892            acquire: false,
893        },
894        Inst::Add {
895            w: Width::B64,
896            op: Alu3 {
897                dst: Op::Gpr(base),
898                lhs: Op::Gpr(base),
899                rhs: Op::Imm(inc),
900            },
901        },
902    ])
903}
904
905fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
906    expect_len(mn, ops, 3)?;
907    if load {
908        Ok(Inst::PairLoad {
909            w,
910            dst1: ops[0],
911            dst2: ops[1],
912            mem: mem_op(mn, &ops[2])?,
913        })
914    } else {
915        Ok(Inst::PairStore {
916            w,
917            src1: ops[1],
918            src2: ops[2],
919            mem: mem_op(mn, &ops[0])?,
920        })
921    }
922}
923
924fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
925    expect_len(mn, ops, 2)?;
926    let dst = gpr_op(mn, &ops[0])?;
927    let src = match &ops[1] {
928        Op::Mem(a) => LeaSrc::Mem(*a),
929        Op::Label(l) => LeaSrc::Label(*l),
930        Op::Extern(e) => LeaSrc::Extern(*e),
931        _ => return Err(bad(mn, "lea needs an address or label source")),
932    };
933    Ok(Inst::Lea { w, dst, src })
934}
935
936fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
937    expect_len(mn, ops, 2)?;
938    // `move reg, extern` takes the symbol's address, like `leap`.
939    if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
940        return Ok(Inst::Lea {
941            w: Width::B64,
942            dst: *dst,
943            src: LeaSrc::Extern(*e),
944        });
945    }
946    Ok(Inst::Mov {
947        w,
948        dst: ops[0],
949        src: ops[1],
950        patch,
951    })
952}
953
954fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
955    expect_len(mn, ops, 2)?;
956    Ok(Inst::MovZero32 {
957        dst: ops[0],
958        src: ops[1],
959    })
960}
961
962fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
963    expect_len(mn, ops, 2)?;
964    Ok(Inst::Load {
965        w,
966        sx: Sx::Zero,
967        dst: gpr_op(mn, &ops[0])?,
968        addr: mem_op(mn, &ops[1])?,
969        acquire: true,
970    })
971}
972
973fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
974    expect_len(mn, ops, 2)?;
975    let addr = mem_op(mn, &ops[0])?;
976    Ok(Inst::Store {
977        w,
978        src: ops[1],
979        addr,
980    })
981}
982
983fn extend_node(
984    mn: Mnemonic,
985    ops: &[Op],
986    from: Width,
987    to: Width,
988    sx: Sx,
989) -> Result<Inst, ExpandError> {
990    expect_len(mn, ops, 2)?;
991    Ok(Inst::Extend {
992        from,
993        to,
994        sx,
995        dst: gpr_op(mn, &ops[0])?,
996        src: ops[1],
997    })
998}
999
1000fn transfer_node(
1001    mn: Mnemonic,
1002    ops: &[Op],
1003    sc: &Scratch,
1004    w: Width,
1005) -> Result<Expansion, ExpandError> {
1006    expect_len(mn, ops, 2)?;
1007    let mut out = Expansion::new();
1008    match &ops[1] {
1009        Op::Gpr(_) => {}
1010        Op::Mem(a) => {
1011            out.push(Inst::Load {
1012                w,
1013                sx: Sx::Zero,
1014                dst: sc.gpr,
1015                addr: *a,
1016                acquire: false,
1017            });
1018        }
1019        Op::Imm(v) => {
1020            out.push(Inst::Mov {
1021                w,
1022                dst: Op::Gpr(sc.gpr),
1023                src: Op::Imm(*v),
1024                patch: false,
1025            });
1026        }
1027        Op::Label(l) => {
1028            out.push(Inst::Lea {
1029                w: Width::B64,
1030                dst: sc.gpr,
1031                src: LeaSrc::Label(*l),
1032            });
1033            let addr = Mem::base(sc.gpr, 0);
1034            out.push(Inst::Load {
1035                w,
1036                sx: Sx::Zero,
1037                dst: sc.gpr,
1038                addr,
1039                acquire: false,
1040            });
1041        }
1042        Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1043        Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1044        Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1045    }
1046    let src = if out.is_empty() {
1047        ops[1]
1048    } else {
1049        Op::Gpr(sc.gpr)
1050    };
1051    out.push(Inst::Mov {
1052        w,
1053        dst: ops[0],
1054        src,
1055        patch: false,
1056    });
1057    Ok(out)
1058}
1059
1060fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1061    expect_len(mn, ops, 2)?;
1062    match (&ops[0], &ops[1]) {
1063        (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1064            k,
1065            dst: *d,
1066            src: *s,
1067        }),
1068        (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1069            k,
1070            dst: *d,
1071            addr: *a,
1072        }),
1073        (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1074            k,
1075            src: *s,
1076            addr: *a,
1077        }),
1078        _ => Err(bad(mn, "float move needs FPR/memory operands")),
1079    }
1080}
1081
1082fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1083    expect_len(mn, ops, 2)?;
1084    if !store {
1085        match (&ops[0], &ops[1]) {
1086            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1087            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1088            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1089                dst: *d,
1090                addr: Mem::base(*s, 0),
1091            }),
1092            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1093        }
1094    } else {
1095        match (&ops[0], &ops[1]) {
1096            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1097            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1098            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1099                dst: *d,
1100                addr: Mem::base(*s, 0),
1101            }),
1102            (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1103            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1104        }
1105    }
1106}
1107
1108/// Checks that every callee-save operand is a GPR in `saved`.
1109///
1110/// Does nothing for every other mnemonic.
1111///
1112/// # Errors
1113///
1114/// [`ExpandError::BadOperand`] if `mnemonic` is `PushCalleeSaves` or
1115/// `PopCalleeSaves` and `ops` contains a non-GPR operand or a GPR whose
1116/// number is not in `saved`.
1117pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1118    if !matches!(
1119        mnemonic,
1120        Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1121    ) {
1122        return Ok(());
1123    }
1124    for op in ops {
1125        match op {
1126            Op::Gpr(g) if saved.contains(&g.0) => {}
1127            _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1128        }
1129    }
1130    Ok(())
1131}
1132
1133/// Bytes one AArch64 `push`/`pop` of `n` quads moves `sp` by: `8 * n`
1134/// rounded up to 16, because `sp` must stay 16-byte aligned whenever it is
1135/// used as a base address.
1136///
1137/// The operands keep the layout they have on the targets with 8-byte slots
1138/// (see [`aligned_push_offset`]), and an odd count leaves one padding word
1139/// above them. A `pop` must name as many operands as the `push` it undoes.
1140pub fn aligned_push_bytes(n: usize) -> i32 {
1141    ((8 * n + 15) & !15) as i32
1142}
1143
1144/// Offset from the lowered `sp` of operand `i` in an `n`-operand AArch64
1145/// `push`/`pop` group. The last operand sits at `[sp]`, as it would after
1146/// `n` separate 8-byte pushes.
1147pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1148    8 * (n - 1 - i) as i32
1149}
1150
1151/// Lowers one `push`/`pop` operand to its [`Inst`].
1152fn push_inst(mn: Mnemonic, op: &Op, push: bool, vector: bool) -> Result<Inst, ExpandError> {
1153    if vector {
1154        let x = fpr_op(mn, op)?;
1155        Ok(if push {
1156            Inst::PushV { src: x }
1157        } else {
1158            Inst::PopV { dst: x }
1159        })
1160    } else {
1161        match op {
1162            Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => Ok(if push {
1163                Inst::Push { op: *op }
1164            } else {
1165                Inst::Pop { op: *op }
1166            }),
1167            _ => Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1168        }
1169    }
1170}
1171
1172fn stack_nodes(
1173    mn: Mnemonic,
1174    ops: &[Op],
1175    push: bool,
1176    vector: bool,
1177) -> Result<Expansion, ExpandError> {
1178    if ops.is_empty() {
1179        return Err(bad(mn, "needs at least one operand"));
1180    }
1181    let mut out = Expansion::new();
1182    // `push a, b` stores `a` then `b`, leaving `b` on top; `pop a, b` must
1183    // unwind LIFO, so it reads `b` before `a` - the reverse of the operand
1184    // list order that `push` uses.
1185    if push {
1186        for op in ops {
1187            out.push(push_inst(mn, op, true, vector)?);
1188        }
1189    } else {
1190        for op in ops.iter().rev() {
1191            out.push(push_inst(mn, op, false, vector)?);
1192        }
1193    }
1194    Ok(out)
1195}
1196
1197fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1198    if ops.len() == 3 {
1199        if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1200            return Ok(Inst::MulImm {
1201                w,
1202                dst: gpr_op(mn, &ops[0])?,
1203                lhs,
1204                imm: *v,
1205            });
1206        }
1207        if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1208            return Ok(Inst::MulImm {
1209                w,
1210                dst: gpr_op(mn, &ops[0])?,
1211                lhs,
1212                imm: *v,
1213            });
1214        }
1215    }
1216    Ok(Inst::Mul {
1217        w,
1218        op: Alu3::from_operands(mn, ops)?,
1219    })
1220}
1221
1222fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1223    expect_len(mn, ops, 3)?;
1224    Ok(Inst::MulWide {
1225        signed,
1226        dst: gpr_op(mn, &ops[0])?,
1227        lhs: gpr_op(mn, &ops[1])?,
1228        rhs: gpr_op(mn, &ops[2])?,
1229    })
1230}
1231
1232fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1233    expect_len(mn, ops, 4)?;
1234    let amount = imm_op(mn, &ops[3])?;
1235    if !(0..=63).contains(&amount) {
1236        return Err(bad(mn, "shift amount out of range"));
1237    }
1238    Ok(Inst::AddShifted {
1239        dst: gpr_op(mn, &ops[0])?,
1240        lhs: gpr_op(mn, &ops[1])?,
1241        rhs: gpr_op(mn, &ops[2])?,
1242        amount: amount as u8,
1243    })
1244}
1245
1246fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1247    expect_len(mn, ops, 2)?;
1248    let inst = if borrow {
1249        Inst::Sbb {
1250            dst: ops[0],
1251            src: ops[1],
1252        }
1253    } else {
1254        Inst::Adc {
1255            dst: ops[0],
1256            src: ops[1],
1257        }
1258    };
1259    Ok(inst)
1260}
1261
1262fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Expansion, ExpandError> {
1263    if ops.len() != 2 && ops.len() != 3 {
1264        return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1265    }
1266    let mut out = Expansion::new();
1267    let (src, dst) = if ops.len() == 3 {
1268        let d = fpr_op(mn, &ops[0])?;
1269        if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1270            // `O1 = O2 op O1`: and/or commute, so fold `O2` into `O1`.
1271            (ops[1], d)
1272        } else {
1273            if !op_eq(&ops[1], &ops[0]) {
1274                match &ops[1] {
1275                    Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1276                    Op::Mem(a) => out.push(Inst::LoadF {
1277                        k,
1278                        dst: d,
1279                        addr: *a,
1280                    }),
1281                    _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1282                }
1283            }
1284            (ops[2], d)
1285        }
1286    } else {
1287        (ops[1], fpr_op(mn, &ops[0])?)
1288    };
1289    out.push(Inst::FBit { or, k, dst, src });
1290    Ok(out)
1291}
1292
1293fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Expansion, ExpandError> {
1294    if ops.len() == 3 {
1295        return Ok(smallvec![Inst::FArith {
1296            op,
1297            k,
1298            arith: FArith3 {
1299                dst: fpr_op(mn, &ops[0])?,
1300                lhs: ops[1],
1301                rhs: Some(ops[2]),
1302            },
1303        }]);
1304    }
1305    Ok(smallvec![Inst::FArith {
1306        op,
1307        k,
1308        arith: FArith3::from_operands(mn, ops)?,
1309    }])
1310}
1311
1312fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1313    expect_len(mn, ops, 2)?;
1314    Ok(Inst::FSqrt {
1315        k,
1316        dst: fpr_op(mn, &ops[0])?,
1317        src: ops[1],
1318    })
1319}
1320
1321fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1322    expect_len(mn, ops, 4)?;
1323    Ok(Inst::Fma {
1324        op,
1325        k,
1326        a: ops[1],
1327        b: ops[2],
1328        c: ops[3],
1329        dst: fpr_op(mn, &ops[0])?,
1330    })
1331}
1332
1333/// Reverses the bytes of `src` into `dst`.
1334fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1335    expect_len(mn, ops, 2)?;
1336    Ok(Inst::Bswap {
1337        w,
1338        dst: ops[0],
1339        src: ops[1],
1340    })
1341}
1342
1343fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1344    expect_len(mn, ops, 2)?;
1345    Ok(Inst::FRound {
1346        k,
1347        mode,
1348        dst: fpr_op(mn, &ops[0])?,
1349        src: ops[1],
1350    })
1351}
1352
1353fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1354    expect_len(mn, ops, 2)?;
1355    Ok(Inst::CvtFI {
1356        k,
1357        w,
1358        dst: gpr_op(mn, &ops[0])?,
1359        src: ops[1],
1360    })
1361}
1362
1363fn cvtif_node(
1364    mn: Mnemonic,
1365    ops: &[Op],
1366    signed: bool,
1367    from: Width,
1368    k: FKind,
1369) -> Result<Inst, ExpandError> {
1370    expect_len(mn, ops, 2)?;
1371    Ok(Inst::CvtIF {
1372        signed,
1373        from,
1374        k,
1375        dst: fpr_op(mn, &ops[0])?,
1376        src: ops[1],
1377    })
1378}
1379
1380fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1381    expect_len(mn, ops, 2)?;
1382    Ok(Inst::CvtFF {
1383        from,
1384        to,
1385        dst: fpr_op(mn, &ops[0])?,
1386        src: ops[1],
1387    })
1388}
1389
1390fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1391    expect_len(mn, ops, 2)?;
1392    Ok(Inst::Bitcast {
1393        dir,
1394        dst: ops[0],
1395        src: ops[1],
1396    })
1397}
1398
1399fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1400    expect_len(mn, ops, 2)?;
1401    let dst = fpr_op(mn, &ops[0])?;
1402    let inst = if negate {
1403        Inst::FNeg {
1404            k,
1405            dst,
1406            src: ops[1],
1407        }
1408    } else {
1409        Inst::FAbs {
1410            k,
1411            dst,
1412            src: ops[1],
1413        }
1414    };
1415    Ok(inst)
1416}
1417
1418fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1419    expect_len(mn, ops, 3)?;
1420    let lane = imm_op(mn, &ops[2])?;
1421    let lanes: i64 = 128 / w.bytes() as i64;
1422    if lane < 0 || lane >= lanes {
1423        return Err(bad(mn, "lane out of range"));
1424    }
1425    Ok(Inst::VecExtract {
1426        w,
1427        dst: gpr_op(mn, &ops[0])?,
1428        src: fpr_op(mn, &ops[1])?,
1429        lane: lane as u8,
1430    })
1431}
1432
1433fn trunc_q_slow(
1434    mn: Mnemonic,
1435    ops: &[Op],
1436    sc: &Scratch,
1437    k: FKind,
1438) -> Result<Expansion, ExpandError> {
1439    expect_len(mn, ops, 2)?;
1440    let (dst, src) = (ops[0], ops[1]);
1441    let slow = Lbl::Fresh(0);
1442    let done = Lbl::Fresh(1);
1443    let gs = Op::Gpr(sc.gpr);
1444    let fs = Op::Fpr(sc.fpr);
1445    let (int64_min, neg_min) = match k {
1446        FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1447        FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1448    };
1449    let dst_gpr = gpr_op(mn, &dst)?;
1450    Ok(smallvec![
1451        Inst::Mov {
1452            w: Width::B64,
1453            dst: gs,
1454            src: Op::Imm(neg_min),
1455            patch: false,
1456        },
1457        Inst::Bitcast {
1458            dir: if k == FKind::F32 {
1459                BitcastDir::I2F
1460            } else {
1461                BitcastDir::Q2D
1462            },
1463            dst: fs,
1464            src: gs,
1465        },
1466        Inst::BranchF {
1467            cc: FCc::Ge,
1468            k,
1469            lhs: src,
1470            rhs: fs,
1471            target: slow,
1472            patch: false,
1473        },
1474        Inst::CvtFI {
1475            k,
1476            w: Width::B64,
1477            dst: dst_gpr,
1478            src,
1479        },
1480        Inst::Jump {
1481            target: Target::Label(done),
1482            patch: false,
1483        },
1484        Inst::Bind(slow),
1485        Inst::Mov {
1486            w: Width::B64,
1487            dst: gs,
1488            src: Op::Imm(int64_min),
1489            patch: false,
1490        },
1491        Inst::Bitcast {
1492            dir: if k == FKind::F32 {
1493                BitcastDir::I2F
1494            } else {
1495                BitcastDir::Q2D
1496            },
1497            dst: fs,
1498            src: gs,
1499        },
1500        Inst::FArith {
1501            op: FAlu::Add,
1502            k,
1503            arith: FArith3 {
1504                dst: sc.fpr,
1505                lhs: src,
1506                rhs: None,
1507            },
1508        },
1509        Inst::CvtFI {
1510            k,
1511            w: Width::B64,
1512            dst: dst_gpr,
1513            src: fs,
1514        },
1515        Inst::Mov {
1516            w: Width::B64,
1517            dst: gs,
1518            src: Op::Imm(0x8000000000000000u64 as i64),
1519            patch: false,
1520        },
1521        Inst::Or {
1522            w: Width::B64,
1523            op: Alu3 {
1524                dst,
1525                lhs: dst,
1526                rhs: gs,
1527            },
1528        },
1529        Inst::Bind(done),
1530    ])
1531}
1532
1533fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Expansion, ExpandError> {
1534    expect_len(mn, ops, 3)?;
1535    let (src, dst) = (ops[1], ops[0]);
1536    let slow = Lbl::Fresh(0);
1537    let done = Lbl::Fresh(1);
1538    let s1 = gpr_op(mn, &ops[2])?;
1539    let s2 = sc.gpr;
1540    let dst_fpr = fpr_op(mn, &dst)?;
1541    Ok(smallvec![
1542        Inst::TestBranch {
1543            w: Width::B64,
1544            value: src,
1545            mask: None,
1546            test: TestKind::Sign,
1547            target: slow,
1548            patch: false,
1549        },
1550        Inst::CvtIF {
1551            signed: true,
1552            from: Width::B64,
1553            k,
1554            dst: dst_fpr,
1555            src,
1556        },
1557        Inst::Jump {
1558            target: Target::Label(done),
1559            patch: false,
1560        },
1561        Inst::Bind(slow),
1562        Inst::Mov {
1563            w: Width::B64,
1564            dst: Op::Gpr(s1),
1565            src,
1566            patch: false,
1567        },
1568        Inst::Mov {
1569            w: Width::B64,
1570            dst: Op::Gpr(s2),
1571            src,
1572            patch: false,
1573        },
1574        Inst::Shift {
1575            w: Width::B64,
1576            op: ShiftOp::Shr,
1577            dst: s1,
1578            cnt: ShiftCnt::Imm(1),
1579        },
1580        Inst::And {
1581            w: Width::B64,
1582            op: Alu3 {
1583                dst: Op::Gpr(s2),
1584                lhs: Op::Gpr(s2),
1585                rhs: Op::Imm(1),
1586            },
1587        },
1588        Inst::Or {
1589            w: Width::B64,
1590            op: Alu3 {
1591                dst: Op::Gpr(s2),
1592                lhs: Op::Gpr(s2),
1593                rhs: Op::Gpr(s1),
1594            },
1595        },
1596        Inst::CvtIF {
1597            signed: true,
1598            from: Width::B64,
1599            k,
1600            dst: dst_fpr,
1601            src: Op::Gpr(s2),
1602        },
1603        Inst::FArith {
1604            op: FAlu::Add,
1605            k,
1606            arith: FArith3 {
1607                dst: dst_fpr,
1608                lhs: Op::Fpr(dst_fpr),
1609                rhs: None,
1610            },
1611        },
1612        Inst::Bind(done),
1613    ])
1614}
1615
1616/// Expands `btd2i` to a convert plus an overflow-sentinel branch.
1617///
1618/// Takes the branch when the low 32 bits equal `0x80000000`.
1619fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
1620    expect_len(mn, ops, 3)?;
1621    Ok(smallvec![
1622        Inst::CvtFI {
1623            k: FKind::F64,
1624            w: Width::B64,
1625            dst: gpr_op(mn, &ops[0])?,
1626            src: ops[1],
1627        },
1628        Inst::Branch {
1629            cc: IntCc::Eq,
1630            w: Width::B32,
1631            lhs: ops[0],
1632            rhs: Op::Imm(-0x8000_0000i64),
1633            target: label_op(mn, &ops[2])?,
1634            patch: false,
1635        },
1636    ])
1637}
1638
1639fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
1640    expect_len(mn, ops, 3)?;
1641    let target = label_op(mn, &ops[2])?;
1642    Ok(smallvec![
1643        Inst::CvtFI {
1644            k: FKind::F64,
1645            w: Width::B64,
1646            dst: gpr_op(mn, &ops[0])?,
1647            src: ops[1],
1648        },
1649        Inst::TestBranch {
1650            w: Width::B64,
1651            value: ops[0],
1652            mask: None,
1653            test: TestKind::Zero,
1654            target,
1655            patch: false,
1656        },
1657        Inst::CvtIF {
1658            signed: true,
1659            from: Width::B64,
1660            k: FKind::F64,
1661            dst: sc.fpr,
1662            src: ops[0],
1663        },
1664        Inst::BranchF {
1665            cc: FCc::NeUn,
1666            k: FKind::F64,
1667            lhs: ops[1],
1668            rhs: Op::Fpr(sc.fpr),
1669            target,
1670            patch: false,
1671        },
1672    ])
1673}
1674
1675/// Zeros a double by computing `dst -= dst`.
1676fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1677    expect_len(mn, ops, 1)?;
1678    let dst = fpr_op(mn, &ops[0])?;
1679    Ok(Inst::FArith {
1680        op: FAlu::Sub,
1681        k: FKind::F64,
1682        arith: FArith3 {
1683            dst,
1684            lhs: Op::Fpr(dst),
1685            rhs: None,
1686        },
1687    })
1688}