Skip to main content

offlineasm_ir/
expand.rs

1//! Expands mnemonic invocations into [`Inst`] sequences.
2
3use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6
7/// Lowers one mnemonic invocation to the [`Inst`] sequence that implements it.
8///
9/// `sc` supplies the scratch registers that multi-instruction expansions
10/// and the destination-aliasing fixup may need. It must not alias any
11/// operand in `ops`.
12/// When `patch` is set, the mnemonic must be a [`patch_kind`] site, and the
13/// returned sequence is shaped so a backend can patch it after emission
14/// (a fixed instruction count, no operand folding).
15///
16/// # Errors
17///
18/// - [`ExpandError::NotPatchable`] if `patch` is set and `mn` has no
19///   [`patch_kind`].
20/// - [`ExpandError::BadOperand`] if `ops` has the wrong count or kind for
21///   `mn`.
22pub fn expand(
23    mn: Mnemonic,
24    ops: &[Op],
25    sc: &Scratch,
26    patch: bool,
27) -> Result<Vec<Inst>, ExpandError> {
28    if patch && patch_kind(mn).is_none() {
29        return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
30    }
31    check_extern_operands(mn, ops, patch)?;
32    let mut insts = expand_inner(mn, ops, sc, patch)?;
33    stage_aliasing_mem(&mut insts, sc);
34    Ok(insts)
35}
36
37/// Rejects external symbols outside the shapes that take one: the target of
38/// `jmp`/`call`, the source of `move`, and the source of a 64-bit `lea`
39/// (`leap`, `globaladdr`, `la`). None of them is a patch site.
40fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
41    let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
42        return Ok(());
43    };
44    if patch {
45        return Err(bad(mn, "an external symbol cannot be a patch site"));
46    }
47    let ok = match mn {
48        Mnemonic::Jmp | Mnemonic::Call => at == 0,
49        Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
50        Mnemonic::Lea {
51            w: Width::B64,
52            form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
53        } => at == 1,
54        _ => false,
55    };
56    if ok {
57        Ok(())
58    } else {
59        Err(bad(
60            mn,
61            "an external symbol is only a `jmp`/`call` target or the source of \
62             `move`/`leap`/`globaladdr`/`la`",
63        ))
64    }
65}
66
67/// Stages aliased memory operands through the scratch registers.
68///
69/// `addis t1, t0, 8[t1]` reads `8[t1]` with the *old* `t1`: all operands are
70/// evaluated before the destination is written. A backend that first lowers
71/// `dst = lhs` into `t1` and then reuses `t1` as the memory base (the
72/// natural 2-operand sequence on every target) reads `8[t1]` after the move
73/// instead and dereferences the wrong address. The fused `Alu3`/op-branch
74/// nodes hand that ordering choice to each backend, so the fix is pinned
75/// down in the front half both assemblers share: the load happens before
76/// the operation, through the expansion scratch register.
77///
78/// The oracle interprets this same sequence, and scratch is clobberable by
79/// design (it is never in the portable register pool, so it cannot alias
80/// `dst` or the operand bases).
81fn stage_aliasing_mem(insts: &mut Vec<Inst>, sc: &Scratch) {
82    let old = std::mem::take(insts);
83    let mut out = Vec::with_capacity(old.len() + 2);
84    for mut inst in old {
85        let loads = stage_one(&mut inst, sc);
86        if let Some(loads) = loads {
87            out.extend(loads);
88        }
89        out.push(inst);
90    }
91    *insts = out;
92}
93
94/// Collects the staging loads for `inst` and rewrites its aliased operands.
95///
96/// Returns `None` when `inst` needs no staging.
97fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Vec<Inst>> {
98    let (w, operands, dst) = match inst {
99        Inst::Add { w, op }
100        | Inst::Sub { w, op }
101        | Inst::Mul { w, op }
102        | Inst::And { w, op }
103        | Inst::Or { w, op }
104        | Inst::Xor { w, op } => (*w, vec![&mut op.lhs, &mut op.rhs], &mut op.dst),
105        Inst::AluS { w, alu, .. } => (*w, vec![&mut alu.lhs, &mut alu.rhs], &mut alu.dst),
106        Inst::OpBranch { w, args, .. } => {
107            let (last, init) = args.split_last_mut()?;
108            (*w, init.iter_mut().collect(), last)
109        }
110        _ => return None,
111    };
112    let Op::Gpr(g) = *dst else {
113        return None;
114    };
115    let slots = [sc.gpr, sc.gpr2];
116    let mut loads = Vec::new();
117    let mut slot = 0usize;
118    for operand in operands {
119        let Op::Mem(m) = operand else {
120            continue;
121        };
122        let aliases = m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g);
123        if !aliases {
124            continue;
125        }
126        // At most two memory operands can alias `dst` (op-branch takes
127        // `[lhs, rhs, dst]`), and both scratch GPRs are available.
128        let &s = slots.get(slot)?;
129        slot += 1;
130        loads.push(Inst::Load {
131            w,
132            sx: Sx::Zero,
133            dst: s,
134            addr: *m,
135            acquire: false,
136        });
137        *operand = Op::Gpr(s);
138    }
139    (!loads.is_empty()).then_some(loads)
140}
141
142/// Returns the patch shape for `mn`.
143///
144/// Returns `None` for mnemonics that cannot be patch sites.
145pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
146    Some(match mn {
147        Mnemonic::Jmp => PatchKind::JumpSite,
148        Mnemonic::Call => PatchKind::CallSite,
149        Mnemonic::Move => PatchKind::MovBlock,
150        Mnemonic::FlagBranch(_)
151        | Mnemonic::OpBranch { .. }
152        | Mnemonic::RiscAluS { .. }
153        | Mnemonic::TestBranch { .. }
154        | Mnemonic::IntBranch { .. }
155        | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
156        _ => return None,
157    })
158}
159
160/// Describes what a `#[patch]` site patches.
161#[derive(Clone, Copy, Debug, PartialEq, Eq)]
162pub enum PatchKind {
163    /// Patches an unconditional jump target.
164    JumpSite,
165    /// Patches a conditional branch target.
166    CondJumpSite,
167    /// Patches a call target.
168    CallSite,
169    /// Patches a 64-bit move slot.
170    MovBlock,
171}
172
173fn float_cc(cc: FCc) -> (FCc, bool) {
174    match cc {
175        FCc::Lt => (FCc::Gt, true),
176        FCc::Le => (FCc::Ge, true),
177        FCc::GtUn => (FCc::GtUn, false),
178        FCc::GeUn => (FCc::GeUn, false),
179        FCc::LtUn => (FCc::GtUn, true),
180        FCc::LeUn => (FCc::GeUn, true),
181        other => (other, false),
182    }
183}
184
185fn test_set_node(
186    mn: Mnemonic,
187    ops: &[Op],
188    w: Width,
189    test: TestKind,
190) -> Result<Vec<Inst>, ExpandError> {
191    if ops.len() != 2 && ops.len() != 3 {
192        return Err(bad(mn, "test-set takes 2 or 3 operands"));
193    }
194    let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
195    Ok(vec![Inst::TestSet {
196        w,
197        value: ops[1],
198        mask,
199        test,
200        dst: ops[0],
201    }])
202}
203
204fn test_branch_node(
205    mn: Mnemonic,
206    ops: &[Op],
207    w: Width,
208    test: TestKind,
209    patch: bool,
210) -> Result<Vec<Inst>, ExpandError> {
211    if ops.len() != 2 && ops.len() != 3 {
212        return Err(bad(mn, "test-branch takes 2 or 3 operands"));
213    }
214    let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
215    let target = label_op(
216        mn,
217        ops.last()
218            .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
219    )?;
220    Ok(vec![Inst::TestBranch {
221        w,
222        value: ops[0],
223        mask,
224        test,
225        target,
226        patch,
227    }])
228}
229
230fn div_rem_node(
231    mn: Mnemonic,
232    ops: &[Op],
233    signed: bool,
234    w: Width,
235    rem: bool,
236) -> Result<Vec<Inst>, ExpandError> {
237    let (dvd, dvs, dst) = match ops {
238        [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
239        [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
240        _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
241    };
242    Ok(vec![if rem {
243        Inst::Rem3 {
244            signed,
245            w,
246            dst,
247            dvd,
248            dvs,
249        }
250    } else {
251        Inst::Div3 {
252            signed,
253            w,
254            dst,
255            dvd,
256            dvs,
257        }
258    }])
259}
260
261fn shift_mem_canon(
262    mn: Mnemonic,
263    w: Width,
264    op: ShiftOp,
265    cnt: &Op,
266    dst: &Mem,
267    sc: &Scratch,
268) -> Result<Vec<Inst>, ExpandError> {
269    let shift_cnt = match cnt {
270        Op::Imm(v) => ShiftCnt::Imm(*v),
271        Op::Gpr(g) => ShiftCnt::Reg(*g),
272        _ => return Err(bad(mn, "shift count must be an immediate or register")),
273    };
274    Ok(vec![
275        Inst::Load {
276            w,
277            sx: Sx::Zero,
278            dst: sc.gpr,
279            addr: *dst,
280            acquire: false,
281        },
282        Inst::Shift {
283            w,
284            op,
285            dst: sc.gpr,
286            cnt: shift_cnt,
287        },
288        Inst::Store {
289            w,
290            src: Op::Gpr(sc.gpr),
291            addr: *dst,
292        },
293    ])
294}
295
296fn shift_node(
297    mn: Mnemonic,
298    w: Width,
299    op: ShiftOp,
300    ops: &[Op],
301    sc: &Scratch,
302) -> Result<Vec<Inst>, ExpandError> {
303    if ops.len() != 2 {
304        return Err(bad(mn, "shift takes 2 operands"));
305    }
306    if let Op::Mem(a) = &ops[0] {
307        return shift_mem_canon(mn, w, op, &ops[1], a, sc);
308    }
309    let dst = gpr_op(mn, &ops[0])?;
310    let cnt = match &ops[1] {
311        Op::Imm(v) => ShiftCnt::Imm(*v),
312        Op::Gpr(g) => ShiftCnt::Reg(*g),
313        _ => return Err(bad(mn, "shift count must be an immediate or register")),
314    };
315    Ok(vec![Inst::Shift { w, op, dst, cnt }])
316}
317
318fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
319    Alu3::from_operands(mn, ops)
320}
321
322fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Op>, ExpandError> {
323    match ops {
324        [dst, rhs] => Ok(vec![*rhs, *dst]),
325        [dst, lhs, rhs] => Ok(vec![*lhs, *rhs, *dst]),
326        _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
327    }
328}
329
330fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
331    match op {
332        Op::Label(l) => Ok(Target::Label(*l)),
333        Op::Gpr(g) => Ok(Target::Reg(*g)),
334        Op::Mem(a) => Ok(Target::Mem(*a)),
335        Op::Imm(v) => Ok(Target::Abs(*v)),
336        Op::Extern(e) => Ok(Target::Extern(*e)),
337        _ => Err(bad(
338            mn,
339            "jump/call needs a label, register, memory, or immediate",
340        )),
341    }
342}
343
344#[allow(clippy::too_many_lines)]
345fn expand_inner(
346    mn: Mnemonic,
347    ops: &[Op],
348    sc: &Scratch,
349    patch: bool,
350) -> Result<Vec<Inst>, ExpandError> {
351    let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
352        expect_len(mn, ops, 1)?;
353        Ok(ops[0])
354    };
355    match mn {
356        Mnemonic::Move => Ok(vec![mov_node(mn, ops, Width::B64, patch)?]),
357        Mnemonic::Load(w) => Ok(vec![mov_node(mn, ops, w, patch)?]),
358        Mnemonic::AtomicLoad(w) => Ok(vec![atomic_load_node(mn, ops, w)?]),
359        Mnemonic::Store(w) => Ok(vec![store_node(mn, ops, w)?]),
360        Mnemonic::Extend { from, to, sx } => Ok(vec![extend_node(mn, ops, from, to, sx)?]),
361        Mnemonic::MovZero32 => Ok(vec![movzero_node(mn, ops)?]),
362        Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
363        Mnemonic::FMove(k) => Ok(vec![fmove_node(mn, ops, k)?]),
364        Mnemonic::VMove { store } => Ok(vec![vmove_node(mn, ops, store)?]),
365        Mnemonic::Lea { w, .. } => Ok(vec![lea_node(mn, ops, w)?]),
366        Mnemonic::LoadQInc => loadqinc_node(mn, ops),
367        Mnemonic::LoadPair(w) => Ok(vec![pair_node(mn, ops, w, true)?]),
368        Mnemonic::StorePair(w) => Ok(vec![pair_node(mn, ops, w, false)?]),
369        Mnemonic::Push => stack_nodes(mn, ops, true, false),
370        Mnemonic::Pop => stack_nodes(mn, ops, false, false),
371        Mnemonic::PushV => stack_nodes(mn, ops, true, true),
372        Mnemonic::PopV => stack_nodes(mn, ops, false, true),
373        Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
374        Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
375        Mnemonic::Peek => Ok(vec![peek_node(mn, ops, true)?]),
376        Mnemonic::Poke => Ok(vec![peek_node(mn, ops, false)?]),
377        Mnemonic::TlsLoad(w) => Ok(vec![tls_node(mn, ops, w, true)?]),
378        Mnemonic::TlsStore(w) => Ok(vec![tls_node(mn, ops, w, false)?]),
379        Mnemonic::TlsLea => Ok(vec![tls_lea_node(mn, ops)?]),
380
381        Mnemonic::Add(w) => Ok(vec![Inst::Add {
382            w,
383            op: alu3(mn, ops)?,
384        }]),
385        Mnemonic::Sub(w) => Ok(vec![Inst::Sub {
386            w,
387            op: alu3(mn, ops)?,
388        }]),
389        Mnemonic::Mul(w) => Ok(vec![mul_node(mn, w, ops)?]),
390        Mnemonic::And(w) => Ok(vec![Inst::And {
391            w,
392            op: alu3(mn, ops)?,
393        }]),
394        Mnemonic::Or(w) => Ok(vec![Inst::Or {
395            w,
396            op: alu3(mn, ops)?,
397        }]),
398        Mnemonic::Xor(w) => Ok(vec![Inst::Xor {
399            w,
400            op: alu3(mn, ops)?,
401        }]),
402        Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
403        Mnemonic::Not(w) => Ok(vec![Inst::Not {
404            w,
405            dst: one(mn, ops)?,
406        }]),
407        Mnemonic::Neg(w) => Ok(vec![Inst::Neg {
408            w,
409            dst: one(mn, ops)?,
410        }]),
411        Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
412        Mnemonic::Bswap(w) => Ok(vec![bswap_node(mn, ops, w)?]),
413        Mnemonic::Adc => Ok(vec![adc_node(mn, ops, false)?]),
414        Mnemonic::Sbb => Ok(vec![adc_node(mn, ops, true)?]),
415        Mnemonic::MulHigh { signed } => Ok(vec![Inst::MulHigh {
416            signed,
417            rhs: one(mn, ops)?,
418        }]),
419        Mnemonic::MulWide { signed } => Ok(vec![mulwide_node(mn, ops, signed)?]),
420        Mnemonic::AluS { op, w } => Ok(vec![Inst::AluS {
421            op,
422            w,
423            alu: alu3(mn, ops)?,
424        }]),
425        Mnemonic::RiscAluS { op, w } => {
426            let has_label =
427                matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
428            if has_label {
429                let flag_op = match op {
430                    AluOp::Add => FlagAlu::Add,
431                    AluOp::Sub => FlagAlu::Sub,
432                    AluOp::Or => FlagAlu::Or,
433                };
434                return Ok(vec![Inst::OpBranch {
435                    op: flag_op,
436                    w,
437                    flag: PredFlag::S,
438                    args: op_branch_args(mn, &ops[..ops.len() - 1])?,
439                    target: label_op(
440                        mn,
441                        ops.last()
442                            .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
443                    )?,
444                    patch,
445                }]);
446            }
447            Ok(vec![Inst::AluS {
448                op,
449                w,
450                alu: alu3(mn, ops)?,
451            }])
452        }
453        Mnemonic::AddShifted => Ok(vec![addshifted_node(mn, ops)?]),
454        Mnemonic::Cqo { quad } => Ok(vec![Inst::Cqo { quad }]),
455        Mnemonic::DivMod1 { signed, w } => Ok(vec![Inst::DivMod1 {
456            signed,
457            w,
458            divisor: one(mn, ops)?,
459        }]),
460        Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
461        Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
462        Mnemonic::IntSet { w, cc } => {
463            expect_len(mn, ops, 3)?;
464            Ok(vec![Inst::IntSet {
465                cc,
466                w,
467                dst: ops[0],
468                lhs: ops[1],
469                rhs: ops[2],
470            }])
471        }
472        Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
473        Mnemonic::Cmov { zero } => Ok(vec![cmov_node(mn, ops, zero)?]),
474        Mnemonic::BitCount { w, trailing } => Ok(vec![cz_node(mn, ops, w, trailing)?]),
475        Mnemonic::Popcnt(w) => Ok(vec![popcnt_node(mn, ops, w)?]),
476        Mnemonic::Bfi => bfi_node(mn, ops, sc),
477        Mnemonic::IntBranch { w, cc } => {
478            expect_len(mn, ops, 3)?;
479            Ok(vec![Inst::Branch {
480                cc,
481                w,
482                lhs: ops[0],
483                rhs: ops[1],
484                target: label_op(mn, &ops[2])?,
485                patch,
486            }])
487        }
488        Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
489        Mnemonic::OpBranch { op, w, flag } => {
490            if ops.len() != 3 && ops.len() != 4 {
491                return Err(bad(mn, "op-branch takes 3 or 4 operands"));
492            }
493            Ok(vec![Inst::OpBranch {
494                op,
495                w,
496                flag,
497                args: op_branch_args(mn, &ops[..ops.len() - 1])?,
498                target: label_op(
499                    mn,
500                    ops.last()
501                        .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
502                )?,
503                patch,
504            }])
505        }
506        Mnemonic::FlagBranch(flag) => Ok(vec![flag_branch_node(mn, ops, flag, patch)?]),
507        Mnemonic::BranchF { cc, k } => {
508            expect_len(mn, ops, 3)?;
509            let (cc, reverse) = float_cc(cc);
510            let (lhs, rhs) = if reverse {
511                (ops[1], ops[0])
512            } else {
513                (ops[0], ops[1])
514            };
515            Ok(vec![Inst::BranchF {
516                cc,
517                k,
518                lhs,
519                rhs,
520                target: label_op(mn, &ops[2])?,
521                patch,
522            }])
523        }
524        Mnemonic::Jmp => {
525            expect_len(mn, ops, 1)?;
526            Ok(vec![Inst::Jump {
527                target: jump_target(mn, &ops[0])?,
528                patch,
529            }])
530        }
531        Mnemonic::Call => {
532            expect_len(mn, ops, 1)?;
533            Ok(vec![Inst::Call {
534                target: jump_target(mn, &ops[0])?,
535                patch,
536            }])
537        }
538        Mnemonic::Ret => {
539            expect_len(mn, ops, 0)?;
540            Ok(vec![Inst::Ret])
541        }
542        Mnemonic::Break => {
543            expect_len(mn, ops, 0)?;
544            Ok(vec![Inst::Break])
545        }
546        Mnemonic::Nop => {
547            expect_len(mn, ops, 0)?;
548            Ok(vec![Inst::Nop])
549        }
550        Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
551        Mnemonic::Fma { op, k } => Ok(vec![fma_node(mn, ops, k, op)?]),
552        Mnemonic::FSqrt(k) => Ok(vec![fsqrt_node(mn, ops, k)?]),
553        Mnemonic::FRound { k, mode } => Ok(vec![fround_node(mn, ops, k, mode)?]),
554        Mnemonic::FAbs { k, negate } => Ok(vec![fabs_node(mn, ops, k, negate)?]),
555        Mnemonic::FSet { cc, k } => {
556            expect_len(mn, ops, 3)?;
557            let (cc, reverse) = float_cc(cc);
558            let (lhs, rhs) = if reverse {
559                (ops[2], ops[1])
560            } else {
561                (ops[1], ops[2])
562            };
563            Ok(vec![Inst::FSet {
564                cc,
565                k,
566                dst: ops[0],
567                lhs,
568                rhs,
569            }])
570        }
571        Mnemonic::CvtFI { k, w } => Ok(vec![cvtfi_node(mn, ops, k, w)?]),
572        Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
573        Mnemonic::CvtIF { signed, from, k } => Ok(vec![cvtif_node(mn, ops, signed, from, k)?]),
574        Mnemonic::CvtFF { from, to } => Ok(vec![cvtff_node(mn, ops, from, to)?]),
575        Mnemonic::Bitcast(dir) => Ok(vec![bitcast_node(mn, ops, dir)?]),
576        Mnemonic::MovDZ => Ok(vec![movdz_node(mn, ops)?]),
577        Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
578        Mnemonic::Btd2i => btd2i_node(mn, ops),
579        Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
580        Mnemonic::VecExtract(w) => Ok(vec![umov_node(mn, ops, w)?]),
581        Mnemonic::AtomicRmw { op, w } => Ok(vec![rmw_node(mn, ops, w, op)?]),
582        Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
583        Mnemonic::AtomicCas { w, branch } => Ok(vec![cas_node(mn, ops, w, branch)?]),
584        Mnemonic::LoadLinked(w) => Ok(vec![ll_node(mn, ops, w)?]),
585        Mnemonic::StoreCond(w) => Ok(vec![sc_node(mn, ops, w)?]),
586        Mnemonic::Fence { full } => Ok(vec![Inst::Fence { full }]),
587        Mnemonic::Pause => Ok(vec![Inst::Pause]),
588        Mnemonic::TagCodePtr
589        | Mnemonic::TagReturnAddress
590        | Mnemonic::UntagReturnAddress
591        | Mnemonic::RemoveCodePtrTag
592        | Mnemonic::UntagArrayPtr
593        | Mnemonic::RemoveArrayPtrTag => Ok(vec![]),
594    }
595}
596
597fn flag_branch_node(
598    mn: Mnemonic,
599    ops: &[Op],
600    flag: PredFlag,
601    patch: bool,
602) -> Result<Inst, ExpandError> {
603    expect_len(mn, ops, 1)?;
604    Ok(Inst::FlagBranch {
605        flag,
606        target: label_op(mn, &ops[0])?,
607        patch,
608    })
609}
610
611fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
612    expect_len(mn, ops, 2)?;
613    Ok(Inst::Popcnt {
614        w,
615        dst: gpr_op(mn, &ops[0])?,
616        src: ops[1],
617    })
618}
619
620fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
621    expect_len(mn, ops, 2)?;
622    let dst = gpr_op(mn, &ops[0])?;
623    let inst = if trailing {
624        Inst::Ctz {
625            w,
626            dst,
627            src: ops[1],
628        }
629    } else {
630        Inst::Clz {
631            w,
632            dst,
633            src: ops[1],
634        }
635    };
636    Ok(inst)
637}
638
639fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
640    expect_len(mn, ops, 2)?;
641    if load {
642        let idx = imm_op(mn, &ops[1])?;
643        Ok(Inst::TlsLoad {
644            w,
645            dst: gpr_op(mn, &ops[0])?,
646            idx,
647        })
648    } else {
649        let idx = imm_op(mn, &ops[0])?;
650        Ok(Inst::TlsStore {
651            w,
652            src: gpr_op(mn, &ops[1])?,
653            idx,
654        })
655    }
656}
657
658fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
659    expect_len(mn, ops, 2)?;
660    Ok(Inst::TlsLea {
661        dst: gpr_op(mn, &ops[0])?,
662        idx: imm_op(mn, &ops[1])?,
663    })
664}
665
666fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
667    expect_len(mn, ops, 2)?;
668
669    Ok(Inst::AtomicRmw {
670        op,
671        w,
672        a: ops[1],
673        b: ops[0],
674    })
675}
676
677fn rmw_not_node(
678    mn: Mnemonic,
679    ops: &[Op],
680    sc: &Scratch,
681    w: Width,
682    op: AtomicOp,
683) -> Result<Vec<Inst>, ExpandError> {
684    expect_len(mn, ops, 2)?;
685    let mut out = Vec::new();
686    // `mem` is ops[0] and `src` is ops[1].
687    let a = match &ops[1] {
688        Op::Imm(v) => Op::Imm(!v),
689        Op::Gpr(g) => {
690            out.push(Inst::Mov {
691                w: Width::B64,
692                dst: Op::Gpr(sc.gpr),
693                src: Op::Gpr(*g),
694                patch: false,
695            });
696            out.push(Inst::Not {
697                w,
698                dst: Op::Gpr(sc.gpr),
699            });
700            Op::Gpr(sc.gpr)
701        }
702        Op::Mem(addr) => {
703            let lw = if w == Width::B64 {
704                Width::B64
705            } else {
706                Width::B32
707            };
708            out.push(Inst::Load {
709                w: lw,
710                sx: Sx::Zero,
711                dst: sc.gpr,
712                addr: *addr,
713                acquire: false,
714            });
715            out.push(Inst::Not {
716                w,
717                dst: Op::Gpr(sc.gpr),
718            });
719            Op::Gpr(sc.gpr)
720        }
721        _ => {
722            return Err(bad(
723                mn,
724                "atomic clear needs an immediate, register, or memory operand",
725            ))
726        }
727    };
728    out.push(Inst::AtomicRmw {
729        op,
730        w,
731        a,
732        b: ops[0],
733    });
734    Ok(out)
735}
736
737fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
738    let n = if branch { 4 } else { 3 };
739    expect_len(mn, ops, n)?;
740
741    Ok(Inst::AtomicCas {
742        w,
743        mem: mem_op(mn, &ops[0])?,
744        src: gpr_op(mn, &ops[2])?,
745        branch: if branch {
746            Some(label_op(mn, &ops[3])?)
747        } else {
748            None
749        },
750    })
751}
752
753fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
754    expect_len(mn, ops, 2)?;
755    Ok(Inst::LoadLinked {
756        w,
757        mem: mem_op(mn, &ops[1])?,
758        dst: gpr_op(mn, &ops[0])?,
759    })
760}
761
762fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
763    expect_len(mn, ops, 3)?;
764    Ok(Inst::StoreCond {
765        w,
766        status: gpr_op(mn, &ops[1])?,
767        src: gpr_op(mn, &ops[2])?,
768        mem: mem_op(mn, &ops[0])?,
769    })
770}
771
772/// Expands `bfiq` to a mask, shift, and OR sequence.
773///
774/// Computes `dst = (dst & ~mask) | ((src & wmask) << lsb)`.
775fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
776    expect_len(mn, ops, 4)?;
777    let lsb = imm_op(mn, &ops[2])?;
778    let width = imm_op(mn, &ops[3])?;
779    if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
780        return Err(bad(mn, "bitfield out of range"));
781    }
782    let wmask: u64 = if width == 64 {
783        u64::MAX
784    } else {
785        (1u64 << width) - 1
786    };
787    let mask: i64 = (wmask << lsb) as i64;
788    let s = sc.gpr;
789    let dst = gpr_op(mn, &ops[0])?;
790
791    Ok(vec![
792        Inst::Mov {
793            w: Width::B64,
794            dst: Op::Gpr(s),
795            src: ops[1],
796            patch: false,
797        },
798        Inst::And {
799            w: Width::B64,
800            op: Alu3 {
801                dst: Op::Gpr(s),
802                lhs: Op::Gpr(s),
803                rhs: Op::Imm(wmask as i64),
804            },
805        },
806        Inst::Shift {
807            w: Width::B64,
808            op: ShiftOp::Shl,
809            dst: s,
810            cnt: ShiftCnt::Imm(lsb),
811        },
812        Inst::And {
813            w: Width::B64,
814            op: Alu3 {
815                dst: Op::Gpr(dst),
816                lhs: Op::Gpr(dst),
817                rhs: Op::Imm(!mask),
818            },
819        },
820        Inst::Or {
821            w: Width::B64,
822            op: Alu3 {
823                dst: Op::Gpr(dst),
824                lhs: Op::Gpr(dst),
825                rhs: Op::Gpr(s),
826            },
827        },
828    ])
829}
830
831fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
832    expect_len(mn, ops, 3)?;
833    Ok(Inst::Cmov {
834        w: Width::B64,
835        zero,
836        dst: gpr_op(mn, &ops[0])?,
837        src: ops[1],
838        test: gpr_op(mn, &ops[2])?,
839    })
840}
841
842/// Expands `peek dst, off` and `poke off, src`, with the destination first
843/// in both. The offset counts quad slots and is scaled here.
844fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
845    expect_len(mn, ops, 2)?;
846    if peek {
847        let disp = slot_disp(mn, &ops[1])?;
848        return Ok(Inst::Peek {
849            disp,
850            dst: gpr_op(mn, &ops[0])?,
851        });
852    }
853    Ok(Inst::Poke {
854        disp: slot_disp(mn, &ops[0])?,
855        src: gpr_op(mn, &ops[1])?,
856    })
857}
858
859fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
860    let idx = imm_op(mn, op)?;
861    let off = idx
862        .checked_mul(8)
863        .ok_or_else(|| bad(mn, "stack offset overflow"))?;
864    if off < i32::MIN as i64 || off > i32::MAX as i64 {
865        return Err(bad(mn, "stack offset out of range"));
866    }
867    Ok(off as i32)
868}
869
870fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
871    expect_len(mn, ops, 3)?;
872    let dst = gpr_op(mn, &ops[0])?;
873    let base = gpr_op(mn, &ops[1])?;
874    let inc = imm_op(mn, &ops[2])?;
875    Ok(vec![
876        Inst::Load {
877            w: Width::B64,
878            sx: Sx::Zero,
879            dst,
880            addr: Mem::base(base, 0),
881            acquire: false,
882        },
883        Inst::Add {
884            w: Width::B64,
885            op: Alu3 {
886                dst: Op::Gpr(base),
887                lhs: Op::Gpr(base),
888                rhs: Op::Imm(inc),
889            },
890        },
891    ])
892}
893
894fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
895    expect_len(mn, ops, 3)?;
896    if load {
897        Ok(Inst::PairLoad {
898            w,
899            dst1: ops[0],
900            dst2: ops[1],
901            mem: mem_op(mn, &ops[2])?,
902        })
903    } else {
904        Ok(Inst::PairStore {
905            w,
906            src1: ops[1],
907            src2: ops[2],
908            mem: mem_op(mn, &ops[0])?,
909        })
910    }
911}
912
913fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
914    expect_len(mn, ops, 2)?;
915    let dst = gpr_op(mn, &ops[0])?;
916    let src = match &ops[1] {
917        Op::Mem(a) => LeaSrc::Mem(*a),
918        Op::Label(l) => LeaSrc::Label(*l),
919        Op::Extern(e) => LeaSrc::Extern(*e),
920        _ => return Err(bad(mn, "lea needs an address or label source")),
921    };
922    Ok(Inst::Lea { w, dst, src })
923}
924
925fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
926    expect_len(mn, ops, 2)?;
927    // `move reg, extern` takes the symbol's address, like `leap`.
928    if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
929        return Ok(Inst::Lea {
930            w: Width::B64,
931            dst: *dst,
932            src: LeaSrc::Extern(*e),
933        });
934    }
935    Ok(Inst::Mov {
936        w,
937        dst: ops[0],
938        src: ops[1],
939        patch,
940    })
941}
942
943fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
944    expect_len(mn, ops, 2)?;
945    Ok(Inst::MovZero32 {
946        dst: ops[0],
947        src: ops[1],
948    })
949}
950
951fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
952    expect_len(mn, ops, 2)?;
953    Ok(Inst::Load {
954        w,
955        sx: Sx::Zero,
956        dst: gpr_op(mn, &ops[0])?,
957        addr: mem_op(mn, &ops[1])?,
958        acquire: true,
959    })
960}
961
962fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
963    expect_len(mn, ops, 2)?;
964    let addr = mem_op(mn, &ops[0])?;
965    Ok(Inst::Store {
966        w,
967        src: ops[1],
968        addr,
969    })
970}
971
972fn extend_node(
973    mn: Mnemonic,
974    ops: &[Op],
975    from: Width,
976    to: Width,
977    sx: Sx,
978) -> Result<Inst, ExpandError> {
979    expect_len(mn, ops, 2)?;
980    Ok(Inst::Extend {
981        from,
982        to,
983        sx,
984        dst: gpr_op(mn, &ops[0])?,
985        src: ops[1],
986    })
987}
988
989fn transfer_node(
990    mn: Mnemonic,
991    ops: &[Op],
992    sc: &Scratch,
993    w: Width,
994) -> Result<Vec<Inst>, ExpandError> {
995    expect_len(mn, ops, 2)?;
996    let mut out = Vec::new();
997    match &ops[1] {
998        Op::Gpr(_) => {}
999        Op::Mem(a) => {
1000            out.push(Inst::Load {
1001                w,
1002                sx: Sx::Zero,
1003                dst: sc.gpr,
1004                addr: *a,
1005                acquire: false,
1006            });
1007        }
1008        Op::Imm(v) => {
1009            out.push(Inst::Mov {
1010                w,
1011                dst: Op::Gpr(sc.gpr),
1012                src: Op::Imm(*v),
1013                patch: false,
1014            });
1015        }
1016        Op::Label(l) => {
1017            out.push(Inst::Lea {
1018                w: Width::B64,
1019                dst: sc.gpr,
1020                src: LeaSrc::Label(*l),
1021            });
1022            let addr = Mem::base(sc.gpr, 0);
1023            out.push(Inst::Load {
1024                w,
1025                sx: Sx::Zero,
1026                dst: sc.gpr,
1027                addr,
1028                acquire: false,
1029            });
1030        }
1031        Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1032        Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1033        Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1034    }
1035    let src = if out.is_empty() {
1036        ops[1]
1037    } else {
1038        Op::Gpr(sc.gpr)
1039    };
1040    out.push(Inst::Mov {
1041        w,
1042        dst: ops[0],
1043        src,
1044        patch: false,
1045    });
1046    Ok(out)
1047}
1048
1049fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1050    expect_len(mn, ops, 2)?;
1051    match (&ops[0], &ops[1]) {
1052        (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1053            k,
1054            dst: *d,
1055            src: *s,
1056        }),
1057        (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1058            k,
1059            dst: *d,
1060            addr: *a,
1061        }),
1062        (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1063            k,
1064            src: *s,
1065            addr: *a,
1066        }),
1067        _ => Err(bad(mn, "float move needs FPR/memory operands")),
1068    }
1069}
1070
1071fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1072    expect_len(mn, ops, 2)?;
1073    if !store {
1074        match (&ops[0], &ops[1]) {
1075            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1076            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1077            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1078                dst: *d,
1079                addr: Mem::base(*s, 0),
1080            }),
1081            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1082        }
1083    } else {
1084        match (&ops[0], &ops[1]) {
1085            (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1086            (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1087            (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1088                dst: *d,
1089                addr: Mem::base(*s, 0),
1090            }),
1091            (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1092            _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1093        }
1094    }
1095}
1096
1097/// Checks that every callee-save operand is a GPR in `saved`.
1098///
1099/// Does nothing for every other mnemonic.
1100///
1101/// # Errors
1102///
1103/// [`ExpandError::BadOperand`] if `mnemonic` is `PushCalleeSaves` or
1104/// `PopCalleeSaves` and `ops` contains a non-GPR operand or a GPR whose
1105/// number is not in `saved`.
1106pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1107    if !matches!(
1108        mnemonic,
1109        Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1110    ) {
1111        return Ok(());
1112    }
1113    for op in ops {
1114        match op {
1115            Op::Gpr(g) if saved.contains(&g.0) => {}
1116            _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1117        }
1118    }
1119    Ok(())
1120}
1121
1122/// Bytes one AArch64 `push`/`pop` of `n` quads moves `sp` by: `8 * n`
1123/// rounded up to 16, because `sp` must stay 16-byte aligned whenever it is
1124/// used as a base address.
1125///
1126/// The operands keep the layout they have on the targets with 8-byte slots
1127/// (see [`aligned_push_offset`]), and an odd count leaves one padding word
1128/// above them. A `pop` must name as many operands as the `push` it undoes.
1129pub fn aligned_push_bytes(n: usize) -> i32 {
1130    ((8 * n + 15) & !15) as i32
1131}
1132
1133/// Offset from the lowered `sp` of operand `i` in an `n`-operand AArch64
1134/// `push`/`pop` group. The last operand sits at `[sp]`, as it would after
1135/// `n` separate 8-byte pushes.
1136pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1137    8 * (n - 1 - i) as i32
1138}
1139
1140fn stack_nodes(
1141    mn: Mnemonic,
1142    ops: &[Op],
1143    push: bool,
1144    vector: bool,
1145) -> Result<Vec<Inst>, ExpandError> {
1146    if ops.is_empty() {
1147        return Err(bad(mn, "needs at least one operand"));
1148    }
1149    let mut out = Vec::with_capacity(ops.len());
1150    // `push a, b` stores `a` then `b`, leaving `b` on top; `pop a, b` must
1151    // unwind LIFO, so it reads `b` before `a` - the reverse of the operand
1152    // list order that `push` uses.
1153    let ordered: Box<dyn Iterator<Item = &Op>> = if push {
1154        Box::new(ops.iter())
1155    } else {
1156        Box::new(ops.iter().rev())
1157    };
1158    for op in ordered {
1159        if vector {
1160            let x = fpr_op(mn, op)?;
1161            out.push(if push {
1162                Inst::PushV { src: x }
1163            } else {
1164                Inst::PopV { dst: x }
1165            });
1166        } else {
1167            match op {
1168                Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => {
1169                    out.push(if push {
1170                        Inst::Push { op: *op }
1171                    } else {
1172                        Inst::Pop { op: *op }
1173                    });
1174                }
1175                _ => return Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1176            }
1177        }
1178    }
1179    Ok(out)
1180}
1181
1182fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1183    if ops.len() == 3 {
1184        if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1185            return Ok(Inst::MulImm {
1186                w,
1187                dst: gpr_op(mn, &ops[0])?,
1188                lhs,
1189                imm: *v,
1190            });
1191        }
1192        if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1193            return Ok(Inst::MulImm {
1194                w,
1195                dst: gpr_op(mn, &ops[0])?,
1196                lhs,
1197                imm: *v,
1198            });
1199        }
1200    }
1201    Ok(Inst::Mul {
1202        w,
1203        op: Alu3::from_operands(mn, ops)?,
1204    })
1205}
1206
1207fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1208    expect_len(mn, ops, 3)?;
1209    Ok(Inst::MulWide {
1210        signed,
1211        dst: gpr_op(mn, &ops[0])?,
1212        lhs: gpr_op(mn, &ops[1])?,
1213        rhs: gpr_op(mn, &ops[2])?,
1214    })
1215}
1216
1217fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1218    expect_len(mn, ops, 4)?;
1219    let amount = imm_op(mn, &ops[3])?;
1220    if !(0..=63).contains(&amount) {
1221        return Err(bad(mn, "shift amount out of range"));
1222    }
1223    Ok(Inst::AddShifted {
1224        dst: gpr_op(mn, &ops[0])?,
1225        lhs: gpr_op(mn, &ops[1])?,
1226        rhs: gpr_op(mn, &ops[2])?,
1227        amount: amount as u8,
1228    })
1229}
1230
1231fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1232    expect_len(mn, ops, 2)?;
1233    let inst = if borrow {
1234        Inst::Sbb {
1235            dst: ops[0],
1236            src: ops[1],
1237        }
1238    } else {
1239        Inst::Adc {
1240            dst: ops[0],
1241            src: ops[1],
1242        }
1243    };
1244    Ok(inst)
1245}
1246
1247fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Vec<Inst>, ExpandError> {
1248    if ops.len() != 2 && ops.len() != 3 {
1249        return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1250    }
1251    let mut out = Vec::new();
1252    let (src, dst) = if ops.len() == 3 {
1253        let d = fpr_op(mn, &ops[0])?;
1254        if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1255            // `O1 = O2 op O1`: and/or commute, so fold `O2` into `O1`.
1256            (ops[1], d)
1257        } else {
1258            if !op_eq(&ops[1], &ops[0]) {
1259                match &ops[1] {
1260                    Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1261                    Op::Mem(a) => out.push(Inst::LoadF {
1262                        k,
1263                        dst: d,
1264                        addr: *a,
1265                    }),
1266                    _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1267                }
1268            }
1269            (ops[2], d)
1270        }
1271    } else {
1272        (ops[1], fpr_op(mn, &ops[0])?)
1273    };
1274    out.push(Inst::FBit { or, k, dst, src });
1275    Ok(out)
1276}
1277
1278fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Vec<Inst>, ExpandError> {
1279    if ops.len() == 3 {
1280        return Ok(vec![Inst::FArith {
1281            op,
1282            k,
1283            arith: FArith3 {
1284                dst: fpr_op(mn, &ops[0])?,
1285                lhs: ops[1],
1286                rhs: Some(ops[2]),
1287            },
1288        }]);
1289    }
1290    Ok(vec![Inst::FArith {
1291        op,
1292        k,
1293        arith: FArith3::from_operands(mn, ops)?,
1294    }])
1295}
1296
1297fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1298    expect_len(mn, ops, 2)?;
1299    Ok(Inst::FSqrt {
1300        k,
1301        dst: fpr_op(mn, &ops[0])?,
1302        src: ops[1],
1303    })
1304}
1305
1306fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1307    expect_len(mn, ops, 4)?;
1308    Ok(Inst::Fma {
1309        op,
1310        k,
1311        a: ops[1],
1312        b: ops[2],
1313        c: ops[3],
1314        dst: fpr_op(mn, &ops[0])?,
1315    })
1316}
1317
1318/// Reverses the bytes of `src` into `dst`.
1319fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1320    expect_len(mn, ops, 2)?;
1321    Ok(Inst::Bswap {
1322        w,
1323        dst: ops[0],
1324        src: ops[1],
1325    })
1326}
1327
1328fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1329    expect_len(mn, ops, 2)?;
1330    Ok(Inst::FRound {
1331        k,
1332        mode,
1333        dst: fpr_op(mn, &ops[0])?,
1334        src: ops[1],
1335    })
1336}
1337
1338fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1339    expect_len(mn, ops, 2)?;
1340    Ok(Inst::CvtFI {
1341        k,
1342        w,
1343        dst: gpr_op(mn, &ops[0])?,
1344        src: ops[1],
1345    })
1346}
1347
1348fn cvtif_node(
1349    mn: Mnemonic,
1350    ops: &[Op],
1351    signed: bool,
1352    from: Width,
1353    k: FKind,
1354) -> Result<Inst, ExpandError> {
1355    expect_len(mn, ops, 2)?;
1356    Ok(Inst::CvtIF {
1357        signed,
1358        from,
1359        k,
1360        dst: fpr_op(mn, &ops[0])?,
1361        src: ops[1],
1362    })
1363}
1364
1365fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1366    expect_len(mn, ops, 2)?;
1367    Ok(Inst::CvtFF {
1368        from,
1369        to,
1370        dst: fpr_op(mn, &ops[0])?,
1371        src: ops[1],
1372    })
1373}
1374
1375fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1376    expect_len(mn, ops, 2)?;
1377    Ok(Inst::Bitcast {
1378        dir,
1379        dst: ops[0],
1380        src: ops[1],
1381    })
1382}
1383
1384fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1385    expect_len(mn, ops, 2)?;
1386    let dst = fpr_op(mn, &ops[0])?;
1387    let inst = if negate {
1388        Inst::FNeg {
1389            k,
1390            dst,
1391            src: ops[1],
1392        }
1393    } else {
1394        Inst::FAbs {
1395            k,
1396            dst,
1397            src: ops[1],
1398        }
1399    };
1400    Ok(inst)
1401}
1402
1403fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1404    expect_len(mn, ops, 3)?;
1405    let lane = imm_op(mn, &ops[2])?;
1406    let lanes: i64 = 128 / w.bytes() as i64;
1407    if lane < 0 || lane >= lanes {
1408        return Err(bad(mn, "lane out of range"));
1409    }
1410    Ok(Inst::VecExtract {
1411        w,
1412        dst: gpr_op(mn, &ops[0])?,
1413        src: fpr_op(mn, &ops[1])?,
1414        lane: lane as u8,
1415    })
1416}
1417
1418fn trunc_q_slow(
1419    mn: Mnemonic,
1420    ops: &[Op],
1421    sc: &Scratch,
1422    k: FKind,
1423) -> Result<Vec<Inst>, ExpandError> {
1424    expect_len(mn, ops, 2)?;
1425    let (dst, src) = (ops[0], ops[1]);
1426    let slow = Lbl::Fresh(0);
1427    let done = Lbl::Fresh(1);
1428    let gs = Op::Gpr(sc.gpr);
1429    let fs = Op::Fpr(sc.fpr);
1430    let (int64_min, neg_min) = match k {
1431        FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1432        FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1433    };
1434    let dst_gpr = gpr_op(mn, &dst)?;
1435    Ok(vec![
1436        Inst::Mov {
1437            w: Width::B64,
1438            dst: gs,
1439            src: Op::Imm(neg_min),
1440            patch: false,
1441        },
1442        Inst::Bitcast {
1443            dir: if k == FKind::F32 {
1444                BitcastDir::I2F
1445            } else {
1446                BitcastDir::Q2D
1447            },
1448            dst: fs,
1449            src: gs,
1450        },
1451        Inst::BranchF {
1452            cc: FCc::Ge,
1453            k,
1454            lhs: src,
1455            rhs: fs,
1456            target: slow,
1457            patch: false,
1458        },
1459        Inst::CvtFI {
1460            k,
1461            w: Width::B64,
1462            dst: dst_gpr,
1463            src,
1464        },
1465        Inst::Jump {
1466            target: Target::Label(done),
1467            patch: false,
1468        },
1469        Inst::Bind(slow),
1470        Inst::Mov {
1471            w: Width::B64,
1472            dst: gs,
1473            src: Op::Imm(int64_min),
1474            patch: false,
1475        },
1476        Inst::Bitcast {
1477            dir: if k == FKind::F32 {
1478                BitcastDir::I2F
1479            } else {
1480                BitcastDir::Q2D
1481            },
1482            dst: fs,
1483            src: gs,
1484        },
1485        Inst::FArith {
1486            op: FAlu::Add,
1487            k,
1488            arith: FArith3 {
1489                dst: sc.fpr,
1490                lhs: src,
1491                rhs: None,
1492            },
1493        },
1494        Inst::CvtFI {
1495            k,
1496            w: Width::B64,
1497            dst: dst_gpr,
1498            src: fs,
1499        },
1500        Inst::Mov {
1501            w: Width::B64,
1502            dst: gs,
1503            src: Op::Imm(0x8000000000000000u64 as i64),
1504            patch: false,
1505        },
1506        Inst::Or {
1507            w: Width::B64,
1508            op: Alu3 {
1509                dst,
1510                lhs: dst,
1511                rhs: gs,
1512            },
1513        },
1514        Inst::Bind(done),
1515    ])
1516}
1517
1518fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Vec<Inst>, ExpandError> {
1519    expect_len(mn, ops, 3)?;
1520    let (src, dst) = (ops[1], ops[0]);
1521    let slow = Lbl::Fresh(0);
1522    let done = Lbl::Fresh(1);
1523    let s1 = gpr_op(mn, &ops[2])?;
1524    let s2 = sc.gpr;
1525    let dst_fpr = fpr_op(mn, &dst)?;
1526    Ok(vec![
1527        Inst::TestBranch {
1528            w: Width::B64,
1529            value: src,
1530            mask: None,
1531            test: TestKind::Sign,
1532            target: slow,
1533            patch: false,
1534        },
1535        Inst::CvtIF {
1536            signed: true,
1537            from: Width::B64,
1538            k,
1539            dst: dst_fpr,
1540            src,
1541        },
1542        Inst::Jump {
1543            target: Target::Label(done),
1544            patch: false,
1545        },
1546        Inst::Bind(slow),
1547        Inst::Mov {
1548            w: Width::B64,
1549            dst: Op::Gpr(s1),
1550            src,
1551            patch: false,
1552        },
1553        Inst::Mov {
1554            w: Width::B64,
1555            dst: Op::Gpr(s2),
1556            src,
1557            patch: false,
1558        },
1559        Inst::Shift {
1560            w: Width::B64,
1561            op: ShiftOp::Shr,
1562            dst: s1,
1563            cnt: ShiftCnt::Imm(1),
1564        },
1565        Inst::And {
1566            w: Width::B64,
1567            op: Alu3 {
1568                dst: Op::Gpr(s2),
1569                lhs: Op::Gpr(s2),
1570                rhs: Op::Imm(1),
1571            },
1572        },
1573        Inst::Or {
1574            w: Width::B64,
1575            op: Alu3 {
1576                dst: Op::Gpr(s2),
1577                lhs: Op::Gpr(s2),
1578                rhs: Op::Gpr(s1),
1579            },
1580        },
1581        Inst::CvtIF {
1582            signed: true,
1583            from: Width::B64,
1584            k,
1585            dst: dst_fpr,
1586            src: Op::Gpr(s2),
1587        },
1588        Inst::FArith {
1589            op: FAlu::Add,
1590            k,
1591            arith: FArith3 {
1592                dst: dst_fpr,
1593                lhs: Op::Fpr(dst_fpr),
1594                rhs: None,
1595            },
1596        },
1597        Inst::Bind(done),
1598    ])
1599}
1600
1601/// Expands `btd2i` to a convert plus an overflow-sentinel branch.
1602///
1603/// Takes the branch when the low 32 bits equal `0x80000000`.
1604fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
1605    expect_len(mn, ops, 3)?;
1606    Ok(vec![
1607        Inst::CvtFI {
1608            k: FKind::F64,
1609            w: Width::B64,
1610            dst: gpr_op(mn, &ops[0])?,
1611            src: ops[1],
1612        },
1613        Inst::Branch {
1614            cc: IntCc::Eq,
1615            w: Width::B32,
1616            lhs: ops[0],
1617            rhs: Op::Imm(-0x8000_0000i64),
1618            target: label_op(mn, &ops[2])?,
1619            patch: false,
1620        },
1621    ])
1622}
1623
1624fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
1625    expect_len(mn, ops, 3)?;
1626    let target = label_op(mn, &ops[2])?;
1627    Ok(vec![
1628        Inst::CvtFI {
1629            k: FKind::F64,
1630            w: Width::B64,
1631            dst: gpr_op(mn, &ops[0])?,
1632            src: ops[1],
1633        },
1634        Inst::TestBranch {
1635            w: Width::B64,
1636            value: ops[0],
1637            mask: None,
1638            test: TestKind::Zero,
1639            target,
1640            patch: false,
1641        },
1642        Inst::CvtIF {
1643            signed: true,
1644            from: Width::B64,
1645            k: FKind::F64,
1646            dst: sc.fpr,
1647            src: ops[0],
1648        },
1649        Inst::BranchF {
1650            cc: FCc::NeUn,
1651            k: FKind::F64,
1652            lhs: ops[1],
1653            rhs: Op::Fpr(sc.fpr),
1654            target,
1655            patch: false,
1656        },
1657    ])
1658}
1659
1660/// Zeros a double by computing `dst -= dst`.
1661fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1662    expect_len(mn, ops, 1)?;
1663    let dst = fpr_op(mn, &ops[0])?;
1664    Ok(Inst::FArith {
1665        op: FAlu::Sub,
1666        k: FKind::F64,
1667        arith: FArith3 {
1668            dst,
1669            lhs: Op::Fpr(dst),
1670            rhs: None,
1671        },
1672    })
1673}