1use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6use smallvec::{smallvec, SmallVec};
7
8pub const EXPANSION_INLINE: usize = 2;
12
13pub type Expansion = SmallVec<[Inst; EXPANSION_INLINE]>;
15
16pub fn expand(
32 mn: Mnemonic,
33 ops: &[Op],
34 sc: &Scratch,
35 patch: bool,
36) -> Result<Expansion, ExpandError> {
37 if patch && patch_kind(mn).is_none() {
38 return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
39 }
40 check_extern_operands(mn, ops, patch)?;
41 let mut insts = expand_inner(mn, ops, sc, patch)?;
42 stage_aliasing_mem(&mut insts, sc);
43 Ok(insts)
44}
45
46fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
50 let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
51 return Ok(());
52 };
53 if patch {
54 return Err(bad(mn, "an external symbol cannot be a patch site"));
55 }
56 let ok = match mn {
57 Mnemonic::Jmp | Mnemonic::Call => at == 0,
58 Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
59 Mnemonic::Lea {
60 w: Width::B64,
61 form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
62 } => at == 1,
63 _ => false,
64 };
65 if ok {
66 Ok(())
67 } else {
68 Err(bad(
69 mn,
70 "an external symbol is only a `jmp`/`call` target or the source of \
71 `move`/`leap`/`globaladdr`/`la`",
72 ))
73 }
74}
75
76fn stage_aliasing_mem(insts: &mut Expansion, sc: &Scratch) {
91 if !insts.iter().any(needs_staging) {
94 return;
95 }
96 let old = std::mem::take(insts);
97 let mut out = Expansion::new();
98 for mut inst in old {
99 if let Some(loads) = stage_one(&mut inst, sc) {
100 out.extend(loads);
101 }
102 out.push(inst);
103 }
104 *insts = out;
105}
106
107fn aliases(m: &Mem, g: Gpr) -> bool {
109 m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g)
110}
111
112fn aliasing_mem<'a>(dst: &Op, operands: impl IntoIterator<Item = &'a Op>) -> bool {
115 let Op::Gpr(g) = *dst else {
116 return false;
117 };
118 operands
119 .into_iter()
120 .any(|op| matches!(op, Op::Mem(m) if aliases(m, g)))
121}
122
123fn needs_staging(inst: &Inst) -> bool {
128 match inst {
129 Inst::Add { op, .. }
130 | Inst::Sub { op, .. }
131 | Inst::Mul { op, .. }
132 | Inst::And { op, .. }
133 | Inst::Or { op, .. }
134 | Inst::Xor { op, .. } => aliasing_mem(&op.dst, [&op.lhs, &op.rhs]),
135 Inst::AluS { alu, .. } => aliasing_mem(&alu.dst, [&alu.lhs, &alu.rhs]),
136 Inst::OpBranch { args, .. } => {
137 let ops = args.as_slice();
138 let Some((dst, srcs)) = ops.split_last() else {
139 return false;
140 };
141 aliasing_mem(dst, srcs.iter())
142 }
143 _ => false,
144 }
145}
146
147fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Expansion> {
151 match inst {
152 Inst::Add { w, op }
153 | Inst::Sub { w, op }
154 | Inst::Mul { w, op }
155 | Inst::And { w, op }
156 | Inst::Or { w, op }
157 | Inst::Xor { w, op } => stage_ops(*w, &mut op.dst, [&mut op.lhs, &mut op.rhs], sc),
158 Inst::AluS { w, alu, .. } => stage_ops(*w, &mut alu.dst, [&mut alu.lhs, &mut alu.rhs], sc),
159 Inst::OpBranch { w, args, .. } => {
160 let (dst, srcs) = args.as_mut_slice().split_last_mut()?;
161 stage_ops(*w, dst, srcs.iter_mut(), sc)
162 }
163 _ => None,
164 }
165}
166
167fn stage_ops<'a>(
169 w: Width,
170 dst: &mut Op,
171 operands: impl IntoIterator<Item = &'a mut Op>,
172 sc: &Scratch,
173) -> Option<Expansion> {
174 let Op::Gpr(g) = *dst else {
175 return None;
176 };
177 let slots = [sc.gpr, sc.gpr2];
178 let mut loads = Expansion::new();
179 let mut slot = 0usize;
180 for operand in operands {
181 let Op::Mem(m) = operand else {
182 continue;
183 };
184 if !aliases(m, g) {
185 continue;
186 }
187 let &s = slots.get(slot)?;
190 slot += 1;
191 loads.push(Inst::Load {
192 w,
193 sx: Sx::Zero,
194 dst: s,
195 addr: *m,
196 acquire: false,
197 });
198 *operand = Op::Gpr(s);
199 }
200 (!loads.is_empty()).then_some(loads)
201}
202
203pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
205 Some(match mn {
206 Mnemonic::Jmp => PatchKind::JumpSite,
207 Mnemonic::Call => PatchKind::CallSite,
208 Mnemonic::Move => PatchKind::MovBlock,
209 Mnemonic::FlagBranch(_)
210 | Mnemonic::OpBranch { .. }
211 | Mnemonic::RiscAluS { .. }
212 | Mnemonic::TestBranch { .. }
213 | Mnemonic::IntBranch { .. }
214 | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
215 _ => return None,
216 })
217}
218
219#[derive(Clone, Copy, Debug, PartialEq, Eq)]
225pub enum PatchKind {
226 JumpSite,
228 CondJumpSite,
230 CallSite,
232 MovBlock,
234}
235
236fn float_cc(cc: FCc) -> (FCc, bool) {
237 match cc {
238 FCc::Lt => (FCc::Gt, true),
239 FCc::Le => (FCc::Ge, true),
240 FCc::GtUn => (FCc::GtUn, false),
241 FCc::GeUn => (FCc::GeUn, false),
242 FCc::LtUn => (FCc::GtUn, true),
243 FCc::LeUn => (FCc::GeUn, true),
244 other => (other, false),
245 }
246}
247
248fn test_set_node(
249 mn: Mnemonic,
250 ops: &[Op],
251 w: Width,
252 test: TestKind,
253) -> Result<Expansion, ExpandError> {
254 if ops.len() != 2 && ops.len() != 3 {
255 return Err(bad(mn, "test-set takes 2 or 3 operands"));
256 }
257 let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
258 Ok(smallvec![Inst::TestSet {
259 w,
260 value: ops[1],
261 mask,
262 test,
263 dst: ops[0],
264 }])
265}
266
267fn test_branch_node(
268 mn: Mnemonic,
269 ops: &[Op],
270 w: Width,
271 test: TestKind,
272 patch: bool,
273) -> Result<Expansion, ExpandError> {
274 if ops.len() != 2 && ops.len() != 3 {
275 return Err(bad(mn, "test-branch takes 2 or 3 operands"));
276 }
277 let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
278 let target = label_op(
279 mn,
280 ops.last()
281 .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
282 )?;
283 Ok(smallvec![Inst::TestBranch {
284 w,
285 value: ops[0],
286 mask,
287 test,
288 target,
289 patch,
290 }])
291}
292
293fn div_rem_node(
294 mn: Mnemonic,
295 ops: &[Op],
296 signed: bool,
297 w: Width,
298 rem: bool,
299) -> Result<Expansion, ExpandError> {
300 let (dvd, dvs, dst) = match ops {
301 [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
302 [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
303 _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
304 };
305 Ok(smallvec![if rem {
306 Inst::Rem3 {
307 signed,
308 w,
309 dst,
310 dvd,
311 dvs,
312 }
313 } else {
314 Inst::Div3 {
315 signed,
316 w,
317 dst,
318 dvd,
319 dvs,
320 }
321 }])
322}
323
324fn shift_mem_canon(
325 mn: Mnemonic,
326 w: Width,
327 op: ShiftOp,
328 cnt: &Op,
329 dst: &Mem,
330 sc: &Scratch,
331) -> Result<Expansion, ExpandError> {
332 let shift_cnt = match cnt {
333 Op::Imm(v) => ShiftCnt::Imm(*v),
334 Op::Gpr(g) => ShiftCnt::Reg(*g),
335 _ => return Err(bad(mn, "shift count must be an immediate or register")),
336 };
337 Ok(smallvec![
338 Inst::Load {
339 w,
340 sx: Sx::Zero,
341 dst: sc.gpr,
342 addr: *dst,
343 acquire: false,
344 },
345 Inst::Shift {
346 w,
347 op,
348 dst: sc.gpr,
349 cnt: shift_cnt,
350 },
351 Inst::Store {
352 w,
353 src: Op::Gpr(sc.gpr),
354 addr: *dst,
355 },
356 ])
357}
358
359fn shift_node(
360 mn: Mnemonic,
361 w: Width,
362 op: ShiftOp,
363 ops: &[Op],
364 sc: &Scratch,
365) -> Result<Expansion, ExpandError> {
366 if ops.len() != 2 {
367 return Err(bad(mn, "shift takes 2 operands"));
368 }
369 if let Op::Mem(a) = &ops[0] {
370 return shift_mem_canon(mn, w, op, &ops[1], a, sc);
371 }
372 let dst = gpr_op(mn, &ops[0])?;
373 let cnt = match &ops[1] {
374 Op::Imm(v) => ShiftCnt::Imm(*v),
375 Op::Gpr(g) => ShiftCnt::Reg(*g),
376 _ => return Err(bad(mn, "shift count must be an immediate or register")),
377 };
378 Ok(smallvec![Inst::Shift { w, op, dst, cnt }])
379}
380
381fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
382 Alu3::from_operands(mn, ops)
383}
384
385fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<BranchArgs, ExpandError> {
386 match ops {
387 [dst, rhs] => Ok(BranchArgs::two(*rhs, *dst)),
388 [dst, lhs, rhs] => Ok(BranchArgs::three(*lhs, *rhs, *dst)),
389 _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
390 }
391}
392
393fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
394 match op {
395 Op::Label(l) => Ok(Target::Label(*l)),
396 Op::Gpr(g) => Ok(Target::Reg(*g)),
397 Op::Mem(a) => Ok(Target::Mem(*a)),
398 Op::Imm(v) => Ok(Target::Abs(*v)),
399 Op::Extern(e) => Ok(Target::Extern(*e)),
400 _ => Err(bad(
401 mn,
402 "jump/call needs a label, register, memory, or immediate",
403 )),
404 }
405}
406
407#[allow(clippy::too_many_lines)]
408fn expand_inner(
409 mn: Mnemonic,
410 ops: &[Op],
411 sc: &Scratch,
412 patch: bool,
413) -> Result<Expansion, ExpandError> {
414 let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
415 expect_len(mn, ops, 1)?;
416 Ok(ops[0])
417 };
418 match mn {
419 Mnemonic::Move => Ok(smallvec![mov_node(mn, ops, Width::B64, patch)?]),
420 Mnemonic::Load(w) => Ok(smallvec![mov_node(mn, ops, w, patch)?]),
421 Mnemonic::AtomicLoad(w) => Ok(smallvec![atomic_load_node(mn, ops, w)?]),
422 Mnemonic::Store(w) => Ok(smallvec![store_node(mn, ops, w)?]),
423 Mnemonic::Extend { from, to, sx } => Ok(smallvec![extend_node(mn, ops, from, to, sx)?]),
424 Mnemonic::MovZero32 => Ok(smallvec![movzero_node(mn, ops)?]),
425 Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
426 Mnemonic::FMove(k) => Ok(smallvec![fmove_node(mn, ops, k)?]),
427 Mnemonic::VMove { store } => Ok(smallvec![vmove_node(mn, ops, store)?]),
428 Mnemonic::Lea { w, .. } => Ok(smallvec![lea_node(mn, ops, w)?]),
429 Mnemonic::LoadQInc => loadqinc_node(mn, ops),
430 Mnemonic::LoadPair(w) => Ok(smallvec![pair_node(mn, ops, w, true)?]),
431 Mnemonic::StorePair(w) => Ok(smallvec![pair_node(mn, ops, w, false)?]),
432 Mnemonic::Push => stack_nodes(mn, ops, true, false),
433 Mnemonic::Pop => stack_nodes(mn, ops, false, false),
434 Mnemonic::PushV => stack_nodes(mn, ops, true, true),
435 Mnemonic::PopV => stack_nodes(mn, ops, false, true),
436 Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
437 Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
438 Mnemonic::Peek => Ok(smallvec![peek_node(mn, ops, true)?]),
439 Mnemonic::Poke => Ok(smallvec![peek_node(mn, ops, false)?]),
440 Mnemonic::TlsLoad(w) => Ok(smallvec![tls_node(mn, ops, w, true)?]),
441 Mnemonic::TlsStore(w) => Ok(smallvec![tls_node(mn, ops, w, false)?]),
442 Mnemonic::TlsLea => Ok(smallvec![tls_lea_node(mn, ops)?]),
443
444 Mnemonic::Add(w) => Ok(smallvec![Inst::Add {
445 w,
446 op: alu3(mn, ops)?,
447 }]),
448 Mnemonic::Sub(w) => Ok(smallvec![Inst::Sub {
449 w,
450 op: alu3(mn, ops)?,
451 }]),
452 Mnemonic::Mul(w) => Ok(smallvec![mul_node(mn, w, ops)?]),
453 Mnemonic::And(w) => Ok(smallvec![Inst::And {
454 w,
455 op: alu3(mn, ops)?,
456 }]),
457 Mnemonic::Or(w) => Ok(smallvec![Inst::Or {
458 w,
459 op: alu3(mn, ops)?,
460 }]),
461 Mnemonic::Xor(w) => Ok(smallvec![Inst::Xor {
462 w,
463 op: alu3(mn, ops)?,
464 }]),
465 Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
466 Mnemonic::Not(w) => Ok(smallvec![Inst::Not {
467 w,
468 dst: one(mn, ops)?,
469 }]),
470 Mnemonic::Neg(w) => Ok(smallvec![Inst::Neg {
471 w,
472 dst: one(mn, ops)?,
473 }]),
474 Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
475 Mnemonic::Bswap(w) => Ok(smallvec![bswap_node(mn, ops, w)?]),
476 Mnemonic::Adc => Ok(smallvec![adc_node(mn, ops, false)?]),
477 Mnemonic::Sbb => Ok(smallvec![adc_node(mn, ops, true)?]),
478 Mnemonic::MulHigh { signed } => Ok(smallvec![Inst::MulHigh {
479 signed,
480 rhs: one(mn, ops)?,
481 }]),
482 Mnemonic::MulWide { signed } => Ok(smallvec![mulwide_node(mn, ops, signed)?]),
483 Mnemonic::AluS { op, w } => Ok(smallvec![Inst::AluS {
484 op,
485 w,
486 alu: alu3(mn, ops)?,
487 }]),
488 Mnemonic::RiscAluS { op, w } => {
489 let has_label =
490 matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
491 if has_label {
492 let flag_op = match op {
493 AluOp::Add => FlagAlu::Add,
494 AluOp::Sub => FlagAlu::Sub,
495 AluOp::Or => FlagAlu::Or,
496 };
497 return Ok(smallvec![Inst::OpBranch {
498 op: flag_op,
499 w,
500 flag: PredFlag::S,
501 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
502 target: label_op(
503 mn,
504 ops.last()
505 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
506 )?,
507 patch,
508 }]);
509 }
510 Ok(smallvec![Inst::AluS {
511 op,
512 w,
513 alu: alu3(mn, ops)?,
514 }])
515 }
516 Mnemonic::AddShifted => Ok(smallvec![addshifted_node(mn, ops)?]),
517 Mnemonic::Cqo { quad } => Ok(smallvec![Inst::Cqo { quad }]),
518 Mnemonic::DivMod1 { signed, w } => Ok(smallvec![Inst::DivMod1 {
519 signed,
520 w,
521 divisor: one(mn, ops)?,
522 }]),
523 Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
524 Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
525 Mnemonic::IntSet { w, cc } => {
526 expect_len(mn, ops, 3)?;
527 Ok(smallvec![Inst::IntSet {
528 cc,
529 w,
530 dst: ops[0],
531 lhs: ops[1],
532 rhs: ops[2],
533 }])
534 }
535 Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
536 Mnemonic::Cmov { zero } => Ok(smallvec![cmov_node(mn, ops, zero)?]),
537 Mnemonic::BitCount { w, trailing } => Ok(smallvec![cz_node(mn, ops, w, trailing)?]),
538 Mnemonic::Popcnt(w) => Ok(smallvec![popcnt_node(mn, ops, w)?]),
539 Mnemonic::Bfi => bfi_node(mn, ops, sc),
540 Mnemonic::IntBranch { w, cc } => {
541 expect_len(mn, ops, 3)?;
542 Ok(smallvec![Inst::Branch {
543 cc,
544 w,
545 lhs: ops[0],
546 rhs: ops[1],
547 target: label_op(mn, &ops[2])?,
548 patch,
549 }])
550 }
551 Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
552 Mnemonic::OpBranch { op, w, flag } => {
553 if ops.len() != 3 && ops.len() != 4 {
554 return Err(bad(mn, "op-branch takes 3 or 4 operands"));
555 }
556 Ok(smallvec![Inst::OpBranch {
557 op,
558 w,
559 flag,
560 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
561 target: label_op(
562 mn,
563 ops.last()
564 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
565 )?,
566 patch,
567 }])
568 }
569 Mnemonic::FlagBranch(flag) => Ok(smallvec![flag_branch_node(mn, ops, flag, patch)?]),
570 Mnemonic::BranchF { cc, k } => {
571 expect_len(mn, ops, 3)?;
572 let (cc, reverse) = float_cc(cc);
573 let (lhs, rhs) = if reverse {
574 (ops[1], ops[0])
575 } else {
576 (ops[0], ops[1])
577 };
578 Ok(smallvec![Inst::BranchF {
579 cc,
580 k,
581 lhs,
582 rhs,
583 target: label_op(mn, &ops[2])?,
584 patch,
585 }])
586 }
587 Mnemonic::Jmp => {
588 expect_len(mn, ops, 1)?;
589 Ok(smallvec![Inst::Jump {
590 target: jump_target(mn, &ops[0])?,
591 patch,
592 }])
593 }
594 Mnemonic::Call => {
595 expect_len(mn, ops, 1)?;
596 Ok(smallvec![Inst::Call {
597 target: jump_target(mn, &ops[0])?,
598 patch,
599 }])
600 }
601 Mnemonic::Ret => {
602 expect_len(mn, ops, 0)?;
603 Ok(smallvec![Inst::Ret])
604 }
605 Mnemonic::Break => {
606 expect_len(mn, ops, 0)?;
607 Ok(smallvec![Inst::Break])
608 }
609 Mnemonic::Nop => {
610 expect_len(mn, ops, 0)?;
611 Ok(smallvec![Inst::Nop])
612 }
613 Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
614 Mnemonic::Fma { op, k } => Ok(smallvec![fma_node(mn, ops, k, op)?]),
615 Mnemonic::FSqrt(k) => Ok(smallvec![fsqrt_node(mn, ops, k)?]),
616 Mnemonic::FRound { k, mode } => Ok(smallvec![fround_node(mn, ops, k, mode)?]),
617 Mnemonic::FAbs { k, negate } => Ok(smallvec![fabs_node(mn, ops, k, negate)?]),
618 Mnemonic::FSet { cc, k } => {
619 expect_len(mn, ops, 3)?;
620 let (cc, reverse) = float_cc(cc);
621 let (lhs, rhs) = if reverse {
622 (ops[2], ops[1])
623 } else {
624 (ops[1], ops[2])
625 };
626 Ok(smallvec![Inst::FSet {
627 cc,
628 k,
629 dst: ops[0],
630 lhs,
631 rhs,
632 }])
633 }
634 Mnemonic::CvtFI { k, w } => Ok(smallvec![cvtfi_node(mn, ops, k, w)?]),
635 Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
636 Mnemonic::CvtIF { signed, from, k } => Ok(smallvec![cvtif_node(mn, ops, signed, from, k)?]),
637 Mnemonic::CvtFF { from, to } => Ok(smallvec![cvtff_node(mn, ops, from, to)?]),
638 Mnemonic::Bitcast(dir) => Ok(smallvec![bitcast_node(mn, ops, dir)?]),
639 Mnemonic::MovDZ => Ok(smallvec![movdz_node(mn, ops)?]),
640 Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
641 Mnemonic::Btd2i => btd2i_node(mn, ops),
642 Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
643 Mnemonic::VecExtract(w) => Ok(smallvec![umov_node(mn, ops, w)?]),
644 Mnemonic::AtomicRmw { op, w } => Ok(smallvec![rmw_node(mn, ops, w, op)?]),
645 Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
646 Mnemonic::AtomicCas { w, branch } => Ok(smallvec![cas_node(mn, ops, w, branch)?]),
647 Mnemonic::LoadLinked(w) => Ok(smallvec![ll_node(mn, ops, w)?]),
648 Mnemonic::StoreCond(w) => Ok(smallvec![sc_node(mn, ops, w)?]),
649 Mnemonic::Fence { full } => Ok(smallvec![Inst::Fence { full }]),
650 Mnemonic::Pause => Ok(smallvec![Inst::Pause]),
651 Mnemonic::TagCodePtr
652 | Mnemonic::TagReturnAddress
653 | Mnemonic::UntagReturnAddress
654 | Mnemonic::RemoveCodePtrTag
655 | Mnemonic::UntagArrayPtr
656 | Mnemonic::RemoveArrayPtrTag => Ok(smallvec![]),
657 }
658}
659
660fn flag_branch_node(
661 mn: Mnemonic,
662 ops: &[Op],
663 flag: PredFlag,
664 patch: bool,
665) -> Result<Inst, ExpandError> {
666 expect_len(mn, ops, 1)?;
667 Ok(Inst::FlagBranch {
668 flag,
669 target: label_op(mn, &ops[0])?,
670 patch,
671 })
672}
673
674fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
675 expect_len(mn, ops, 2)?;
676 Ok(Inst::Popcnt {
677 w,
678 dst: gpr_op(mn, &ops[0])?,
679 src: ops[1],
680 })
681}
682
683fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
684 expect_len(mn, ops, 2)?;
685 let dst = gpr_op(mn, &ops[0])?;
686 let inst = if trailing {
687 Inst::Ctz {
688 w,
689 dst,
690 src: ops[1],
691 }
692 } else {
693 Inst::Clz {
694 w,
695 dst,
696 src: ops[1],
697 }
698 };
699 Ok(inst)
700}
701
702fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
703 expect_len(mn, ops, 2)?;
704 if load {
705 let idx = imm_op(mn, &ops[1])?;
706 Ok(Inst::TlsLoad {
707 w,
708 dst: gpr_op(mn, &ops[0])?,
709 idx,
710 })
711 } else {
712 let idx = imm_op(mn, &ops[0])?;
713 Ok(Inst::TlsStore {
714 w,
715 src: gpr_op(mn, &ops[1])?,
716 idx,
717 })
718 }
719}
720
721fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
722 expect_len(mn, ops, 2)?;
723 Ok(Inst::TlsLea {
724 dst: gpr_op(mn, &ops[0])?,
725 idx: imm_op(mn, &ops[1])?,
726 })
727}
728
729fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
730 expect_len(mn, ops, 2)?;
731
732 Ok(Inst::AtomicRmw {
733 op,
734 w,
735 a: ops[1],
736 b: ops[0],
737 })
738}
739
740fn rmw_not_node(
741 mn: Mnemonic,
742 ops: &[Op],
743 sc: &Scratch,
744 w: Width,
745 op: AtomicOp,
746) -> Result<Expansion, ExpandError> {
747 expect_len(mn, ops, 2)?;
748 let mut out = Expansion::new();
749 let a = match &ops[1] {
751 Op::Imm(v) => Op::Imm(!v),
752 Op::Gpr(g) => {
753 out.push(Inst::Mov {
754 w: Width::B64,
755 dst: Op::Gpr(sc.gpr),
756 src: Op::Gpr(*g),
757 patch: false,
758 });
759 out.push(Inst::Not {
760 w,
761 dst: Op::Gpr(sc.gpr),
762 });
763 Op::Gpr(sc.gpr)
764 }
765 Op::Mem(addr) => {
766 let lw = if w == Width::B64 {
767 Width::B64
768 } else {
769 Width::B32
770 };
771 out.push(Inst::Load {
772 w: lw,
773 sx: Sx::Zero,
774 dst: sc.gpr,
775 addr: *addr,
776 acquire: false,
777 });
778 out.push(Inst::Not {
779 w,
780 dst: Op::Gpr(sc.gpr),
781 });
782 Op::Gpr(sc.gpr)
783 }
784 _ => {
785 return Err(bad(
786 mn,
787 "atomic clear needs an immediate, register, or memory operand",
788 ))
789 }
790 };
791 out.push(Inst::AtomicRmw {
792 op,
793 w,
794 a,
795 b: ops[0],
796 });
797 Ok(out)
798}
799
800fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
801 let n = if branch { 4 } else { 3 };
802 expect_len(mn, ops, n)?;
803
804 Ok(Inst::AtomicCas {
805 w,
806 mem: mem_op(mn, &ops[0])?,
807 src: gpr_op(mn, &ops[2])?,
808 branch: if branch {
809 Some(label_op(mn, &ops[3])?)
810 } else {
811 None
812 },
813 })
814}
815
816fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
817 expect_len(mn, ops, 2)?;
818 Ok(Inst::LoadLinked {
819 w,
820 mem: mem_op(mn, &ops[1])?,
821 dst: gpr_op(mn, &ops[0])?,
822 })
823}
824
825fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
826 expect_len(mn, ops, 3)?;
827 Ok(Inst::StoreCond {
828 w,
829 status: gpr_op(mn, &ops[1])?,
830 src: gpr_op(mn, &ops[2])?,
831 mem: mem_op(mn, &ops[0])?,
832 })
833}
834
835fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
839 expect_len(mn, ops, 4)?;
840 let lsb = imm_op(mn, &ops[2])?;
841 let width = imm_op(mn, &ops[3])?;
842 if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
843 return Err(bad(mn, "bitfield out of range"));
844 }
845 let wmask: u64 = if width == 64 {
846 u64::MAX
847 } else {
848 (1u64 << width) - 1
849 };
850 let mask: i64 = (wmask << lsb) as i64;
851 let s = sc.gpr;
852 let dst = gpr_op(mn, &ops[0])?;
853
854 Ok(smallvec![
855 Inst::Mov {
856 w: Width::B64,
857 dst: Op::Gpr(s),
858 src: ops[1],
859 patch: false,
860 },
861 Inst::And {
862 w: Width::B64,
863 op: Alu3 {
864 dst: Op::Gpr(s),
865 lhs: Op::Gpr(s),
866 rhs: Op::Imm(wmask as i64),
867 },
868 },
869 Inst::Shift {
870 w: Width::B64,
871 op: ShiftOp::Shl,
872 dst: s,
873 cnt: ShiftCnt::Imm(lsb),
874 },
875 Inst::And {
876 w: Width::B64,
877 op: Alu3 {
878 dst: Op::Gpr(dst),
879 lhs: Op::Gpr(dst),
880 rhs: Op::Imm(!mask),
881 },
882 },
883 Inst::Or {
884 w: Width::B64,
885 op: Alu3 {
886 dst: Op::Gpr(dst),
887 lhs: Op::Gpr(dst),
888 rhs: Op::Gpr(s),
889 },
890 },
891 ])
892}
893
894fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
895 expect_len(mn, ops, 3)?;
896 Ok(Inst::Cmov {
897 w: Width::B64,
898 zero,
899 dst: gpr_op(mn, &ops[0])?,
900 src: ops[1],
901 test: gpr_op(mn, &ops[2])?,
902 })
903}
904
905fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
908 expect_len(mn, ops, 2)?;
909 if peek {
910 let disp = slot_disp(mn, &ops[1])?;
911 return Ok(Inst::Peek {
912 disp,
913 dst: gpr_op(mn, &ops[0])?,
914 });
915 }
916 Ok(Inst::Poke {
917 disp: slot_disp(mn, &ops[0])?,
918 src: gpr_op(mn, &ops[1])?,
919 })
920}
921
922fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
923 let idx = imm_op(mn, op)?;
924 let off = idx
925 .checked_mul(8)
926 .ok_or_else(|| bad(mn, "stack offset overflow"))?;
927 if off < i32::MIN as i64 || off > i32::MAX as i64 {
928 return Err(bad(mn, "stack offset out of range"));
929 }
930 Ok(off as i32)
931}
932
933fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
934 expect_len(mn, ops, 3)?;
935 let dst = gpr_op(mn, &ops[0])?;
936 let base = gpr_op(mn, &ops[1])?;
937 let inc = imm_op(mn, &ops[2])?;
938 Ok(smallvec![
939 Inst::Load {
940 w: Width::B64,
941 sx: Sx::Zero,
942 dst,
943 addr: Mem::base(base, 0),
944 acquire: false,
945 },
946 Inst::Add {
947 w: Width::B64,
948 op: Alu3 {
949 dst: Op::Gpr(base),
950 lhs: Op::Gpr(base),
951 rhs: Op::Imm(inc),
952 },
953 },
954 ])
955}
956
957fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
958 expect_len(mn, ops, 3)?;
959 if load {
960 Ok(Inst::PairLoad {
961 w,
962 dst1: ops[0],
963 dst2: ops[1],
964 mem: mem_op(mn, &ops[2])?,
965 })
966 } else {
967 Ok(Inst::PairStore {
968 w,
969 src1: ops[1],
970 src2: ops[2],
971 mem: mem_op(mn, &ops[0])?,
972 })
973 }
974}
975
976fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
977 expect_len(mn, ops, 2)?;
978 let dst = gpr_op(mn, &ops[0])?;
979 let src = match &ops[1] {
980 Op::Mem(a) => LeaSrc::Mem(*a),
981 Op::Label(l) => LeaSrc::Label(*l),
982 Op::Extern(e) => LeaSrc::Extern(*e),
983 _ => return Err(bad(mn, "lea needs an address or label source")),
984 };
985 Ok(Inst::Lea { w, dst, src })
986}
987
988fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
989 expect_len(mn, ops, 2)?;
990 if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
992 return Ok(Inst::Lea {
993 w: Width::B64,
994 dst: *dst,
995 src: LeaSrc::Extern(*e),
996 });
997 }
998 Ok(Inst::Mov {
999 w,
1000 dst: ops[0],
1001 src: ops[1],
1002 patch,
1003 })
1004}
1005
1006fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1007 expect_len(mn, ops, 2)?;
1008 Ok(Inst::MovZero32 {
1009 dst: ops[0],
1010 src: ops[1],
1011 })
1012}
1013
1014fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1015 expect_len(mn, ops, 2)?;
1016 Ok(Inst::Load {
1017 w,
1018 sx: Sx::Zero,
1019 dst: gpr_op(mn, &ops[0])?,
1020 addr: mem_op(mn, &ops[1])?,
1021 acquire: true,
1022 })
1023}
1024
1025fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1026 expect_len(mn, ops, 2)?;
1027 let addr = mem_op(mn, &ops[0])?;
1028 Ok(Inst::Store {
1029 w,
1030 src: ops[1],
1031 addr,
1032 })
1033}
1034
1035fn extend_node(
1036 mn: Mnemonic,
1037 ops: &[Op],
1038 from: Width,
1039 to: Width,
1040 sx: Sx,
1041) -> Result<Inst, ExpandError> {
1042 expect_len(mn, ops, 2)?;
1043 Ok(Inst::Extend {
1044 from,
1045 to,
1046 sx,
1047 dst: gpr_op(mn, &ops[0])?,
1048 src: ops[1],
1049 })
1050}
1051
1052fn transfer_node(
1053 mn: Mnemonic,
1054 ops: &[Op],
1055 sc: &Scratch,
1056 w: Width,
1057) -> Result<Expansion, ExpandError> {
1058 expect_len(mn, ops, 2)?;
1059 let mut out = Expansion::new();
1060 match &ops[1] {
1061 Op::Gpr(_) => {}
1062 Op::Mem(a) => {
1063 out.push(Inst::Load {
1064 w,
1065 sx: Sx::Zero,
1066 dst: sc.gpr,
1067 addr: *a,
1068 acquire: false,
1069 });
1070 }
1071 Op::Imm(v) => {
1072 out.push(Inst::Mov {
1073 w,
1074 dst: Op::Gpr(sc.gpr),
1075 src: Op::Imm(*v),
1076 patch: false,
1077 });
1078 }
1079 Op::Label(l) => {
1080 out.push(Inst::Lea {
1081 w: Width::B64,
1082 dst: sc.gpr,
1083 src: LeaSrc::Label(*l),
1084 });
1085 let addr = Mem::base(sc.gpr, 0);
1086 out.push(Inst::Load {
1087 w,
1088 sx: Sx::Zero,
1089 dst: sc.gpr,
1090 addr,
1091 acquire: false,
1092 });
1093 }
1094 Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1095 Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1096 Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1097 }
1098 let src = if out.is_empty() {
1099 ops[1]
1100 } else {
1101 Op::Gpr(sc.gpr)
1102 };
1103 out.push(Inst::Mov {
1104 w,
1105 dst: ops[0],
1106 src,
1107 patch: false,
1108 });
1109 Ok(out)
1110}
1111
1112fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1113 expect_len(mn, ops, 2)?;
1114 match (&ops[0], &ops[1]) {
1115 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1116 k,
1117 dst: *d,
1118 src: *s,
1119 }),
1120 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1121 k,
1122 dst: *d,
1123 addr: *a,
1124 }),
1125 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1126 k,
1127 src: *s,
1128 addr: *a,
1129 }),
1130 _ => Err(bad(mn, "float move needs FPR/memory operands")),
1131 }
1132}
1133
1134fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1135 expect_len(mn, ops, 2)?;
1136 if !store {
1137 match (&ops[0], &ops[1]) {
1138 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1139 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1140 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1141 dst: *d,
1142 addr: Mem::base(*s, 0),
1143 }),
1144 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1145 }
1146 } else {
1147 match (&ops[0], &ops[1]) {
1148 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1149 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1150 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1151 dst: *d,
1152 addr: Mem::base(*s, 0),
1153 }),
1154 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1155 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1156 }
1157 }
1158}
1159
1160#[inline]
1170pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1171 if !matches!(
1172 mnemonic,
1173 Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1174 ) {
1175 return Ok(());
1176 }
1177 for op in ops {
1178 match op {
1179 Op::Gpr(g) if saved.contains(&g.0) => {}
1180 _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1181 }
1182 }
1183 Ok(())
1184}
1185
1186pub fn aligned_push_bytes(n: usize) -> i32 {
1194 ((8 * n + 15) & !15) as i32
1195}
1196
1197pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1201 8 * (n - 1 - i) as i32
1202}
1203
1204fn push_inst(mn: Mnemonic, op: &Op, push: bool, vector: bool) -> Result<Inst, ExpandError> {
1206 if vector {
1207 let x = fpr_op(mn, op)?;
1208 Ok(if push {
1209 Inst::PushV { src: x }
1210 } else {
1211 Inst::PopV { dst: x }
1212 })
1213 } else {
1214 match op {
1215 Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => Ok(if push {
1216 Inst::Push { op: *op }
1217 } else {
1218 Inst::Pop { op: *op }
1219 }),
1220 _ => Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1221 }
1222 }
1223}
1224
1225fn stack_nodes(
1226 mn: Mnemonic,
1227 ops: &[Op],
1228 push: bool,
1229 vector: bool,
1230) -> Result<Expansion, ExpandError> {
1231 if ops.is_empty() {
1232 return Err(bad(mn, "needs at least one operand"));
1233 }
1234 let mut out = Expansion::new();
1235 if push {
1239 for op in ops {
1240 out.push(push_inst(mn, op, true, vector)?);
1241 }
1242 } else {
1243 for op in ops.iter().rev() {
1244 out.push(push_inst(mn, op, false, vector)?);
1245 }
1246 }
1247 Ok(out)
1248}
1249
1250fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1251 if ops.len() == 3 {
1252 if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1253 return Ok(Inst::MulImm {
1254 w,
1255 dst: gpr_op(mn, &ops[0])?,
1256 lhs,
1257 imm: *v,
1258 });
1259 }
1260 if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1261 return Ok(Inst::MulImm {
1262 w,
1263 dst: gpr_op(mn, &ops[0])?,
1264 lhs,
1265 imm: *v,
1266 });
1267 }
1268 }
1269 Ok(Inst::Mul {
1270 w,
1271 op: Alu3::from_operands(mn, ops)?,
1272 })
1273}
1274
1275fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1276 expect_len(mn, ops, 3)?;
1277 Ok(Inst::MulWide {
1278 signed,
1279 dst: gpr_op(mn, &ops[0])?,
1280 lhs: gpr_op(mn, &ops[1])?,
1281 rhs: gpr_op(mn, &ops[2])?,
1282 })
1283}
1284
1285fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1286 expect_len(mn, ops, 4)?;
1287 let amount = imm_op(mn, &ops[3])?;
1288 if !(0..=63).contains(&amount) {
1289 return Err(bad(mn, "shift amount out of range"));
1290 }
1291 Ok(Inst::AddShifted {
1292 dst: gpr_op(mn, &ops[0])?,
1293 lhs: gpr_op(mn, &ops[1])?,
1294 rhs: gpr_op(mn, &ops[2])?,
1295 amount: amount as u8,
1296 })
1297}
1298
1299fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1300 expect_len(mn, ops, 2)?;
1301 let inst = if borrow {
1302 Inst::Sbb {
1303 dst: ops[0],
1304 src: ops[1],
1305 }
1306 } else {
1307 Inst::Adc {
1308 dst: ops[0],
1309 src: ops[1],
1310 }
1311 };
1312 Ok(inst)
1313}
1314
1315fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Expansion, ExpandError> {
1316 if ops.len() != 2 && ops.len() != 3 {
1317 return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1318 }
1319 let mut out = Expansion::new();
1320 let (src, dst) = if ops.len() == 3 {
1321 let d = fpr_op(mn, &ops[0])?;
1322 if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1323 (ops[1], d)
1325 } else {
1326 if !op_eq(&ops[1], &ops[0]) {
1327 match &ops[1] {
1328 Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1329 Op::Mem(a) => out.push(Inst::LoadF {
1330 k,
1331 dst: d,
1332 addr: *a,
1333 }),
1334 _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1335 }
1336 }
1337 (ops[2], d)
1338 }
1339 } else {
1340 (ops[1], fpr_op(mn, &ops[0])?)
1341 };
1342 out.push(Inst::FBit { or, k, dst, src });
1343 Ok(out)
1344}
1345
1346fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Expansion, ExpandError> {
1347 if ops.len() == 3 {
1348 return Ok(smallvec![Inst::FArith {
1349 op,
1350 k,
1351 arith: FArith3 {
1352 dst: fpr_op(mn, &ops[0])?,
1353 lhs: ops[1],
1354 rhs: Some(ops[2]),
1355 },
1356 }]);
1357 }
1358 Ok(smallvec![Inst::FArith {
1359 op,
1360 k,
1361 arith: FArith3::from_operands(mn, ops)?,
1362 }])
1363}
1364
1365fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1366 expect_len(mn, ops, 2)?;
1367 Ok(Inst::FSqrt {
1368 k,
1369 dst: fpr_op(mn, &ops[0])?,
1370 src: ops[1],
1371 })
1372}
1373
1374fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1375 expect_len(mn, ops, 4)?;
1376 Ok(Inst::Fma {
1377 op,
1378 k,
1379 a: ops[1],
1380 b: ops[2],
1381 c: ops[3],
1382 dst: fpr_op(mn, &ops[0])?,
1383 })
1384}
1385
1386fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1388 expect_len(mn, ops, 2)?;
1389 Ok(Inst::Bswap {
1390 w,
1391 dst: ops[0],
1392 src: ops[1],
1393 })
1394}
1395
1396fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1397 expect_len(mn, ops, 2)?;
1398 Ok(Inst::FRound {
1399 k,
1400 mode,
1401 dst: fpr_op(mn, &ops[0])?,
1402 src: ops[1],
1403 })
1404}
1405
1406fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1407 expect_len(mn, ops, 2)?;
1408 Ok(Inst::CvtFI {
1409 k,
1410 w,
1411 dst: gpr_op(mn, &ops[0])?,
1412 src: ops[1],
1413 })
1414}
1415
1416fn cvtif_node(
1417 mn: Mnemonic,
1418 ops: &[Op],
1419 signed: bool,
1420 from: Width,
1421 k: FKind,
1422) -> Result<Inst, ExpandError> {
1423 expect_len(mn, ops, 2)?;
1424 Ok(Inst::CvtIF {
1425 signed,
1426 from,
1427 k,
1428 dst: fpr_op(mn, &ops[0])?,
1429 src: ops[1],
1430 })
1431}
1432
1433fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1434 expect_len(mn, ops, 2)?;
1435 Ok(Inst::CvtFF {
1436 from,
1437 to,
1438 dst: fpr_op(mn, &ops[0])?,
1439 src: ops[1],
1440 })
1441}
1442
1443fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1444 expect_len(mn, ops, 2)?;
1445 Ok(Inst::Bitcast {
1446 dir,
1447 dst: ops[0],
1448 src: ops[1],
1449 })
1450}
1451
1452fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1453 expect_len(mn, ops, 2)?;
1454 let dst = fpr_op(mn, &ops[0])?;
1455 let inst = if negate {
1456 Inst::FNeg {
1457 k,
1458 dst,
1459 src: ops[1],
1460 }
1461 } else {
1462 Inst::FAbs {
1463 k,
1464 dst,
1465 src: ops[1],
1466 }
1467 };
1468 Ok(inst)
1469}
1470
1471fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1472 expect_len(mn, ops, 3)?;
1473 let lane = imm_op(mn, &ops[2])?;
1474 let lanes: i64 = 128 / w.bytes() as i64;
1475 if lane < 0 || lane >= lanes {
1476 return Err(bad(mn, "lane out of range"));
1477 }
1478 Ok(Inst::VecExtract {
1479 w,
1480 dst: gpr_op(mn, &ops[0])?,
1481 src: fpr_op(mn, &ops[1])?,
1482 lane: lane as u8,
1483 })
1484}
1485
1486fn trunc_q_slow(
1487 mn: Mnemonic,
1488 ops: &[Op],
1489 sc: &Scratch,
1490 k: FKind,
1491) -> Result<Expansion, ExpandError> {
1492 expect_len(mn, ops, 2)?;
1493 let (dst, src) = (ops[0], ops[1]);
1494 let slow = Lbl::Fresh(0);
1495 let done = Lbl::Fresh(1);
1496 let gs = Op::Gpr(sc.gpr);
1497 let fs = Op::Fpr(sc.fpr);
1498 let (int64_min, neg_min) = match k {
1499 FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1500 FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1501 };
1502 let dst_gpr = gpr_op(mn, &dst)?;
1503 Ok(smallvec![
1504 Inst::Mov {
1505 w: Width::B64,
1506 dst: gs,
1507 src: Op::Imm(neg_min),
1508 patch: false,
1509 },
1510 Inst::Bitcast {
1511 dir: if k == FKind::F32 {
1512 BitcastDir::I2F
1513 } else {
1514 BitcastDir::Q2D
1515 },
1516 dst: fs,
1517 src: gs,
1518 },
1519 Inst::BranchF {
1520 cc: FCc::Ge,
1521 k,
1522 lhs: src,
1523 rhs: fs,
1524 target: slow,
1525 patch: false,
1526 },
1527 Inst::CvtFI {
1528 k,
1529 w: Width::B64,
1530 dst: dst_gpr,
1531 src,
1532 },
1533 Inst::Jump {
1534 target: Target::Label(done),
1535 patch: false,
1536 },
1537 Inst::Bind(slow),
1538 Inst::Mov {
1539 w: Width::B64,
1540 dst: gs,
1541 src: Op::Imm(int64_min),
1542 patch: false,
1543 },
1544 Inst::Bitcast {
1545 dir: if k == FKind::F32 {
1546 BitcastDir::I2F
1547 } else {
1548 BitcastDir::Q2D
1549 },
1550 dst: fs,
1551 src: gs,
1552 },
1553 Inst::FArith {
1554 op: FAlu::Add,
1555 k,
1556 arith: FArith3 {
1557 dst: sc.fpr,
1558 lhs: src,
1559 rhs: None,
1560 },
1561 },
1562 Inst::CvtFI {
1563 k,
1564 w: Width::B64,
1565 dst: dst_gpr,
1566 src: fs,
1567 },
1568 Inst::Mov {
1569 w: Width::B64,
1570 dst: gs,
1571 src: Op::Imm(0x8000000000000000u64 as i64),
1572 patch: false,
1573 },
1574 Inst::Or {
1575 w: Width::B64,
1576 op: Alu3 {
1577 dst,
1578 lhs: dst,
1579 rhs: gs,
1580 },
1581 },
1582 Inst::Bind(done),
1583 ])
1584}
1585
1586fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Expansion, ExpandError> {
1587 expect_len(mn, ops, 3)?;
1588 let (src, dst) = (ops[1], ops[0]);
1589 let slow = Lbl::Fresh(0);
1590 let done = Lbl::Fresh(1);
1591 let s1 = gpr_op(mn, &ops[2])?;
1592 let s2 = sc.gpr;
1593 let dst_fpr = fpr_op(mn, &dst)?;
1594 Ok(smallvec![
1595 Inst::TestBranch {
1596 w: Width::B64,
1597 value: src,
1598 mask: None,
1599 test: TestKind::Sign,
1600 target: slow,
1601 patch: false,
1602 },
1603 Inst::CvtIF {
1604 signed: true,
1605 from: Width::B64,
1606 k,
1607 dst: dst_fpr,
1608 src,
1609 },
1610 Inst::Jump {
1611 target: Target::Label(done),
1612 patch: false,
1613 },
1614 Inst::Bind(slow),
1615 Inst::Mov {
1616 w: Width::B64,
1617 dst: Op::Gpr(s1),
1618 src,
1619 patch: false,
1620 },
1621 Inst::Mov {
1622 w: Width::B64,
1623 dst: Op::Gpr(s2),
1624 src,
1625 patch: false,
1626 },
1627 Inst::Shift {
1628 w: Width::B64,
1629 op: ShiftOp::Shr,
1630 dst: s1,
1631 cnt: ShiftCnt::Imm(1),
1632 },
1633 Inst::And {
1634 w: Width::B64,
1635 op: Alu3 {
1636 dst: Op::Gpr(s2),
1637 lhs: Op::Gpr(s2),
1638 rhs: Op::Imm(1),
1639 },
1640 },
1641 Inst::Or {
1642 w: Width::B64,
1643 op: Alu3 {
1644 dst: Op::Gpr(s2),
1645 lhs: Op::Gpr(s2),
1646 rhs: Op::Gpr(s1),
1647 },
1648 },
1649 Inst::CvtIF {
1650 signed: true,
1651 from: Width::B64,
1652 k,
1653 dst: dst_fpr,
1654 src: Op::Gpr(s2),
1655 },
1656 Inst::FArith {
1657 op: FAlu::Add,
1658 k,
1659 arith: FArith3 {
1660 dst: dst_fpr,
1661 lhs: Op::Fpr(dst_fpr),
1662 rhs: None,
1663 },
1664 },
1665 Inst::Bind(done),
1666 ])
1667}
1668
1669fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
1673 expect_len(mn, ops, 3)?;
1674 Ok(smallvec![
1675 Inst::CvtFI {
1676 k: FKind::F64,
1677 w: Width::B64,
1678 dst: gpr_op(mn, &ops[0])?,
1679 src: ops[1],
1680 },
1681 Inst::Branch {
1682 cc: IntCc::Eq,
1683 w: Width::B32,
1684 lhs: ops[0],
1685 rhs: Op::Imm(-0x8000_0000i64),
1686 target: label_op(mn, &ops[2])?,
1687 patch: false,
1688 },
1689 ])
1690}
1691
1692fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
1693 expect_len(mn, ops, 3)?;
1694 let target = label_op(mn, &ops[2])?;
1695 Ok(smallvec![
1696 Inst::CvtFI {
1697 k: FKind::F64,
1698 w: Width::B64,
1699 dst: gpr_op(mn, &ops[0])?,
1700 src: ops[1],
1701 },
1702 Inst::TestBranch {
1703 w: Width::B64,
1704 value: ops[0],
1705 mask: None,
1706 test: TestKind::Zero,
1707 target,
1708 patch: false,
1709 },
1710 Inst::CvtIF {
1711 signed: true,
1712 from: Width::B64,
1713 k: FKind::F64,
1714 dst: sc.fpr,
1715 src: ops[0],
1716 },
1717 Inst::BranchF {
1718 cc: FCc::NeUn,
1719 k: FKind::F64,
1720 lhs: ops[1],
1721 rhs: Op::Fpr(sc.fpr),
1722 target,
1723 patch: false,
1724 },
1725 ])
1726}
1727
1728fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1730 expect_len(mn, ops, 1)?;
1731 let dst = fpr_op(mn, &ops[0])?;
1732 Ok(Inst::FArith {
1733 op: FAlu::Sub,
1734 k: FKind::F64,
1735 arith: FArith3 {
1736 dst,
1737 lhs: Op::Fpr(dst),
1738 rhs: None,
1739 },
1740 })
1741}