1use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6use smallvec::{smallvec, SmallVec};
7
8pub const EXPANSION_INLINE: usize = 2;
12
13pub type Expansion = SmallVec<[Inst; EXPANSION_INLINE]>;
15
16pub fn expand(
32 mn: Mnemonic,
33 ops: &[Op],
34 sc: &Scratch,
35 patch: bool,
36) -> Result<Expansion, ExpandError> {
37 if patch && patch_kind(mn).is_none() {
38 return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
39 }
40 check_extern_operands(mn, ops, patch)?;
41 let mut insts = expand_inner(mn, ops, sc, patch)?;
42 stage_aliasing_mem(&mut insts, sc);
43 Ok(insts)
44}
45
46fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
50 let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
51 return Ok(());
52 };
53 if patch {
54 return Err(bad(mn, "an external symbol cannot be a patch site"));
55 }
56 let ok = match mn {
57 Mnemonic::Jmp | Mnemonic::Call => at == 0,
58 Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
59 Mnemonic::Lea {
60 w: Width::B64,
61 form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
62 } => at == 1,
63 _ => false,
64 };
65 if ok {
66 Ok(())
67 } else {
68 Err(bad(
69 mn,
70 "an external symbol is only a `jmp`/`call` target or the source of \
71 `move`/`leap`/`globaladdr`/`la`",
72 ))
73 }
74}
75
76fn stage_aliasing_mem(insts: &mut Expansion, sc: &Scratch) {
91 let old = std::mem::take(insts);
92 let mut out = Expansion::new();
93 for mut inst in old {
94 let loads = stage_one(&mut inst, sc);
95 if let Some(loads) = loads {
96 out.extend(loads);
97 }
98 out.push(inst);
99 }
100 *insts = out;
101}
102
103fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Expansion> {
107 let (w, operands, dst) = match inst {
108 Inst::Add { w, op }
109 | Inst::Sub { w, op }
110 | Inst::Mul { w, op }
111 | Inst::And { w, op }
112 | Inst::Or { w, op }
113 | Inst::Xor { w, op } => (*w, vec![&mut op.lhs, &mut op.rhs], &mut op.dst),
114 Inst::AluS { w, alu, .. } => (*w, vec![&mut alu.lhs, &mut alu.rhs], &mut alu.dst),
115 Inst::OpBranch { w, args, .. } => {
116 let (last, init) = args.as_mut_slice().split_last_mut()?;
117 (*w, init.iter_mut().collect(), last)
118 }
119 _ => return None,
120 };
121 let Op::Gpr(g) = *dst else {
122 return None;
123 };
124 let slots = [sc.gpr, sc.gpr2];
125 let mut loads = Expansion::new();
126 let mut slot = 0usize;
127 for operand in operands {
128 let Op::Mem(m) = operand else {
129 continue;
130 };
131 let aliases = m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g);
132 if !aliases {
133 continue;
134 }
135 let &s = slots.get(slot)?;
138 slot += 1;
139 loads.push(Inst::Load {
140 w,
141 sx: Sx::Zero,
142 dst: s,
143 addr: *m,
144 acquire: false,
145 });
146 *operand = Op::Gpr(s);
147 }
148 (!loads.is_empty()).then_some(loads)
149}
150
151pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
153 Some(match mn {
154 Mnemonic::Jmp => PatchKind::JumpSite,
155 Mnemonic::Call => PatchKind::CallSite,
156 Mnemonic::Move => PatchKind::MovBlock,
157 Mnemonic::FlagBranch(_)
158 | Mnemonic::OpBranch { .. }
159 | Mnemonic::RiscAluS { .. }
160 | Mnemonic::TestBranch { .. }
161 | Mnemonic::IntBranch { .. }
162 | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
163 _ => return None,
164 })
165}
166
167#[derive(Clone, Copy, Debug, PartialEq, Eq)]
173pub enum PatchKind {
174 JumpSite,
176 CondJumpSite,
178 CallSite,
180 MovBlock,
182}
183
184fn float_cc(cc: FCc) -> (FCc, bool) {
185 match cc {
186 FCc::Lt => (FCc::Gt, true),
187 FCc::Le => (FCc::Ge, true),
188 FCc::GtUn => (FCc::GtUn, false),
189 FCc::GeUn => (FCc::GeUn, false),
190 FCc::LtUn => (FCc::GtUn, true),
191 FCc::LeUn => (FCc::GeUn, true),
192 other => (other, false),
193 }
194}
195
196fn test_set_node(
197 mn: Mnemonic,
198 ops: &[Op],
199 w: Width,
200 test: TestKind,
201) -> Result<Expansion, ExpandError> {
202 if ops.len() != 2 && ops.len() != 3 {
203 return Err(bad(mn, "test-set takes 2 or 3 operands"));
204 }
205 let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
206 Ok(smallvec![Inst::TestSet {
207 w,
208 value: ops[1],
209 mask,
210 test,
211 dst: ops[0],
212 }])
213}
214
215fn test_branch_node(
216 mn: Mnemonic,
217 ops: &[Op],
218 w: Width,
219 test: TestKind,
220 patch: bool,
221) -> Result<Expansion, ExpandError> {
222 if ops.len() != 2 && ops.len() != 3 {
223 return Err(bad(mn, "test-branch takes 2 or 3 operands"));
224 }
225 let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
226 let target = label_op(
227 mn,
228 ops.last()
229 .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
230 )?;
231 Ok(smallvec![Inst::TestBranch {
232 w,
233 value: ops[0],
234 mask,
235 test,
236 target,
237 patch,
238 }])
239}
240
241fn div_rem_node(
242 mn: Mnemonic,
243 ops: &[Op],
244 signed: bool,
245 w: Width,
246 rem: bool,
247) -> Result<Expansion, ExpandError> {
248 let (dvd, dvs, dst) = match ops {
249 [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
250 [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
251 _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
252 };
253 Ok(smallvec![if rem {
254 Inst::Rem3 {
255 signed,
256 w,
257 dst,
258 dvd,
259 dvs,
260 }
261 } else {
262 Inst::Div3 {
263 signed,
264 w,
265 dst,
266 dvd,
267 dvs,
268 }
269 }])
270}
271
272fn shift_mem_canon(
273 mn: Mnemonic,
274 w: Width,
275 op: ShiftOp,
276 cnt: &Op,
277 dst: &Mem,
278 sc: &Scratch,
279) -> Result<Expansion, ExpandError> {
280 let shift_cnt = match cnt {
281 Op::Imm(v) => ShiftCnt::Imm(*v),
282 Op::Gpr(g) => ShiftCnt::Reg(*g),
283 _ => return Err(bad(mn, "shift count must be an immediate or register")),
284 };
285 Ok(smallvec![
286 Inst::Load {
287 w,
288 sx: Sx::Zero,
289 dst: sc.gpr,
290 addr: *dst,
291 acquire: false,
292 },
293 Inst::Shift {
294 w,
295 op,
296 dst: sc.gpr,
297 cnt: shift_cnt,
298 },
299 Inst::Store {
300 w,
301 src: Op::Gpr(sc.gpr),
302 addr: *dst,
303 },
304 ])
305}
306
307fn shift_node(
308 mn: Mnemonic,
309 w: Width,
310 op: ShiftOp,
311 ops: &[Op],
312 sc: &Scratch,
313) -> Result<Expansion, ExpandError> {
314 if ops.len() != 2 {
315 return Err(bad(mn, "shift takes 2 operands"));
316 }
317 if let Op::Mem(a) = &ops[0] {
318 return shift_mem_canon(mn, w, op, &ops[1], a, sc);
319 }
320 let dst = gpr_op(mn, &ops[0])?;
321 let cnt = match &ops[1] {
322 Op::Imm(v) => ShiftCnt::Imm(*v),
323 Op::Gpr(g) => ShiftCnt::Reg(*g),
324 _ => return Err(bad(mn, "shift count must be an immediate or register")),
325 };
326 Ok(smallvec![Inst::Shift { w, op, dst, cnt }])
327}
328
329fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
330 Alu3::from_operands(mn, ops)
331}
332
333fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<BranchArgs, ExpandError> {
334 match ops {
335 [dst, rhs] => Ok(BranchArgs::two(*rhs, *dst)),
336 [dst, lhs, rhs] => Ok(BranchArgs::three(*lhs, *rhs, *dst)),
337 _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
338 }
339}
340
341fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
342 match op {
343 Op::Label(l) => Ok(Target::Label(*l)),
344 Op::Gpr(g) => Ok(Target::Reg(*g)),
345 Op::Mem(a) => Ok(Target::Mem(*a)),
346 Op::Imm(v) => Ok(Target::Abs(*v)),
347 Op::Extern(e) => Ok(Target::Extern(*e)),
348 _ => Err(bad(
349 mn,
350 "jump/call needs a label, register, memory, or immediate",
351 )),
352 }
353}
354
355#[allow(clippy::too_many_lines)]
356fn expand_inner(
357 mn: Mnemonic,
358 ops: &[Op],
359 sc: &Scratch,
360 patch: bool,
361) -> Result<Expansion, ExpandError> {
362 let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
363 expect_len(mn, ops, 1)?;
364 Ok(ops[0])
365 };
366 match mn {
367 Mnemonic::Move => Ok(smallvec![mov_node(mn, ops, Width::B64, patch)?]),
368 Mnemonic::Load(w) => Ok(smallvec![mov_node(mn, ops, w, patch)?]),
369 Mnemonic::AtomicLoad(w) => Ok(smallvec![atomic_load_node(mn, ops, w)?]),
370 Mnemonic::Store(w) => Ok(smallvec![store_node(mn, ops, w)?]),
371 Mnemonic::Extend { from, to, sx } => Ok(smallvec![extend_node(mn, ops, from, to, sx)?]),
372 Mnemonic::MovZero32 => Ok(smallvec![movzero_node(mn, ops)?]),
373 Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
374 Mnemonic::FMove(k) => Ok(smallvec![fmove_node(mn, ops, k)?]),
375 Mnemonic::VMove { store } => Ok(smallvec![vmove_node(mn, ops, store)?]),
376 Mnemonic::Lea { w, .. } => Ok(smallvec![lea_node(mn, ops, w)?]),
377 Mnemonic::LoadQInc => loadqinc_node(mn, ops),
378 Mnemonic::LoadPair(w) => Ok(smallvec![pair_node(mn, ops, w, true)?]),
379 Mnemonic::StorePair(w) => Ok(smallvec![pair_node(mn, ops, w, false)?]),
380 Mnemonic::Push => stack_nodes(mn, ops, true, false),
381 Mnemonic::Pop => stack_nodes(mn, ops, false, false),
382 Mnemonic::PushV => stack_nodes(mn, ops, true, true),
383 Mnemonic::PopV => stack_nodes(mn, ops, false, true),
384 Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
385 Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
386 Mnemonic::Peek => Ok(smallvec![peek_node(mn, ops, true)?]),
387 Mnemonic::Poke => Ok(smallvec![peek_node(mn, ops, false)?]),
388 Mnemonic::TlsLoad(w) => Ok(smallvec![tls_node(mn, ops, w, true)?]),
389 Mnemonic::TlsStore(w) => Ok(smallvec![tls_node(mn, ops, w, false)?]),
390 Mnemonic::TlsLea => Ok(smallvec![tls_lea_node(mn, ops)?]),
391
392 Mnemonic::Add(w) => Ok(smallvec![Inst::Add {
393 w,
394 op: alu3(mn, ops)?,
395 }]),
396 Mnemonic::Sub(w) => Ok(smallvec![Inst::Sub {
397 w,
398 op: alu3(mn, ops)?,
399 }]),
400 Mnemonic::Mul(w) => Ok(smallvec![mul_node(mn, w, ops)?]),
401 Mnemonic::And(w) => Ok(smallvec![Inst::And {
402 w,
403 op: alu3(mn, ops)?,
404 }]),
405 Mnemonic::Or(w) => Ok(smallvec![Inst::Or {
406 w,
407 op: alu3(mn, ops)?,
408 }]),
409 Mnemonic::Xor(w) => Ok(smallvec![Inst::Xor {
410 w,
411 op: alu3(mn, ops)?,
412 }]),
413 Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
414 Mnemonic::Not(w) => Ok(smallvec![Inst::Not {
415 w,
416 dst: one(mn, ops)?,
417 }]),
418 Mnemonic::Neg(w) => Ok(smallvec![Inst::Neg {
419 w,
420 dst: one(mn, ops)?,
421 }]),
422 Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
423 Mnemonic::Bswap(w) => Ok(smallvec![bswap_node(mn, ops, w)?]),
424 Mnemonic::Adc => Ok(smallvec![adc_node(mn, ops, false)?]),
425 Mnemonic::Sbb => Ok(smallvec![adc_node(mn, ops, true)?]),
426 Mnemonic::MulHigh { signed } => Ok(smallvec![Inst::MulHigh {
427 signed,
428 rhs: one(mn, ops)?,
429 }]),
430 Mnemonic::MulWide { signed } => Ok(smallvec![mulwide_node(mn, ops, signed)?]),
431 Mnemonic::AluS { op, w } => Ok(smallvec![Inst::AluS {
432 op,
433 w,
434 alu: alu3(mn, ops)?,
435 }]),
436 Mnemonic::RiscAluS { op, w } => {
437 let has_label =
438 matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
439 if has_label {
440 let flag_op = match op {
441 AluOp::Add => FlagAlu::Add,
442 AluOp::Sub => FlagAlu::Sub,
443 AluOp::Or => FlagAlu::Or,
444 };
445 return Ok(smallvec![Inst::OpBranch {
446 op: flag_op,
447 w,
448 flag: PredFlag::S,
449 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
450 target: label_op(
451 mn,
452 ops.last()
453 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
454 )?,
455 patch,
456 }]);
457 }
458 Ok(smallvec![Inst::AluS {
459 op,
460 w,
461 alu: alu3(mn, ops)?,
462 }])
463 }
464 Mnemonic::AddShifted => Ok(smallvec![addshifted_node(mn, ops)?]),
465 Mnemonic::Cqo { quad } => Ok(smallvec![Inst::Cqo { quad }]),
466 Mnemonic::DivMod1 { signed, w } => Ok(smallvec![Inst::DivMod1 {
467 signed,
468 w,
469 divisor: one(mn, ops)?,
470 }]),
471 Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
472 Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
473 Mnemonic::IntSet { w, cc } => {
474 expect_len(mn, ops, 3)?;
475 Ok(smallvec![Inst::IntSet {
476 cc,
477 w,
478 dst: ops[0],
479 lhs: ops[1],
480 rhs: ops[2],
481 }])
482 }
483 Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
484 Mnemonic::Cmov { zero } => Ok(smallvec![cmov_node(mn, ops, zero)?]),
485 Mnemonic::BitCount { w, trailing } => Ok(smallvec![cz_node(mn, ops, w, trailing)?]),
486 Mnemonic::Popcnt(w) => Ok(smallvec![popcnt_node(mn, ops, w)?]),
487 Mnemonic::Bfi => bfi_node(mn, ops, sc),
488 Mnemonic::IntBranch { w, cc } => {
489 expect_len(mn, ops, 3)?;
490 Ok(smallvec![Inst::Branch {
491 cc,
492 w,
493 lhs: ops[0],
494 rhs: ops[1],
495 target: label_op(mn, &ops[2])?,
496 patch,
497 }])
498 }
499 Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
500 Mnemonic::OpBranch { op, w, flag } => {
501 if ops.len() != 3 && ops.len() != 4 {
502 return Err(bad(mn, "op-branch takes 3 or 4 operands"));
503 }
504 Ok(smallvec![Inst::OpBranch {
505 op,
506 w,
507 flag,
508 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
509 target: label_op(
510 mn,
511 ops.last()
512 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
513 )?,
514 patch,
515 }])
516 }
517 Mnemonic::FlagBranch(flag) => Ok(smallvec![flag_branch_node(mn, ops, flag, patch)?]),
518 Mnemonic::BranchF { cc, k } => {
519 expect_len(mn, ops, 3)?;
520 let (cc, reverse) = float_cc(cc);
521 let (lhs, rhs) = if reverse {
522 (ops[1], ops[0])
523 } else {
524 (ops[0], ops[1])
525 };
526 Ok(smallvec![Inst::BranchF {
527 cc,
528 k,
529 lhs,
530 rhs,
531 target: label_op(mn, &ops[2])?,
532 patch,
533 }])
534 }
535 Mnemonic::Jmp => {
536 expect_len(mn, ops, 1)?;
537 Ok(smallvec![Inst::Jump {
538 target: jump_target(mn, &ops[0])?,
539 patch,
540 }])
541 }
542 Mnemonic::Call => {
543 expect_len(mn, ops, 1)?;
544 Ok(smallvec![Inst::Call {
545 target: jump_target(mn, &ops[0])?,
546 patch,
547 }])
548 }
549 Mnemonic::Ret => {
550 expect_len(mn, ops, 0)?;
551 Ok(smallvec![Inst::Ret])
552 }
553 Mnemonic::Break => {
554 expect_len(mn, ops, 0)?;
555 Ok(smallvec![Inst::Break])
556 }
557 Mnemonic::Nop => {
558 expect_len(mn, ops, 0)?;
559 Ok(smallvec![Inst::Nop])
560 }
561 Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
562 Mnemonic::Fma { op, k } => Ok(smallvec![fma_node(mn, ops, k, op)?]),
563 Mnemonic::FSqrt(k) => Ok(smallvec![fsqrt_node(mn, ops, k)?]),
564 Mnemonic::FRound { k, mode } => Ok(smallvec![fround_node(mn, ops, k, mode)?]),
565 Mnemonic::FAbs { k, negate } => Ok(smallvec![fabs_node(mn, ops, k, negate)?]),
566 Mnemonic::FSet { cc, k } => {
567 expect_len(mn, ops, 3)?;
568 let (cc, reverse) = float_cc(cc);
569 let (lhs, rhs) = if reverse {
570 (ops[2], ops[1])
571 } else {
572 (ops[1], ops[2])
573 };
574 Ok(smallvec![Inst::FSet {
575 cc,
576 k,
577 dst: ops[0],
578 lhs,
579 rhs,
580 }])
581 }
582 Mnemonic::CvtFI { k, w } => Ok(smallvec![cvtfi_node(mn, ops, k, w)?]),
583 Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
584 Mnemonic::CvtIF { signed, from, k } => Ok(smallvec![cvtif_node(mn, ops, signed, from, k)?]),
585 Mnemonic::CvtFF { from, to } => Ok(smallvec![cvtff_node(mn, ops, from, to)?]),
586 Mnemonic::Bitcast(dir) => Ok(smallvec![bitcast_node(mn, ops, dir)?]),
587 Mnemonic::MovDZ => Ok(smallvec![movdz_node(mn, ops)?]),
588 Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
589 Mnemonic::Btd2i => btd2i_node(mn, ops),
590 Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
591 Mnemonic::VecExtract(w) => Ok(smallvec![umov_node(mn, ops, w)?]),
592 Mnemonic::AtomicRmw { op, w } => Ok(smallvec![rmw_node(mn, ops, w, op)?]),
593 Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
594 Mnemonic::AtomicCas { w, branch } => Ok(smallvec![cas_node(mn, ops, w, branch)?]),
595 Mnemonic::LoadLinked(w) => Ok(smallvec![ll_node(mn, ops, w)?]),
596 Mnemonic::StoreCond(w) => Ok(smallvec![sc_node(mn, ops, w)?]),
597 Mnemonic::Fence { full } => Ok(smallvec![Inst::Fence { full }]),
598 Mnemonic::Pause => Ok(smallvec![Inst::Pause]),
599 Mnemonic::TagCodePtr
600 | Mnemonic::TagReturnAddress
601 | Mnemonic::UntagReturnAddress
602 | Mnemonic::RemoveCodePtrTag
603 | Mnemonic::UntagArrayPtr
604 | Mnemonic::RemoveArrayPtrTag => Ok(smallvec![]),
605 }
606}
607
608fn flag_branch_node(
609 mn: Mnemonic,
610 ops: &[Op],
611 flag: PredFlag,
612 patch: bool,
613) -> Result<Inst, ExpandError> {
614 expect_len(mn, ops, 1)?;
615 Ok(Inst::FlagBranch {
616 flag,
617 target: label_op(mn, &ops[0])?,
618 patch,
619 })
620}
621
622fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
623 expect_len(mn, ops, 2)?;
624 Ok(Inst::Popcnt {
625 w,
626 dst: gpr_op(mn, &ops[0])?,
627 src: ops[1],
628 })
629}
630
631fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
632 expect_len(mn, ops, 2)?;
633 let dst = gpr_op(mn, &ops[0])?;
634 let inst = if trailing {
635 Inst::Ctz {
636 w,
637 dst,
638 src: ops[1],
639 }
640 } else {
641 Inst::Clz {
642 w,
643 dst,
644 src: ops[1],
645 }
646 };
647 Ok(inst)
648}
649
650fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
651 expect_len(mn, ops, 2)?;
652 if load {
653 let idx = imm_op(mn, &ops[1])?;
654 Ok(Inst::TlsLoad {
655 w,
656 dst: gpr_op(mn, &ops[0])?,
657 idx,
658 })
659 } else {
660 let idx = imm_op(mn, &ops[0])?;
661 Ok(Inst::TlsStore {
662 w,
663 src: gpr_op(mn, &ops[1])?,
664 idx,
665 })
666 }
667}
668
669fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
670 expect_len(mn, ops, 2)?;
671 Ok(Inst::TlsLea {
672 dst: gpr_op(mn, &ops[0])?,
673 idx: imm_op(mn, &ops[1])?,
674 })
675}
676
677fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
678 expect_len(mn, ops, 2)?;
679
680 Ok(Inst::AtomicRmw {
681 op,
682 w,
683 a: ops[1],
684 b: ops[0],
685 })
686}
687
688fn rmw_not_node(
689 mn: Mnemonic,
690 ops: &[Op],
691 sc: &Scratch,
692 w: Width,
693 op: AtomicOp,
694) -> Result<Expansion, ExpandError> {
695 expect_len(mn, ops, 2)?;
696 let mut out = Expansion::new();
697 let a = match &ops[1] {
699 Op::Imm(v) => Op::Imm(!v),
700 Op::Gpr(g) => {
701 out.push(Inst::Mov {
702 w: Width::B64,
703 dst: Op::Gpr(sc.gpr),
704 src: Op::Gpr(*g),
705 patch: false,
706 });
707 out.push(Inst::Not {
708 w,
709 dst: Op::Gpr(sc.gpr),
710 });
711 Op::Gpr(sc.gpr)
712 }
713 Op::Mem(addr) => {
714 let lw = if w == Width::B64 {
715 Width::B64
716 } else {
717 Width::B32
718 };
719 out.push(Inst::Load {
720 w: lw,
721 sx: Sx::Zero,
722 dst: sc.gpr,
723 addr: *addr,
724 acquire: false,
725 });
726 out.push(Inst::Not {
727 w,
728 dst: Op::Gpr(sc.gpr),
729 });
730 Op::Gpr(sc.gpr)
731 }
732 _ => {
733 return Err(bad(
734 mn,
735 "atomic clear needs an immediate, register, or memory operand",
736 ))
737 }
738 };
739 out.push(Inst::AtomicRmw {
740 op,
741 w,
742 a,
743 b: ops[0],
744 });
745 Ok(out)
746}
747
748fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
749 let n = if branch { 4 } else { 3 };
750 expect_len(mn, ops, n)?;
751
752 Ok(Inst::AtomicCas {
753 w,
754 mem: mem_op(mn, &ops[0])?,
755 src: gpr_op(mn, &ops[2])?,
756 branch: if branch {
757 Some(label_op(mn, &ops[3])?)
758 } else {
759 None
760 },
761 })
762}
763
764fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
765 expect_len(mn, ops, 2)?;
766 Ok(Inst::LoadLinked {
767 w,
768 mem: mem_op(mn, &ops[1])?,
769 dst: gpr_op(mn, &ops[0])?,
770 })
771}
772
773fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
774 expect_len(mn, ops, 3)?;
775 Ok(Inst::StoreCond {
776 w,
777 status: gpr_op(mn, &ops[1])?,
778 src: gpr_op(mn, &ops[2])?,
779 mem: mem_op(mn, &ops[0])?,
780 })
781}
782
783fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
787 expect_len(mn, ops, 4)?;
788 let lsb = imm_op(mn, &ops[2])?;
789 let width = imm_op(mn, &ops[3])?;
790 if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
791 return Err(bad(mn, "bitfield out of range"));
792 }
793 let wmask: u64 = if width == 64 {
794 u64::MAX
795 } else {
796 (1u64 << width) - 1
797 };
798 let mask: i64 = (wmask << lsb) as i64;
799 let s = sc.gpr;
800 let dst = gpr_op(mn, &ops[0])?;
801
802 Ok(smallvec![
803 Inst::Mov {
804 w: Width::B64,
805 dst: Op::Gpr(s),
806 src: ops[1],
807 patch: false,
808 },
809 Inst::And {
810 w: Width::B64,
811 op: Alu3 {
812 dst: Op::Gpr(s),
813 lhs: Op::Gpr(s),
814 rhs: Op::Imm(wmask as i64),
815 },
816 },
817 Inst::Shift {
818 w: Width::B64,
819 op: ShiftOp::Shl,
820 dst: s,
821 cnt: ShiftCnt::Imm(lsb),
822 },
823 Inst::And {
824 w: Width::B64,
825 op: Alu3 {
826 dst: Op::Gpr(dst),
827 lhs: Op::Gpr(dst),
828 rhs: Op::Imm(!mask),
829 },
830 },
831 Inst::Or {
832 w: Width::B64,
833 op: Alu3 {
834 dst: Op::Gpr(dst),
835 lhs: Op::Gpr(dst),
836 rhs: Op::Gpr(s),
837 },
838 },
839 ])
840}
841
842fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
843 expect_len(mn, ops, 3)?;
844 Ok(Inst::Cmov {
845 w: Width::B64,
846 zero,
847 dst: gpr_op(mn, &ops[0])?,
848 src: ops[1],
849 test: gpr_op(mn, &ops[2])?,
850 })
851}
852
853fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
856 expect_len(mn, ops, 2)?;
857 if peek {
858 let disp = slot_disp(mn, &ops[1])?;
859 return Ok(Inst::Peek {
860 disp,
861 dst: gpr_op(mn, &ops[0])?,
862 });
863 }
864 Ok(Inst::Poke {
865 disp: slot_disp(mn, &ops[0])?,
866 src: gpr_op(mn, &ops[1])?,
867 })
868}
869
870fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
871 let idx = imm_op(mn, op)?;
872 let off = idx
873 .checked_mul(8)
874 .ok_or_else(|| bad(mn, "stack offset overflow"))?;
875 if off < i32::MIN as i64 || off > i32::MAX as i64 {
876 return Err(bad(mn, "stack offset out of range"));
877 }
878 Ok(off as i32)
879}
880
881fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
882 expect_len(mn, ops, 3)?;
883 let dst = gpr_op(mn, &ops[0])?;
884 let base = gpr_op(mn, &ops[1])?;
885 let inc = imm_op(mn, &ops[2])?;
886 Ok(smallvec![
887 Inst::Load {
888 w: Width::B64,
889 sx: Sx::Zero,
890 dst,
891 addr: Mem::base(base, 0),
892 acquire: false,
893 },
894 Inst::Add {
895 w: Width::B64,
896 op: Alu3 {
897 dst: Op::Gpr(base),
898 lhs: Op::Gpr(base),
899 rhs: Op::Imm(inc),
900 },
901 },
902 ])
903}
904
905fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
906 expect_len(mn, ops, 3)?;
907 if load {
908 Ok(Inst::PairLoad {
909 w,
910 dst1: ops[0],
911 dst2: ops[1],
912 mem: mem_op(mn, &ops[2])?,
913 })
914 } else {
915 Ok(Inst::PairStore {
916 w,
917 src1: ops[1],
918 src2: ops[2],
919 mem: mem_op(mn, &ops[0])?,
920 })
921 }
922}
923
924fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
925 expect_len(mn, ops, 2)?;
926 let dst = gpr_op(mn, &ops[0])?;
927 let src = match &ops[1] {
928 Op::Mem(a) => LeaSrc::Mem(*a),
929 Op::Label(l) => LeaSrc::Label(*l),
930 Op::Extern(e) => LeaSrc::Extern(*e),
931 _ => return Err(bad(mn, "lea needs an address or label source")),
932 };
933 Ok(Inst::Lea { w, dst, src })
934}
935
936fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
937 expect_len(mn, ops, 2)?;
938 if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
940 return Ok(Inst::Lea {
941 w: Width::B64,
942 dst: *dst,
943 src: LeaSrc::Extern(*e),
944 });
945 }
946 Ok(Inst::Mov {
947 w,
948 dst: ops[0],
949 src: ops[1],
950 patch,
951 })
952}
953
954fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
955 expect_len(mn, ops, 2)?;
956 Ok(Inst::MovZero32 {
957 dst: ops[0],
958 src: ops[1],
959 })
960}
961
962fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
963 expect_len(mn, ops, 2)?;
964 Ok(Inst::Load {
965 w,
966 sx: Sx::Zero,
967 dst: gpr_op(mn, &ops[0])?,
968 addr: mem_op(mn, &ops[1])?,
969 acquire: true,
970 })
971}
972
973fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
974 expect_len(mn, ops, 2)?;
975 let addr = mem_op(mn, &ops[0])?;
976 Ok(Inst::Store {
977 w,
978 src: ops[1],
979 addr,
980 })
981}
982
983fn extend_node(
984 mn: Mnemonic,
985 ops: &[Op],
986 from: Width,
987 to: Width,
988 sx: Sx,
989) -> Result<Inst, ExpandError> {
990 expect_len(mn, ops, 2)?;
991 Ok(Inst::Extend {
992 from,
993 to,
994 sx,
995 dst: gpr_op(mn, &ops[0])?,
996 src: ops[1],
997 })
998}
999
1000fn transfer_node(
1001 mn: Mnemonic,
1002 ops: &[Op],
1003 sc: &Scratch,
1004 w: Width,
1005) -> Result<Expansion, ExpandError> {
1006 expect_len(mn, ops, 2)?;
1007 let mut out = Expansion::new();
1008 match &ops[1] {
1009 Op::Gpr(_) => {}
1010 Op::Mem(a) => {
1011 out.push(Inst::Load {
1012 w,
1013 sx: Sx::Zero,
1014 dst: sc.gpr,
1015 addr: *a,
1016 acquire: false,
1017 });
1018 }
1019 Op::Imm(v) => {
1020 out.push(Inst::Mov {
1021 w,
1022 dst: Op::Gpr(sc.gpr),
1023 src: Op::Imm(*v),
1024 patch: false,
1025 });
1026 }
1027 Op::Label(l) => {
1028 out.push(Inst::Lea {
1029 w: Width::B64,
1030 dst: sc.gpr,
1031 src: LeaSrc::Label(*l),
1032 });
1033 let addr = Mem::base(sc.gpr, 0);
1034 out.push(Inst::Load {
1035 w,
1036 sx: Sx::Zero,
1037 dst: sc.gpr,
1038 addr,
1039 acquire: false,
1040 });
1041 }
1042 Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1043 Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1044 Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1045 }
1046 let src = if out.is_empty() {
1047 ops[1]
1048 } else {
1049 Op::Gpr(sc.gpr)
1050 };
1051 out.push(Inst::Mov {
1052 w,
1053 dst: ops[0],
1054 src,
1055 patch: false,
1056 });
1057 Ok(out)
1058}
1059
1060fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1061 expect_len(mn, ops, 2)?;
1062 match (&ops[0], &ops[1]) {
1063 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1064 k,
1065 dst: *d,
1066 src: *s,
1067 }),
1068 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1069 k,
1070 dst: *d,
1071 addr: *a,
1072 }),
1073 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1074 k,
1075 src: *s,
1076 addr: *a,
1077 }),
1078 _ => Err(bad(mn, "float move needs FPR/memory operands")),
1079 }
1080}
1081
1082fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1083 expect_len(mn, ops, 2)?;
1084 if !store {
1085 match (&ops[0], &ops[1]) {
1086 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1087 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1088 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1089 dst: *d,
1090 addr: Mem::base(*s, 0),
1091 }),
1092 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1093 }
1094 } else {
1095 match (&ops[0], &ops[1]) {
1096 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1097 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1098 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1099 dst: *d,
1100 addr: Mem::base(*s, 0),
1101 }),
1102 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1103 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1104 }
1105 }
1106}
1107
1108pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1118 if !matches!(
1119 mnemonic,
1120 Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1121 ) {
1122 return Ok(());
1123 }
1124 for op in ops {
1125 match op {
1126 Op::Gpr(g) if saved.contains(&g.0) => {}
1127 _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1128 }
1129 }
1130 Ok(())
1131}
1132
1133pub fn aligned_push_bytes(n: usize) -> i32 {
1141 ((8 * n + 15) & !15) as i32
1142}
1143
1144pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1148 8 * (n - 1 - i) as i32
1149}
1150
1151fn push_inst(mn: Mnemonic, op: &Op, push: bool, vector: bool) -> Result<Inst, ExpandError> {
1153 if vector {
1154 let x = fpr_op(mn, op)?;
1155 Ok(if push {
1156 Inst::PushV { src: x }
1157 } else {
1158 Inst::PopV { dst: x }
1159 })
1160 } else {
1161 match op {
1162 Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => Ok(if push {
1163 Inst::Push { op: *op }
1164 } else {
1165 Inst::Pop { op: *op }
1166 }),
1167 _ => Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1168 }
1169 }
1170}
1171
1172fn stack_nodes(
1173 mn: Mnemonic,
1174 ops: &[Op],
1175 push: bool,
1176 vector: bool,
1177) -> Result<Expansion, ExpandError> {
1178 if ops.is_empty() {
1179 return Err(bad(mn, "needs at least one operand"));
1180 }
1181 let mut out = Expansion::new();
1182 if push {
1186 for op in ops {
1187 out.push(push_inst(mn, op, true, vector)?);
1188 }
1189 } else {
1190 for op in ops.iter().rev() {
1191 out.push(push_inst(mn, op, false, vector)?);
1192 }
1193 }
1194 Ok(out)
1195}
1196
1197fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1198 if ops.len() == 3 {
1199 if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1200 return Ok(Inst::MulImm {
1201 w,
1202 dst: gpr_op(mn, &ops[0])?,
1203 lhs,
1204 imm: *v,
1205 });
1206 }
1207 if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1208 return Ok(Inst::MulImm {
1209 w,
1210 dst: gpr_op(mn, &ops[0])?,
1211 lhs,
1212 imm: *v,
1213 });
1214 }
1215 }
1216 Ok(Inst::Mul {
1217 w,
1218 op: Alu3::from_operands(mn, ops)?,
1219 })
1220}
1221
1222fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1223 expect_len(mn, ops, 3)?;
1224 Ok(Inst::MulWide {
1225 signed,
1226 dst: gpr_op(mn, &ops[0])?,
1227 lhs: gpr_op(mn, &ops[1])?,
1228 rhs: gpr_op(mn, &ops[2])?,
1229 })
1230}
1231
1232fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1233 expect_len(mn, ops, 4)?;
1234 let amount = imm_op(mn, &ops[3])?;
1235 if !(0..=63).contains(&amount) {
1236 return Err(bad(mn, "shift amount out of range"));
1237 }
1238 Ok(Inst::AddShifted {
1239 dst: gpr_op(mn, &ops[0])?,
1240 lhs: gpr_op(mn, &ops[1])?,
1241 rhs: gpr_op(mn, &ops[2])?,
1242 amount: amount as u8,
1243 })
1244}
1245
1246fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1247 expect_len(mn, ops, 2)?;
1248 let inst = if borrow {
1249 Inst::Sbb {
1250 dst: ops[0],
1251 src: ops[1],
1252 }
1253 } else {
1254 Inst::Adc {
1255 dst: ops[0],
1256 src: ops[1],
1257 }
1258 };
1259 Ok(inst)
1260}
1261
1262fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Expansion, ExpandError> {
1263 if ops.len() != 2 && ops.len() != 3 {
1264 return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1265 }
1266 let mut out = Expansion::new();
1267 let (src, dst) = if ops.len() == 3 {
1268 let d = fpr_op(mn, &ops[0])?;
1269 if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1270 (ops[1], d)
1272 } else {
1273 if !op_eq(&ops[1], &ops[0]) {
1274 match &ops[1] {
1275 Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1276 Op::Mem(a) => out.push(Inst::LoadF {
1277 k,
1278 dst: d,
1279 addr: *a,
1280 }),
1281 _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1282 }
1283 }
1284 (ops[2], d)
1285 }
1286 } else {
1287 (ops[1], fpr_op(mn, &ops[0])?)
1288 };
1289 out.push(Inst::FBit { or, k, dst, src });
1290 Ok(out)
1291}
1292
1293fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Expansion, ExpandError> {
1294 if ops.len() == 3 {
1295 return Ok(smallvec![Inst::FArith {
1296 op,
1297 k,
1298 arith: FArith3 {
1299 dst: fpr_op(mn, &ops[0])?,
1300 lhs: ops[1],
1301 rhs: Some(ops[2]),
1302 },
1303 }]);
1304 }
1305 Ok(smallvec![Inst::FArith {
1306 op,
1307 k,
1308 arith: FArith3::from_operands(mn, ops)?,
1309 }])
1310}
1311
1312fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1313 expect_len(mn, ops, 2)?;
1314 Ok(Inst::FSqrt {
1315 k,
1316 dst: fpr_op(mn, &ops[0])?,
1317 src: ops[1],
1318 })
1319}
1320
1321fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1322 expect_len(mn, ops, 4)?;
1323 Ok(Inst::Fma {
1324 op,
1325 k,
1326 a: ops[1],
1327 b: ops[2],
1328 c: ops[3],
1329 dst: fpr_op(mn, &ops[0])?,
1330 })
1331}
1332
1333fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1335 expect_len(mn, ops, 2)?;
1336 Ok(Inst::Bswap {
1337 w,
1338 dst: ops[0],
1339 src: ops[1],
1340 })
1341}
1342
1343fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1344 expect_len(mn, ops, 2)?;
1345 Ok(Inst::FRound {
1346 k,
1347 mode,
1348 dst: fpr_op(mn, &ops[0])?,
1349 src: ops[1],
1350 })
1351}
1352
1353fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1354 expect_len(mn, ops, 2)?;
1355 Ok(Inst::CvtFI {
1356 k,
1357 w,
1358 dst: gpr_op(mn, &ops[0])?,
1359 src: ops[1],
1360 })
1361}
1362
1363fn cvtif_node(
1364 mn: Mnemonic,
1365 ops: &[Op],
1366 signed: bool,
1367 from: Width,
1368 k: FKind,
1369) -> Result<Inst, ExpandError> {
1370 expect_len(mn, ops, 2)?;
1371 Ok(Inst::CvtIF {
1372 signed,
1373 from,
1374 k,
1375 dst: fpr_op(mn, &ops[0])?,
1376 src: ops[1],
1377 })
1378}
1379
1380fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1381 expect_len(mn, ops, 2)?;
1382 Ok(Inst::CvtFF {
1383 from,
1384 to,
1385 dst: fpr_op(mn, &ops[0])?,
1386 src: ops[1],
1387 })
1388}
1389
1390fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1391 expect_len(mn, ops, 2)?;
1392 Ok(Inst::Bitcast {
1393 dir,
1394 dst: ops[0],
1395 src: ops[1],
1396 })
1397}
1398
1399fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1400 expect_len(mn, ops, 2)?;
1401 let dst = fpr_op(mn, &ops[0])?;
1402 let inst = if negate {
1403 Inst::FNeg {
1404 k,
1405 dst,
1406 src: ops[1],
1407 }
1408 } else {
1409 Inst::FAbs {
1410 k,
1411 dst,
1412 src: ops[1],
1413 }
1414 };
1415 Ok(inst)
1416}
1417
1418fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1419 expect_len(mn, ops, 3)?;
1420 let lane = imm_op(mn, &ops[2])?;
1421 let lanes: i64 = 128 / w.bytes() as i64;
1422 if lane < 0 || lane >= lanes {
1423 return Err(bad(mn, "lane out of range"));
1424 }
1425 Ok(Inst::VecExtract {
1426 w,
1427 dst: gpr_op(mn, &ops[0])?,
1428 src: fpr_op(mn, &ops[1])?,
1429 lane: lane as u8,
1430 })
1431}
1432
1433fn trunc_q_slow(
1434 mn: Mnemonic,
1435 ops: &[Op],
1436 sc: &Scratch,
1437 k: FKind,
1438) -> Result<Expansion, ExpandError> {
1439 expect_len(mn, ops, 2)?;
1440 let (dst, src) = (ops[0], ops[1]);
1441 let slow = Lbl::Fresh(0);
1442 let done = Lbl::Fresh(1);
1443 let gs = Op::Gpr(sc.gpr);
1444 let fs = Op::Fpr(sc.fpr);
1445 let (int64_min, neg_min) = match k {
1446 FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1447 FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1448 };
1449 let dst_gpr = gpr_op(mn, &dst)?;
1450 Ok(smallvec![
1451 Inst::Mov {
1452 w: Width::B64,
1453 dst: gs,
1454 src: Op::Imm(neg_min),
1455 patch: false,
1456 },
1457 Inst::Bitcast {
1458 dir: if k == FKind::F32 {
1459 BitcastDir::I2F
1460 } else {
1461 BitcastDir::Q2D
1462 },
1463 dst: fs,
1464 src: gs,
1465 },
1466 Inst::BranchF {
1467 cc: FCc::Ge,
1468 k,
1469 lhs: src,
1470 rhs: fs,
1471 target: slow,
1472 patch: false,
1473 },
1474 Inst::CvtFI {
1475 k,
1476 w: Width::B64,
1477 dst: dst_gpr,
1478 src,
1479 },
1480 Inst::Jump {
1481 target: Target::Label(done),
1482 patch: false,
1483 },
1484 Inst::Bind(slow),
1485 Inst::Mov {
1486 w: Width::B64,
1487 dst: gs,
1488 src: Op::Imm(int64_min),
1489 patch: false,
1490 },
1491 Inst::Bitcast {
1492 dir: if k == FKind::F32 {
1493 BitcastDir::I2F
1494 } else {
1495 BitcastDir::Q2D
1496 },
1497 dst: fs,
1498 src: gs,
1499 },
1500 Inst::FArith {
1501 op: FAlu::Add,
1502 k,
1503 arith: FArith3 {
1504 dst: sc.fpr,
1505 lhs: src,
1506 rhs: None,
1507 },
1508 },
1509 Inst::CvtFI {
1510 k,
1511 w: Width::B64,
1512 dst: dst_gpr,
1513 src: fs,
1514 },
1515 Inst::Mov {
1516 w: Width::B64,
1517 dst: gs,
1518 src: Op::Imm(0x8000000000000000u64 as i64),
1519 patch: false,
1520 },
1521 Inst::Or {
1522 w: Width::B64,
1523 op: Alu3 {
1524 dst,
1525 lhs: dst,
1526 rhs: gs,
1527 },
1528 },
1529 Inst::Bind(done),
1530 ])
1531}
1532
1533fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Expansion, ExpandError> {
1534 expect_len(mn, ops, 3)?;
1535 let (src, dst) = (ops[1], ops[0]);
1536 let slow = Lbl::Fresh(0);
1537 let done = Lbl::Fresh(1);
1538 let s1 = gpr_op(mn, &ops[2])?;
1539 let s2 = sc.gpr;
1540 let dst_fpr = fpr_op(mn, &dst)?;
1541 Ok(smallvec![
1542 Inst::TestBranch {
1543 w: Width::B64,
1544 value: src,
1545 mask: None,
1546 test: TestKind::Sign,
1547 target: slow,
1548 patch: false,
1549 },
1550 Inst::CvtIF {
1551 signed: true,
1552 from: Width::B64,
1553 k,
1554 dst: dst_fpr,
1555 src,
1556 },
1557 Inst::Jump {
1558 target: Target::Label(done),
1559 patch: false,
1560 },
1561 Inst::Bind(slow),
1562 Inst::Mov {
1563 w: Width::B64,
1564 dst: Op::Gpr(s1),
1565 src,
1566 patch: false,
1567 },
1568 Inst::Mov {
1569 w: Width::B64,
1570 dst: Op::Gpr(s2),
1571 src,
1572 patch: false,
1573 },
1574 Inst::Shift {
1575 w: Width::B64,
1576 op: ShiftOp::Shr,
1577 dst: s1,
1578 cnt: ShiftCnt::Imm(1),
1579 },
1580 Inst::And {
1581 w: Width::B64,
1582 op: Alu3 {
1583 dst: Op::Gpr(s2),
1584 lhs: Op::Gpr(s2),
1585 rhs: Op::Imm(1),
1586 },
1587 },
1588 Inst::Or {
1589 w: Width::B64,
1590 op: Alu3 {
1591 dst: Op::Gpr(s2),
1592 lhs: Op::Gpr(s2),
1593 rhs: Op::Gpr(s1),
1594 },
1595 },
1596 Inst::CvtIF {
1597 signed: true,
1598 from: Width::B64,
1599 k,
1600 dst: dst_fpr,
1601 src: Op::Gpr(s2),
1602 },
1603 Inst::FArith {
1604 op: FAlu::Add,
1605 k,
1606 arith: FArith3 {
1607 dst: dst_fpr,
1608 lhs: Op::Fpr(dst_fpr),
1609 rhs: None,
1610 },
1611 },
1612 Inst::Bind(done),
1613 ])
1614}
1615
1616fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Expansion, ExpandError> {
1620 expect_len(mn, ops, 3)?;
1621 Ok(smallvec![
1622 Inst::CvtFI {
1623 k: FKind::F64,
1624 w: Width::B64,
1625 dst: gpr_op(mn, &ops[0])?,
1626 src: ops[1],
1627 },
1628 Inst::Branch {
1629 cc: IntCc::Eq,
1630 w: Width::B32,
1631 lhs: ops[0],
1632 rhs: Op::Imm(-0x8000_0000i64),
1633 target: label_op(mn, &ops[2])?,
1634 patch: false,
1635 },
1636 ])
1637}
1638
1639fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Expansion, ExpandError> {
1640 expect_len(mn, ops, 3)?;
1641 let target = label_op(mn, &ops[2])?;
1642 Ok(smallvec![
1643 Inst::CvtFI {
1644 k: FKind::F64,
1645 w: Width::B64,
1646 dst: gpr_op(mn, &ops[0])?,
1647 src: ops[1],
1648 },
1649 Inst::TestBranch {
1650 w: Width::B64,
1651 value: ops[0],
1652 mask: None,
1653 test: TestKind::Zero,
1654 target,
1655 patch: false,
1656 },
1657 Inst::CvtIF {
1658 signed: true,
1659 from: Width::B64,
1660 k: FKind::F64,
1661 dst: sc.fpr,
1662 src: ops[0],
1663 },
1664 Inst::BranchF {
1665 cc: FCc::NeUn,
1666 k: FKind::F64,
1667 lhs: ops[1],
1668 rhs: Op::Fpr(sc.fpr),
1669 target,
1670 patch: false,
1671 },
1672 ])
1673}
1674
1675fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1677 expect_len(mn, ops, 1)?;
1678 let dst = fpr_op(mn, &ops[0])?;
1679 Ok(Inst::FArith {
1680 op: FAlu::Sub,
1681 k: FKind::F64,
1682 arith: FArith3 {
1683 dst,
1684 lhs: Op::Fpr(dst),
1685 rhs: None,
1686 },
1687 })
1688}