1use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6
7pub fn expand(
23 mn: Mnemonic,
24 ops: &[Op],
25 sc: &Scratch,
26 patch: bool,
27) -> Result<Vec<Inst>, ExpandError> {
28 if patch && patch_kind(mn).is_none() {
29 return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
30 }
31 check_extern_operands(mn, ops, patch)?;
32 let mut insts = expand_inner(mn, ops, sc, patch)?;
33 stage_aliasing_mem(&mut insts, sc);
34 Ok(insts)
35}
36
37fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
41 let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
42 return Ok(());
43 };
44 if patch {
45 return Err(bad(mn, "an external symbol cannot be a patch site"));
46 }
47 let ok = match mn {
48 Mnemonic::Jmp | Mnemonic::Call => at == 0,
49 Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
50 Mnemonic::Lea {
51 w: Width::B64,
52 form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
53 } => at == 1,
54 _ => false,
55 };
56 if ok {
57 Ok(())
58 } else {
59 Err(bad(
60 mn,
61 "an external symbol is only a `jmp`/`call` target or the source of \
62 `move`/`leap`/`globaladdr`/`la`",
63 ))
64 }
65}
66
67fn stage_aliasing_mem(insts: &mut Vec<Inst>, sc: &Scratch) {
82 let old = std::mem::take(insts);
83 let mut out = Vec::with_capacity(old.len() + 2);
84 for mut inst in old {
85 let loads = stage_one(&mut inst, sc);
86 if let Some(loads) = loads {
87 out.extend(loads);
88 }
89 out.push(inst);
90 }
91 *insts = out;
92}
93
94fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Vec<Inst>> {
98 let (w, operands, dst) = match inst {
99 Inst::Add { w, op }
100 | Inst::Sub { w, op }
101 | Inst::Mul { w, op }
102 | Inst::And { w, op }
103 | Inst::Or { w, op }
104 | Inst::Xor { w, op } => (*w, vec![&mut op.lhs, &mut op.rhs], &mut op.dst),
105 Inst::AluS { w, alu, .. } => (*w, vec![&mut alu.lhs, &mut alu.rhs], &mut alu.dst),
106 Inst::OpBranch { w, args, .. } => {
107 let (last, init) = args.split_last_mut()?;
108 (*w, init.iter_mut().collect(), last)
109 }
110 _ => return None,
111 };
112 let Op::Gpr(g) = *dst else {
113 return None;
114 };
115 let slots = [sc.gpr, sc.gpr2];
116 let mut loads = Vec::new();
117 let mut slot = 0usize;
118 for operand in operands {
119 let Op::Mem(m) = operand else {
120 continue;
121 };
122 let aliases = m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g);
123 if !aliases {
124 continue;
125 }
126 let &s = slots.get(slot)?;
129 slot += 1;
130 loads.push(Inst::Load {
131 w,
132 sx: Sx::Zero,
133 dst: s,
134 addr: *m,
135 acquire: false,
136 });
137 *operand = Op::Gpr(s);
138 }
139 (!loads.is_empty()).then_some(loads)
140}
141
142pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
146 Some(match mn {
147 Mnemonic::Jmp => PatchKind::JumpSite,
148 Mnemonic::Call => PatchKind::CallSite,
149 Mnemonic::Move => PatchKind::MovBlock,
150 Mnemonic::FlagBranch(_)
151 | Mnemonic::OpBranch { .. }
152 | Mnemonic::RiscAluS { .. }
153 | Mnemonic::TestBranch { .. }
154 | Mnemonic::IntBranch { .. }
155 | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
156 _ => return None,
157 })
158}
159
160#[derive(Clone, Copy, Debug, PartialEq, Eq)]
162pub enum PatchKind {
163 JumpSite,
165 CondJumpSite,
167 CallSite,
169 MovBlock,
171}
172
173fn float_cc(cc: FCc) -> (FCc, bool) {
174 match cc {
175 FCc::Lt => (FCc::Gt, true),
176 FCc::Le => (FCc::Ge, true),
177 FCc::GtUn => (FCc::GtUn, false),
178 FCc::GeUn => (FCc::GeUn, false),
179 FCc::LtUn => (FCc::GtUn, true),
180 FCc::LeUn => (FCc::GeUn, true),
181 other => (other, false),
182 }
183}
184
185fn test_set_node(
186 mn: Mnemonic,
187 ops: &[Op],
188 w: Width,
189 test: TestKind,
190) -> Result<Vec<Inst>, ExpandError> {
191 if ops.len() != 2 && ops.len() != 3 {
192 return Err(bad(mn, "test-set takes 2 or 3 operands"));
193 }
194 let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
195 Ok(vec![Inst::TestSet {
196 w,
197 value: ops[1],
198 mask,
199 test,
200 dst: ops[0],
201 }])
202}
203
204fn test_branch_node(
205 mn: Mnemonic,
206 ops: &[Op],
207 w: Width,
208 test: TestKind,
209 patch: bool,
210) -> Result<Vec<Inst>, ExpandError> {
211 if ops.len() != 2 && ops.len() != 3 {
212 return Err(bad(mn, "test-branch takes 2 or 3 operands"));
213 }
214 let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
215 let target = label_op(
216 mn,
217 ops.last()
218 .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
219 )?;
220 Ok(vec![Inst::TestBranch {
221 w,
222 value: ops[0],
223 mask,
224 test,
225 target,
226 patch,
227 }])
228}
229
230fn div_rem_node(
231 mn: Mnemonic,
232 ops: &[Op],
233 signed: bool,
234 w: Width,
235 rem: bool,
236) -> Result<Vec<Inst>, ExpandError> {
237 let (dvd, dvs, dst) = match ops {
238 [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
239 [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
240 _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
241 };
242 Ok(vec![if rem {
243 Inst::Rem3 {
244 signed,
245 w,
246 dst,
247 dvd,
248 dvs,
249 }
250 } else {
251 Inst::Div3 {
252 signed,
253 w,
254 dst,
255 dvd,
256 dvs,
257 }
258 }])
259}
260
261fn shift_mem_canon(
262 mn: Mnemonic,
263 w: Width,
264 op: ShiftOp,
265 cnt: &Op,
266 dst: &Mem,
267 sc: &Scratch,
268) -> Result<Vec<Inst>, ExpandError> {
269 let shift_cnt = match cnt {
270 Op::Imm(v) => ShiftCnt::Imm(*v),
271 Op::Gpr(g) => ShiftCnt::Reg(*g),
272 _ => return Err(bad(mn, "shift count must be an immediate or register")),
273 };
274 Ok(vec![
275 Inst::Load {
276 w,
277 sx: Sx::Zero,
278 dst: sc.gpr,
279 addr: *dst,
280 acquire: false,
281 },
282 Inst::Shift {
283 w,
284 op,
285 dst: sc.gpr,
286 cnt: shift_cnt,
287 },
288 Inst::Store {
289 w,
290 src: Op::Gpr(sc.gpr),
291 addr: *dst,
292 },
293 ])
294}
295
296fn shift_node(
297 mn: Mnemonic,
298 w: Width,
299 op: ShiftOp,
300 ops: &[Op],
301 sc: &Scratch,
302) -> Result<Vec<Inst>, ExpandError> {
303 if ops.len() != 2 {
304 return Err(bad(mn, "shift takes 2 operands"));
305 }
306 if let Op::Mem(a) = &ops[0] {
307 return shift_mem_canon(mn, w, op, &ops[1], a, sc);
308 }
309 let dst = gpr_op(mn, &ops[0])?;
310 let cnt = match &ops[1] {
311 Op::Imm(v) => ShiftCnt::Imm(*v),
312 Op::Gpr(g) => ShiftCnt::Reg(*g),
313 _ => return Err(bad(mn, "shift count must be an immediate or register")),
314 };
315 Ok(vec![Inst::Shift { w, op, dst, cnt }])
316}
317
318fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
319 Alu3::from_operands(mn, ops)
320}
321
322fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Op>, ExpandError> {
323 match ops {
324 [dst, rhs] => Ok(vec![*rhs, *dst]),
325 [dst, lhs, rhs] => Ok(vec![*lhs, *rhs, *dst]),
326 _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
327 }
328}
329
330fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
331 match op {
332 Op::Label(l) => Ok(Target::Label(*l)),
333 Op::Gpr(g) => Ok(Target::Reg(*g)),
334 Op::Mem(a) => Ok(Target::Mem(*a)),
335 Op::Imm(v) => Ok(Target::Abs(*v)),
336 Op::Extern(e) => Ok(Target::Extern(*e)),
337 _ => Err(bad(
338 mn,
339 "jump/call needs a label, register, memory, or immediate",
340 )),
341 }
342}
343
344#[allow(clippy::too_many_lines)]
345fn expand_inner(
346 mn: Mnemonic,
347 ops: &[Op],
348 sc: &Scratch,
349 patch: bool,
350) -> Result<Vec<Inst>, ExpandError> {
351 let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
352 expect_len(mn, ops, 1)?;
353 Ok(ops[0])
354 };
355 match mn {
356 Mnemonic::Move => Ok(vec![mov_node(mn, ops, Width::B64, patch)?]),
357 Mnemonic::Load(w) => Ok(vec![mov_node(mn, ops, w, patch)?]),
358 Mnemonic::AtomicLoad(w) => Ok(vec![atomic_load_node(mn, ops, w)?]),
359 Mnemonic::Store(w) => Ok(vec![store_node(mn, ops, w)?]),
360 Mnemonic::Extend { from, to, sx } => Ok(vec![extend_node(mn, ops, from, to, sx)?]),
361 Mnemonic::MovZero32 => Ok(vec![movzero_node(mn, ops)?]),
362 Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
363 Mnemonic::FMove(k) => Ok(vec![fmove_node(mn, ops, k)?]),
364 Mnemonic::VMove { store } => Ok(vec![vmove_node(mn, ops, store)?]),
365 Mnemonic::Lea { w, .. } => Ok(vec![lea_node(mn, ops, w)?]),
366 Mnemonic::LoadQInc => loadqinc_node(mn, ops),
367 Mnemonic::LoadPair(w) => Ok(vec![pair_node(mn, ops, w, true)?]),
368 Mnemonic::StorePair(w) => Ok(vec![pair_node(mn, ops, w, false)?]),
369 Mnemonic::Push => stack_nodes(mn, ops, true, false),
370 Mnemonic::Pop => stack_nodes(mn, ops, false, false),
371 Mnemonic::PushV => stack_nodes(mn, ops, true, true),
372 Mnemonic::PopV => stack_nodes(mn, ops, false, true),
373 Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
374 Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
375 Mnemonic::Peek => Ok(vec![peek_node(mn, ops, true)?]),
376 Mnemonic::Poke => Ok(vec![peek_node(mn, ops, false)?]),
377 Mnemonic::TlsLoad(w) => Ok(vec![tls_node(mn, ops, w, true)?]),
378 Mnemonic::TlsStore(w) => Ok(vec![tls_node(mn, ops, w, false)?]),
379 Mnemonic::TlsLea => Ok(vec![tls_lea_node(mn, ops)?]),
380
381 Mnemonic::Add(w) => Ok(vec![Inst::Add {
382 w,
383 op: alu3(mn, ops)?,
384 }]),
385 Mnemonic::Sub(w) => Ok(vec![Inst::Sub {
386 w,
387 op: alu3(mn, ops)?,
388 }]),
389 Mnemonic::Mul(w) => Ok(vec![mul_node(mn, w, ops)?]),
390 Mnemonic::And(w) => Ok(vec![Inst::And {
391 w,
392 op: alu3(mn, ops)?,
393 }]),
394 Mnemonic::Or(w) => Ok(vec![Inst::Or {
395 w,
396 op: alu3(mn, ops)?,
397 }]),
398 Mnemonic::Xor(w) => Ok(vec![Inst::Xor {
399 w,
400 op: alu3(mn, ops)?,
401 }]),
402 Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
403 Mnemonic::Not(w) => Ok(vec![Inst::Not {
404 w,
405 dst: one(mn, ops)?,
406 }]),
407 Mnemonic::Neg(w) => Ok(vec![Inst::Neg {
408 w,
409 dst: one(mn, ops)?,
410 }]),
411 Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
412 Mnemonic::Bswap(w) => Ok(vec![bswap_node(mn, ops, w)?]),
413 Mnemonic::Adc => Ok(vec![adc_node(mn, ops, false)?]),
414 Mnemonic::Sbb => Ok(vec![adc_node(mn, ops, true)?]),
415 Mnemonic::MulHigh { signed } => Ok(vec![Inst::MulHigh {
416 signed,
417 rhs: one(mn, ops)?,
418 }]),
419 Mnemonic::MulWide { signed } => Ok(vec![mulwide_node(mn, ops, signed)?]),
420 Mnemonic::AluS { op, w } => Ok(vec![Inst::AluS {
421 op,
422 w,
423 alu: alu3(mn, ops)?,
424 }]),
425 Mnemonic::RiscAluS { op, w } => {
426 let has_label =
427 matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
428 if has_label {
429 let flag_op = match op {
430 AluOp::Add => FlagAlu::Add,
431 AluOp::Sub => FlagAlu::Sub,
432 AluOp::Or => FlagAlu::Or,
433 };
434 return Ok(vec![Inst::OpBranch {
435 op: flag_op,
436 w,
437 flag: PredFlag::S,
438 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
439 target: label_op(
440 mn,
441 ops.last()
442 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
443 )?,
444 patch,
445 }]);
446 }
447 Ok(vec![Inst::AluS {
448 op,
449 w,
450 alu: alu3(mn, ops)?,
451 }])
452 }
453 Mnemonic::AddShifted => Ok(vec![addshifted_node(mn, ops)?]),
454 Mnemonic::Cqo { quad } => Ok(vec![Inst::Cqo { quad }]),
455 Mnemonic::DivMod1 { signed, w } => Ok(vec![Inst::DivMod1 {
456 signed,
457 w,
458 divisor: one(mn, ops)?,
459 }]),
460 Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
461 Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
462 Mnemonic::IntSet { w, cc } => {
463 expect_len(mn, ops, 3)?;
464 Ok(vec![Inst::IntSet {
465 cc,
466 w,
467 dst: ops[0],
468 lhs: ops[1],
469 rhs: ops[2],
470 }])
471 }
472 Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
473 Mnemonic::Cmov { zero } => Ok(vec![cmov_node(mn, ops, zero)?]),
474 Mnemonic::BitCount { w, trailing } => Ok(vec![cz_node(mn, ops, w, trailing)?]),
475 Mnemonic::Popcnt(w) => Ok(vec![popcnt_node(mn, ops, w)?]),
476 Mnemonic::Bfi => bfi_node(mn, ops, sc),
477 Mnemonic::IntBranch { w, cc } => {
478 expect_len(mn, ops, 3)?;
479 Ok(vec![Inst::Branch {
480 cc,
481 w,
482 lhs: ops[0],
483 rhs: ops[1],
484 target: label_op(mn, &ops[2])?,
485 patch,
486 }])
487 }
488 Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
489 Mnemonic::OpBranch { op, w, flag } => {
490 if ops.len() != 3 && ops.len() != 4 {
491 return Err(bad(mn, "op-branch takes 3 or 4 operands"));
492 }
493 Ok(vec![Inst::OpBranch {
494 op,
495 w,
496 flag,
497 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
498 target: label_op(
499 mn,
500 ops.last()
501 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
502 )?,
503 patch,
504 }])
505 }
506 Mnemonic::FlagBranch(flag) => Ok(vec![flag_branch_node(mn, ops, flag, patch)?]),
507 Mnemonic::BranchF { cc, k } => {
508 expect_len(mn, ops, 3)?;
509 let (cc, reverse) = float_cc(cc);
510 let (lhs, rhs) = if reverse {
511 (ops[1], ops[0])
512 } else {
513 (ops[0], ops[1])
514 };
515 Ok(vec![Inst::BranchF {
516 cc,
517 k,
518 lhs,
519 rhs,
520 target: label_op(mn, &ops[2])?,
521 patch,
522 }])
523 }
524 Mnemonic::Jmp => {
525 expect_len(mn, ops, 1)?;
526 Ok(vec![Inst::Jump {
527 target: jump_target(mn, &ops[0])?,
528 patch,
529 }])
530 }
531 Mnemonic::Call => {
532 expect_len(mn, ops, 1)?;
533 Ok(vec![Inst::Call {
534 target: jump_target(mn, &ops[0])?,
535 patch,
536 }])
537 }
538 Mnemonic::Ret => {
539 expect_len(mn, ops, 0)?;
540 Ok(vec![Inst::Ret])
541 }
542 Mnemonic::Break => {
543 expect_len(mn, ops, 0)?;
544 Ok(vec![Inst::Break])
545 }
546 Mnemonic::Nop => {
547 expect_len(mn, ops, 0)?;
548 Ok(vec![Inst::Nop])
549 }
550 Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
551 Mnemonic::Fma { op, k } => Ok(vec![fma_node(mn, ops, k, op)?]),
552 Mnemonic::FSqrt(k) => Ok(vec![fsqrt_node(mn, ops, k)?]),
553 Mnemonic::FRound { k, mode } => Ok(vec![fround_node(mn, ops, k, mode)?]),
554 Mnemonic::FAbs { k, negate } => Ok(vec![fabs_node(mn, ops, k, negate)?]),
555 Mnemonic::FSet { cc, k } => {
556 expect_len(mn, ops, 3)?;
557 let (cc, reverse) = float_cc(cc);
558 let (lhs, rhs) = if reverse {
559 (ops[2], ops[1])
560 } else {
561 (ops[1], ops[2])
562 };
563 Ok(vec![Inst::FSet {
564 cc,
565 k,
566 dst: ops[0],
567 lhs,
568 rhs,
569 }])
570 }
571 Mnemonic::CvtFI { k, w } => Ok(vec![cvtfi_node(mn, ops, k, w)?]),
572 Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
573 Mnemonic::CvtIF { signed, from, k } => Ok(vec![cvtif_node(mn, ops, signed, from, k)?]),
574 Mnemonic::CvtFF { from, to } => Ok(vec![cvtff_node(mn, ops, from, to)?]),
575 Mnemonic::Bitcast(dir) => Ok(vec![bitcast_node(mn, ops, dir)?]),
576 Mnemonic::MovDZ => Ok(vec![movdz_node(mn, ops)?]),
577 Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
578 Mnemonic::Btd2i => btd2i_node(mn, ops),
579 Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
580 Mnemonic::VecExtract(w) => Ok(vec![umov_node(mn, ops, w)?]),
581 Mnemonic::AtomicRmw { op, w } => Ok(vec![rmw_node(mn, ops, w, op)?]),
582 Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
583 Mnemonic::AtomicCas { w, branch } => Ok(vec![cas_node(mn, ops, w, branch)?]),
584 Mnemonic::LoadLinked(w) => Ok(vec![ll_node(mn, ops, w)?]),
585 Mnemonic::StoreCond(w) => Ok(vec![sc_node(mn, ops, w)?]),
586 Mnemonic::Fence { full } => Ok(vec![Inst::Fence { full }]),
587 Mnemonic::Pause => Ok(vec![Inst::Pause]),
588 Mnemonic::TagCodePtr
589 | Mnemonic::TagReturnAddress
590 | Mnemonic::UntagReturnAddress
591 | Mnemonic::RemoveCodePtrTag
592 | Mnemonic::UntagArrayPtr
593 | Mnemonic::RemoveArrayPtrTag => Ok(vec![]),
594 }
595}
596
597fn flag_branch_node(
598 mn: Mnemonic,
599 ops: &[Op],
600 flag: PredFlag,
601 patch: bool,
602) -> Result<Inst, ExpandError> {
603 expect_len(mn, ops, 1)?;
604 Ok(Inst::FlagBranch {
605 flag,
606 target: label_op(mn, &ops[0])?,
607 patch,
608 })
609}
610
611fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
612 expect_len(mn, ops, 2)?;
613 Ok(Inst::Popcnt {
614 w,
615 dst: gpr_op(mn, &ops[0])?,
616 src: ops[1],
617 })
618}
619
620fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
621 expect_len(mn, ops, 2)?;
622 let dst = gpr_op(mn, &ops[0])?;
623 let inst = if trailing {
624 Inst::Ctz {
625 w,
626 dst,
627 src: ops[1],
628 }
629 } else {
630 Inst::Clz {
631 w,
632 dst,
633 src: ops[1],
634 }
635 };
636 Ok(inst)
637}
638
639fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
640 expect_len(mn, ops, 2)?;
641 if load {
642 let idx = imm_op(mn, &ops[1])?;
643 Ok(Inst::TlsLoad {
644 w,
645 dst: gpr_op(mn, &ops[0])?,
646 idx,
647 })
648 } else {
649 let idx = imm_op(mn, &ops[0])?;
650 Ok(Inst::TlsStore {
651 w,
652 src: gpr_op(mn, &ops[1])?,
653 idx,
654 })
655 }
656}
657
658fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
659 expect_len(mn, ops, 2)?;
660 Ok(Inst::TlsLea {
661 dst: gpr_op(mn, &ops[0])?,
662 idx: imm_op(mn, &ops[1])?,
663 })
664}
665
666fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
667 expect_len(mn, ops, 2)?;
668
669 Ok(Inst::AtomicRmw {
670 op,
671 w,
672 a: ops[1],
673 b: ops[0],
674 })
675}
676
677fn rmw_not_node(
678 mn: Mnemonic,
679 ops: &[Op],
680 sc: &Scratch,
681 w: Width,
682 op: AtomicOp,
683) -> Result<Vec<Inst>, ExpandError> {
684 expect_len(mn, ops, 2)?;
685 let mut out = Vec::new();
686 let a = match &ops[1] {
688 Op::Imm(v) => Op::Imm(!v),
689 Op::Gpr(g) => {
690 out.push(Inst::Mov {
691 w: Width::B64,
692 dst: Op::Gpr(sc.gpr),
693 src: Op::Gpr(*g),
694 patch: false,
695 });
696 out.push(Inst::Not {
697 w,
698 dst: Op::Gpr(sc.gpr),
699 });
700 Op::Gpr(sc.gpr)
701 }
702 Op::Mem(addr) => {
703 let lw = if w == Width::B64 {
704 Width::B64
705 } else {
706 Width::B32
707 };
708 out.push(Inst::Load {
709 w: lw,
710 sx: Sx::Zero,
711 dst: sc.gpr,
712 addr: *addr,
713 acquire: false,
714 });
715 out.push(Inst::Not {
716 w,
717 dst: Op::Gpr(sc.gpr),
718 });
719 Op::Gpr(sc.gpr)
720 }
721 _ => {
722 return Err(bad(
723 mn,
724 "atomic clear needs an immediate, register, or memory operand",
725 ))
726 }
727 };
728 out.push(Inst::AtomicRmw {
729 op,
730 w,
731 a,
732 b: ops[0],
733 });
734 Ok(out)
735}
736
737fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
738 let n = if branch { 4 } else { 3 };
739 expect_len(mn, ops, n)?;
740
741 Ok(Inst::AtomicCas {
742 w,
743 mem: mem_op(mn, &ops[0])?,
744 src: gpr_op(mn, &ops[2])?,
745 branch: if branch {
746 Some(label_op(mn, &ops[3])?)
747 } else {
748 None
749 },
750 })
751}
752
753fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
754 expect_len(mn, ops, 2)?;
755 Ok(Inst::LoadLinked {
756 w,
757 mem: mem_op(mn, &ops[1])?,
758 dst: gpr_op(mn, &ops[0])?,
759 })
760}
761
762fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
763 expect_len(mn, ops, 3)?;
764 Ok(Inst::StoreCond {
765 w,
766 status: gpr_op(mn, &ops[1])?,
767 src: gpr_op(mn, &ops[2])?,
768 mem: mem_op(mn, &ops[0])?,
769 })
770}
771
772fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
776 expect_len(mn, ops, 4)?;
777 let lsb = imm_op(mn, &ops[2])?;
778 let width = imm_op(mn, &ops[3])?;
779 if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
780 return Err(bad(mn, "bitfield out of range"));
781 }
782 let wmask: u64 = if width == 64 {
783 u64::MAX
784 } else {
785 (1u64 << width) - 1
786 };
787 let mask: i64 = (wmask << lsb) as i64;
788 let s = sc.gpr;
789 let dst = gpr_op(mn, &ops[0])?;
790
791 Ok(vec![
792 Inst::Mov {
793 w: Width::B64,
794 dst: Op::Gpr(s),
795 src: ops[1],
796 patch: false,
797 },
798 Inst::And {
799 w: Width::B64,
800 op: Alu3 {
801 dst: Op::Gpr(s),
802 lhs: Op::Gpr(s),
803 rhs: Op::Imm(wmask as i64),
804 },
805 },
806 Inst::Shift {
807 w: Width::B64,
808 op: ShiftOp::Shl,
809 dst: s,
810 cnt: ShiftCnt::Imm(lsb),
811 },
812 Inst::And {
813 w: Width::B64,
814 op: Alu3 {
815 dst: Op::Gpr(dst),
816 lhs: Op::Gpr(dst),
817 rhs: Op::Imm(!mask),
818 },
819 },
820 Inst::Or {
821 w: Width::B64,
822 op: Alu3 {
823 dst: Op::Gpr(dst),
824 lhs: Op::Gpr(dst),
825 rhs: Op::Gpr(s),
826 },
827 },
828 ])
829}
830
831fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
832 expect_len(mn, ops, 3)?;
833 Ok(Inst::Cmov {
834 w: Width::B64,
835 zero,
836 dst: gpr_op(mn, &ops[0])?,
837 src: ops[1],
838 test: gpr_op(mn, &ops[2])?,
839 })
840}
841
842fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
845 expect_len(mn, ops, 2)?;
846 if peek {
847 let disp = slot_disp(mn, &ops[1])?;
848 return Ok(Inst::Peek {
849 disp,
850 dst: gpr_op(mn, &ops[0])?,
851 });
852 }
853 Ok(Inst::Poke {
854 disp: slot_disp(mn, &ops[0])?,
855 src: gpr_op(mn, &ops[1])?,
856 })
857}
858
859fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
860 let idx = imm_op(mn, op)?;
861 let off = idx
862 .checked_mul(8)
863 .ok_or_else(|| bad(mn, "stack offset overflow"))?;
864 if off < i32::MIN as i64 || off > i32::MAX as i64 {
865 return Err(bad(mn, "stack offset out of range"));
866 }
867 Ok(off as i32)
868}
869
870fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
871 expect_len(mn, ops, 3)?;
872 let dst = gpr_op(mn, &ops[0])?;
873 let base = gpr_op(mn, &ops[1])?;
874 let inc = imm_op(mn, &ops[2])?;
875 Ok(vec![
876 Inst::Load {
877 w: Width::B64,
878 sx: Sx::Zero,
879 dst,
880 addr: Mem::base(base, 0),
881 acquire: false,
882 },
883 Inst::Add {
884 w: Width::B64,
885 op: Alu3 {
886 dst: Op::Gpr(base),
887 lhs: Op::Gpr(base),
888 rhs: Op::Imm(inc),
889 },
890 },
891 ])
892}
893
894fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
895 expect_len(mn, ops, 3)?;
896 if load {
897 Ok(Inst::PairLoad {
898 w,
899 dst1: ops[0],
900 dst2: ops[1],
901 mem: mem_op(mn, &ops[2])?,
902 })
903 } else {
904 Ok(Inst::PairStore {
905 w,
906 src1: ops[1],
907 src2: ops[2],
908 mem: mem_op(mn, &ops[0])?,
909 })
910 }
911}
912
913fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
914 expect_len(mn, ops, 2)?;
915 let dst = gpr_op(mn, &ops[0])?;
916 let src = match &ops[1] {
917 Op::Mem(a) => LeaSrc::Mem(*a),
918 Op::Label(l) => LeaSrc::Label(*l),
919 Op::Extern(e) => LeaSrc::Extern(*e),
920 _ => return Err(bad(mn, "lea needs an address or label source")),
921 };
922 Ok(Inst::Lea { w, dst, src })
923}
924
925fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
926 expect_len(mn, ops, 2)?;
927 if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
929 return Ok(Inst::Lea {
930 w: Width::B64,
931 dst: *dst,
932 src: LeaSrc::Extern(*e),
933 });
934 }
935 Ok(Inst::Mov {
936 w,
937 dst: ops[0],
938 src: ops[1],
939 patch,
940 })
941}
942
943fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
944 expect_len(mn, ops, 2)?;
945 Ok(Inst::MovZero32 {
946 dst: ops[0],
947 src: ops[1],
948 })
949}
950
951fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
952 expect_len(mn, ops, 2)?;
953 Ok(Inst::Load {
954 w,
955 sx: Sx::Zero,
956 dst: gpr_op(mn, &ops[0])?,
957 addr: mem_op(mn, &ops[1])?,
958 acquire: true,
959 })
960}
961
962fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
963 expect_len(mn, ops, 2)?;
964 let addr = mem_op(mn, &ops[0])?;
965 Ok(Inst::Store {
966 w,
967 src: ops[1],
968 addr,
969 })
970}
971
972fn extend_node(
973 mn: Mnemonic,
974 ops: &[Op],
975 from: Width,
976 to: Width,
977 sx: Sx,
978) -> Result<Inst, ExpandError> {
979 expect_len(mn, ops, 2)?;
980 Ok(Inst::Extend {
981 from,
982 to,
983 sx,
984 dst: gpr_op(mn, &ops[0])?,
985 src: ops[1],
986 })
987}
988
989fn transfer_node(
990 mn: Mnemonic,
991 ops: &[Op],
992 sc: &Scratch,
993 w: Width,
994) -> Result<Vec<Inst>, ExpandError> {
995 expect_len(mn, ops, 2)?;
996 let mut out = Vec::new();
997 match &ops[1] {
998 Op::Gpr(_) => {}
999 Op::Mem(a) => {
1000 out.push(Inst::Load {
1001 w,
1002 sx: Sx::Zero,
1003 dst: sc.gpr,
1004 addr: *a,
1005 acquire: false,
1006 });
1007 }
1008 Op::Imm(v) => {
1009 out.push(Inst::Mov {
1010 w,
1011 dst: Op::Gpr(sc.gpr),
1012 src: Op::Imm(*v),
1013 patch: false,
1014 });
1015 }
1016 Op::Label(l) => {
1017 out.push(Inst::Lea {
1018 w: Width::B64,
1019 dst: sc.gpr,
1020 src: LeaSrc::Label(*l),
1021 });
1022 let addr = Mem::base(sc.gpr, 0);
1023 out.push(Inst::Load {
1024 w,
1025 sx: Sx::Zero,
1026 dst: sc.gpr,
1027 addr,
1028 acquire: false,
1029 });
1030 }
1031 Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1032 Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1033 Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1034 }
1035 let src = if out.is_empty() {
1036 ops[1]
1037 } else {
1038 Op::Gpr(sc.gpr)
1039 };
1040 out.push(Inst::Mov {
1041 w,
1042 dst: ops[0],
1043 src,
1044 patch: false,
1045 });
1046 Ok(out)
1047}
1048
1049fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1050 expect_len(mn, ops, 2)?;
1051 match (&ops[0], &ops[1]) {
1052 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1053 k,
1054 dst: *d,
1055 src: *s,
1056 }),
1057 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1058 k,
1059 dst: *d,
1060 addr: *a,
1061 }),
1062 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1063 k,
1064 src: *s,
1065 addr: *a,
1066 }),
1067 _ => Err(bad(mn, "float move needs FPR/memory operands")),
1068 }
1069}
1070
1071fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1072 expect_len(mn, ops, 2)?;
1073 if !store {
1074 match (&ops[0], &ops[1]) {
1075 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1076 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1077 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1078 dst: *d,
1079 addr: Mem::base(*s, 0),
1080 }),
1081 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1082 }
1083 } else {
1084 match (&ops[0], &ops[1]) {
1085 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1086 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1087 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1088 dst: *d,
1089 addr: Mem::base(*s, 0),
1090 }),
1091 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1092 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1093 }
1094 }
1095}
1096
1097pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1107 if !matches!(
1108 mnemonic,
1109 Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1110 ) {
1111 return Ok(());
1112 }
1113 for op in ops {
1114 match op {
1115 Op::Gpr(g) if saved.contains(&g.0) => {}
1116 _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1117 }
1118 }
1119 Ok(())
1120}
1121
1122pub fn aligned_push_bytes(n: usize) -> i32 {
1130 ((8 * n + 15) & !15) as i32
1131}
1132
1133pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1137 8 * (n - 1 - i) as i32
1138}
1139
1140fn stack_nodes(
1141 mn: Mnemonic,
1142 ops: &[Op],
1143 push: bool,
1144 vector: bool,
1145) -> Result<Vec<Inst>, ExpandError> {
1146 if ops.is_empty() {
1147 return Err(bad(mn, "needs at least one operand"));
1148 }
1149 let mut out = Vec::with_capacity(ops.len());
1150 let ordered: Box<dyn Iterator<Item = &Op>> = if push {
1154 Box::new(ops.iter())
1155 } else {
1156 Box::new(ops.iter().rev())
1157 };
1158 for op in ordered {
1159 if vector {
1160 let x = fpr_op(mn, op)?;
1161 out.push(if push {
1162 Inst::PushV { src: x }
1163 } else {
1164 Inst::PopV { dst: x }
1165 });
1166 } else {
1167 match op {
1168 Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => {
1169 out.push(if push {
1170 Inst::Push { op: *op }
1171 } else {
1172 Inst::Pop { op: *op }
1173 });
1174 }
1175 _ => return Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1176 }
1177 }
1178 }
1179 Ok(out)
1180}
1181
1182fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1183 if ops.len() == 3 {
1184 if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1185 return Ok(Inst::MulImm {
1186 w,
1187 dst: gpr_op(mn, &ops[0])?,
1188 lhs,
1189 imm: *v,
1190 });
1191 }
1192 if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1193 return Ok(Inst::MulImm {
1194 w,
1195 dst: gpr_op(mn, &ops[0])?,
1196 lhs,
1197 imm: *v,
1198 });
1199 }
1200 }
1201 Ok(Inst::Mul {
1202 w,
1203 op: Alu3::from_operands(mn, ops)?,
1204 })
1205}
1206
1207fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1208 expect_len(mn, ops, 3)?;
1209 Ok(Inst::MulWide {
1210 signed,
1211 dst: gpr_op(mn, &ops[0])?,
1212 lhs: gpr_op(mn, &ops[1])?,
1213 rhs: gpr_op(mn, &ops[2])?,
1214 })
1215}
1216
1217fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1218 expect_len(mn, ops, 4)?;
1219 let amount = imm_op(mn, &ops[3])?;
1220 if !(0..=63).contains(&amount) {
1221 return Err(bad(mn, "shift amount out of range"));
1222 }
1223 Ok(Inst::AddShifted {
1224 dst: gpr_op(mn, &ops[0])?,
1225 lhs: gpr_op(mn, &ops[1])?,
1226 rhs: gpr_op(mn, &ops[2])?,
1227 amount: amount as u8,
1228 })
1229}
1230
1231fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1232 expect_len(mn, ops, 2)?;
1233 let inst = if borrow {
1234 Inst::Sbb {
1235 dst: ops[0],
1236 src: ops[1],
1237 }
1238 } else {
1239 Inst::Adc {
1240 dst: ops[0],
1241 src: ops[1],
1242 }
1243 };
1244 Ok(inst)
1245}
1246
1247fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Vec<Inst>, ExpandError> {
1248 if ops.len() != 2 && ops.len() != 3 {
1249 return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1250 }
1251 let mut out = Vec::new();
1252 let (src, dst) = if ops.len() == 3 {
1253 let d = fpr_op(mn, &ops[0])?;
1254 if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1255 (ops[1], d)
1257 } else {
1258 if !op_eq(&ops[1], &ops[0]) {
1259 match &ops[1] {
1260 Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1261 Op::Mem(a) => out.push(Inst::LoadF {
1262 k,
1263 dst: d,
1264 addr: *a,
1265 }),
1266 _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1267 }
1268 }
1269 (ops[2], d)
1270 }
1271 } else {
1272 (ops[1], fpr_op(mn, &ops[0])?)
1273 };
1274 out.push(Inst::FBit { or, k, dst, src });
1275 Ok(out)
1276}
1277
1278fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Vec<Inst>, ExpandError> {
1279 if ops.len() == 3 {
1280 return Ok(vec![Inst::FArith {
1281 op,
1282 k,
1283 arith: FArith3 {
1284 dst: fpr_op(mn, &ops[0])?,
1285 lhs: ops[1],
1286 rhs: Some(ops[2]),
1287 },
1288 }]);
1289 }
1290 Ok(vec![Inst::FArith {
1291 op,
1292 k,
1293 arith: FArith3::from_operands(mn, ops)?,
1294 }])
1295}
1296
1297fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1298 expect_len(mn, ops, 2)?;
1299 Ok(Inst::FSqrt {
1300 k,
1301 dst: fpr_op(mn, &ops[0])?,
1302 src: ops[1],
1303 })
1304}
1305
1306fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1307 expect_len(mn, ops, 4)?;
1308 Ok(Inst::Fma {
1309 op,
1310 k,
1311 a: ops[1],
1312 b: ops[2],
1313 c: ops[3],
1314 dst: fpr_op(mn, &ops[0])?,
1315 })
1316}
1317
1318fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1320 expect_len(mn, ops, 2)?;
1321 Ok(Inst::Bswap {
1322 w,
1323 dst: ops[0],
1324 src: ops[1],
1325 })
1326}
1327
1328fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1329 expect_len(mn, ops, 2)?;
1330 Ok(Inst::FRound {
1331 k,
1332 mode,
1333 dst: fpr_op(mn, &ops[0])?,
1334 src: ops[1],
1335 })
1336}
1337
1338fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1339 expect_len(mn, ops, 2)?;
1340 Ok(Inst::CvtFI {
1341 k,
1342 w,
1343 dst: gpr_op(mn, &ops[0])?,
1344 src: ops[1],
1345 })
1346}
1347
1348fn cvtif_node(
1349 mn: Mnemonic,
1350 ops: &[Op],
1351 signed: bool,
1352 from: Width,
1353 k: FKind,
1354) -> Result<Inst, ExpandError> {
1355 expect_len(mn, ops, 2)?;
1356 Ok(Inst::CvtIF {
1357 signed,
1358 from,
1359 k,
1360 dst: fpr_op(mn, &ops[0])?,
1361 src: ops[1],
1362 })
1363}
1364
1365fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1366 expect_len(mn, ops, 2)?;
1367 Ok(Inst::CvtFF {
1368 from,
1369 to,
1370 dst: fpr_op(mn, &ops[0])?,
1371 src: ops[1],
1372 })
1373}
1374
1375fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1376 expect_len(mn, ops, 2)?;
1377 Ok(Inst::Bitcast {
1378 dir,
1379 dst: ops[0],
1380 src: ops[1],
1381 })
1382}
1383
1384fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1385 expect_len(mn, ops, 2)?;
1386 let dst = fpr_op(mn, &ops[0])?;
1387 let inst = if negate {
1388 Inst::FNeg {
1389 k,
1390 dst,
1391 src: ops[1],
1392 }
1393 } else {
1394 Inst::FAbs {
1395 k,
1396 dst,
1397 src: ops[1],
1398 }
1399 };
1400 Ok(inst)
1401}
1402
1403fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1404 expect_len(mn, ops, 3)?;
1405 let lane = imm_op(mn, &ops[2])?;
1406 let lanes: i64 = 128 / w.bytes() as i64;
1407 if lane < 0 || lane >= lanes {
1408 return Err(bad(mn, "lane out of range"));
1409 }
1410 Ok(Inst::VecExtract {
1411 w,
1412 dst: gpr_op(mn, &ops[0])?,
1413 src: fpr_op(mn, &ops[1])?,
1414 lane: lane as u8,
1415 })
1416}
1417
1418fn trunc_q_slow(
1419 mn: Mnemonic,
1420 ops: &[Op],
1421 sc: &Scratch,
1422 k: FKind,
1423) -> Result<Vec<Inst>, ExpandError> {
1424 expect_len(mn, ops, 2)?;
1425 let (dst, src) = (ops[0], ops[1]);
1426 let slow = Lbl::Fresh(0);
1427 let done = Lbl::Fresh(1);
1428 let gs = Op::Gpr(sc.gpr);
1429 let fs = Op::Fpr(sc.fpr);
1430 let (int64_min, neg_min) = match k {
1431 FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1432 FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1433 };
1434 let dst_gpr = gpr_op(mn, &dst)?;
1435 Ok(vec![
1436 Inst::Mov {
1437 w: Width::B64,
1438 dst: gs,
1439 src: Op::Imm(neg_min),
1440 patch: false,
1441 },
1442 Inst::Bitcast {
1443 dir: if k == FKind::F32 {
1444 BitcastDir::I2F
1445 } else {
1446 BitcastDir::Q2D
1447 },
1448 dst: fs,
1449 src: gs,
1450 },
1451 Inst::BranchF {
1452 cc: FCc::Ge,
1453 k,
1454 lhs: src,
1455 rhs: fs,
1456 target: slow,
1457 patch: false,
1458 },
1459 Inst::CvtFI {
1460 k,
1461 w: Width::B64,
1462 dst: dst_gpr,
1463 src,
1464 },
1465 Inst::Jump {
1466 target: Target::Label(done),
1467 patch: false,
1468 },
1469 Inst::Bind(slow),
1470 Inst::Mov {
1471 w: Width::B64,
1472 dst: gs,
1473 src: Op::Imm(int64_min),
1474 patch: false,
1475 },
1476 Inst::Bitcast {
1477 dir: if k == FKind::F32 {
1478 BitcastDir::I2F
1479 } else {
1480 BitcastDir::Q2D
1481 },
1482 dst: fs,
1483 src: gs,
1484 },
1485 Inst::FArith {
1486 op: FAlu::Add,
1487 k,
1488 arith: FArith3 {
1489 dst: sc.fpr,
1490 lhs: src,
1491 rhs: None,
1492 },
1493 },
1494 Inst::CvtFI {
1495 k,
1496 w: Width::B64,
1497 dst: dst_gpr,
1498 src: fs,
1499 },
1500 Inst::Mov {
1501 w: Width::B64,
1502 dst: gs,
1503 src: Op::Imm(0x8000000000000000u64 as i64),
1504 patch: false,
1505 },
1506 Inst::Or {
1507 w: Width::B64,
1508 op: Alu3 {
1509 dst,
1510 lhs: dst,
1511 rhs: gs,
1512 },
1513 },
1514 Inst::Bind(done),
1515 ])
1516}
1517
1518fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Vec<Inst>, ExpandError> {
1519 expect_len(mn, ops, 3)?;
1520 let (src, dst) = (ops[1], ops[0]);
1521 let slow = Lbl::Fresh(0);
1522 let done = Lbl::Fresh(1);
1523 let s1 = gpr_op(mn, &ops[2])?;
1524 let s2 = sc.gpr;
1525 let dst_fpr = fpr_op(mn, &dst)?;
1526 Ok(vec![
1527 Inst::TestBranch {
1528 w: Width::B64,
1529 value: src,
1530 mask: None,
1531 test: TestKind::Sign,
1532 target: slow,
1533 patch: false,
1534 },
1535 Inst::CvtIF {
1536 signed: true,
1537 from: Width::B64,
1538 k,
1539 dst: dst_fpr,
1540 src,
1541 },
1542 Inst::Jump {
1543 target: Target::Label(done),
1544 patch: false,
1545 },
1546 Inst::Bind(slow),
1547 Inst::Mov {
1548 w: Width::B64,
1549 dst: Op::Gpr(s1),
1550 src,
1551 patch: false,
1552 },
1553 Inst::Mov {
1554 w: Width::B64,
1555 dst: Op::Gpr(s2),
1556 src,
1557 patch: false,
1558 },
1559 Inst::Shift {
1560 w: Width::B64,
1561 op: ShiftOp::Shr,
1562 dst: s1,
1563 cnt: ShiftCnt::Imm(1),
1564 },
1565 Inst::And {
1566 w: Width::B64,
1567 op: Alu3 {
1568 dst: Op::Gpr(s2),
1569 lhs: Op::Gpr(s2),
1570 rhs: Op::Imm(1),
1571 },
1572 },
1573 Inst::Or {
1574 w: Width::B64,
1575 op: Alu3 {
1576 dst: Op::Gpr(s2),
1577 lhs: Op::Gpr(s2),
1578 rhs: Op::Gpr(s1),
1579 },
1580 },
1581 Inst::CvtIF {
1582 signed: true,
1583 from: Width::B64,
1584 k,
1585 dst: dst_fpr,
1586 src: Op::Gpr(s2),
1587 },
1588 Inst::FArith {
1589 op: FAlu::Add,
1590 k,
1591 arith: FArith3 {
1592 dst: dst_fpr,
1593 lhs: Op::Fpr(dst_fpr),
1594 rhs: None,
1595 },
1596 },
1597 Inst::Bind(done),
1598 ])
1599}
1600
1601fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
1605 expect_len(mn, ops, 3)?;
1606 Ok(vec![
1607 Inst::CvtFI {
1608 k: FKind::F64,
1609 w: Width::B64,
1610 dst: gpr_op(mn, &ops[0])?,
1611 src: ops[1],
1612 },
1613 Inst::Branch {
1614 cc: IntCc::Eq,
1615 w: Width::B32,
1616 lhs: ops[0],
1617 rhs: Op::Imm(-0x8000_0000i64),
1618 target: label_op(mn, &ops[2])?,
1619 patch: false,
1620 },
1621 ])
1622}
1623
1624fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
1625 expect_len(mn, ops, 3)?;
1626 let target = label_op(mn, &ops[2])?;
1627 Ok(vec![
1628 Inst::CvtFI {
1629 k: FKind::F64,
1630 w: Width::B64,
1631 dst: gpr_op(mn, &ops[0])?,
1632 src: ops[1],
1633 },
1634 Inst::TestBranch {
1635 w: Width::B64,
1636 value: ops[0],
1637 mask: None,
1638 test: TestKind::Zero,
1639 target,
1640 patch: false,
1641 },
1642 Inst::CvtIF {
1643 signed: true,
1644 from: Width::B64,
1645 k: FKind::F64,
1646 dst: sc.fpr,
1647 src: ops[0],
1648 },
1649 Inst::BranchF {
1650 cc: FCc::NeUn,
1651 k: FKind::F64,
1652 lhs: ops[1],
1653 rhs: Op::Fpr(sc.fpr),
1654 target,
1655 patch: false,
1656 },
1657 ])
1658}
1659
1660fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1662 expect_len(mn, ops, 1)?;
1663 let dst = fpr_op(mn, &ops[0])?;
1664 Ok(Inst::FArith {
1665 op: FAlu::Sub,
1666 k: FKind::F64,
1667 arith: FArith3 {
1668 dst,
1669 lhs: Op::Fpr(dst),
1670 rhs: None,
1671 },
1672 })
1673}