1use super::inst::*;
4use super::mnemonic::Mnemonic;
5use super::*;
6
7pub fn expand(
23 mn: Mnemonic,
24 ops: &[Op],
25 sc: &Scratch,
26 patch: bool,
27) -> Result<Vec<Inst>, ExpandError> {
28 if patch && patch_kind(mn).is_none() {
29 return Err(ExpandError::NotPatchable(mn.as_str().to_string()));
30 }
31 check_extern_operands(mn, ops, patch)?;
32 let mut insts = expand_inner(mn, ops, sc, patch)?;
33 stage_aliasing_mem(&mut insts, sc);
34 Ok(insts)
35}
36
37fn check_extern_operands(mn: Mnemonic, ops: &[Op], patch: bool) -> Result<(), ExpandError> {
41 let Some(at) = ops.iter().position(|o| matches!(o, Op::Extern(_))) else {
42 return Ok(());
43 };
44 if patch {
45 return Err(bad(mn, "an external symbol cannot be a patch site"));
46 }
47 let ok = match mn {
48 Mnemonic::Jmp | Mnemonic::Call => at == 0,
49 Mnemonic::Move => at == 1 && matches!(ops[0], Op::Gpr(_)),
50 Mnemonic::Lea {
51 w: Width::B64,
52 form: LeaForm::Plain | LeaForm::Global | LeaForm::La,
53 } => at == 1,
54 _ => false,
55 };
56 if ok {
57 Ok(())
58 } else {
59 Err(bad(
60 mn,
61 "an external symbol is only a `jmp`/`call` target or the source of \
62 `move`/`leap`/`globaladdr`/`la`",
63 ))
64 }
65}
66
67fn stage_aliasing_mem(insts: &mut Vec<Inst>, sc: &Scratch) {
82 let old = std::mem::take(insts);
83 let mut out = Vec::with_capacity(old.len() + 2);
84 for mut inst in old {
85 let loads = stage_one(&mut inst, sc);
86 if let Some(loads) = loads {
87 out.extend(loads);
88 }
89 out.push(inst);
90 }
91 *insts = out;
92}
93
94fn stage_one(inst: &mut Inst, sc: &Scratch) -> Option<Vec<Inst>> {
98 let (w, operands, dst) = match inst {
99 Inst::Add { w, op }
100 | Inst::Sub { w, op }
101 | Inst::Mul { w, op }
102 | Inst::And { w, op }
103 | Inst::Or { w, op }
104 | Inst::Xor { w, op } => (*w, vec![&mut op.lhs, &mut op.rhs], &mut op.dst),
105 Inst::AluS { w, alu, .. } => (*w, vec![&mut alu.lhs, &mut alu.rhs], &mut alu.dst),
106 Inst::OpBranch { w, args, .. } => {
107 let (last, init) = args.split_last_mut()?;
108 (*w, init.iter_mut().collect(), last)
109 }
110 _ => return None,
111 };
112 let Op::Gpr(g) = *dst else {
113 return None;
114 };
115 let slots = [sc.gpr, sc.gpr2];
116 let mut loads = Vec::new();
117 let mut slot = 0usize;
118 for operand in operands {
119 let Op::Mem(m) = operand else {
120 continue;
121 };
122 let aliases = m.base == Some(g) || matches!(m.index, Some((idx, _)) if idx == g);
123 if !aliases {
124 continue;
125 }
126 let &s = slots.get(slot)?;
129 slot += 1;
130 loads.push(Inst::Load {
131 w,
132 sx: Sx::Zero,
133 dst: s,
134 addr: *m,
135 acquire: false,
136 });
137 *operand = Op::Gpr(s);
138 }
139 (!loads.is_empty()).then_some(loads)
140}
141
142pub fn patch_kind(mn: Mnemonic) -> Option<PatchKind> {
144 Some(match mn {
145 Mnemonic::Jmp => PatchKind::JumpSite,
146 Mnemonic::Call => PatchKind::CallSite,
147 Mnemonic::Move => PatchKind::MovBlock,
148 Mnemonic::FlagBranch(_)
149 | Mnemonic::OpBranch { .. }
150 | Mnemonic::RiscAluS { .. }
151 | Mnemonic::TestBranch { .. }
152 | Mnemonic::IntBranch { .. }
153 | Mnemonic::BranchF { .. } => PatchKind::CondJumpSite,
154 _ => return None,
155 })
156}
157
158#[derive(Clone, Copy, Debug, PartialEq, Eq)]
164pub enum PatchKind {
165 JumpSite,
167 CondJumpSite,
169 CallSite,
171 MovBlock,
173}
174
175fn float_cc(cc: FCc) -> (FCc, bool) {
176 match cc {
177 FCc::Lt => (FCc::Gt, true),
178 FCc::Le => (FCc::Ge, true),
179 FCc::GtUn => (FCc::GtUn, false),
180 FCc::GeUn => (FCc::GeUn, false),
181 FCc::LtUn => (FCc::GtUn, true),
182 FCc::LeUn => (FCc::GeUn, true),
183 other => (other, false),
184 }
185}
186
187fn test_set_node(
188 mn: Mnemonic,
189 ops: &[Op],
190 w: Width,
191 test: TestKind,
192) -> Result<Vec<Inst>, ExpandError> {
193 if ops.len() != 2 && ops.len() != 3 {
194 return Err(bad(mn, "test-set takes 2 or 3 operands"));
195 }
196 let mask = if ops.len() == 3 { Some(ops[2]) } else { None };
197 Ok(vec![Inst::TestSet {
198 w,
199 value: ops[1],
200 mask,
201 test,
202 dst: ops[0],
203 }])
204}
205
206fn test_branch_node(
207 mn: Mnemonic,
208 ops: &[Op],
209 w: Width,
210 test: TestKind,
211 patch: bool,
212) -> Result<Vec<Inst>, ExpandError> {
213 if ops.len() != 2 && ops.len() != 3 {
214 return Err(bad(mn, "test-branch takes 2 or 3 operands"));
215 }
216 let mask = if ops.len() == 3 { Some(ops[1]) } else { None };
217 let target = label_op(
218 mn,
219 ops.last()
220 .ok_or_else(|| bad(mn, "test-branch takes 2 or 3 operands"))?,
221 )?;
222 Ok(vec![Inst::TestBranch {
223 w,
224 value: ops[0],
225 mask,
226 test,
227 target,
228 patch,
229 }])
230}
231
232fn div_rem_node(
233 mn: Mnemonic,
234 ops: &[Op],
235 signed: bool,
236 w: Width,
237 rem: bool,
238) -> Result<Vec<Inst>, ExpandError> {
239 let (dvd, dvs, dst) = match ops {
240 [dst, dvs] => (*dst, *dvs, gpr_op(mn, dst)?),
241 [dst, dvd, dvs] => (*dvd, *dvs, gpr_op(mn, dst)?),
242 _ => return Err(bad(mn, "divide takes 2 or 3 operands")),
243 };
244 Ok(vec![if rem {
245 Inst::Rem3 {
246 signed,
247 w,
248 dst,
249 dvd,
250 dvs,
251 }
252 } else {
253 Inst::Div3 {
254 signed,
255 w,
256 dst,
257 dvd,
258 dvs,
259 }
260 }])
261}
262
263fn shift_mem_canon(
264 mn: Mnemonic,
265 w: Width,
266 op: ShiftOp,
267 cnt: &Op,
268 dst: &Mem,
269 sc: &Scratch,
270) -> Result<Vec<Inst>, ExpandError> {
271 let shift_cnt = match cnt {
272 Op::Imm(v) => ShiftCnt::Imm(*v),
273 Op::Gpr(g) => ShiftCnt::Reg(*g),
274 _ => return Err(bad(mn, "shift count must be an immediate or register")),
275 };
276 Ok(vec![
277 Inst::Load {
278 w,
279 sx: Sx::Zero,
280 dst: sc.gpr,
281 addr: *dst,
282 acquire: false,
283 },
284 Inst::Shift {
285 w,
286 op,
287 dst: sc.gpr,
288 cnt: shift_cnt,
289 },
290 Inst::Store {
291 w,
292 src: Op::Gpr(sc.gpr),
293 addr: *dst,
294 },
295 ])
296}
297
298fn shift_node(
299 mn: Mnemonic,
300 w: Width,
301 op: ShiftOp,
302 ops: &[Op],
303 sc: &Scratch,
304) -> Result<Vec<Inst>, ExpandError> {
305 if ops.len() != 2 {
306 return Err(bad(mn, "shift takes 2 operands"));
307 }
308 if let Op::Mem(a) = &ops[0] {
309 return shift_mem_canon(mn, w, op, &ops[1], a, sc);
310 }
311 let dst = gpr_op(mn, &ops[0])?;
312 let cnt = match &ops[1] {
313 Op::Imm(v) => ShiftCnt::Imm(*v),
314 Op::Gpr(g) => ShiftCnt::Reg(*g),
315 _ => return Err(bad(mn, "shift count must be an immediate or register")),
316 };
317 Ok(vec![Inst::Shift { w, op, dst, cnt }])
318}
319
320fn alu3(mn: Mnemonic, ops: &[Op]) -> Result<Alu3, ExpandError> {
321 Alu3::from_operands(mn, ops)
322}
323
324fn op_branch_args(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Op>, ExpandError> {
325 match ops {
326 [dst, rhs] => Ok(vec![*rhs, *dst]),
327 [dst, lhs, rhs] => Ok(vec![*lhs, *rhs, *dst]),
328 _ => Err(bad(mn, "op-branch takes 3 or 4 operands")),
329 }
330}
331
332fn jump_target(mn: Mnemonic, op: &Op) -> Result<Target, ExpandError> {
333 match op {
334 Op::Label(l) => Ok(Target::Label(*l)),
335 Op::Gpr(g) => Ok(Target::Reg(*g)),
336 Op::Mem(a) => Ok(Target::Mem(*a)),
337 Op::Imm(v) => Ok(Target::Abs(*v)),
338 Op::Extern(e) => Ok(Target::Extern(*e)),
339 _ => Err(bad(
340 mn,
341 "jump/call needs a label, register, memory, or immediate",
342 )),
343 }
344}
345
346#[allow(clippy::too_many_lines)]
347fn expand_inner(
348 mn: Mnemonic,
349 ops: &[Op],
350 sc: &Scratch,
351 patch: bool,
352) -> Result<Vec<Inst>, ExpandError> {
353 let one = |mn: Mnemonic, ops: &[Op]| -> Result<Op, ExpandError> {
354 expect_len(mn, ops, 1)?;
355 Ok(ops[0])
356 };
357 match mn {
358 Mnemonic::Move => Ok(vec![mov_node(mn, ops, Width::B64, patch)?]),
359 Mnemonic::Load(w) => Ok(vec![mov_node(mn, ops, w, patch)?]),
360 Mnemonic::AtomicLoad(w) => Ok(vec![atomic_load_node(mn, ops, w)?]),
361 Mnemonic::Store(w) => Ok(vec![store_node(mn, ops, w)?]),
362 Mnemonic::Extend { from, to, sx } => Ok(vec![extend_node(mn, ops, from, to, sx)?]),
363 Mnemonic::MovZero32 => Ok(vec![movzero_node(mn, ops)?]),
364 Mnemonic::Transfer(w) => transfer_node(mn, ops, sc, w),
365 Mnemonic::FMove(k) => Ok(vec![fmove_node(mn, ops, k)?]),
366 Mnemonic::VMove { store } => Ok(vec![vmove_node(mn, ops, store)?]),
367 Mnemonic::Lea { w, .. } => Ok(vec![lea_node(mn, ops, w)?]),
368 Mnemonic::LoadQInc => loadqinc_node(mn, ops),
369 Mnemonic::LoadPair(w) => Ok(vec![pair_node(mn, ops, w, true)?]),
370 Mnemonic::StorePair(w) => Ok(vec![pair_node(mn, ops, w, false)?]),
371 Mnemonic::Push => stack_nodes(mn, ops, true, false),
372 Mnemonic::Pop => stack_nodes(mn, ops, false, false),
373 Mnemonic::PushV => stack_nodes(mn, ops, true, true),
374 Mnemonic::PopV => stack_nodes(mn, ops, false, true),
375 Mnemonic::PushCalleeSaves => stack_nodes(mn, ops, true, false),
376 Mnemonic::PopCalleeSaves => stack_nodes(mn, ops, false, false),
377 Mnemonic::Peek => Ok(vec![peek_node(mn, ops, true)?]),
378 Mnemonic::Poke => Ok(vec![peek_node(mn, ops, false)?]),
379 Mnemonic::TlsLoad(w) => Ok(vec![tls_node(mn, ops, w, true)?]),
380 Mnemonic::TlsStore(w) => Ok(vec![tls_node(mn, ops, w, false)?]),
381 Mnemonic::TlsLea => Ok(vec![tls_lea_node(mn, ops)?]),
382
383 Mnemonic::Add(w) => Ok(vec![Inst::Add {
384 w,
385 op: alu3(mn, ops)?,
386 }]),
387 Mnemonic::Sub(w) => Ok(vec![Inst::Sub {
388 w,
389 op: alu3(mn, ops)?,
390 }]),
391 Mnemonic::Mul(w) => Ok(vec![mul_node(mn, w, ops)?]),
392 Mnemonic::And(w) => Ok(vec![Inst::And {
393 w,
394 op: alu3(mn, ops)?,
395 }]),
396 Mnemonic::Or(w) => Ok(vec![Inst::Or {
397 w,
398 op: alu3(mn, ops)?,
399 }]),
400 Mnemonic::Xor(w) => Ok(vec![Inst::Xor {
401 w,
402 op: alu3(mn, ops)?,
403 }]),
404 Mnemonic::FBit { or, k } => fbit_node(mn, ops, k, or),
405 Mnemonic::Not(w) => Ok(vec![Inst::Not {
406 w,
407 dst: one(mn, ops)?,
408 }]),
409 Mnemonic::Neg(w) => Ok(vec![Inst::Neg {
410 w,
411 dst: one(mn, ops)?,
412 }]),
413 Mnemonic::Shift { w, op } => shift_node(mn, w, op, ops, sc),
414 Mnemonic::Bswap(w) => Ok(vec![bswap_node(mn, ops, w)?]),
415 Mnemonic::Adc => Ok(vec![adc_node(mn, ops, false)?]),
416 Mnemonic::Sbb => Ok(vec![adc_node(mn, ops, true)?]),
417 Mnemonic::MulHigh { signed } => Ok(vec![Inst::MulHigh {
418 signed,
419 rhs: one(mn, ops)?,
420 }]),
421 Mnemonic::MulWide { signed } => Ok(vec![mulwide_node(mn, ops, signed)?]),
422 Mnemonic::AluS { op, w } => Ok(vec![Inst::AluS {
423 op,
424 w,
425 alu: alu3(mn, ops)?,
426 }]),
427 Mnemonic::RiscAluS { op, w } => {
428 let has_label =
429 matches!(ops.last(), Some(Op::Label(_))) && (ops.len() == 3 || ops.len() == 4);
430 if has_label {
431 let flag_op = match op {
432 AluOp::Add => FlagAlu::Add,
433 AluOp::Sub => FlagAlu::Sub,
434 AluOp::Or => FlagAlu::Or,
435 };
436 return Ok(vec![Inst::OpBranch {
437 op: flag_op,
438 w,
439 flag: PredFlag::S,
440 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
441 target: label_op(
442 mn,
443 ops.last()
444 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
445 )?,
446 patch,
447 }]);
448 }
449 Ok(vec![Inst::AluS {
450 op,
451 w,
452 alu: alu3(mn, ops)?,
453 }])
454 }
455 Mnemonic::AddShifted => Ok(vec![addshifted_node(mn, ops)?]),
456 Mnemonic::Cqo { quad } => Ok(vec![Inst::Cqo { quad }]),
457 Mnemonic::DivMod1 { signed, w } => Ok(vec![Inst::DivMod1 {
458 signed,
459 w,
460 divisor: one(mn, ops)?,
461 }]),
462 Mnemonic::Div { signed, w } => div_rem_node(mn, ops, signed, w, false),
463 Mnemonic::Rem { signed, w } => div_rem_node(mn, ops, signed, w, true),
464 Mnemonic::IntSet { w, cc } => {
465 expect_len(mn, ops, 3)?;
466 Ok(vec![Inst::IntSet {
467 cc,
468 w,
469 dst: ops[0],
470 lhs: ops[1],
471 rhs: ops[2],
472 }])
473 }
474 Mnemonic::TestSet { w, test } => test_set_node(mn, ops, w, test),
475 Mnemonic::Cmov { zero } => Ok(vec![cmov_node(mn, ops, zero)?]),
476 Mnemonic::BitCount { w, trailing } => Ok(vec![cz_node(mn, ops, w, trailing)?]),
477 Mnemonic::Popcnt(w) => Ok(vec![popcnt_node(mn, ops, w)?]),
478 Mnemonic::Bfi => bfi_node(mn, ops, sc),
479 Mnemonic::IntBranch { w, cc } => {
480 expect_len(mn, ops, 3)?;
481 Ok(vec![Inst::Branch {
482 cc,
483 w,
484 lhs: ops[0],
485 rhs: ops[1],
486 target: label_op(mn, &ops[2])?,
487 patch,
488 }])
489 }
490 Mnemonic::TestBranch { w, test } => test_branch_node(mn, ops, w, test, patch),
491 Mnemonic::OpBranch { op, w, flag } => {
492 if ops.len() != 3 && ops.len() != 4 {
493 return Err(bad(mn, "op-branch takes 3 or 4 operands"));
494 }
495 Ok(vec![Inst::OpBranch {
496 op,
497 w,
498 flag,
499 args: op_branch_args(mn, &ops[..ops.len() - 1])?,
500 target: label_op(
501 mn,
502 ops.last()
503 .ok_or_else(|| bad(mn, "op-branch takes 3 or 4 operands"))?,
504 )?,
505 patch,
506 }])
507 }
508 Mnemonic::FlagBranch(flag) => Ok(vec![flag_branch_node(mn, ops, flag, patch)?]),
509 Mnemonic::BranchF { cc, k } => {
510 expect_len(mn, ops, 3)?;
511 let (cc, reverse) = float_cc(cc);
512 let (lhs, rhs) = if reverse {
513 (ops[1], ops[0])
514 } else {
515 (ops[0], ops[1])
516 };
517 Ok(vec![Inst::BranchF {
518 cc,
519 k,
520 lhs,
521 rhs,
522 target: label_op(mn, &ops[2])?,
523 patch,
524 }])
525 }
526 Mnemonic::Jmp => {
527 expect_len(mn, ops, 1)?;
528 Ok(vec![Inst::Jump {
529 target: jump_target(mn, &ops[0])?,
530 patch,
531 }])
532 }
533 Mnemonic::Call => {
534 expect_len(mn, ops, 1)?;
535 Ok(vec![Inst::Call {
536 target: jump_target(mn, &ops[0])?,
537 patch,
538 }])
539 }
540 Mnemonic::Ret => {
541 expect_len(mn, ops, 0)?;
542 Ok(vec![Inst::Ret])
543 }
544 Mnemonic::Break => {
545 expect_len(mn, ops, 0)?;
546 Ok(vec![Inst::Break])
547 }
548 Mnemonic::Nop => {
549 expect_len(mn, ops, 0)?;
550 Ok(vec![Inst::Nop])
551 }
552 Mnemonic::FArith { op, k } => farith_node(mn, ops, k, op),
553 Mnemonic::Fma { op, k } => Ok(vec![fma_node(mn, ops, k, op)?]),
554 Mnemonic::FSqrt(k) => Ok(vec![fsqrt_node(mn, ops, k)?]),
555 Mnemonic::FRound { k, mode } => Ok(vec![fround_node(mn, ops, k, mode)?]),
556 Mnemonic::FAbs { k, negate } => Ok(vec![fabs_node(mn, ops, k, negate)?]),
557 Mnemonic::FSet { cc, k } => {
558 expect_len(mn, ops, 3)?;
559 let (cc, reverse) = float_cc(cc);
560 let (lhs, rhs) = if reverse {
561 (ops[2], ops[1])
562 } else {
563 (ops[1], ops[2])
564 };
565 Ok(vec![Inst::FSet {
566 cc,
567 k,
568 dst: ops[0],
569 lhs,
570 rhs,
571 }])
572 }
573 Mnemonic::CvtFI { k, w } => Ok(vec![cvtfi_node(mn, ops, k, w)?]),
574 Mnemonic::CvtQuad(k) => quad_to_fp(mn, ops, sc, k),
575 Mnemonic::CvtIF { signed, from, k } => Ok(vec![cvtif_node(mn, ops, signed, from, k)?]),
576 Mnemonic::CvtFF { from, to } => Ok(vec![cvtff_node(mn, ops, from, to)?]),
577 Mnemonic::Bitcast(dir) => Ok(vec![bitcast_node(mn, ops, dir)?]),
578 Mnemonic::MovDZ => Ok(vec![movdz_node(mn, ops)?]),
579 Mnemonic::TruncQSlow(k) => trunc_q_slow(mn, ops, sc, k),
580 Mnemonic::Btd2i => btd2i_node(mn, ops),
581 Mnemonic::Bcd2i => bcd2i_node(mn, ops, sc),
582 Mnemonic::VecExtract(w) => Ok(vec![umov_node(mn, ops, w)?]),
583 Mnemonic::AtomicRmw { op, w } => Ok(vec![rmw_node(mn, ops, w, op)?]),
584 Mnemonic::AtomicRmwNot(w) => rmw_not_node(mn, ops, sc, w, AtomicOp::And),
585 Mnemonic::AtomicCas { w, branch } => Ok(vec![cas_node(mn, ops, w, branch)?]),
586 Mnemonic::LoadLinked(w) => Ok(vec![ll_node(mn, ops, w)?]),
587 Mnemonic::StoreCond(w) => Ok(vec![sc_node(mn, ops, w)?]),
588 Mnemonic::Fence { full } => Ok(vec![Inst::Fence { full }]),
589 Mnemonic::Pause => Ok(vec![Inst::Pause]),
590 Mnemonic::TagCodePtr
591 | Mnemonic::TagReturnAddress
592 | Mnemonic::UntagReturnAddress
593 | Mnemonic::RemoveCodePtrTag
594 | Mnemonic::UntagArrayPtr
595 | Mnemonic::RemoveArrayPtrTag => Ok(vec![]),
596 }
597}
598
599fn flag_branch_node(
600 mn: Mnemonic,
601 ops: &[Op],
602 flag: PredFlag,
603 patch: bool,
604) -> Result<Inst, ExpandError> {
605 expect_len(mn, ops, 1)?;
606 Ok(Inst::FlagBranch {
607 flag,
608 target: label_op(mn, &ops[0])?,
609 patch,
610 })
611}
612
613fn popcnt_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
614 expect_len(mn, ops, 2)?;
615 Ok(Inst::Popcnt {
616 w,
617 dst: gpr_op(mn, &ops[0])?,
618 src: ops[1],
619 })
620}
621
622fn cz_node(mn: Mnemonic, ops: &[Op], w: Width, trailing: bool) -> Result<Inst, ExpandError> {
623 expect_len(mn, ops, 2)?;
624 let dst = gpr_op(mn, &ops[0])?;
625 let inst = if trailing {
626 Inst::Ctz {
627 w,
628 dst,
629 src: ops[1],
630 }
631 } else {
632 Inst::Clz {
633 w,
634 dst,
635 src: ops[1],
636 }
637 };
638 Ok(inst)
639}
640
641fn tls_node(mn: Mnemonic, ops: &[Op], w: Width, load: bool) -> Result<Inst, ExpandError> {
642 expect_len(mn, ops, 2)?;
643 if load {
644 let idx = imm_op(mn, &ops[1])?;
645 Ok(Inst::TlsLoad {
646 w,
647 dst: gpr_op(mn, &ops[0])?,
648 idx,
649 })
650 } else {
651 let idx = imm_op(mn, &ops[0])?;
652 Ok(Inst::TlsStore {
653 w,
654 src: gpr_op(mn, &ops[1])?,
655 idx,
656 })
657 }
658}
659
660fn tls_lea_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
661 expect_len(mn, ops, 2)?;
662 Ok(Inst::TlsLea {
663 dst: gpr_op(mn, &ops[0])?,
664 idx: imm_op(mn, &ops[1])?,
665 })
666}
667
668fn rmw_node(mn: Mnemonic, ops: &[Op], w: Width, op: AtomicOp) -> Result<Inst, ExpandError> {
669 expect_len(mn, ops, 2)?;
670
671 Ok(Inst::AtomicRmw {
672 op,
673 w,
674 a: ops[1],
675 b: ops[0],
676 })
677}
678
679fn rmw_not_node(
680 mn: Mnemonic,
681 ops: &[Op],
682 sc: &Scratch,
683 w: Width,
684 op: AtomicOp,
685) -> Result<Vec<Inst>, ExpandError> {
686 expect_len(mn, ops, 2)?;
687 let mut out = Vec::new();
688 let a = match &ops[1] {
690 Op::Imm(v) => Op::Imm(!v),
691 Op::Gpr(g) => {
692 out.push(Inst::Mov {
693 w: Width::B64,
694 dst: Op::Gpr(sc.gpr),
695 src: Op::Gpr(*g),
696 patch: false,
697 });
698 out.push(Inst::Not {
699 w,
700 dst: Op::Gpr(sc.gpr),
701 });
702 Op::Gpr(sc.gpr)
703 }
704 Op::Mem(addr) => {
705 let lw = if w == Width::B64 {
706 Width::B64
707 } else {
708 Width::B32
709 };
710 out.push(Inst::Load {
711 w: lw,
712 sx: Sx::Zero,
713 dst: sc.gpr,
714 addr: *addr,
715 acquire: false,
716 });
717 out.push(Inst::Not {
718 w,
719 dst: Op::Gpr(sc.gpr),
720 });
721 Op::Gpr(sc.gpr)
722 }
723 _ => {
724 return Err(bad(
725 mn,
726 "atomic clear needs an immediate, register, or memory operand",
727 ))
728 }
729 };
730 out.push(Inst::AtomicRmw {
731 op,
732 w,
733 a,
734 b: ops[0],
735 });
736 Ok(out)
737}
738
739fn cas_node(mn: Mnemonic, ops: &[Op], w: Width, branch: bool) -> Result<Inst, ExpandError> {
740 let n = if branch { 4 } else { 3 };
741 expect_len(mn, ops, n)?;
742
743 Ok(Inst::AtomicCas {
744 w,
745 mem: mem_op(mn, &ops[0])?,
746 src: gpr_op(mn, &ops[2])?,
747 branch: if branch {
748 Some(label_op(mn, &ops[3])?)
749 } else {
750 None
751 },
752 })
753}
754
755fn ll_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
756 expect_len(mn, ops, 2)?;
757 Ok(Inst::LoadLinked {
758 w,
759 mem: mem_op(mn, &ops[1])?,
760 dst: gpr_op(mn, &ops[0])?,
761 })
762}
763
764fn sc_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
765 expect_len(mn, ops, 3)?;
766 Ok(Inst::StoreCond {
767 w,
768 status: gpr_op(mn, &ops[1])?,
769 src: gpr_op(mn, &ops[2])?,
770 mem: mem_op(mn, &ops[0])?,
771 })
772}
773
774fn bfi_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
778 expect_len(mn, ops, 4)?;
779 let lsb = imm_op(mn, &ops[2])?;
780 let width = imm_op(mn, &ops[3])?;
781 if !(1..=64).contains(&width) || lsb < 0 || lsb + width > 64 {
782 return Err(bad(mn, "bitfield out of range"));
783 }
784 let wmask: u64 = if width == 64 {
785 u64::MAX
786 } else {
787 (1u64 << width) - 1
788 };
789 let mask: i64 = (wmask << lsb) as i64;
790 let s = sc.gpr;
791 let dst = gpr_op(mn, &ops[0])?;
792
793 Ok(vec![
794 Inst::Mov {
795 w: Width::B64,
796 dst: Op::Gpr(s),
797 src: ops[1],
798 patch: false,
799 },
800 Inst::And {
801 w: Width::B64,
802 op: Alu3 {
803 dst: Op::Gpr(s),
804 lhs: Op::Gpr(s),
805 rhs: Op::Imm(wmask as i64),
806 },
807 },
808 Inst::Shift {
809 w: Width::B64,
810 op: ShiftOp::Shl,
811 dst: s,
812 cnt: ShiftCnt::Imm(lsb),
813 },
814 Inst::And {
815 w: Width::B64,
816 op: Alu3 {
817 dst: Op::Gpr(dst),
818 lhs: Op::Gpr(dst),
819 rhs: Op::Imm(!mask),
820 },
821 },
822 Inst::Or {
823 w: Width::B64,
824 op: Alu3 {
825 dst: Op::Gpr(dst),
826 lhs: Op::Gpr(dst),
827 rhs: Op::Gpr(s),
828 },
829 },
830 ])
831}
832
833fn cmov_node(mn: Mnemonic, ops: &[Op], zero: bool) -> Result<Inst, ExpandError> {
834 expect_len(mn, ops, 3)?;
835 Ok(Inst::Cmov {
836 w: Width::B64,
837 zero,
838 dst: gpr_op(mn, &ops[0])?,
839 src: ops[1],
840 test: gpr_op(mn, &ops[2])?,
841 })
842}
843
844fn peek_node(mn: Mnemonic, ops: &[Op], peek: bool) -> Result<Inst, ExpandError> {
847 expect_len(mn, ops, 2)?;
848 if peek {
849 let disp = slot_disp(mn, &ops[1])?;
850 return Ok(Inst::Peek {
851 disp,
852 dst: gpr_op(mn, &ops[0])?,
853 });
854 }
855 Ok(Inst::Poke {
856 disp: slot_disp(mn, &ops[0])?,
857 src: gpr_op(mn, &ops[1])?,
858 })
859}
860
861fn slot_disp(mn: Mnemonic, op: &Op) -> Result<i32, ExpandError> {
862 let idx = imm_op(mn, op)?;
863 let off = idx
864 .checked_mul(8)
865 .ok_or_else(|| bad(mn, "stack offset overflow"))?;
866 if off < i32::MIN as i64 || off > i32::MAX as i64 {
867 return Err(bad(mn, "stack offset out of range"));
868 }
869 Ok(off as i32)
870}
871
872fn loadqinc_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
873 expect_len(mn, ops, 3)?;
874 let dst = gpr_op(mn, &ops[0])?;
875 let base = gpr_op(mn, &ops[1])?;
876 let inc = imm_op(mn, &ops[2])?;
877 Ok(vec![
878 Inst::Load {
879 w: Width::B64,
880 sx: Sx::Zero,
881 dst,
882 addr: Mem::base(base, 0),
883 acquire: false,
884 },
885 Inst::Add {
886 w: Width::B64,
887 op: Alu3 {
888 dst: Op::Gpr(base),
889 lhs: Op::Gpr(base),
890 rhs: Op::Imm(inc),
891 },
892 },
893 ])
894}
895
896fn pair_node(mn: Mnemonic, ops: &[Op], w: PairW, load: bool) -> Result<Inst, ExpandError> {
897 expect_len(mn, ops, 3)?;
898 if load {
899 Ok(Inst::PairLoad {
900 w,
901 dst1: ops[0],
902 dst2: ops[1],
903 mem: mem_op(mn, &ops[2])?,
904 })
905 } else {
906 Ok(Inst::PairStore {
907 w,
908 src1: ops[1],
909 src2: ops[2],
910 mem: mem_op(mn, &ops[0])?,
911 })
912 }
913}
914
915fn lea_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
916 expect_len(mn, ops, 2)?;
917 let dst = gpr_op(mn, &ops[0])?;
918 let src = match &ops[1] {
919 Op::Mem(a) => LeaSrc::Mem(*a),
920 Op::Label(l) => LeaSrc::Label(*l),
921 Op::Extern(e) => LeaSrc::Extern(*e),
922 _ => return Err(bad(mn, "lea needs an address or label source")),
923 };
924 Ok(Inst::Lea { w, dst, src })
925}
926
927fn mov_node(mn: Mnemonic, ops: &[Op], w: Width, patch: bool) -> Result<Inst, ExpandError> {
928 expect_len(mn, ops, 2)?;
929 if let (Op::Gpr(dst), Op::Extern(e)) = (&ops[0], &ops[1]) {
931 return Ok(Inst::Lea {
932 w: Width::B64,
933 dst: *dst,
934 src: LeaSrc::Extern(*e),
935 });
936 }
937 Ok(Inst::Mov {
938 w,
939 dst: ops[0],
940 src: ops[1],
941 patch,
942 })
943}
944
945fn movzero_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
946 expect_len(mn, ops, 2)?;
947 Ok(Inst::MovZero32 {
948 dst: ops[0],
949 src: ops[1],
950 })
951}
952
953fn atomic_load_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
954 expect_len(mn, ops, 2)?;
955 Ok(Inst::Load {
956 w,
957 sx: Sx::Zero,
958 dst: gpr_op(mn, &ops[0])?,
959 addr: mem_op(mn, &ops[1])?,
960 acquire: true,
961 })
962}
963
964fn store_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
965 expect_len(mn, ops, 2)?;
966 let addr = mem_op(mn, &ops[0])?;
967 Ok(Inst::Store {
968 w,
969 src: ops[1],
970 addr,
971 })
972}
973
974fn extend_node(
975 mn: Mnemonic,
976 ops: &[Op],
977 from: Width,
978 to: Width,
979 sx: Sx,
980) -> Result<Inst, ExpandError> {
981 expect_len(mn, ops, 2)?;
982 Ok(Inst::Extend {
983 from,
984 to,
985 sx,
986 dst: gpr_op(mn, &ops[0])?,
987 src: ops[1],
988 })
989}
990
991fn transfer_node(
992 mn: Mnemonic,
993 ops: &[Op],
994 sc: &Scratch,
995 w: Width,
996) -> Result<Vec<Inst>, ExpandError> {
997 expect_len(mn, ops, 2)?;
998 let mut out = Vec::new();
999 match &ops[1] {
1000 Op::Gpr(_) => {}
1001 Op::Mem(a) => {
1002 out.push(Inst::Load {
1003 w,
1004 sx: Sx::Zero,
1005 dst: sc.gpr,
1006 addr: *a,
1007 acquire: false,
1008 });
1009 }
1010 Op::Imm(v) => {
1011 out.push(Inst::Mov {
1012 w,
1013 dst: Op::Gpr(sc.gpr),
1014 src: Op::Imm(*v),
1015 patch: false,
1016 });
1017 }
1018 Op::Label(l) => {
1019 out.push(Inst::Lea {
1020 w: Width::B64,
1021 dst: sc.gpr,
1022 src: LeaSrc::Label(*l),
1023 });
1024 let addr = Mem::base(sc.gpr, 0);
1025 out.push(Inst::Load {
1026 w,
1027 sx: Sx::Zero,
1028 dst: sc.gpr,
1029 addr,
1030 acquire: false,
1031 });
1032 }
1033 Op::Fpr(_) => return Err(bad(mn, "cannot transfer from an FPR")),
1034 Op::Sym(_) => return Err(bad(mn, "cannot transfer from a symbolic operand")),
1035 Op::Extern(_) => return Err(bad(mn, "cannot transfer from an external symbol")),
1036 }
1037 let src = if out.is_empty() {
1038 ops[1]
1039 } else {
1040 Op::Gpr(sc.gpr)
1041 };
1042 out.push(Inst::Mov {
1043 w,
1044 dst: ops[0],
1045 src,
1046 patch: false,
1047 });
1048 Ok(out)
1049}
1050
1051fn fmove_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1052 expect_len(mn, ops, 2)?;
1053 match (&ops[0], &ops[1]) {
1054 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::FMov {
1055 k,
1056 dst: *d,
1057 src: *s,
1058 }),
1059 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadF {
1060 k,
1061 dst: *d,
1062 addr: *a,
1063 }),
1064 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreF {
1065 k,
1066 src: *s,
1067 addr: *a,
1068 }),
1069 _ => Err(bad(mn, "float move needs FPR/memory operands")),
1070 }
1071}
1072
1073fn vmove_node(mn: Mnemonic, ops: &[Op], store: bool) -> Result<Inst, ExpandError> {
1074 expect_len(mn, ops, 2)?;
1075 if !store {
1076 match (&ops[0], &ops[1]) {
1077 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1078 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1079 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1080 dst: *d,
1081 addr: Mem::base(*s, 0),
1082 }),
1083 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1084 }
1085 } else {
1086 match (&ops[0], &ops[1]) {
1087 (Op::Fpr(d), Op::Fpr(s)) => Ok(Inst::VMov { dst: *d, src: *s }),
1088 (Op::Fpr(d), Op::Mem(a)) => Ok(Inst::LoadV { dst: *d, addr: *a }),
1089 (Op::Fpr(d), Op::Gpr(s)) => Ok(Inst::LoadV {
1090 dst: *d,
1091 addr: Mem::base(*s, 0),
1092 }),
1093 (Op::Mem(a), Op::Fpr(s)) => Ok(Inst::StoreV { src: *s, addr: *a }),
1094 _ => Err(bad(mn, "vector move needs FPR/memory operands")),
1095 }
1096 }
1097}
1098
1099pub fn check_callee_saved(mnemonic: Mnemonic, ops: &[Op], saved: &[u8]) -> Result<(), ExpandError> {
1109 if !matches!(
1110 mnemonic,
1111 Mnemonic::PushCalleeSaves | Mnemonic::PopCalleeSaves
1112 ) {
1113 return Ok(());
1114 }
1115 for op in ops {
1116 match op {
1117 Op::Gpr(g) if saved.contains(&g.0) => {}
1118 _ => return Err(bad(mnemonic, "needs callee-saved GPRs")),
1119 }
1120 }
1121 Ok(())
1122}
1123
1124pub fn aligned_push_bytes(n: usize) -> i32 {
1132 ((8 * n + 15) & !15) as i32
1133}
1134
1135pub fn aligned_push_offset(n: usize, i: usize) -> i32 {
1139 8 * (n - 1 - i) as i32
1140}
1141
1142fn stack_nodes(
1143 mn: Mnemonic,
1144 ops: &[Op],
1145 push: bool,
1146 vector: bool,
1147) -> Result<Vec<Inst>, ExpandError> {
1148 if ops.is_empty() {
1149 return Err(bad(mn, "needs at least one operand"));
1150 }
1151 let mut out = Vec::with_capacity(ops.len());
1152 let ordered: Box<dyn Iterator<Item = &Op>> = if push {
1156 Box::new(ops.iter())
1157 } else {
1158 Box::new(ops.iter().rev())
1159 };
1160 for op in ordered {
1161 if vector {
1162 let x = fpr_op(mn, op)?;
1163 out.push(if push {
1164 Inst::PushV { src: x }
1165 } else {
1166 Inst::PopV { dst: x }
1167 });
1168 } else {
1169 match op {
1170 Op::Gpr(_) | Op::Mem(_) | Op::Imm(_) => {
1171 out.push(if push {
1172 Inst::Push { op: *op }
1173 } else {
1174 Inst::Pop { op: *op }
1175 });
1176 }
1177 _ => return Err(bad(mn, "push/pop needs registers, memory, or immediates")),
1178 }
1179 }
1180 }
1181 Ok(out)
1182}
1183
1184fn mul_node(mn: Mnemonic, w: Width, ops: &[Op]) -> Result<Inst, ExpandError> {
1185 if ops.len() == 3 {
1186 if let (Op::Imm(v), Ok(lhs)) = (&ops[1], gpr_op(mn, &ops[2])) {
1187 return Ok(Inst::MulImm {
1188 w,
1189 dst: gpr_op(mn, &ops[0])?,
1190 lhs,
1191 imm: *v,
1192 });
1193 }
1194 if let (Ok(lhs), Op::Imm(v)) = (gpr_op(mn, &ops[1]), &ops[2]) {
1195 return Ok(Inst::MulImm {
1196 w,
1197 dst: gpr_op(mn, &ops[0])?,
1198 lhs,
1199 imm: *v,
1200 });
1201 }
1202 }
1203 Ok(Inst::Mul {
1204 w,
1205 op: Alu3::from_operands(mn, ops)?,
1206 })
1207}
1208
1209fn mulwide_node(mn: Mnemonic, ops: &[Op], signed: bool) -> Result<Inst, ExpandError> {
1210 expect_len(mn, ops, 3)?;
1211 Ok(Inst::MulWide {
1212 signed,
1213 dst: gpr_op(mn, &ops[0])?,
1214 lhs: gpr_op(mn, &ops[1])?,
1215 rhs: gpr_op(mn, &ops[2])?,
1216 })
1217}
1218
1219fn addshifted_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1220 expect_len(mn, ops, 4)?;
1221 let amount = imm_op(mn, &ops[3])?;
1222 if !(0..=63).contains(&amount) {
1223 return Err(bad(mn, "shift amount out of range"));
1224 }
1225 Ok(Inst::AddShifted {
1226 dst: gpr_op(mn, &ops[0])?,
1227 lhs: gpr_op(mn, &ops[1])?,
1228 rhs: gpr_op(mn, &ops[2])?,
1229 amount: amount as u8,
1230 })
1231}
1232
1233fn adc_node(mn: Mnemonic, ops: &[Op], borrow: bool) -> Result<Inst, ExpandError> {
1234 expect_len(mn, ops, 2)?;
1235 let inst = if borrow {
1236 Inst::Sbb {
1237 dst: ops[0],
1238 src: ops[1],
1239 }
1240 } else {
1241 Inst::Adc {
1242 dst: ops[0],
1243 src: ops[1],
1244 }
1245 };
1246 Ok(inst)
1247}
1248
1249fn fbit_node(mn: Mnemonic, ops: &[Op], k: FKind, or: bool) -> Result<Vec<Inst>, ExpandError> {
1250 if ops.len() != 2 && ops.len() != 3 {
1251 return Err(bad(mn, "fp bitwise takes 2 or 3 operands"));
1252 }
1253 let mut out = Vec::new();
1254 let (src, dst) = if ops.len() == 3 {
1255 let d = fpr_op(mn, &ops[0])?;
1256 if op_eq(&ops[2], &ops[0]) && !op_eq(&ops[1], &ops[0]) {
1257 (ops[1], d)
1259 } else {
1260 if !op_eq(&ops[1], &ops[0]) {
1261 match &ops[1] {
1262 Op::Fpr(s) => out.push(Inst::FMov { k, dst: d, src: *s }),
1263 Op::Mem(a) => out.push(Inst::LoadF {
1264 k,
1265 dst: d,
1266 addr: *a,
1267 }),
1268 _ => return Err(bad(mn, "fp bitwise needs FPR/memory operands")),
1269 }
1270 }
1271 (ops[2], d)
1272 }
1273 } else {
1274 (ops[1], fpr_op(mn, &ops[0])?)
1275 };
1276 out.push(Inst::FBit { or, k, dst, src });
1277 Ok(out)
1278}
1279
1280fn farith_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FAlu) -> Result<Vec<Inst>, ExpandError> {
1281 if ops.len() == 3 {
1282 return Ok(vec![Inst::FArith {
1283 op,
1284 k,
1285 arith: FArith3 {
1286 dst: fpr_op(mn, &ops[0])?,
1287 lhs: ops[1],
1288 rhs: Some(ops[2]),
1289 },
1290 }]);
1291 }
1292 Ok(vec![Inst::FArith {
1293 op,
1294 k,
1295 arith: FArith3::from_operands(mn, ops)?,
1296 }])
1297}
1298
1299fn fsqrt_node(mn: Mnemonic, ops: &[Op], k: FKind) -> Result<Inst, ExpandError> {
1300 expect_len(mn, ops, 2)?;
1301 Ok(Inst::FSqrt {
1302 k,
1303 dst: fpr_op(mn, &ops[0])?,
1304 src: ops[1],
1305 })
1306}
1307
1308fn fma_node(mn: Mnemonic, ops: &[Op], k: FKind, op: FmaOp) -> Result<Inst, ExpandError> {
1309 expect_len(mn, ops, 4)?;
1310 Ok(Inst::Fma {
1311 op,
1312 k,
1313 a: ops[1],
1314 b: ops[2],
1315 c: ops[3],
1316 dst: fpr_op(mn, &ops[0])?,
1317 })
1318}
1319
1320fn bswap_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1322 expect_len(mn, ops, 2)?;
1323 Ok(Inst::Bswap {
1324 w,
1325 dst: ops[0],
1326 src: ops[1],
1327 })
1328}
1329
1330fn fround_node(mn: Mnemonic, ops: &[Op], k: FKind, mode: RoundMode) -> Result<Inst, ExpandError> {
1331 expect_len(mn, ops, 2)?;
1332 Ok(Inst::FRound {
1333 k,
1334 mode,
1335 dst: fpr_op(mn, &ops[0])?,
1336 src: ops[1],
1337 })
1338}
1339
1340fn cvtfi_node(mn: Mnemonic, ops: &[Op], k: FKind, w: Width) -> Result<Inst, ExpandError> {
1341 expect_len(mn, ops, 2)?;
1342 Ok(Inst::CvtFI {
1343 k,
1344 w,
1345 dst: gpr_op(mn, &ops[0])?,
1346 src: ops[1],
1347 })
1348}
1349
1350fn cvtif_node(
1351 mn: Mnemonic,
1352 ops: &[Op],
1353 signed: bool,
1354 from: Width,
1355 k: FKind,
1356) -> Result<Inst, ExpandError> {
1357 expect_len(mn, ops, 2)?;
1358 Ok(Inst::CvtIF {
1359 signed,
1360 from,
1361 k,
1362 dst: fpr_op(mn, &ops[0])?,
1363 src: ops[1],
1364 })
1365}
1366
1367fn cvtff_node(mn: Mnemonic, ops: &[Op], from: FKind, to: FKind) -> Result<Inst, ExpandError> {
1368 expect_len(mn, ops, 2)?;
1369 Ok(Inst::CvtFF {
1370 from,
1371 to,
1372 dst: fpr_op(mn, &ops[0])?,
1373 src: ops[1],
1374 })
1375}
1376
1377fn bitcast_node(mn: Mnemonic, ops: &[Op], dir: BitcastDir) -> Result<Inst, ExpandError> {
1378 expect_len(mn, ops, 2)?;
1379 Ok(Inst::Bitcast {
1380 dir,
1381 dst: ops[0],
1382 src: ops[1],
1383 })
1384}
1385
1386fn fabs_node(mn: Mnemonic, ops: &[Op], k: FKind, negate: bool) -> Result<Inst, ExpandError> {
1387 expect_len(mn, ops, 2)?;
1388 let dst = fpr_op(mn, &ops[0])?;
1389 let inst = if negate {
1390 Inst::FNeg {
1391 k,
1392 dst,
1393 src: ops[1],
1394 }
1395 } else {
1396 Inst::FAbs {
1397 k,
1398 dst,
1399 src: ops[1],
1400 }
1401 };
1402 Ok(inst)
1403}
1404
1405fn umov_node(mn: Mnemonic, ops: &[Op], w: Width) -> Result<Inst, ExpandError> {
1406 expect_len(mn, ops, 3)?;
1407 let lane = imm_op(mn, &ops[2])?;
1408 let lanes: i64 = 128 / w.bytes() as i64;
1409 if lane < 0 || lane >= lanes {
1410 return Err(bad(mn, "lane out of range"));
1411 }
1412 Ok(Inst::VecExtract {
1413 w,
1414 dst: gpr_op(mn, &ops[0])?,
1415 src: fpr_op(mn, &ops[1])?,
1416 lane: lane as u8,
1417 })
1418}
1419
1420fn trunc_q_slow(
1421 mn: Mnemonic,
1422 ops: &[Op],
1423 sc: &Scratch,
1424 k: FKind,
1425) -> Result<Vec<Inst>, ExpandError> {
1426 expect_len(mn, ops, 2)?;
1427 let (dst, src) = (ops[0], ops[1]);
1428 let slow = Lbl::Fresh(0);
1429 let done = Lbl::Fresh(1);
1430 let gs = Op::Gpr(sc.gpr);
1431 let fs = Op::Fpr(sc.fpr);
1432 let (int64_min, neg_min) = match k {
1433 FKind::F32 => (0xdf000000i64, 0x5f000000i64),
1434 FKind::F64 => (0xc3e0000000000000u64 as i64, 0x43e0000000000000u64 as i64),
1435 };
1436 let dst_gpr = gpr_op(mn, &dst)?;
1437 Ok(vec![
1438 Inst::Mov {
1439 w: Width::B64,
1440 dst: gs,
1441 src: Op::Imm(neg_min),
1442 patch: false,
1443 },
1444 Inst::Bitcast {
1445 dir: if k == FKind::F32 {
1446 BitcastDir::I2F
1447 } else {
1448 BitcastDir::Q2D
1449 },
1450 dst: fs,
1451 src: gs,
1452 },
1453 Inst::BranchF {
1454 cc: FCc::Ge,
1455 k,
1456 lhs: src,
1457 rhs: fs,
1458 target: slow,
1459 patch: false,
1460 },
1461 Inst::CvtFI {
1462 k,
1463 w: Width::B64,
1464 dst: dst_gpr,
1465 src,
1466 },
1467 Inst::Jump {
1468 target: Target::Label(done),
1469 patch: false,
1470 },
1471 Inst::Bind(slow),
1472 Inst::Mov {
1473 w: Width::B64,
1474 dst: gs,
1475 src: Op::Imm(int64_min),
1476 patch: false,
1477 },
1478 Inst::Bitcast {
1479 dir: if k == FKind::F32 {
1480 BitcastDir::I2F
1481 } else {
1482 BitcastDir::Q2D
1483 },
1484 dst: fs,
1485 src: gs,
1486 },
1487 Inst::FArith {
1488 op: FAlu::Add,
1489 k,
1490 arith: FArith3 {
1491 dst: sc.fpr,
1492 lhs: src,
1493 rhs: None,
1494 },
1495 },
1496 Inst::CvtFI {
1497 k,
1498 w: Width::B64,
1499 dst: dst_gpr,
1500 src: fs,
1501 },
1502 Inst::Mov {
1503 w: Width::B64,
1504 dst: gs,
1505 src: Op::Imm(0x8000000000000000u64 as i64),
1506 patch: false,
1507 },
1508 Inst::Or {
1509 w: Width::B64,
1510 op: Alu3 {
1511 dst,
1512 lhs: dst,
1513 rhs: gs,
1514 },
1515 },
1516 Inst::Bind(done),
1517 ])
1518}
1519
1520fn quad_to_fp(mn: Mnemonic, ops: &[Op], sc: &Scratch, k: FKind) -> Result<Vec<Inst>, ExpandError> {
1521 expect_len(mn, ops, 3)?;
1522 let (src, dst) = (ops[1], ops[0]);
1523 let slow = Lbl::Fresh(0);
1524 let done = Lbl::Fresh(1);
1525 let s1 = gpr_op(mn, &ops[2])?;
1526 let s2 = sc.gpr;
1527 let dst_fpr = fpr_op(mn, &dst)?;
1528 Ok(vec![
1529 Inst::TestBranch {
1530 w: Width::B64,
1531 value: src,
1532 mask: None,
1533 test: TestKind::Sign,
1534 target: slow,
1535 patch: false,
1536 },
1537 Inst::CvtIF {
1538 signed: true,
1539 from: Width::B64,
1540 k,
1541 dst: dst_fpr,
1542 src,
1543 },
1544 Inst::Jump {
1545 target: Target::Label(done),
1546 patch: false,
1547 },
1548 Inst::Bind(slow),
1549 Inst::Mov {
1550 w: Width::B64,
1551 dst: Op::Gpr(s1),
1552 src,
1553 patch: false,
1554 },
1555 Inst::Mov {
1556 w: Width::B64,
1557 dst: Op::Gpr(s2),
1558 src,
1559 patch: false,
1560 },
1561 Inst::Shift {
1562 w: Width::B64,
1563 op: ShiftOp::Shr,
1564 dst: s1,
1565 cnt: ShiftCnt::Imm(1),
1566 },
1567 Inst::And {
1568 w: Width::B64,
1569 op: Alu3 {
1570 dst: Op::Gpr(s2),
1571 lhs: Op::Gpr(s2),
1572 rhs: Op::Imm(1),
1573 },
1574 },
1575 Inst::Or {
1576 w: Width::B64,
1577 op: Alu3 {
1578 dst: Op::Gpr(s2),
1579 lhs: Op::Gpr(s2),
1580 rhs: Op::Gpr(s1),
1581 },
1582 },
1583 Inst::CvtIF {
1584 signed: true,
1585 from: Width::B64,
1586 k,
1587 dst: dst_fpr,
1588 src: Op::Gpr(s2),
1589 },
1590 Inst::FArith {
1591 op: FAlu::Add,
1592 k,
1593 arith: FArith3 {
1594 dst: dst_fpr,
1595 lhs: Op::Fpr(dst_fpr),
1596 rhs: None,
1597 },
1598 },
1599 Inst::Bind(done),
1600 ])
1601}
1602
1603fn btd2i_node(mn: Mnemonic, ops: &[Op]) -> Result<Vec<Inst>, ExpandError> {
1607 expect_len(mn, ops, 3)?;
1608 Ok(vec![
1609 Inst::CvtFI {
1610 k: FKind::F64,
1611 w: Width::B64,
1612 dst: gpr_op(mn, &ops[0])?,
1613 src: ops[1],
1614 },
1615 Inst::Branch {
1616 cc: IntCc::Eq,
1617 w: Width::B32,
1618 lhs: ops[0],
1619 rhs: Op::Imm(-0x8000_0000i64),
1620 target: label_op(mn, &ops[2])?,
1621 patch: false,
1622 },
1623 ])
1624}
1625
1626fn bcd2i_node(mn: Mnemonic, ops: &[Op], sc: &Scratch) -> Result<Vec<Inst>, ExpandError> {
1627 expect_len(mn, ops, 3)?;
1628 let target = label_op(mn, &ops[2])?;
1629 Ok(vec![
1630 Inst::CvtFI {
1631 k: FKind::F64,
1632 w: Width::B64,
1633 dst: gpr_op(mn, &ops[0])?,
1634 src: ops[1],
1635 },
1636 Inst::TestBranch {
1637 w: Width::B64,
1638 value: ops[0],
1639 mask: None,
1640 test: TestKind::Zero,
1641 target,
1642 patch: false,
1643 },
1644 Inst::CvtIF {
1645 signed: true,
1646 from: Width::B64,
1647 k: FKind::F64,
1648 dst: sc.fpr,
1649 src: ops[0],
1650 },
1651 Inst::BranchF {
1652 cc: FCc::NeUn,
1653 k: FKind::F64,
1654 lhs: ops[1],
1655 rhs: Op::Fpr(sc.fpr),
1656 target,
1657 patch: false,
1658 },
1659 ])
1660}
1661
1662fn movdz_node(mn: Mnemonic, ops: &[Op]) -> Result<Inst, ExpandError> {
1664 expect_len(mn, ops, 1)?;
1665 let dst = fpr_op(mn, &ops[0])?;
1666 Ok(Inst::FArith {
1667 op: FAlu::Sub,
1668 k: FKind::F64,
1669 arith: FArith3 {
1670 dst,
1671 lhs: Op::Fpr(dst),
1672 rhs: None,
1673 },
1674 })
1675}