Skip to main content

offlineasm_ir/
inst.rs

1use super::*;
2
3#[derive(Clone, PartialEq, Eq, Debug)]
4pub struct Alu3 {
5    pub dst: Op,
6    pub lhs: Op,
7    pub rhs: Op,
8}
9
10impl Alu3 {
11    /// Builds an `Alu3` from a 2- or 3-operand invocation.
12    ///
13    /// `dst` doubles as `lhs` in the 2-operand form.
14    ///
15    /// # Errors
16    ///
17    /// [`ExpandError::BadOperand`] if `ops` has neither 2 nor 3 elements.
18    pub fn from_operands(mn: Mnemonic, ops: &[Op]) -> Result<Self, ExpandError> {
19        match ops {
20            [dst, rhs] => Ok(Alu3 {
21                dst: *dst,
22                lhs: *dst,
23                rhs: *rhs,
24            }),
25            [dst, lhs, rhs] => Ok(Alu3 {
26                dst: *dst,
27                lhs: *lhs,
28                rhs: *rhs,
29            }),
30            _ => Err(bad(mn, "takes 2 or 3 operands")),
31        }
32    }
33}
34
35#[derive(Clone, PartialEq, Eq, Debug)]
36pub enum Inst {
37    Mov {
38        w: Width,
39        dst: Op,
40        src: Op,
41        patch: bool,
42    },
43    Bind(Lbl),
44    MovZero32 {
45        dst: Op,
46        src: Op,
47    },
48    Bswap {
49        w: Width,
50        dst: Op,
51        src: Op,
52    },
53    Load {
54        w: Width,
55        sx: Sx,
56        dst: Gpr,
57        addr: Mem,
58        acquire: bool,
59    },
60    Store {
61        w: Width,
62        src: Op,
63        addr: Mem,
64    },
65    LoadF {
66        k: FKind,
67        dst: Fpr,
68        addr: Mem,
69    },
70    StoreF {
71        k: FKind,
72        src: Fpr,
73        addr: Mem,
74    },
75    FMov {
76        k: FKind,
77        dst: Fpr,
78        src: Fpr,
79    },
80    LoadV {
81        dst: Fpr,
82        addr: Mem,
83    },
84    StoreV {
85        src: Fpr,
86        addr: Mem,
87    },
88    VMov {
89        dst: Fpr,
90        src: Fpr,
91    },
92    Lea {
93        w: Width,
94        dst: Gpr,
95        src: LeaSrc,
96    },
97    Push {
98        op: Op,
99    },
100    Pop {
101        op: Op,
102    },
103    PushV {
104        src: Fpr,
105    },
106    PopV {
107        dst: Fpr,
108    },
109    Peek {
110        disp: i32,
111        dst: Gpr,
112    },
113    Poke {
114        disp: i32,
115        src: Gpr,
116    },
117
118    Add {
119        w: Width,
120        op: Alu3,
121    },
122    Sub {
123        w: Width,
124        op: Alu3,
125    },
126    Mul {
127        w: Width,
128        op: Alu3,
129    },
130    And {
131        w: Width,
132        op: Alu3,
133    },
134    Or {
135        w: Width,
136        op: Alu3,
137    },
138    Xor {
139        w: Width,
140        op: Alu3,
141    },
142    Not {
143        w: Width,
144        dst: Op,
145    },
146    Neg {
147        w: Width,
148        dst: Op,
149    },
150    Shift {
151        w: Width,
152        op: ShiftOp,
153        dst: Gpr,
154        cnt: ShiftCnt,
155    },
156    MulImm {
157        w: Width,
158        dst: Gpr,
159        lhs: Gpr,
160        imm: i64,
161    },
162    AluS {
163        op: AluOp,
164        w: Width,
165        alu: Alu3,
166    },
167    Adc {
168        dst: Op,
169        src: Op,
170    },
171    Sbb {
172        dst: Op,
173        src: Op,
174    },
175    MulWide {
176        signed: bool,
177        dst: Gpr,
178        lhs: Gpr,
179        rhs: Gpr,
180    },
181    MulHigh {
182        signed: bool,
183        rhs: Op,
184    },
185    DivMod1 {
186        signed: bool,
187        w: Width,
188        divisor: Op,
189    },
190    Div3 {
191        signed: bool,
192        w: Width,
193        dst: Gpr,
194        dvd: Op,
195        dvs: Op,
196    },
197    Rem3 {
198        signed: bool,
199        w: Width,
200        dst: Gpr,
201        dvd: Op,
202        dvs: Op,
203    },
204    Extend {
205        from: Width,
206        to: Width,
207        sx: Sx,
208        dst: Gpr,
209        src: Op,
210    },
211    Ctz {
212        w: Width,
213        dst: Gpr,
214        src: Op,
215    },
216    Clz {
217        w: Width,
218        dst: Gpr,
219        src: Op,
220    },
221    Popcnt {
222        w: Width,
223        dst: Gpr,
224        src: Op,
225    },
226    Cqo {
227        quad: bool,
228    },
229    AddShifted {
230        dst: Gpr,
231        lhs: Gpr,
232        rhs: Gpr,
233        amount: u8,
234    },
235    Cmov {
236        w: Width,
237        zero: bool,
238        dst: Gpr,
239        src: Op,
240        test: Gpr,
241    },
242    VecExtract {
243        w: Width,
244        dst: Gpr,
245        src: Fpr,
246        lane: u8,
247    },
248
249    Branch {
250        cc: IntCc,
251        w: Width,
252        lhs: Op,
253        rhs: Op,
254        target: Lbl,
255        patch: bool,
256    },
257
258    TestBranch {
259        w: Width,
260        value: Op,
261        mask: Option<Op>,
262        test: TestKind,
263        target: Lbl,
264        patch: bool,
265    },
266
267    OpBranch {
268        op: FlagAlu,
269        w: Width,
270        flag: PredFlag,
271        args: Vec<Op>,
272        target: Lbl,
273        patch: bool,
274    },
275
276    FlagBranch {
277        flag: PredFlag,
278        target: Lbl,
279        patch: bool,
280    },
281    Jump {
282        target: Target,
283        patch: bool,
284    },
285    Call {
286        target: Target,
287        patch: bool,
288    },
289    Ret,
290    /// Traps. `break` lowers to `int3`, `brk`, or `ebreak`.
291    Break,
292    Nop,
293
294    /// Compares integers and writes 0 or 1 to `dst`.
295    IntSet {
296        cc: IntCc,
297        w: Width,
298        dst: Op,
299        lhs: Op,
300        rhs: Op,
301    },
302    /// Tests a value and writes 0 or 1 to `dst`.
303    TestSet {
304        w: Width,
305        value: Op,
306        mask: Option<Op>,
307        test: TestKind,
308        dst: Op,
309    },
310
311    /// Runs scalar float arithmetic such as `addf` through `divd`.
312    FArith {
313        op: FAlu,
314        k: FKind,
315        arith: FArith3,
316    },
317    /// Runs a fused multiply-add. `dst` is `op(a, b, c)`.
318    Fma {
319        op: FmaOp,
320        k: FKind,
321        a: Op,
322        b: Op,
323        c: Op,
324        dst: Fpr,
325    },
326    FSqrt {
327        k: FKind,
328        dst: Fpr,
329        src: Op,
330    },
331    FRound {
332        k: FKind,
333        mode: RoundMode,
334        dst: Fpr,
335        src: Op,
336    },
337    FAbs {
338        k: FKind,
339        dst: Fpr,
340        src: Op,
341    },
342    FNeg {
343        k: FKind,
344        dst: Fpr,
345        src: Op,
346    },
347    /// Runs bitwise logic on float bits.
348    FBit {
349        or: bool,
350        k: FKind,
351        dst: Fpr,
352        src: Op,
353    },
354    /// Truncates a float to an integer when the value is in range.
355    ///
356    /// Out-of-range results are the backend's native saturate value.
357    CvtFI {
358        k: FKind,
359        w: Width,
360        dst: Gpr,
361        src: Op,
362    },
363    /// Converts an integer to a float exactly.
364    CvtIF {
365        signed: bool,
366        from: Width,
367        k: FKind,
368        dst: Fpr,
369        src: Op,
370    },
371    /// Narrows or widens a float.
372    CvtFF {
373        from: FKind,
374        to: FKind,
375        dst: Fpr,
376        src: Op,
377    },
378    /// Reinterprets bits between integer and float.
379    Bitcast {
380        dir: BitcastDir,
381        dst: Op,
382        src: Op,
383    },
384    /// Compares floats and branches on the result.
385    BranchF {
386        cc: FCc,
387        k: FKind,
388        lhs: Op,
389        rhs: Op,
390        target: Lbl,
391        patch: bool,
392    },
393    /// Compares floats and writes 0 or 1.
394    FSet {
395        cc: FCc,
396        k: FKind,
397        dst: Op,
398        lhs: Op,
399        rhs: Op,
400    },
401
402    /// Orders memory. `memfence` is a full barrier and `fence` is inner-shareable.
403    Fence {
404        full: bool,
405    },
406    /// Hints a spin loop. Emits `PAUSE` on x86 and RISC-V, `YIELD` on AArch64.
407    Pause,
408    /// Loads a thread-local slot into a register.
409    ///
410    /// Slots are 8 bytes wide. Sub-word widths zero-extend, matching
411    /// `loadb`/`loadh`/`loadi`.
412    TlsLoad {
413        w: Width,
414        dst: Gpr,
415        idx: i64,
416    },
417    /// Stores a register into a thread-local slot.
418    ///
419    /// Only the low `w` bytes are written; the rest of the slot is
420    /// preserved.
421    TlsStore {
422        w: Width,
423        src: Gpr,
424        idx: i64,
425    },
426    /// Materializes the address of a thread-local slot.
427    TlsLea {
428        dst: Gpr,
429        idx: i64,
430    },
431    /// Runs an atomic read-modify-write.
432    AtomicRmw {
433        op: AtomicOp,
434        w: Width,
435        a: Op,
436        b: Op,
437    },
438    /// Compares and swaps atomically. The expected value lives in `t0`.
439    ///
440    /// `branch` holds the `batomicweakcas*` target when present.
441    AtomicCas {
442        w: Width,
443        mem: Mem,
444        src: Gpr,
445        branch: Option<Lbl>,
446    },
447    /// Loads linked for a later store-conditional.
448    LoadLinked {
449        w: Width,
450        mem: Mem,
451        dst: Gpr,
452    },
453    /// Stores conditionally. Writes 0 or 1 to `status`.
454    StoreCond {
455        w: Width,
456        status: Gpr,
457        src: Gpr,
458        mem: Mem,
459    },
460    /// Loads a pair of values from one address.
461    PairLoad {
462        w: PairW,
463        dst1: Op,
464        dst2: Op,
465        mem: Mem,
466    },
467    /// Stores a pair of values to one address.
468    PairStore {
469        w: PairW,
470        src1: Op,
471        src2: Op,
472        mem: Mem,
473    },
474}
475
476/// Selects sign or zero extension.
477#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
478pub enum Sx {
479    Zero,
480    Sign,
481}
482
483/// Holds a shift count as an immediate or a register.
484#[derive(Clone, PartialEq, Eq, Debug)]
485pub enum ShiftCnt {
486    Imm(i64),
487    Reg(Gpr),
488}
489
490/// Holds a `lea` source: memory, a label, or an external symbol.
491#[derive(Clone, PartialEq, Eq, Debug)]
492pub enum LeaSrc {
493    Mem(Mem),
494    Label(Lbl),
495    Extern(Extern),
496}
497
498/// Names the ALU op fused into [`Inst::OpBranch`].
499#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
500pub enum FlagAlu {
501    Add,
502    Sub,
503    Mul,
504    Or,
505}
506
507/// Names an ALU op that sets the pending flag.
508#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
509pub enum AluOp {
510    Add,
511    Sub,
512    Or,
513}
514
515/// Names a scalar floating-point operation.
516#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
517pub enum FAlu {
518    Add,
519    Sub,
520    Mul,
521    Div,
522    Min,
523    Max,
524}
525
526impl FAlu {
527    /// Reports whether `self` commutes.
528    pub const fn is_commutative(self) -> bool {
529        matches!(self, FAlu::Add | FAlu::Mul | FAlu::Min | FAlu::Max)
530    }
531}
532
533/// Names a fused multiply-add operation.
534#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
535pub enum FmaOp {
536    /// `a*b + c`.
537    Add,
538    /// `a*b - c`.
539    Sub,
540    /// `-(a*b) + c`.
541    NegAdd,
542    /// `-(a*b) - c`.
543    NegSub,
544}
545
546#[derive(Clone, PartialEq, Eq, Debug)]
547pub struct FArith3 {
548    pub dst: Fpr,
549    pub lhs: Op,
550    pub rhs: Option<Op>,
551}
552
553impl FArith3 {
554    /// Builds an `FArith3` from a 2-operand (`dst, lhs`) mnemonic
555    /// invocation. `rhs` stays `None`.
556    ///
557    /// # Errors
558    ///
559    /// [`ExpandError::BadOperand`] if `ops` does not have exactly 2
560    /// elements, or if the first is not [`Op::Fpr`].
561    pub fn from_operands(mn: Mnemonic, ops: &[Op]) -> Result<Self, ExpandError> {
562        match ops {
563            [dst, lhs] => Ok(FArith3 {
564                dst: fpr_op(mn, dst)?,
565                lhs: *lhs,
566                rhs: None,
567            }),
568            _ => Err(bad(mn, "takes 2 operands")),
569        }
570    }
571}
572
573/// Names a bitcast direction.
574#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
575pub enum BitcastDir {
576    Q2D,
577    D2Q,
578    I2F,
579    F2I,
580}
581
582/// Names an atomic read-modify-write operation.
583#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
584pub enum AtomicOp {
585    Add,
586    Sub,
587    Xchg,
588    And,
589    Or,
590    Xor,
591}
592
593/// Names a paired-access width.
594#[derive(Clone, Copy, PartialEq, Eq, Hash, Debug)]
595pub enum PairW {
596    B32,
597    B64,
598    F64,
599    V128,
600}
601
602/// Builds an [`ExpandError::BadOperand`] for `mn`.
603///
604/// `msg` describes what was wrong with its operands.
605pub fn bad(mn: Mnemonic, msg: impl Into<String>) -> ExpandError {
606    ExpandError::BadOperand {
607        mnemonic: mn.as_str().to_string(),
608        msg: msg.into(),
609    }
610}
611
612/// Checks that `ops` has exactly `n` elements.
613///
614/// # Errors
615///
616/// [`ExpandError::BadOperand`] if `ops.len() != n`.
617pub fn expect_len(mn: Mnemonic, ops: &[Op], n: usize) -> Result<(), ExpandError> {
618    if ops.len() == n {
619        Ok(())
620    } else {
621        Err(bad(
622            mn,
623            format!("expected {} operands, got {}", n, ops.len()),
624        ))
625    }
626}
627
628/// Returns the immediate in `op`.
629///
630/// # Errors
631///
632/// [`ExpandError::BadOperand`] if `op` is not [`Op::Imm`].
633pub fn imm_op(mn: Mnemonic, op: &Op) -> Result<i64, ExpandError> {
634    match op {
635        Op::Imm(v) => Ok(*v),
636        _ => Err(bad(mn, "expected immediate")),
637    }
638}
639
640/// Returns the GPR in `op`.
641///
642/// # Errors
643///
644/// [`ExpandError::BadOperand`] if `op` is not [`Op::Gpr`].
645pub fn gpr_op(mn: Mnemonic, op: &Op) -> Result<Gpr, ExpandError> {
646    match op {
647        Op::Gpr(g) => Ok(*g),
648        _ => Err(bad(mn, "expected GPR")),
649    }
650}
651
652/// Returns the FPR in `op`.
653///
654/// # Errors
655///
656/// [`ExpandError::BadOperand`] if `op` is not [`Op::Fpr`].
657pub fn fpr_op(mn: Mnemonic, op: &Op) -> Result<Fpr, ExpandError> {
658    match op {
659        Op::Fpr(x) => Ok(*x),
660        _ => Err(bad(mn, "expected FPR")),
661    }
662}
663
664/// Returns the address in `op`.
665///
666/// # Errors
667///
668/// [`ExpandError::BadOperand`] if `op` is not [`Op::Mem`].
669pub fn mem_op(mn: Mnemonic, op: &Op) -> Result<Mem, ExpandError> {
670    match op {
671        Op::Mem(a) => Ok(*a),
672        _ => Err(bad(mn, "expected memory")),
673    }
674}
675
676/// Returns the label in `op`.
677///
678/// # Errors
679///
680/// [`ExpandError::BadOperand`] if `op` is not [`Op::Label`].
681pub fn label_op(mn: Mnemonic, op: &Op) -> Result<Lbl, ExpandError> {
682    match op {
683        Op::Label(l) => Ok(*l),
684        _ => Err(bad(mn, "expected label")),
685    }
686}
687
688/// Compares two operands for equality.
689///
690/// Only immediates and registers compare equal. Memory, label, and symbolic
691/// operands always compare unequal.
692pub fn op_eq(a: &Op, b: &Op) -> bool {
693    match (a, b) {
694        (Op::Imm(x), Op::Imm(y)) => x == y,
695        (Op::Gpr(x), Op::Gpr(y)) => x == y,
696        (Op::Fpr(x), Op::Fpr(y)) => x == y,
697        _ => false,
698    }
699}
700
701impl Inst {
702    /// Visits every label this instruction binds, branches to, or names.
703    ///
704    /// Frontends use this to resolve the opaque handles of one expansion
705    /// (notably the `Fresh` labels `expand` mints) before printing or
706    /// encoding the sequence.
707    pub fn visit_labels(&self, f: &mut dyn FnMut(Lbl)) {
708        fn visit_op(f: &mut dyn FnMut(Lbl), o: &Op) {
709            if let Op::Label(l) = o {
710                f(*l);
711            }
712        }
713        fn visit_target(f: &mut dyn FnMut(Lbl), t: &Target) {
714            if let Target::Label(l) = t {
715                f(*l);
716            }
717        }
718        fn visit_lea(f: &mut dyn FnMut(Lbl), s: &LeaSrc) {
719            if let LeaSrc::Label(l) = s {
720                f(*l);
721            }
722        }
723        fn visit_ops(f: &mut dyn FnMut(Lbl), ops: &[Op]) {
724            for o in ops {
725                visit_op(f, o);
726            }
727        }
728        match self {
729            Inst::Mov { dst, src, .. }
730            | Inst::MovZero32 { dst, src }
731            | Inst::Bswap { dst, src, .. }
732            | Inst::Adc { dst, src }
733            | Inst::Sbb { dst, src }
734            | Inst::Bitcast { dst, src, .. } => {
735                visit_op(f, dst);
736                visit_op(f, src);
737            }
738            Inst::IntSet { dst, lhs, rhs, .. } | Inst::FSet { dst, lhs, rhs, .. } => {
739                visit_op(f, dst);
740                visit_op(f, lhs);
741                visit_op(f, rhs);
742            }
743            Inst::Bind(l) => f(*l),
744            Inst::Lea { src, .. } => visit_lea(f, src),
745            Inst::Store { src, .. } => visit_op(f, src),
746            Inst::Push { op } | Inst::Pop { op } => visit_op(f, op),
747            Inst::Add { op: a, .. }
748            | Inst::Sub { op: a, .. }
749            | Inst::Mul { op: a, .. }
750            | Inst::And { op: a, .. }
751            | Inst::Or { op: a, .. }
752            | Inst::Xor { op: a, .. } => {
753                visit_op(f, &a.dst);
754                visit_op(f, &a.lhs);
755                visit_op(f, &a.rhs);
756            }
757            Inst::AluS { alu, .. } => {
758                visit_op(f, &alu.dst);
759                visit_op(f, &alu.lhs);
760                visit_op(f, &alu.rhs);
761            }
762            Inst::Not { dst, .. } | Inst::Neg { dst, .. } => visit_op(f, dst),
763            Inst::MulHigh { rhs, .. } => visit_op(f, rhs),
764            Inst::DivMod1 { divisor, .. } => visit_op(f, divisor),
765            Inst::Div3 { dvd, dvs, .. } | Inst::Rem3 { dvd, dvs, .. } => {
766                visit_op(f, dvd);
767                visit_op(f, dvs);
768            }
769            Inst::Branch {
770                lhs, rhs, target, ..
771            } => {
772                visit_op(f, lhs);
773                visit_op(f, rhs);
774                f(*target);
775            }
776            Inst::TestBranch {
777                value,
778                mask,
779                target,
780                ..
781            } => {
782                visit_op(f, value);
783                if let Some(m) = mask {
784                    visit_op(f, m);
785                }
786                f(*target);
787            }
788            Inst::OpBranch { args, target, .. } => {
789                visit_ops(f, args);
790                f(*target);
791            }
792            Inst::FlagBranch { target, .. } => f(*target),
793            Inst::Jump { target, .. } | Inst::Call { target, .. } => visit_target(f, target),
794            Inst::TestSet {
795                value, mask, dst, ..
796            } => {
797                visit_op(f, value);
798                if let Some(m) = mask {
799                    visit_op(f, m);
800                }
801                visit_op(f, dst);
802            }
803            Inst::FArith { arith, .. } => {
804                visit_op(f, &arith.lhs);
805                if let Some(r) = &arith.rhs {
806                    visit_op(f, r);
807                }
808            }
809            Inst::Fma { a, b, c, .. } => {
810                visit_op(f, a);
811                visit_op(f, b);
812                visit_op(f, c);
813            }
814            Inst::FSqrt { src, .. }
815            | Inst::FRound { src, .. }
816            | Inst::FAbs { src, .. }
817            | Inst::FNeg { src, .. }
818            | Inst::FBit { src, .. }
819            | Inst::CvtFI { src, .. }
820            | Inst::CvtIF { src, .. }
821            | Inst::CvtFF { src, .. } => visit_op(f, src),
822            Inst::BranchF {
823                lhs, rhs, target, ..
824            } => {
825                visit_op(f, lhs);
826                visit_op(f, rhs);
827                f(*target);
828            }
829            Inst::AtomicRmw { a, b, .. } => {
830                visit_op(f, a);
831                visit_op(f, b);
832            }
833            Inst::Load { .. }
834            | Inst::LoadF { .. }
835            | Inst::StoreF { .. }
836            | Inst::FMov { .. }
837            | Inst::LoadV { .. }
838            | Inst::StoreV { .. }
839            | Inst::VMov { .. }
840            | Inst::PushV { .. }
841            | Inst::PopV { .. }
842            | Inst::Peek { .. }
843            | Inst::Poke { .. }
844            | Inst::Shift { .. }
845            | Inst::MulImm { .. }
846            | Inst::Extend { .. }
847            | Inst::Ctz { .. }
848            | Inst::Clz { .. }
849            | Inst::Popcnt { .. }
850            | Inst::Cqo { .. }
851            | Inst::MulWide { .. }
852            | Inst::AddShifted { .. }
853            | Inst::Cmov { .. }
854            | Inst::VecExtract { .. }
855            | Inst::Fence { .. }
856            | Inst::Pause
857            | Inst::TlsLoad { .. }
858            | Inst::TlsStore { .. }
859            | Inst::TlsLea { .. }
860            | Inst::AtomicCas { .. }
861            | Inst::LoadLinked { .. }
862            | Inst::StoreCond { .. }
863            | Inst::PairLoad { .. }
864            | Inst::PairStore { .. }
865            | Inst::Ret
866            | Inst::Break
867            | Inst::Nop => {}
868        }
869    }
870}