cranelift_codegen/isa/x64/inst/
args.rs

1//! Instruction operand sub-components (aka "parts"): definitions and printing.
2
3use super::regs::{self};
4use crate::ir::MemFlags;
5use crate::ir::condcodes::{FloatCC, IntCC};
6use crate::ir::types::*;
7use crate::isa::x64::inst::Inst;
8use crate::isa::x64::inst::regs::pretty_print_reg;
9use crate::machinst::*;
10use smallvec::{SmallVec, smallvec};
11use std::fmt;
12use std::string::String;
13
14/// An extension trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
15pub trait ToWritableReg {
16    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
17    fn to_writable_reg(&self) -> Writable<Reg>;
18}
19
20/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.
21pub trait FromWritableReg: Sized {
22    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.
23    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
24}
25
26/// A macro for defining a newtype of `Reg` that enforces some invariant about
27/// the wrapped `Reg` (such as that it is of a particular register class).
28macro_rules! newtype_of_reg {
29    (
30        $newtype_reg:ident,
31        $newtype_writable_reg:ident,
32        $newtype_option_writable_reg:ident,
33        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
34        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
35        |$check_reg:ident| $check:expr
36    ) => {
37        /// A newtype wrapper around `Reg`.
38        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
39        pub struct $newtype_reg(Reg);
40
41        impl PartialEq<Reg> for $newtype_reg {
42            fn eq(&self, other: &Reg) -> bool {
43                self.0 == *other
44            }
45        }
46
47        impl From<$newtype_reg> for Reg {
48            fn from(r: $newtype_reg) -> Self {
49                r.0
50            }
51        }
52
53        impl $newtype_reg {
54            /// Create this newtype from the given register, or return `None` if the register
55            /// is not a valid instance of this newtype.
56            pub fn new($check_reg: Reg) -> Option<Self> {
57                if $check {
58                    Some(Self($check_reg))
59                } else {
60                    None
61                }
62            }
63
64            /// Like `Self::new(r).unwrap()` but with a better panic message on
65            /// failure.
66            pub fn unwrap_new($check_reg: Reg) -> Self {
67                if $check {
68                    Self($check_reg)
69                } else {
70                    panic!(
71                        "cannot construct {} from register {:?} with register class {:?}",
72                        stringify!($newtype_reg),
73                        $check_reg,
74                        $check_reg.class(),
75                    )
76                }
77            }
78
79            /// Get this newtype's underlying `Reg`.
80            pub fn to_reg(self) -> Reg {
81                self.0
82            }
83        }
84
85        // Convenience impl so that people working with this newtype can use it
86        // "just like" a plain `Reg`.
87        //
88        // NB: We cannot implement `DerefMut` because that would let people do
89        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the
90        // invariants that `Gpr` provides.
91        impl std::ops::Deref for $newtype_reg {
92            type Target = Reg;
93
94            fn deref(&self) -> &Reg {
95                &self.0
96            }
97        }
98
99        /// If you know what you're doing, you can explicitly mutably borrow the
100        /// underlying `Reg`. Don't make it point to the wrong type of register
101        /// please.
102        impl AsMut<Reg> for $newtype_reg {
103            fn as_mut(&mut self) -> &mut Reg {
104                &mut self.0
105            }
106        }
107
108        /// Writable Gpr.
109        pub type $newtype_writable_reg = Writable<$newtype_reg>;
110
111        #[allow(dead_code)] // Used by some newtypes and not others.
112        /// Optional writable Gpr.
113        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
114
115        impl ToWritableReg for $newtype_writable_reg {
116            fn to_writable_reg(&self) -> Writable<Reg> {
117                Writable::from_reg(self.to_reg().to_reg())
118            }
119        }
120
121        impl FromWritableReg for $newtype_writable_reg {
122            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {
123                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))
124            }
125        }
126
127        $(
128            /// A newtype wrapper around `RegMem` for general-purpose registers.
129            #[derive(Clone, Debug)]
130            pub struct $newtype_reg_mem(RegMem);
131
132            impl From<$newtype_reg_mem> for RegMem {
133                fn from(rm: $newtype_reg_mem) -> Self {
134                    rm.0
135                }
136            }
137            impl<'a> From<&'a $newtype_reg_mem> for &'a RegMem {
138                fn from(rm: &'a $newtype_reg_mem) -> &'a RegMem {
139                    &rm.0
140                }
141            }
142
143            impl From<$newtype_reg> for $newtype_reg_mem {
144                fn from(r: $newtype_reg) -> Self {
145                    $newtype_reg_mem(RegMem::reg(r.into()))
146                }
147            }
148
149            impl $newtype_reg_mem {
150                /// Construct a `RegMem` newtype from the given `RegMem`, or return
151                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
152                /// newtype.
153                pub fn new(rm: RegMem) -> Option<Self> {
154                    match rm {
155                        RegMem::Mem { addr } => {
156                            let mut _allow = true;
157                            $(
158                                if $aligned {
159                                    _allow = addr.aligned();
160                                }
161                            )?
162                            if _allow {
163                                Some(Self(RegMem::Mem { addr }))
164                            } else {
165                                None
166                            }
167                        }
168                        RegMem::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
169                    }
170                }
171
172                /// Like `Self::new(rm).unwrap()` but with better panic messages
173                /// in case of failure.
174                pub fn unwrap_new(rm: RegMem) -> Self {
175                    match rm {
176                        RegMem::Mem { addr } => {
177                            $(
178                                if $aligned && !addr.aligned() {
179                                    panic!(
180                                        "cannot create {} from an unaligned memory address: {addr:?}",
181                                        stringify!($newtype_reg_mem),
182                                    );
183                                }
184                            )?
185                            Self(RegMem::Mem { addr })
186                        }
187                        RegMem::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
188                    }
189                }
190
191                /// Convert this newtype into its underlying `RegMem`.
192                pub fn to_reg_mem(self) -> RegMem {
193                    self.0
194                }
195
196                #[allow(dead_code)] // Used by some newtypes and not others.
197                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
198                    self.0.get_operands(collector);
199                }
200            }
201            impl PrettyPrint for $newtype_reg_mem {
202                fn pretty_print(&self, size: u8) -> String {
203                    self.0.pretty_print(size)
204                }
205            }
206        )*
207
208        $(
209            /// A newtype wrapper around `RegMemImm`.
210            #[derive(Clone, Debug)]
211            pub struct $newtype_reg_mem_imm(RegMemImm);
212
213            impl From<$newtype_reg_mem_imm> for RegMemImm {
214                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
215                    rmi.0
216                }
217            }
218            impl<'a> From<&'a $newtype_reg_mem_imm> for &'a RegMemImm {
219                fn from(rmi: &'a $newtype_reg_mem_imm) -> &'a RegMemImm {
220                    &rmi.0
221                }
222            }
223
224            impl From<$newtype_reg> for $newtype_reg_mem_imm {
225                fn from(r: $newtype_reg) -> Self {
226                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
227                }
228            }
229
230            impl $newtype_reg_mem_imm {
231                /// Construct this newtype from the given `RegMemImm`, or return
232                /// `None` if the `RegMemImm` is not a valid instance of this
233                /// newtype.
234                pub fn new(rmi: RegMemImm) -> Option<Self> {
235                    match rmi {
236                        RegMemImm::Imm { .. } => Some(Self(rmi)),
237                        RegMemImm::Mem { addr } => {
238                            let mut _allow = true;
239                            $(
240                                if $aligned_imm {
241                                    _allow = addr.aligned();
242                                }
243                            )?
244                            if _allow {
245                                Some(Self(RegMemImm::Mem { addr }))
246                            } else {
247                                None
248                            }
249                        }
250                        RegMemImm::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
251                    }
252                }
253
254                /// Like `Self::new(rmi).unwrap()` but with better panic
255                /// messages in case of failure.
256                pub fn unwrap_new(rmi: RegMemImm) -> Self {
257                    match rmi {
258                        RegMemImm::Imm { .. } => Self(rmi),
259                        RegMemImm::Mem { addr } => {
260                            $(
261                                if $aligned_imm && !addr.aligned() {
262                                    panic!(
263                                        "cannot construct {} from unaligned memory address: {:?}",
264                                        stringify!($newtype_reg_mem_imm),
265                                        addr,
266                                    );
267                                }
268                            )?
269                            Self(RegMemImm::Mem { addr })
270
271                        }
272                        RegMemImm::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
273                    }
274                }
275
276                /// Convert this newtype into its underlying `RegMemImm`.
277                #[allow(dead_code)] // Used by some newtypes and not others.
278                pub fn to_reg_mem_imm(self) -> RegMemImm {
279                    self.0
280                }
281
282                #[allow(dead_code)] // Used by some newtypes and not others.
283                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
284                    self.0.get_operands(collector);
285                }
286            }
287
288            impl PrettyPrint for $newtype_reg_mem_imm {
289                fn pretty_print(&self, size: u8) -> String {
290                    self.0.pretty_print(size)
291                }
292            }
293        )*
294    };
295}
296
297// Define a newtype of `Reg` for general-purpose registers.
298newtype_of_reg!(
299    Gpr,
300    WritableGpr,
301    OptionWritableGpr,
302    reg_mem: (GprMem),
303    reg_mem_imm: (GprMemImm),
304    |reg| reg.class() == RegClass::Int
305);
306
307// Define a newtype of `Reg` for XMM registers.
308newtype_of_reg!(
309    Xmm,
310    WritableXmm,
311    OptionWritableXmm,
312    reg_mem: (XmmMem, XmmMemAligned aligned:true),
313    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
314    |reg| reg.class() == RegClass::Float
315);
316
317// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
318// constructors here.
319
320// Re-export the type from the ISLE generated code.
321pub use crate::isa::x64::lower::isle::generated_code::Amode;
322
323impl Amode {
324    /// Create an immediate sign-extended and register addressing mode.
325    pub fn imm_reg(simm32: i32, base: Reg) -> Self {
326        debug_assert!(base.class() == RegClass::Int);
327        Self::ImmReg {
328            simm32,
329            base,
330            flags: MemFlags::trusted(),
331        }
332    }
333
334    /// Create a sign-extended-32-to-64 with register and shift addressing mode.
335    pub fn imm_reg_reg_shift(simm32: i32, base: Gpr, index: Gpr, shift: u8) -> Self {
336        debug_assert!(base.class() == RegClass::Int);
337        debug_assert!(index.class() == RegClass::Int);
338        debug_assert!(shift <= 3);
339        Self::ImmRegRegShift {
340            simm32,
341            base,
342            index,
343            shift,
344            flags: MemFlags::trusted(),
345        }
346    }
347
348    pub(crate) fn rip_relative(target: MachLabel) -> Self {
349        Self::RipRelative { target }
350    }
351
352    /// Set the specified [MemFlags] to the [Amode].
353    pub fn with_flags(&self, flags: MemFlags) -> Self {
354        match self {
355            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
356                simm32,
357                base,
358                flags,
359            },
360            &Self::ImmRegRegShift {
361                simm32,
362                base,
363                index,
364                shift,
365                ..
366            } => Self::ImmRegRegShift {
367                simm32,
368                base,
369                index,
370                shift,
371                flags,
372            },
373            _ => panic!("Amode {self:?} cannot take memflags"),
374        }
375    }
376
377    /// Add the registers mentioned by `self` to `collector`.
378    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
379        match self {
380            Amode::ImmReg { base, .. } => {
381                if *base != regs::rbp() && *base != regs::rsp() {
382                    collector.reg_use(base);
383                }
384            }
385            Amode::ImmRegRegShift { base, index, .. } => {
386                debug_assert_ne!(base.to_reg(), regs::rbp());
387                debug_assert_ne!(base.to_reg(), regs::rsp());
388                collector.reg_use(base);
389                debug_assert_ne!(index.to_reg(), regs::rbp());
390                debug_assert_ne!(index.to_reg(), regs::rsp());
391                collector.reg_use(index);
392            }
393            Amode::RipRelative { .. } => {
394                // RIP isn't involved in regalloc.
395            }
396        }
397    }
398
399    /// Same as `get_operands`, but add the registers in the "late" phase.
400    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
401        match self {
402            Amode::ImmReg { base, .. } => {
403                collector.reg_late_use(base);
404            }
405            Amode::ImmRegRegShift { base, index, .. } => {
406                collector.reg_late_use(base);
407                collector.reg_late_use(index);
408            }
409            Amode::RipRelative { .. } => {
410                // RIP isn't involved in regalloc.
411            }
412        }
413    }
414
415    pub(crate) fn get_flags(&self) -> MemFlags {
416        match self {
417            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,
418            Amode::RipRelative { .. } => MemFlags::trusted(),
419        }
420    }
421
422    /// Offset the amode by a fixed offset.
423    pub(crate) fn offset(&self, offset: i32) -> Self {
424        let mut ret = self.clone();
425        match &mut ret {
426            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
427            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
428            _ => panic!("Cannot offset amode: {self:?}"),
429        }
430        ret
431    }
432
433    pub(crate) fn aligned(&self) -> bool {
434        self.get_flags().aligned()
435    }
436}
437
438impl PrettyPrint for Amode {
439    fn pretty_print(&self, _size: u8) -> String {
440        match self {
441            Amode::ImmReg { simm32, base, .. } => {
442                // Note: size is always 8; the address is 64 bits,
443                // even if the addressed operand is smaller.
444                format!("{}({})", *simm32, pretty_print_reg(*base, 8))
445            }
446            Amode::ImmRegRegShift {
447                simm32,
448                base,
449                index,
450                shift,
451                ..
452            } => format!(
453                "{}({},{},{})",
454                *simm32,
455                pretty_print_reg(base.to_reg(), 8),
456                pretty_print_reg(index.to_reg(), 8),
457                1 << shift
458            ),
459            Amode::RipRelative { target } => format!("label{}(%rip)", target.as_u32()),
460        }
461    }
462}
463
464/// A Memory Address. These denote a 64-bit value only.
465/// Used for usual addressing modes as well as addressing modes used during compilation, when the
466/// moving SP offset is not known.
467#[derive(Clone, Debug)]
468pub enum SyntheticAmode {
469    /// A real amode.
470    Real(Amode),
471
472    /// A (virtual) offset into the incoming argument area.
473    IncomingArg {
474        /// The downward offset from the start of the incoming argument area.
475        offset: u32,
476    },
477
478    /// A (virtual) offset to the slot area of the function frame, which lies just above the
479    /// outgoing arguments.
480    SlotOffset {
481        /// The offset into the slot area.
482        simm32: i32,
483    },
484
485    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.
486    ConstantOffset(VCodeConstant),
487}
488
489impl SyntheticAmode {
490    /// Create a real addressing mode.
491    pub fn real(amode: Amode) -> Self {
492        Self::Real(amode)
493    }
494
495    pub(crate) fn slot_offset(simm32: i32) -> Self {
496        SyntheticAmode::SlotOffset { simm32 }
497    }
498
499    /// Add the registers mentioned by `self` to `collector`.
500    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
501        match self {
502            SyntheticAmode::Real(addr) => addr.get_operands(collector),
503            SyntheticAmode::IncomingArg { .. } => {
504                // Nothing to do; the base is known and isn't involved in regalloc.
505            }
506            SyntheticAmode::SlotOffset { .. } => {
507                // Nothing to do; the base is SP and isn't involved in regalloc.
508            }
509            SyntheticAmode::ConstantOffset(_) => {}
510        }
511    }
512
513    /// Same as `get_operands`, but add the register in the "late" phase.
514    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
515        match self {
516            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
517            SyntheticAmode::IncomingArg { .. } => {
518                // Nothing to do; the base is known and isn't involved in regalloc.
519            }
520            SyntheticAmode::SlotOffset { .. } => {
521                // Nothing to do; the base is SP and isn't involved in regalloc.
522            }
523            SyntheticAmode::ConstantOffset(_) => {}
524        }
525    }
526
527    pub(crate) fn finalize(&self, frame: &FrameLayout, buffer: &mut MachBuffer<Inst>) -> Amode {
528        match self {
529            SyntheticAmode::Real(addr) => addr.clone(),
530            SyntheticAmode::IncomingArg { offset } => {
531                // NOTE: this could be made relative to RSP by adding additional
532                // offsets from the frame_layout.
533                let args_max_fp_offset = frame.tail_args_size + frame.setup_area_size;
534                Amode::imm_reg(
535                    i32::try_from(args_max_fp_offset - offset).unwrap(),
536                    regs::rbp(),
537                )
538            }
539            SyntheticAmode::SlotOffset { simm32 } => {
540                let off = *simm32 as i64 + i64::from(frame.outgoing_args_size);
541                Amode::imm_reg(off.try_into().expect("invalid sp offset"), regs::rsp())
542            }
543            SyntheticAmode::ConstantOffset(c) => {
544                Amode::rip_relative(buffer.get_label_for_constant(*c))
545            }
546        }
547    }
548
549    pub(crate) fn aligned(&self) -> bool {
550        match self {
551            SyntheticAmode::Real(addr) => addr.aligned(),
552            &SyntheticAmode::IncomingArg { .. }
553            | SyntheticAmode::SlotOffset { .. }
554            | SyntheticAmode::ConstantOffset { .. } => true,
555        }
556    }
557}
558
559impl From<Amode> for SyntheticAmode {
560    fn from(amode: Amode) -> SyntheticAmode {
561        SyntheticAmode::Real(amode)
562    }
563}
564
565impl From<VCodeConstant> for SyntheticAmode {
566    fn from(c: VCodeConstant) -> SyntheticAmode {
567        SyntheticAmode::ConstantOffset(c)
568    }
569}
570
571impl PrettyPrint for SyntheticAmode {
572    fn pretty_print(&self, _size: u8) -> String {
573        match self {
574            // See note in `Amode` regarding constant size of `8`.
575            SyntheticAmode::Real(addr) => addr.pretty_print(8),
576            &SyntheticAmode::IncomingArg { offset } => {
577                format!("rbp(stack args max - {offset})")
578            }
579            SyntheticAmode::SlotOffset { simm32 } => {
580                format!("rsp({} + virtual offset)", *simm32)
581            }
582            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),
583        }
584    }
585}
586
587/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can
588/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only
589/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by
590/// `simm32` is its sign-extension out to 64 bits.
591#[derive(Clone, Debug)]
592pub enum RegMemImm {
593    /// A register operand.
594    Reg {
595        /// The underlying register.
596        reg: Reg,
597    },
598    /// A memory operand.
599    Mem {
600        /// The memory address.
601        addr: SyntheticAmode,
602    },
603    /// An immediate operand.
604    Imm {
605        /// The immediate value.
606        simm32: u32,
607    },
608}
609
610impl RegMemImm {
611    /// Create a register operand.
612    pub fn reg(reg: Reg) -> Self {
613        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
614        Self::Reg { reg }
615    }
616
617    /// Create a memory operand.
618    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
619        Self::Mem { addr: addr.into() }
620    }
621
622    /// Create an immediate operand.
623    pub fn imm(simm32: u32) -> Self {
624        Self::Imm { simm32 }
625    }
626
627    /// Asserts that in register mode, the reg class is the one that's expected.
628    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
629        if let Self::Reg { reg } = self {
630            debug_assert_eq!(reg.class(), expected_reg_class);
631        }
632    }
633
634    /// Add the regs mentioned by `self` to `collector`.
635    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
636        match self {
637            Self::Reg { reg } => collector.reg_use(reg),
638            Self::Mem { addr } => addr.get_operands(collector),
639            Self::Imm { .. } => {}
640        }
641    }
642}
643
644impl From<RegMem> for RegMemImm {
645    fn from(rm: RegMem) -> RegMemImm {
646        match rm {
647            RegMem::Reg { reg } => RegMemImm::Reg { reg },
648            RegMem::Mem { addr } => RegMemImm::Mem { addr },
649        }
650    }
651}
652
653impl From<Reg> for RegMemImm {
654    fn from(reg: Reg) -> Self {
655        RegMemImm::Reg { reg }
656    }
657}
658
659impl PrettyPrint for RegMemImm {
660    fn pretty_print(&self, size: u8) -> String {
661        match self {
662            Self::Reg { reg } => pretty_print_reg(*reg, size),
663            Self::Mem { addr } => addr.pretty_print(size),
664            Self::Imm { simm32 } => format!("${}", *simm32 as i32),
665        }
666    }
667}
668
669/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
670/// 32, 64, or 128 bit value.
671#[derive(Clone, Debug)]
672pub enum RegMem {
673    /// A register operand.
674    Reg {
675        /// The underlying register.
676        reg: Reg,
677    },
678    /// A memory operand.
679    Mem {
680        /// The memory address.
681        addr: SyntheticAmode,
682    },
683}
684
685impl RegMem {
686    /// Create a register operand.
687    pub fn reg(reg: Reg) -> Self {
688        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
689        Self::Reg { reg }
690    }
691
692    /// Create a memory operand.
693    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
694        Self::Mem { addr: addr.into() }
695    }
696    /// Asserts that in register mode, the reg class is the one that's expected.
697    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
698        if let Self::Reg { reg } = self {
699            debug_assert_eq!(reg.class(), expected_reg_class);
700        }
701    }
702    /// Add the regs mentioned by `self` to `collector`.
703    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
704        match self {
705            RegMem::Reg { reg } => collector.reg_use(reg),
706            RegMem::Mem { addr, .. } => addr.get_operands(collector),
707        }
708    }
709}
710
711impl From<Reg> for RegMem {
712    fn from(reg: Reg) -> RegMem {
713        RegMem::Reg { reg }
714    }
715}
716
717impl From<Writable<Reg>> for RegMem {
718    fn from(r: Writable<Reg>) -> Self {
719        RegMem::reg(r.to_reg())
720    }
721}
722
723impl PrettyPrint for RegMem {
724    fn pretty_print(&self, size: u8) -> String {
725        match self {
726            RegMem::Reg { reg } => pretty_print_reg(*reg, size),
727            RegMem::Mem { addr, .. } => addr.pretty_print(size),
728        }
729    }
730}
731
732#[derive(Clone, Copy, PartialEq)]
733/// Comparison operations.
734pub enum CmpOpcode {
735    /// CMP instruction: compute `a - b` and set flags from result.
736    Cmp,
737    /// TEST instruction: compute `a & b` and set flags from result.
738    Test,
739}
740
741#[derive(Debug)]
742pub(crate) enum InstructionSet {
743    SSE,
744    SSE2,
745    CMPXCHG16b,
746    SSSE3,
747    SSE41,
748    SSE42,
749    Popcnt,
750    Lzcnt,
751    BMI1,
752    #[allow(dead_code)] // never constructed (yet).
753    BMI2,
754    FMA,
755    AVX,
756    AVX2,
757    AVX512BITALG,
758    AVX512DQ,
759    AVX512F,
760    AVX512VBMI,
761    AVX512VL,
762}
763
764/// Some SSE operations requiring 2 operands r/m and r.
765#[derive(Clone, Copy, PartialEq)]
766#[allow(dead_code)] // some variants here aren't used just yet
767#[allow(missing_docs)]
768pub enum SseOpcode {
769    Blendvpd,
770    Blendvps,
771    Comiss,
772    Comisd,
773    Cmpps,
774    Cmppd,
775    Cmpss,
776    Cmpsd,
777    Insertps,
778    Movlhps,
779    Pabsb,
780    Pabsw,
781    Pabsd,
782    Packssdw,
783    Packsswb,
784    Packusdw,
785    Packuswb,
786    Palignr,
787    Pavgb,
788    Pavgw,
789    Pblendvb,
790    Pcmpeqb,
791    Pcmpeqw,
792    Pcmpeqd,
793    Pcmpeqq,
794    Pcmpgtb,
795    Pcmpgtw,
796    Pcmpgtd,
797    Pcmpgtq,
798    Pmaddubsw,
799    Pmaddwd,
800    Pshufb,
801    Pshufd,
802    Ptest,
803    Rcpss,
804    Roundps,
805    Roundpd,
806    Roundss,
807    Roundsd,
808    Rsqrtss,
809    Shufps,
810    Ucomiss,
811    Ucomisd,
812    Pshuflw,
813    Pshufhw,
814    Pblendw,
815    Movddup,
816}
817
818impl SseOpcode {
819    /// Which `InstructionSet` is the first supporting this opcode?
820    pub(crate) fn available_from(&self) -> InstructionSet {
821        use InstructionSet::*;
822        match self {
823            SseOpcode::Comiss
824            | SseOpcode::Cmpps
825            | SseOpcode::Cmpss
826            | SseOpcode::Movlhps
827            | SseOpcode::Rcpss
828            | SseOpcode::Rsqrtss
829            | SseOpcode::Shufps
830            | SseOpcode::Ucomiss => SSE,
831
832            SseOpcode::Cmppd
833            | SseOpcode::Cmpsd
834            | SseOpcode::Comisd
835            | SseOpcode::Packssdw
836            | SseOpcode::Packsswb
837            | SseOpcode::Packuswb
838            | SseOpcode::Pavgb
839            | SseOpcode::Pavgw
840            | SseOpcode::Pcmpeqb
841            | SseOpcode::Pcmpeqw
842            | SseOpcode::Pcmpeqd
843            | SseOpcode::Pcmpgtb
844            | SseOpcode::Pcmpgtw
845            | SseOpcode::Pcmpgtd
846            | SseOpcode::Pmaddwd
847            | SseOpcode::Pshufd
848            | SseOpcode::Ucomisd
849            | SseOpcode::Pshuflw
850            | SseOpcode::Pshufhw => SSE2,
851
852            SseOpcode::Pabsb
853            | SseOpcode::Pabsw
854            | SseOpcode::Pabsd
855            | SseOpcode::Palignr
856            | SseOpcode::Pshufb
857            | SseOpcode::Pmaddubsw
858            | SseOpcode::Movddup => SSSE3,
859
860            SseOpcode::Blendvpd
861            | SseOpcode::Blendvps
862            | SseOpcode::Insertps
863            | SseOpcode::Packusdw
864            | SseOpcode::Pblendvb
865            | SseOpcode::Pcmpeqq
866            | SseOpcode::Ptest
867            | SseOpcode::Roundps
868            | SseOpcode::Roundpd
869            | SseOpcode::Roundss
870            | SseOpcode::Roundsd
871            | SseOpcode::Pblendw => SSE41,
872
873            SseOpcode::Pcmpgtq => SSE42,
874        }
875    }
876
877    /// Returns the src operand size for an instruction.
878    pub(crate) fn src_size(&self) -> u8 {
879        match self {
880            _ => 8,
881        }
882    }
883
884    /// Is `src2` with this opcode a scalar, as for lane insertions?
885    pub(crate) fn has_scalar_src2(self) -> bool {
886        false
887    }
888}
889
890impl fmt::Debug for SseOpcode {
891    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
892        let name = match self {
893            SseOpcode::Blendvpd => "blendvpd",
894            SseOpcode::Blendvps => "blendvps",
895            SseOpcode::Cmpps => "cmpps",
896            SseOpcode::Cmppd => "cmppd",
897            SseOpcode::Cmpss => "cmpss",
898            SseOpcode::Cmpsd => "cmpsd",
899            SseOpcode::Comiss => "comiss",
900            SseOpcode::Comisd => "comisd",
901            SseOpcode::Insertps => "insertps",
902            SseOpcode::Movlhps => "movlhps",
903            SseOpcode::Pabsb => "pabsb",
904            SseOpcode::Pabsw => "pabsw",
905            SseOpcode::Pabsd => "pabsd",
906            SseOpcode::Packssdw => "packssdw",
907            SseOpcode::Packsswb => "packsswb",
908            SseOpcode::Packusdw => "packusdw",
909            SseOpcode::Packuswb => "packuswb",
910            SseOpcode::Palignr => "palignr",
911            SseOpcode::Pavgb => "pavgb",
912            SseOpcode::Pavgw => "pavgw",
913            SseOpcode::Pblendvb => "pblendvb",
914            SseOpcode::Pcmpeqb => "pcmpeqb",
915            SseOpcode::Pcmpeqw => "pcmpeqw",
916            SseOpcode::Pcmpeqd => "pcmpeqd",
917            SseOpcode::Pcmpeqq => "pcmpeqq",
918            SseOpcode::Pcmpgtb => "pcmpgtb",
919            SseOpcode::Pcmpgtw => "pcmpgtw",
920            SseOpcode::Pcmpgtd => "pcmpgtd",
921            SseOpcode::Pcmpgtq => "pcmpgtq",
922            SseOpcode::Pmaddubsw => "pmaddubsw",
923            SseOpcode::Pmaddwd => "pmaddwd",
924            SseOpcode::Pshufb => "pshufb",
925            SseOpcode::Pshufd => "pshufd",
926            SseOpcode::Ptest => "ptest",
927            SseOpcode::Rcpss => "rcpss",
928            SseOpcode::Roundps => "roundps",
929            SseOpcode::Roundpd => "roundpd",
930            SseOpcode::Roundss => "roundss",
931            SseOpcode::Roundsd => "roundsd",
932            SseOpcode::Rsqrtss => "rsqrtss",
933            SseOpcode::Shufps => "shufps",
934            SseOpcode::Ucomiss => "ucomiss",
935            SseOpcode::Ucomisd => "ucomisd",
936            SseOpcode::Pshuflw => "pshuflw",
937            SseOpcode::Pshufhw => "pshufhw",
938            SseOpcode::Pblendw => "pblendw",
939            SseOpcode::Movddup => "movddup",
940        };
941        write!(fmt, "{name}")
942    }
943}
944
945impl fmt::Display for SseOpcode {
946    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
947        fmt::Debug::fmt(self, f)
948    }
949}
950
951pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
952
953impl AvxOpcode {
954    /// Which `InstructionSet`s support the opcode?
955    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
956        match self {
957            AvxOpcode::Vfmadd213ss
958            | AvxOpcode::Vfmadd213sd
959            | AvxOpcode::Vfmadd213ps
960            | AvxOpcode::Vfmadd213pd
961            | AvxOpcode::Vfmadd132ss
962            | AvxOpcode::Vfmadd132sd
963            | AvxOpcode::Vfmadd132ps
964            | AvxOpcode::Vfmadd132pd
965            | AvxOpcode::Vfnmadd213ss
966            | AvxOpcode::Vfnmadd213sd
967            | AvxOpcode::Vfnmadd213ps
968            | AvxOpcode::Vfnmadd213pd
969            | AvxOpcode::Vfnmadd132ss
970            | AvxOpcode::Vfnmadd132sd
971            | AvxOpcode::Vfnmadd132ps
972            | AvxOpcode::Vfnmadd132pd
973            | AvxOpcode::Vfmsub213ss
974            | AvxOpcode::Vfmsub213sd
975            | AvxOpcode::Vfmsub213ps
976            | AvxOpcode::Vfmsub213pd
977            | AvxOpcode::Vfmsub132ss
978            | AvxOpcode::Vfmsub132sd
979            | AvxOpcode::Vfmsub132ps
980            | AvxOpcode::Vfmsub132pd
981            | AvxOpcode::Vfnmsub213ss
982            | AvxOpcode::Vfnmsub213sd
983            | AvxOpcode::Vfnmsub213ps
984            | AvxOpcode::Vfnmsub213pd
985            | AvxOpcode::Vfnmsub132ss
986            | AvxOpcode::Vfnmsub132sd
987            | AvxOpcode::Vfnmsub132ps
988            | AvxOpcode::Vfnmsub132pd => smallvec![InstructionSet::FMA],
989            AvxOpcode::Vminps
990            | AvxOpcode::Vminpd
991            | AvxOpcode::Vmaxps
992            | AvxOpcode::Vmaxpd
993            | AvxOpcode::Vandnps
994            | AvxOpcode::Vandnpd
995            | AvxOpcode::Vpandn
996            | AvxOpcode::Vcmpps
997            | AvxOpcode::Vcmppd
998            | AvxOpcode::Vpsrlw
999            | AvxOpcode::Vpsrld
1000            | AvxOpcode::Vpsrlq
1001            | AvxOpcode::Vpaddb
1002            | AvxOpcode::Vpaddw
1003            | AvxOpcode::Vpaddd
1004            | AvxOpcode::Vpaddq
1005            | AvxOpcode::Vpaddsb
1006            | AvxOpcode::Vpaddsw
1007            | AvxOpcode::Vpaddusb
1008            | AvxOpcode::Vpaddusw
1009            | AvxOpcode::Vpsubb
1010            | AvxOpcode::Vpsubw
1011            | AvxOpcode::Vpsubd
1012            | AvxOpcode::Vpsubq
1013            | AvxOpcode::Vpsubsb
1014            | AvxOpcode::Vpsubsw
1015            | AvxOpcode::Vpsubusb
1016            | AvxOpcode::Vpsubusw
1017            | AvxOpcode::Vpavgb
1018            | AvxOpcode::Vpavgw
1019            | AvxOpcode::Vpand
1020            | AvxOpcode::Vandps
1021            | AvxOpcode::Vandpd
1022            | AvxOpcode::Vpor
1023            | AvxOpcode::Vorps
1024            | AvxOpcode::Vorpd
1025            | AvxOpcode::Vpxor
1026            | AvxOpcode::Vxorps
1027            | AvxOpcode::Vxorpd
1028            | AvxOpcode::Vpmullw
1029            | AvxOpcode::Vpmulld
1030            | AvxOpcode::Vpmulhw
1031            | AvxOpcode::Vpmulhd
1032            | AvxOpcode::Vpmulhrsw
1033            | AvxOpcode::Vpmulhuw
1034            | AvxOpcode::Vpmuldq
1035            | AvxOpcode::Vpmuludq
1036            | AvxOpcode::Vpunpckhwd
1037            | AvxOpcode::Vpunpcklwd
1038            | AvxOpcode::Vunpcklps
1039            | AvxOpcode::Vunpckhps
1040            | AvxOpcode::Vaddps
1041            | AvxOpcode::Vaddpd
1042            | AvxOpcode::Vsubps
1043            | AvxOpcode::Vsubpd
1044            | AvxOpcode::Vmulps
1045            | AvxOpcode::Vmulpd
1046            | AvxOpcode::Vdivps
1047            | AvxOpcode::Vdivpd
1048            | AvxOpcode::Vpcmpeqb
1049            | AvxOpcode::Vpcmpeqw
1050            | AvxOpcode::Vpcmpeqd
1051            | AvxOpcode::Vpcmpeqq
1052            | AvxOpcode::Vpcmpgtb
1053            | AvxOpcode::Vpcmpgtw
1054            | AvxOpcode::Vpcmpgtd
1055            | AvxOpcode::Vpcmpgtq
1056            | AvxOpcode::Vblendvps
1057            | AvxOpcode::Vblendvpd
1058            | AvxOpcode::Vpblendvb
1059            | AvxOpcode::Vmovlhps
1060            | AvxOpcode::Vpminsb
1061            | AvxOpcode::Vpminsw
1062            | AvxOpcode::Vpminsd
1063            | AvxOpcode::Vpminub
1064            | AvxOpcode::Vpminuw
1065            | AvxOpcode::Vpminud
1066            | AvxOpcode::Vpmaxsb
1067            | AvxOpcode::Vpmaxsw
1068            | AvxOpcode::Vpmaxsd
1069            | AvxOpcode::Vpmaxub
1070            | AvxOpcode::Vpmaxuw
1071            | AvxOpcode::Vpmaxud
1072            | AvxOpcode::Vpunpcklbw
1073            | AvxOpcode::Vpunpckhbw
1074            | AvxOpcode::Vpacksswb
1075            | AvxOpcode::Vpackssdw
1076            | AvxOpcode::Vpackuswb
1077            | AvxOpcode::Vpackusdw
1078            | AvxOpcode::Vpalignr
1079            | AvxOpcode::Vpinsrb
1080            | AvxOpcode::Vpinsrw
1081            | AvxOpcode::Vpinsrd
1082            | AvxOpcode::Vpinsrq
1083            | AvxOpcode::Vpmaddwd
1084            | AvxOpcode::Vpmaddubsw
1085            | AvxOpcode::Vinsertps
1086            | AvxOpcode::Vpshufb
1087            | AvxOpcode::Vshufps
1088            | AvxOpcode::Vpsllw
1089            | AvxOpcode::Vpslld
1090            | AvxOpcode::Vpsllq
1091            | AvxOpcode::Vpsraw
1092            | AvxOpcode::Vpsrad
1093            | AvxOpcode::Vpmovsxbw
1094            | AvxOpcode::Vpmovzxbw
1095            | AvxOpcode::Vpmovsxwd
1096            | AvxOpcode::Vpmovzxwd
1097            | AvxOpcode::Vpmovsxdq
1098            | AvxOpcode::Vpmovzxdq
1099            | AvxOpcode::Vaddss
1100            | AvxOpcode::Vaddsd
1101            | AvxOpcode::Vmulss
1102            | AvxOpcode::Vmulsd
1103            | AvxOpcode::Vsubss
1104            | AvxOpcode::Vsubsd
1105            | AvxOpcode::Vdivss
1106            | AvxOpcode::Vdivsd
1107            | AvxOpcode::Vpabsb
1108            | AvxOpcode::Vpabsw
1109            | AvxOpcode::Vpabsd
1110            | AvxOpcode::Vminss
1111            | AvxOpcode::Vminsd
1112            | AvxOpcode::Vmaxss
1113            | AvxOpcode::Vmaxsd
1114            | AvxOpcode::Vsqrtps
1115            | AvxOpcode::Vsqrtpd
1116            | AvxOpcode::Vroundpd
1117            | AvxOpcode::Vroundps
1118            | AvxOpcode::Vcvtdq2pd
1119            | AvxOpcode::Vcvtdq2ps
1120            | AvxOpcode::Vcvtpd2ps
1121            | AvxOpcode::Vcvtps2pd
1122            | AvxOpcode::Vcvttpd2dq
1123            | AvxOpcode::Vcvttps2dq
1124            | AvxOpcode::Vphaddw
1125            | AvxOpcode::Vphaddd
1126            | AvxOpcode::Vpunpckldq
1127            | AvxOpcode::Vpunpckhdq
1128            | AvxOpcode::Vpunpcklqdq
1129            | AvxOpcode::Vpunpckhqdq
1130            | AvxOpcode::Vpshuflw
1131            | AvxOpcode::Vpshufhw
1132            | AvxOpcode::Vpshufd
1133            | AvxOpcode::Vmovss
1134            | AvxOpcode::Vmovsd
1135            | AvxOpcode::Vmovups
1136            | AvxOpcode::Vmovupd
1137            | AvxOpcode::Vmovdqu
1138            | AvxOpcode::Vpextrb
1139            | AvxOpcode::Vpextrw
1140            | AvxOpcode::Vpextrd
1141            | AvxOpcode::Vpextrq
1142            | AvxOpcode::Vpblendw
1143            | AvxOpcode::Vmovddup
1144            | AvxOpcode::Vbroadcastss
1145            | AvxOpcode::Vmovd
1146            | AvxOpcode::Vmovq
1147            | AvxOpcode::Vmovmskps
1148            | AvxOpcode::Vmovmskpd
1149            | AvxOpcode::Vpmovmskb
1150            | AvxOpcode::Vcvtsi2ss
1151            | AvxOpcode::Vcvtsi2sd
1152            | AvxOpcode::Vcvtss2sd
1153            | AvxOpcode::Vcvtsd2ss
1154            | AvxOpcode::Vsqrtss
1155            | AvxOpcode::Vsqrtsd
1156            | AvxOpcode::Vroundss
1157            | AvxOpcode::Vroundsd
1158            | AvxOpcode::Vunpcklpd
1159            | AvxOpcode::Vptest
1160            | AvxOpcode::Vucomiss
1161            | AvxOpcode::Vucomisd => {
1162                smallvec![InstructionSet::AVX]
1163            }
1164
1165            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {
1166                smallvec![InstructionSet::AVX2]
1167            }
1168        }
1169    }
1170
1171    /// Is the opcode known to be commutative?
1172    ///
1173    /// Note that this method is not exhaustive, and there may be commutative
1174    /// opcodes that we don't recognize as commutative.
1175    pub(crate) fn is_commutative(&self) -> bool {
1176        match *self {
1177            AvxOpcode::Vpaddb
1178            | AvxOpcode::Vpaddw
1179            | AvxOpcode::Vpaddd
1180            | AvxOpcode::Vpaddq
1181            | AvxOpcode::Vpaddsb
1182            | AvxOpcode::Vpaddsw
1183            | AvxOpcode::Vpaddusb
1184            | AvxOpcode::Vpaddusw
1185            | AvxOpcode::Vpand
1186            | AvxOpcode::Vandps
1187            | AvxOpcode::Vandpd
1188            | AvxOpcode::Vpor
1189            | AvxOpcode::Vorps
1190            | AvxOpcode::Vorpd
1191            | AvxOpcode::Vpxor
1192            | AvxOpcode::Vxorps
1193            | AvxOpcode::Vxorpd
1194            | AvxOpcode::Vpmuldq
1195            | AvxOpcode::Vpmuludq
1196            | AvxOpcode::Vaddps
1197            | AvxOpcode::Vaddpd
1198            | AvxOpcode::Vmulps
1199            | AvxOpcode::Vmulpd
1200            | AvxOpcode::Vpcmpeqb
1201            | AvxOpcode::Vpcmpeqw
1202            | AvxOpcode::Vpcmpeqd
1203            | AvxOpcode::Vpcmpeqq
1204            | AvxOpcode::Vaddss
1205            | AvxOpcode::Vaddsd
1206            | AvxOpcode::Vmulss
1207            | AvxOpcode::Vmulsd => true,
1208            _ => false,
1209        }
1210    }
1211}
1212
1213impl fmt::Display for AvxOpcode {
1214    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1215        format!("{self:?}").to_lowercase().fmt(f)
1216    }
1217}
1218
1219#[derive(Copy, Clone, PartialEq)]
1220#[allow(missing_docs)]
1221pub enum Avx512TupleType {
1222    Full,
1223    FullMem,
1224    Mem128,
1225}
1226
1227pub use crate::isa::x64::lower::isle::generated_code::Avx512Opcode;
1228
1229impl Avx512Opcode {
1230    /// Which `InstructionSet`s support the opcode?
1231    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1232        match self {
1233            Avx512Opcode::Vcvtudq2ps
1234            | Avx512Opcode::Vpabsq
1235            | Avx512Opcode::Vpsraq
1236            | Avx512Opcode::VpsraqImm => {
1237                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]
1238            }
1239            Avx512Opcode::Vpermi2b => {
1240                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]
1241            }
1242            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],
1243            Avx512Opcode::Vpopcntb => {
1244                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]
1245            }
1246        }
1247    }
1248
1249    /// What is the "TupleType" of this opcode, which affects the scaling factor
1250    /// for 8-bit displacements when this instruction uses memory operands.
1251    ///
1252    /// This can be found in the encoding table for each instruction and is
1253    /// interpreted according to Table 2-34 and 2-35 in the Intel instruction
1254    /// manual.
1255    pub fn tuple_type(&self) -> Avx512TupleType {
1256        use Avx512Opcode::*;
1257        use Avx512TupleType::*;
1258
1259        match self {
1260            Vcvtudq2ps | Vpabsq | Vpmullq | VpsraqImm => Full,
1261            Vpermi2b | Vpopcntb => FullMem,
1262            Vpsraq => Mem128,
1263        }
1264    }
1265}
1266
1267impl fmt::Display for Avx512Opcode {
1268    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1269        let s = format!("{self:?}");
1270        f.write_str(&s.to_lowercase())
1271    }
1272}
1273
1274/// This defines the ways a value can be extended: either signed- or zero-extension, or none for
1275/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which
1276/// values can be extended.
1277#[allow(dead_code)]
1278#[derive(Clone, PartialEq)]
1279pub enum ExtKind {
1280    /// No extension.
1281    None,
1282    /// Sign-extend.
1283    SignExtend,
1284    /// Zero-extend.
1285    ZeroExtend,
1286}
1287
1288/// These indicate ways of extending (widening) a value, using the Intel
1289/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64
1290#[derive(Clone, PartialEq)]
1291pub enum ExtMode {
1292    /// Byte -> Longword.
1293    BL,
1294    /// Byte -> Quadword.
1295    BQ,
1296    /// Word -> Longword.
1297    WL,
1298    /// Word -> Quadword.
1299    WQ,
1300    /// Longword -> Quadword.
1301    LQ,
1302}
1303
1304impl ExtMode {
1305    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.
1306    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {
1307        match (from_bits, to_bits) {
1308            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
1309            (1, 64) | (8, 64) => Some(ExtMode::BQ),
1310            (16, 32) => Some(ExtMode::WL),
1311            (16, 64) => Some(ExtMode::WQ),
1312            (32, 64) => Some(ExtMode::LQ),
1313            _ => None,
1314        }
1315    }
1316}
1317
1318impl fmt::Debug for ExtMode {
1319    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1320        let name = match self {
1321            ExtMode::BL => "bl",
1322            ExtMode::BQ => "bq",
1323            ExtMode::WL => "wl",
1324            ExtMode::WQ => "wq",
1325            ExtMode::LQ => "lq",
1326        };
1327        write!(fmt, "{name}")
1328    }
1329}
1330
1331impl fmt::Display for ExtMode {
1332    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1333        fmt::Debug::fmt(self, f)
1334    }
1335}
1336
1337/// These indicate condition code tests.  Not all are represented since not all are useful in
1338/// compiler-generated code.
1339#[derive(Copy, Clone, PartialEq, Eq)]
1340#[repr(u8)]
1341pub enum CC {
1342    ///  overflow
1343    O = 0,
1344    /// no overflow
1345    NO = 1,
1346
1347    /// < unsigned
1348    B = 2,
1349    /// >= unsigned
1350    NB = 3,
1351
1352    /// zero
1353    Z = 4,
1354    /// not-zero
1355    NZ = 5,
1356
1357    /// <= unsigned
1358    BE = 6,
1359    /// > unsigned
1360    NBE = 7,
1361
1362    /// negative
1363    S = 8,
1364    /// not-negative
1365    NS = 9,
1366
1367    /// < signed
1368    L = 12,
1369    /// >= signed
1370    NL = 13,
1371
1372    /// <= signed
1373    LE = 14,
1374    /// > signed
1375    NLE = 15,
1376
1377    /// parity
1378    P = 10,
1379
1380    /// not parity
1381    NP = 11,
1382}
1383
1384impl CC {
1385    pub(crate) fn from_intcc(intcc: IntCC) -> Self {
1386        match intcc {
1387            IntCC::Equal => CC::Z,
1388            IntCC::NotEqual => CC::NZ,
1389            IntCC::SignedGreaterThanOrEqual => CC::NL,
1390            IntCC::SignedGreaterThan => CC::NLE,
1391            IntCC::SignedLessThanOrEqual => CC::LE,
1392            IntCC::SignedLessThan => CC::L,
1393            IntCC::UnsignedGreaterThanOrEqual => CC::NB,
1394            IntCC::UnsignedGreaterThan => CC::NBE,
1395            IntCC::UnsignedLessThanOrEqual => CC::BE,
1396            IntCC::UnsignedLessThan => CC::B,
1397        }
1398    }
1399
1400    pub(crate) fn invert(&self) -> Self {
1401        match self {
1402            CC::O => CC::NO,
1403            CC::NO => CC::O,
1404
1405            CC::B => CC::NB,
1406            CC::NB => CC::B,
1407
1408            CC::Z => CC::NZ,
1409            CC::NZ => CC::Z,
1410
1411            CC::BE => CC::NBE,
1412            CC::NBE => CC::BE,
1413
1414            CC::S => CC::NS,
1415            CC::NS => CC::S,
1416
1417            CC::L => CC::NL,
1418            CC::NL => CC::L,
1419
1420            CC::LE => CC::NLE,
1421            CC::NLE => CC::LE,
1422
1423            CC::P => CC::NP,
1424            CC::NP => CC::P,
1425        }
1426    }
1427
1428    pub(crate) fn get_enc(self) -> u8 {
1429        self as u8
1430    }
1431}
1432
1433impl fmt::Debug for CC {
1434    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1435        let name = match self {
1436            CC::O => "o",
1437            CC::NO => "no",
1438            CC::B => "b",
1439            CC::NB => "nb",
1440            CC::Z => "z",
1441            CC::NZ => "nz",
1442            CC::BE => "be",
1443            CC::NBE => "nbe",
1444            CC::S => "s",
1445            CC::NS => "ns",
1446            CC::L => "l",
1447            CC::NL => "nl",
1448            CC::LE => "le",
1449            CC::NLE => "nle",
1450            CC::P => "p",
1451            CC::NP => "np",
1452        };
1453        write!(fmt, "{name}")
1454    }
1455}
1456
1457impl fmt::Display for CC {
1458    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1459        fmt::Debug::fmt(self, f)
1460    }
1461}
1462
1463/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,
1464/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS
1465/// whereas [FcmpImm] is used as an immediate.
1466#[derive(Clone, Copy)]
1467pub enum FcmpImm {
1468    /// Equal comparison.
1469    Equal = 0x00,
1470    /// Less than comparison.
1471    LessThan = 0x01,
1472    /// Less than or equal comparison.
1473    LessThanOrEqual = 0x02,
1474    /// Unordered.
1475    Unordered = 0x03,
1476    /// Not equal comparison.
1477    NotEqual = 0x04,
1478    /// Unordered of greater than or equal comparison.
1479    UnorderedOrGreaterThanOrEqual = 0x05,
1480    /// Unordered or greater than comparison.
1481    UnorderedOrGreaterThan = 0x06,
1482    /// Ordered.
1483    Ordered = 0x07,
1484}
1485
1486impl FcmpImm {
1487    pub(crate) fn encode(self) -> u8 {
1488        self as u8
1489    }
1490}
1491
1492impl From<FloatCC> for FcmpImm {
1493    fn from(cond: FloatCC) -> Self {
1494        match cond {
1495            FloatCC::Equal => FcmpImm::Equal,
1496            FloatCC::LessThan => FcmpImm::LessThan,
1497            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,
1498            FloatCC::Unordered => FcmpImm::Unordered,
1499            FloatCC::NotEqual => FcmpImm::NotEqual,
1500            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,
1501            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,
1502            FloatCC::Ordered => FcmpImm::Ordered,
1503            _ => panic!("unable to create comparison predicate for {cond}"),
1504        }
1505    }
1506}
1507
1508/// Encode the rounding modes used as part of the Rounding Control field.
1509/// Note, these rounding immediates only consider the rounding control field
1510/// (i.e. the rounding mode) which only take up the first two bits when encoded.
1511/// However the rounding immediate which this field helps make up, also includes
1512/// bits 3 and 4 which define the rounding select and precision mask respectively.
1513/// These two bits are not defined here and are implicitly set to zero when encoded.
1514#[derive(Clone, Copy)]
1515pub enum RoundImm {
1516    /// Round to nearest mode.
1517    RoundNearest = 0x00,
1518    /// Round down mode.
1519    RoundDown = 0x01,
1520    /// Round up mode.
1521    RoundUp = 0x02,
1522    /// Round to zero mode.
1523    RoundZero = 0x03,
1524}
1525
1526impl RoundImm {
1527    pub(crate) fn encode(self) -> u8 {
1528        self as u8
1529    }
1530}
1531
1532/// An operand's size in bits.
1533#[derive(Clone, Copy, PartialEq)]
1534pub enum OperandSize {
1535    /// 8-bit.
1536    Size8,
1537    /// 16-bit.
1538    Size16,
1539    /// 32-bit.
1540    Size32,
1541    /// 64-bit.
1542    Size64,
1543}
1544
1545impl OperandSize {
1546    pub(crate) fn from_bytes(num_bytes: u32) -> Self {
1547        match num_bytes {
1548            1 => OperandSize::Size8,
1549            2 => OperandSize::Size16,
1550            4 => OperandSize::Size32,
1551            8 => OperandSize::Size64,
1552            _ => unreachable!("Invalid OperandSize: {}", num_bytes),
1553        }
1554    }
1555
1556    // Computes the OperandSize for a given type.
1557    // For vectors, the OperandSize of the lanes is returned.
1558    pub(crate) fn from_ty(ty: Type) -> Self {
1559        Self::from_bytes(ty.lane_type().bytes())
1560    }
1561
1562    // Check that the value of self is one of the allowed sizes.
1563    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {
1564        sizes.iter().any(|val| *self == *val)
1565    }
1566
1567    pub(crate) fn to_bytes(&self) -> u8 {
1568        match self {
1569            Self::Size8 => 1,
1570            Self::Size16 => 2,
1571            Self::Size32 => 4,
1572            Self::Size64 => 8,
1573        }
1574    }
1575
1576    pub(crate) fn to_bits(&self) -> u8 {
1577        self.to_bytes() * 8
1578    }
1579
1580    pub(crate) fn to_type(&self) -> Type {
1581        match self {
1582            Self::Size8 => I8,
1583            Self::Size16 => I16,
1584            Self::Size32 => I32,
1585            Self::Size64 => I64,
1586        }
1587    }
1588}
1589
1590/// An x64 memory fence kind.
1591#[derive(Clone)]
1592#[allow(dead_code)]
1593pub enum FenceKind {
1594    /// `mfence` instruction ("Memory Fence")
1595    MFence,
1596    /// `lfence` instruction ("Load Fence")
1597    LFence,
1598    /// `sfence` instruction ("Store Fence")
1599    SFence,
1600}
cranelift_codegen/isa/x64/inst/args.rs

cranelift_codegen/isa/x64/inst/
args.rs
