cranelift_codegen/isa/x64/inst/
args.rs

1//! Instruction operand sub-components (aka "parts"): definitions and printing.
2
3use super::regs::{self};
4use crate::ir::condcodes::{FloatCC, IntCC};
5use crate::ir::types::*;
6use crate::ir::MemFlags;
7use crate::isa::x64::inst::regs::pretty_print_reg;
8use crate::isa::x64::inst::Inst;
9use crate::machinst::*;
10use smallvec::{smallvec, SmallVec};
11use std::fmt;
12use std::string::String;
13
14pub use crate::isa::x64::lower::isle::generated_code::DivSignedness;
15
16/// An extension trait for converting `Writable{Xmm,Gpr}` to `Writable<Reg>`.
17pub trait ToWritableReg {
18    /// Convert `Writable{Xmm,Gpr}` to `Writable<Reg>`.
19    fn to_writable_reg(&self) -> Writable<Reg>;
20}
21
22/// An extension trait for converting `Writable<Reg>` to `Writable{Xmm,Gpr}`.
23pub trait FromWritableReg: Sized {
24    /// Convert `Writable<Reg>` to `Writable{Xmm,Gpr}`.
25    fn from_writable_reg(w: Writable<Reg>) -> Option<Self>;
26}
27
28/// A macro for defining a newtype of `Reg` that enforces some invariant about
29/// the wrapped `Reg` (such as that it is of a particular register class).
30macro_rules! newtype_of_reg {
31    (
32        $newtype_reg:ident,
33        $newtype_writable_reg:ident,
34        $newtype_option_writable_reg:ident,
35        reg_mem: ($($newtype_reg_mem:ident $(aligned:$aligned:ident)?),*),
36        reg_mem_imm: ($($newtype_reg_mem_imm:ident $(aligned:$aligned_imm:ident)?),*),
37        $newtype_imm8_reg:ident,
38        |$check_reg:ident| $check:expr
39    ) => {
40        /// A newtype wrapper around `Reg`.
41        #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
42        pub struct $newtype_reg(Reg);
43
44        impl PartialEq<Reg> for $newtype_reg {
45            fn eq(&self, other: &Reg) -> bool {
46                self.0 == *other
47            }
48        }
49
50        impl From<$newtype_reg> for Reg {
51            fn from(r: $newtype_reg) -> Self {
52                r.0
53            }
54        }
55
56        impl $newtype_reg {
57            /// Create this newtype from the given register, or return `None` if the register
58            /// is not a valid instance of this newtype.
59            pub fn new($check_reg: Reg) -> Option<Self> {
60                if $check {
61                    Some(Self($check_reg))
62                } else {
63                    None
64                }
65            }
66
67            /// Like `Self::new(r).unwrap()` but with a better panic message on
68            /// failure.
69            pub fn unwrap_new($check_reg: Reg) -> Self {
70                if $check {
71                    Self($check_reg)
72                } else {
73                    panic!(
74                        "cannot construct {} from register {:?} with register class {:?}",
75                        stringify!($newtype_reg),
76                        $check_reg,
77                        $check_reg.class(),
78                    )
79                }
80            }
81
82            /// Get this newtype's underlying `Reg`.
83            pub fn to_reg(self) -> Reg {
84                self.0
85            }
86        }
87
88        // Convenience impl so that people working with this newtype can use it
89        // "just like" a plain `Reg`.
90        //
91        // NB: We cannot implement `DerefMut` because that would let people do
92        // nasty stuff like `*my_gpr.deref_mut() = some_xmm_reg`, breaking the
93        // invariants that `Gpr` provides.
94        impl std::ops::Deref for $newtype_reg {
95            type Target = Reg;
96
97            fn deref(&self) -> &Reg {
98                &self.0
99            }
100        }
101
102        /// If you know what you're doing, you can explicitly mutably borrow the
103        /// underlying `Reg`. Don't make it point to the wrong type of register
104        /// please.
105        impl AsMut<Reg> for $newtype_reg {
106            fn as_mut(&mut self) -> &mut Reg {
107                &mut self.0
108            }
109        }
110
111        /// Writable Gpr.
112        pub type $newtype_writable_reg = Writable<$newtype_reg>;
113
114        #[allow(dead_code)] // Used by some newtypes and not others.
115        /// Optional writable Gpr.
116        pub type $newtype_option_writable_reg = Option<Writable<$newtype_reg>>;
117
118        impl ToWritableReg for $newtype_writable_reg {
119            fn to_writable_reg(&self) -> Writable<Reg> {
120                Writable::from_reg(self.to_reg().to_reg())
121            }
122        }
123
124        impl FromWritableReg for $newtype_writable_reg {
125            fn from_writable_reg(w: Writable<Reg>) -> Option<Self> {
126                Some(Writable::from_reg($newtype_reg::new(w.to_reg())?))
127            }
128        }
129
130        $(
131            /// A newtype wrapper around `RegMem` for general-purpose registers.
132            #[derive(Clone, Debug)]
133            pub struct $newtype_reg_mem(RegMem);
134
135            impl From<$newtype_reg_mem> for RegMem {
136                fn from(rm: $newtype_reg_mem) -> Self {
137                    rm.0
138                }
139            }
140            impl<'a> From<&'a $newtype_reg_mem> for &'a RegMem {
141                fn from(rm: &'a $newtype_reg_mem) -> &'a RegMem {
142                    &rm.0
143                }
144            }
145
146            impl From<$newtype_reg> for $newtype_reg_mem {
147                fn from(r: $newtype_reg) -> Self {
148                    $newtype_reg_mem(RegMem::reg(r.into()))
149                }
150            }
151
152            impl $newtype_reg_mem {
153                /// Construct a `RegMem` newtype from the given `RegMem`, or return
154                /// `None` if the `RegMem` is not a valid instance of this `RegMem`
155                /// newtype.
156                pub fn new(rm: RegMem) -> Option<Self> {
157                    match rm {
158                        RegMem::Mem { addr } => {
159                            let mut _allow = true;
160                            $(
161                                if $aligned {
162                                    _allow = addr.aligned();
163                                }
164                            )?
165                            if _allow {
166                                Some(Self(RegMem::Mem { addr }))
167                            } else {
168                                None
169                            }
170                        }
171                        RegMem::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
172                    }
173                }
174
175                /// Like `Self::new(rm).unwrap()` but with better panic messages
176                /// in case of failure.
177                pub fn unwrap_new(rm: RegMem) -> Self {
178                    match rm {
179                        RegMem::Mem { addr } => {
180                            $(
181                                if $aligned && !addr.aligned() {
182                                    panic!(
183                                        "cannot create {} from an unaligned memory address: {addr:?}",
184                                        stringify!($newtype_reg_mem),
185                                    );
186                                }
187                            )?
188                            Self(RegMem::Mem { addr })
189                        }
190                        RegMem::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
191                    }
192                }
193
194                /// Convert this newtype into its underlying `RegMem`.
195                pub fn to_reg_mem(self) -> RegMem {
196                    self.0
197                }
198
199                #[allow(dead_code)] // Used by some newtypes and not others.
200                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
201                    self.0.get_operands(collector);
202                }
203            }
204            impl PrettyPrint for $newtype_reg_mem {
205                fn pretty_print(&self, size: u8) -> String {
206                    self.0.pretty_print(size)
207                }
208            }
209        )*
210
211        $(
212            /// A newtype wrapper around `RegMemImm`.
213            #[derive(Clone, Debug)]
214            pub struct $newtype_reg_mem_imm(RegMemImm);
215
216            impl From<$newtype_reg_mem_imm> for RegMemImm {
217                fn from(rmi: $newtype_reg_mem_imm) -> RegMemImm {
218                    rmi.0
219                }
220            }
221            impl<'a> From<&'a $newtype_reg_mem_imm> for &'a RegMemImm {
222                fn from(rmi: &'a $newtype_reg_mem_imm) -> &'a RegMemImm {
223                    &rmi.0
224                }
225            }
226
227            impl From<$newtype_reg> for $newtype_reg_mem_imm {
228                fn from(r: $newtype_reg) -> Self {
229                    $newtype_reg_mem_imm(RegMemImm::reg(r.into()))
230                }
231            }
232
233            impl $newtype_reg_mem_imm {
234                /// Construct this newtype from the given `RegMemImm`, or return
235                /// `None` if the `RegMemImm` is not a valid instance of this
236                /// newtype.
237                pub fn new(rmi: RegMemImm) -> Option<Self> {
238                    match rmi {
239                        RegMemImm::Imm { .. } => Some(Self(rmi)),
240                        RegMemImm::Mem { addr } => {
241                            let mut _allow = true;
242                            $(
243                                if $aligned_imm {
244                                    _allow = addr.aligned();
245                                }
246                            )?
247                            if _allow {
248                                Some(Self(RegMemImm::Mem { addr }))
249                            } else {
250                                None
251                            }
252                        }
253                        RegMemImm::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
254                    }
255                }
256
257                /// Like `Self::new(rmi).unwrap()` but with better panic
258                /// messages in case of failure.
259                pub fn unwrap_new(rmi: RegMemImm) -> Self {
260                    match rmi {
261                        RegMemImm::Imm { .. } => Self(rmi),
262                        RegMemImm::Mem { addr } => {
263                            $(
264                                if $aligned_imm && !addr.aligned() {
265                                    panic!(
266                                        "cannot construct {} from unaligned memory address: {:?}",
267                                        stringify!($newtype_reg_mem_imm),
268                                        addr,
269                                    );
270                                }
271                            )?
272                            Self(RegMemImm::Mem { addr })
273
274                        }
275                        RegMemImm::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
276                    }
277                }
278
279                /// Convert this newtype into its underlying `RegMemImm`.
280                #[allow(dead_code)] // Used by some newtypes and not others.
281                pub fn to_reg_mem_imm(self) -> RegMemImm {
282                    self.0
283                }
284
285                #[allow(dead_code)] // Used by some newtypes and not others.
286                pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
287                    self.0.get_operands(collector);
288                }
289            }
290
291            impl PrettyPrint for $newtype_reg_mem_imm {
292                fn pretty_print(&self, size: u8) -> String {
293                    self.0.pretty_print(size)
294                }
295            }
296        )*
297
298        /// A newtype wrapper around `Imm8Reg`.
299        #[derive(Clone, Debug)]
300        #[allow(dead_code)] // Used by some newtypes and not others.
301        pub struct $newtype_imm8_reg(Imm8Reg);
302
303        impl From<$newtype_reg> for $newtype_imm8_reg {
304            fn from(r: $newtype_reg) -> Self {
305                Self(Imm8Reg::Reg { reg: r.to_reg() })
306            }
307        }
308
309        impl $newtype_imm8_reg {
310            /// Construct this newtype from the given `Imm8Reg`, or return
311            /// `None` if the `Imm8Reg` is not a valid instance of this newtype.
312            #[allow(dead_code)] // Used by some newtypes and not others.
313            pub fn new(imm8_reg: Imm8Reg) -> Option<Self> {
314                match imm8_reg {
315                    Imm8Reg::Imm8 { .. } => Some(Self(imm8_reg)),
316                    Imm8Reg::Reg { reg } => Some($newtype_reg::new(reg)?.into()),
317                }
318            }
319
320            /// Like `Self::new(imm8_reg).unwrap()` but with better panic
321            /// messages on failure.
322            pub fn unwrap_new(imm8_reg: Imm8Reg) -> Self {
323                match imm8_reg {
324                    Imm8Reg::Imm8 { .. } => Self(imm8_reg),
325                    Imm8Reg::Reg { reg } => $newtype_reg::unwrap_new(reg).into(),
326                }
327            }
328
329            /// Borrow this newtype as its underlying `Imm8Reg`.
330            #[allow(dead_code)] // Used by some newtypes and not others.
331            pub fn as_imm8_reg(&self) -> &Imm8Reg {
332                &self.0
333            }
334
335            /// Borrow this newtype as its underlying `Imm8Reg`.
336            #[allow(dead_code)] // Used by some newtypes and not others.
337            pub fn as_imm8_reg_mut(&mut self) -> &mut Imm8Reg {
338                &mut self.0
339            }
340        }
341    };
342}
343
344// Define a newtype of `Reg` for general-purpose registers.
345newtype_of_reg!(
346    Gpr,
347    WritableGpr,
348    OptionWritableGpr,
349    reg_mem: (GprMem),
350    reg_mem_imm: (GprMemImm),
351    Imm8Gpr,
352    |reg| reg.class() == RegClass::Int
353);
354
355// Define a newtype of `Reg` for XMM registers.
356newtype_of_reg!(
357    Xmm,
358    WritableXmm,
359    OptionWritableXmm,
360    reg_mem: (XmmMem, XmmMemAligned aligned:true),
361    reg_mem_imm: (XmmMemImm, XmmMemAlignedImm aligned:true),
362    Imm8Xmm,
363    |reg| reg.class() == RegClass::Float
364);
365
366// N.B.: `Amode` is defined in `inst.isle`. We add some convenience
367// constructors here.
368
369// Re-export the type from the ISLE generated code.
370pub use crate::isa::x64::lower::isle::generated_code::Amode;
371
372impl Amode {
373    /// Create an immediate sign-extended and register addressing mode.
374    pub fn imm_reg(simm32: i32, base: Reg) -> Self {
375        debug_assert!(base.class() == RegClass::Int);
376        Self::ImmReg {
377            simm32,
378            base,
379            flags: MemFlags::trusted(),
380        }
381    }
382
383    /// Create a sign-extended-32-to-64 with register and shift addressing mode.
384    pub fn imm_reg_reg_shift(simm32: i32, base: Gpr, index: Gpr, shift: u8) -> Self {
385        debug_assert!(base.class() == RegClass::Int);
386        debug_assert!(index.class() == RegClass::Int);
387        debug_assert!(shift <= 3);
388        Self::ImmRegRegShift {
389            simm32,
390            base,
391            index,
392            shift,
393            flags: MemFlags::trusted(),
394        }
395    }
396
397    pub(crate) fn rip_relative(target: MachLabel) -> Self {
398        Self::RipRelative { target }
399    }
400
401    /// Set the specified [MemFlags] to the [Amode].
402    pub fn with_flags(&self, flags: MemFlags) -> Self {
403        match self {
404            &Self::ImmReg { simm32, base, .. } => Self::ImmReg {
405                simm32,
406                base,
407                flags,
408            },
409            &Self::ImmRegRegShift {
410                simm32,
411                base,
412                index,
413                shift,
414                ..
415            } => Self::ImmRegRegShift {
416                simm32,
417                base,
418                index,
419                shift,
420                flags,
421            },
422            _ => panic!("Amode {self:?} cannot take memflags"),
423        }
424    }
425
426    /// Add the registers mentioned by `self` to `collector`.
427    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
428        match self {
429            Amode::ImmReg { base, .. } => {
430                if *base != regs::rbp() && *base != regs::rsp() {
431                    collector.reg_use(base);
432                }
433            }
434            Amode::ImmRegRegShift { base, index, .. } => {
435                debug_assert_ne!(base.to_reg(), regs::rbp());
436                debug_assert_ne!(base.to_reg(), regs::rsp());
437                collector.reg_use(base);
438                debug_assert_ne!(index.to_reg(), regs::rbp());
439                debug_assert_ne!(index.to_reg(), regs::rsp());
440                collector.reg_use(index);
441            }
442            Amode::RipRelative { .. } => {
443                // RIP isn't involved in regalloc.
444            }
445        }
446    }
447
448    /// Same as `get_operands`, but add the registers in the "late" phase.
449    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
450        match self {
451            Amode::ImmReg { base, .. } => {
452                collector.reg_late_use(base);
453            }
454            Amode::ImmRegRegShift { base, index, .. } => {
455                collector.reg_late_use(base);
456                collector.reg_late_use(index);
457            }
458            Amode::RipRelative { .. } => {
459                // RIP isn't involved in regalloc.
460            }
461        }
462    }
463
464    pub(crate) fn get_flags(&self) -> MemFlags {
465        match self {
466            Amode::ImmReg { flags, .. } | Amode::ImmRegRegShift { flags, .. } => *flags,
467            Amode::RipRelative { .. } => MemFlags::trusted(),
468        }
469    }
470
471    /// Offset the amode by a fixed offset.
472    pub(crate) fn offset(&self, offset: i32) -> Self {
473        let mut ret = self.clone();
474        match &mut ret {
475            &mut Amode::ImmReg { ref mut simm32, .. } => *simm32 += offset,
476            &mut Amode::ImmRegRegShift { ref mut simm32, .. } => *simm32 += offset,
477            _ => panic!("Cannot offset amode: {self:?}"),
478        }
479        ret
480    }
481
482    pub(crate) fn aligned(&self) -> bool {
483        self.get_flags().aligned()
484    }
485}
486
487impl PrettyPrint for Amode {
488    fn pretty_print(&self, _size: u8) -> String {
489        match self {
490            Amode::ImmReg { simm32, base, .. } => {
491                // Note: size is always 8; the address is 64 bits,
492                // even if the addressed operand is smaller.
493                format!("{}({})", *simm32, pretty_print_reg(*base, 8))
494            }
495            Amode::ImmRegRegShift {
496                simm32,
497                base,
498                index,
499                shift,
500                ..
501            } => format!(
502                "{}({},{},{})",
503                *simm32,
504                pretty_print_reg(base.to_reg(), 8),
505                pretty_print_reg(index.to_reg(), 8),
506                1 << shift
507            ),
508            Amode::RipRelative { target } => format!("label{}(%rip)", target.as_u32()),
509        }
510    }
511}
512
513/// A Memory Address. These denote a 64-bit value only.
514/// Used for usual addressing modes as well as addressing modes used during compilation, when the
515/// moving SP offset is not known.
516#[derive(Clone, Debug)]
517pub enum SyntheticAmode {
518    /// A real amode.
519    Real(Amode),
520
521    /// A (virtual) offset into the incoming argument area.
522    IncomingArg {
523        /// The downward offset from the start of the incoming argument area.
524        offset: u32,
525    },
526
527    /// A (virtual) offset to the slot area of the function frame, which lies just above the
528    /// outgoing arguments.
529    SlotOffset {
530        /// The offset into the slot area.
531        simm32: i32,
532    },
533
534    /// A virtual offset to a constant that will be emitted in the constant section of the buffer.
535    ConstantOffset(VCodeConstant),
536}
537
538impl SyntheticAmode {
539    /// Create a real addressing mode.
540    pub fn real(amode: Amode) -> Self {
541        Self::Real(amode)
542    }
543
544    pub(crate) fn slot_offset(simm32: i32) -> Self {
545        SyntheticAmode::SlotOffset { simm32 }
546    }
547
548    /// Add the registers mentioned by `self` to `collector`.
549    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
550        match self {
551            SyntheticAmode::Real(addr) => addr.get_operands(collector),
552            SyntheticAmode::IncomingArg { .. } => {
553                // Nothing to do; the base is known and isn't involved in regalloc.
554            }
555            SyntheticAmode::SlotOffset { .. } => {
556                // Nothing to do; the base is SP and isn't involved in regalloc.
557            }
558            SyntheticAmode::ConstantOffset(_) => {}
559        }
560    }
561
562    /// Same as `get_operands`, but add the register in the "late" phase.
563    pub(crate) fn get_operands_late(&mut self, collector: &mut impl OperandVisitor) {
564        match self {
565            SyntheticAmode::Real(addr) => addr.get_operands_late(collector),
566            SyntheticAmode::IncomingArg { .. } => {
567                // Nothing to do; the base is known and isn't involved in regalloc.
568            }
569            SyntheticAmode::SlotOffset { .. } => {
570                // Nothing to do; the base is SP and isn't involved in regalloc.
571            }
572            SyntheticAmode::ConstantOffset(_) => {}
573        }
574    }
575
576    pub(crate) fn finalize(&self, frame: &FrameLayout, buffer: &mut MachBuffer<Inst>) -> Amode {
577        match self {
578            SyntheticAmode::Real(addr) => addr.clone(),
579            SyntheticAmode::IncomingArg { offset } => {
580                // NOTE: this could be made relative to RSP by adding additional
581                // offsets from the frame_layout.
582                let args_max_fp_offset = frame.tail_args_size + frame.setup_area_size;
583                Amode::imm_reg(
584                    i32::try_from(args_max_fp_offset - offset).unwrap(),
585                    regs::rbp(),
586                )
587            }
588            SyntheticAmode::SlotOffset { simm32 } => {
589                let off = *simm32 as i64 + i64::from(frame.outgoing_args_size);
590                Amode::imm_reg(off.try_into().expect("invalid sp offset"), regs::rsp())
591            }
592            SyntheticAmode::ConstantOffset(c) => {
593                Amode::rip_relative(buffer.get_label_for_constant(*c))
594            }
595        }
596    }
597
598    pub(crate) fn aligned(&self) -> bool {
599        match self {
600            SyntheticAmode::Real(addr) => addr.aligned(),
601            &SyntheticAmode::IncomingArg { .. }
602            | SyntheticAmode::SlotOffset { .. }
603            | SyntheticAmode::ConstantOffset { .. } => true,
604        }
605    }
606}
607
608impl Into<SyntheticAmode> for Amode {
609    fn into(self) -> SyntheticAmode {
610        SyntheticAmode::Real(self)
611    }
612}
613
614impl Into<SyntheticAmode> for VCodeConstant {
615    fn into(self) -> SyntheticAmode {
616        SyntheticAmode::ConstantOffset(self)
617    }
618}
619
620impl PrettyPrint for SyntheticAmode {
621    fn pretty_print(&self, _size: u8) -> String {
622        match self {
623            // See note in `Amode` regarding constant size of `8`.
624            SyntheticAmode::Real(addr) => addr.pretty_print(8),
625            &SyntheticAmode::IncomingArg { offset } => {
626                format!("rbp(stack args max - {offset})")
627            }
628            SyntheticAmode::SlotOffset { simm32 } => {
629                format!("rsp({} + virtual offset)", *simm32)
630            }
631            SyntheticAmode::ConstantOffset(c) => format!("const({})", c.as_u32()),
632        }
633    }
634}
635
636/// An operand which is either an integer Register, a value in Memory or an Immediate.  This can
637/// denote an 8, 16, 32 or 64 bit value.  For the Immediate form, in the 8- and 16-bit case, only
638/// the lower 8 or 16 bits of `simm32` is relevant.  In the 64-bit case, the value denoted by
639/// `simm32` is its sign-extension out to 64 bits.
640#[derive(Clone, Debug)]
641pub enum RegMemImm {
642    /// A register operand.
643    Reg {
644        /// The underlying register.
645        reg: Reg,
646    },
647    /// A memory operand.
648    Mem {
649        /// The memory address.
650        addr: SyntheticAmode,
651    },
652    /// An immediate operand.
653    Imm {
654        /// The immediate value.
655        simm32: u32,
656    },
657}
658
659impl RegMemImm {
660    /// Create a register operand.
661    pub fn reg(reg: Reg) -> Self {
662        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
663        Self::Reg { reg }
664    }
665
666    /// Create a memory operand.
667    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
668        Self::Mem { addr: addr.into() }
669    }
670
671    /// Create an immediate operand.
672    pub fn imm(simm32: u32) -> Self {
673        Self::Imm { simm32 }
674    }
675
676    /// Asserts that in register mode, the reg class is the one that's expected.
677    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
678        if let Self::Reg { reg } = self {
679            debug_assert_eq!(reg.class(), expected_reg_class);
680        }
681    }
682
683    /// Add the regs mentioned by `self` to `collector`.
684    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
685        match self {
686            Self::Reg { reg } => collector.reg_use(reg),
687            Self::Mem { addr } => addr.get_operands(collector),
688            Self::Imm { .. } => {}
689        }
690    }
691}
692
693impl From<RegMem> for RegMemImm {
694    fn from(rm: RegMem) -> RegMemImm {
695        match rm {
696            RegMem::Reg { reg } => RegMemImm::Reg { reg },
697            RegMem::Mem { addr } => RegMemImm::Mem { addr },
698        }
699    }
700}
701
702impl From<Reg> for RegMemImm {
703    fn from(reg: Reg) -> Self {
704        RegMemImm::Reg { reg }
705    }
706}
707
708impl PrettyPrint for RegMemImm {
709    fn pretty_print(&self, size: u8) -> String {
710        match self {
711            Self::Reg { reg } => pretty_print_reg(*reg, size),
712            Self::Mem { addr } => addr.pretty_print(size),
713            Self::Imm { simm32 } => format!("${}", *simm32 as i32),
714        }
715    }
716}
717
718/// An operand which is either an 8-bit integer immediate or a register.
719#[derive(Clone, Debug)]
720pub enum Imm8Reg {
721    /// 8-bit immediate operand.
722    Imm8 {
723        /// The 8-bit immediate value.
724        imm: u8,
725    },
726    /// A register operand.
727    Reg {
728        /// The underlying register.
729        reg: Reg,
730    },
731}
732
733impl From<u8> for Imm8Reg {
734    fn from(imm: u8) -> Self {
735        Self::Imm8 { imm }
736    }
737}
738
739impl From<Reg> for Imm8Reg {
740    fn from(reg: Reg) -> Self {
741        Self::Reg { reg }
742    }
743}
744
745/// An operand which is either an integer Register or a value in Memory.  This can denote an 8, 16,
746/// 32, 64, or 128 bit value.
747#[derive(Clone, Debug)]
748pub enum RegMem {
749    /// A register operand.
750    Reg {
751        /// The underlying register.
752        reg: Reg,
753    },
754    /// A memory operand.
755    Mem {
756        /// The memory address.
757        addr: SyntheticAmode,
758    },
759}
760
761impl RegMem {
762    /// Create a register operand.
763    pub fn reg(reg: Reg) -> Self {
764        debug_assert!(reg.class() == RegClass::Int || reg.class() == RegClass::Float);
765        Self::Reg { reg }
766    }
767
768    /// Create a memory operand.
769    pub fn mem(addr: impl Into<SyntheticAmode>) -> Self {
770        Self::Mem { addr: addr.into() }
771    }
772    /// Asserts that in register mode, the reg class is the one that's expected.
773    pub(crate) fn assert_regclass_is(&self, expected_reg_class: RegClass) {
774        if let Self::Reg { reg } = self {
775            debug_assert_eq!(reg.class(), expected_reg_class);
776        }
777    }
778    /// Add the regs mentioned by `self` to `collector`.
779    pub(crate) fn get_operands(&mut self, collector: &mut impl OperandVisitor) {
780        match self {
781            RegMem::Reg { reg } => collector.reg_use(reg),
782            RegMem::Mem { addr, .. } => addr.get_operands(collector),
783        }
784    }
785}
786
787impl From<Reg> for RegMem {
788    fn from(reg: Reg) -> RegMem {
789        RegMem::Reg { reg }
790    }
791}
792
793impl From<Writable<Reg>> for RegMem {
794    fn from(r: Writable<Reg>) -> Self {
795        RegMem::reg(r.to_reg())
796    }
797}
798
799impl PrettyPrint for RegMem {
800    fn pretty_print(&self, size: u8) -> String {
801        match self {
802            RegMem::Reg { reg } => pretty_print_reg(*reg, size),
803            RegMem::Mem { addr, .. } => addr.pretty_print(size),
804        }
805    }
806}
807
808pub use crate::isa::x64::lower::isle::generated_code::AluRmROpcode;
809
810impl AluRmROpcode {
811    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
812        match self {
813            AluRmROpcode::Andn => smallvec![InstructionSet::BMI1],
814            AluRmROpcode::Sarx | AluRmROpcode::Shrx | AluRmROpcode::Shlx | AluRmROpcode::Bzhi => {
815                smallvec![InstructionSet::BMI2]
816            }
817        }
818    }
819}
820
821impl fmt::Display for AluRmROpcode {
822    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
823        f.write_str(&format!("{self:?}").to_lowercase())
824    }
825}
826
827#[derive(Clone, PartialEq)]
828/// Unary operations requiring register or memory and register operands.
829pub enum UnaryRmROpcode {
830    /// Bit-scan reverse.
831    Bsr,
832    /// Bit-scan forward.
833    Bsf,
834    /// Counts leading zeroes (Leading Zero CouNT).
835    Lzcnt,
836    /// Counts trailing zeroes (Trailing Zero CouNT).
837    Tzcnt,
838    /// Counts the number of ones (POPulation CouNT).
839    Popcnt,
840}
841
842impl UnaryRmROpcode {
843    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
844        match self {
845            UnaryRmROpcode::Bsr | UnaryRmROpcode::Bsf => smallvec![],
846            UnaryRmROpcode::Lzcnt => smallvec![InstructionSet::Lzcnt],
847            UnaryRmROpcode::Tzcnt => smallvec![InstructionSet::BMI1],
848            UnaryRmROpcode::Popcnt => smallvec![InstructionSet::Popcnt],
849        }
850    }
851}
852
853impl fmt::Debug for UnaryRmROpcode {
854    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
855        match self {
856            UnaryRmROpcode::Bsr => write!(fmt, "bsr"),
857            UnaryRmROpcode::Bsf => write!(fmt, "bsf"),
858            UnaryRmROpcode::Lzcnt => write!(fmt, "lzcnt"),
859            UnaryRmROpcode::Tzcnt => write!(fmt, "tzcnt"),
860            UnaryRmROpcode::Popcnt => write!(fmt, "popcnt"),
861        }
862    }
863}
864
865impl fmt::Display for UnaryRmROpcode {
866    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
867        fmt::Debug::fmt(self, f)
868    }
869}
870
871pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRVexOpcode;
872
873impl UnaryRmRVexOpcode {
874    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
875        match self {
876            UnaryRmRVexOpcode::Blsi | UnaryRmRVexOpcode::Blsmsk | UnaryRmRVexOpcode::Blsr => {
877                smallvec![InstructionSet::BMI1]
878            }
879        }
880    }
881}
882
883impl fmt::Display for UnaryRmRVexOpcode {
884    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
885        f.write_str(&format!("{self:?}").to_lowercase())
886    }
887}
888
889pub use crate::isa::x64::lower::isle::generated_code::UnaryRmRImmVexOpcode;
890
891impl UnaryRmRImmVexOpcode {
892    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
893        match self {
894            UnaryRmRImmVexOpcode::Rorx => {
895                smallvec![InstructionSet::BMI2]
896            }
897        }
898    }
899}
900
901impl fmt::Display for UnaryRmRImmVexOpcode {
902    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
903        f.write_str(&format!("{self:?}").to_lowercase())
904    }
905}
906
907#[derive(Clone, Copy, PartialEq)]
908/// Comparison operations.
909pub enum CmpOpcode {
910    /// CMP instruction: compute `a - b` and set flags from result.
911    Cmp,
912    /// TEST instruction: compute `a & b` and set flags from result.
913    Test,
914}
915
916#[derive(Debug)]
917pub(crate) enum InstructionSet {
918    SSE,
919    SSE2,
920    CMPXCHG16b,
921    SSSE3,
922    SSE41,
923    SSE42,
924    Popcnt,
925    Lzcnt,
926    BMI1,
927    #[allow(dead_code)] // never constructed (yet).
928    BMI2,
929    FMA,
930    AVX,
931    AVX2,
932    AVX512BITALG,
933    AVX512DQ,
934    AVX512F,
935    AVX512VBMI,
936    AVX512VL,
937}
938
939/// Some SSE operations requiring 2 operands r/m and r.
940#[derive(Clone, Copy, PartialEq)]
941#[allow(dead_code)] // some variants here aren't used just yet
942#[allow(missing_docs)]
943pub enum SseOpcode {
944    Andps,
945    Andpd,
946    Andnps,
947    Andnpd,
948    Blendvpd,
949    Blendvps,
950    Comiss,
951    Comisd,
952    Cmpps,
953    Cmppd,
954    Cmpss,
955    Cmpsd,
956    Cvtdq2ps,
957    Cvtdq2pd,
958    Cvtpd2ps,
959    Cvtps2pd,
960    Cvtsd2ss,
961    Cvtsd2si,
962    Cvtsi2ss,
963    Cvtsi2sd,
964    Cvtss2si,
965    Cvtss2sd,
966    Cvttpd2dq,
967    Cvttps2dq,
968    Cvttss2si,
969    Cvttsd2si,
970    Divps,
971    Divpd,
972    Divss,
973    Divsd,
974    Insertps,
975    Maxps,
976    Maxpd,
977    Maxss,
978    Maxsd,
979    Minps,
980    Minpd,
981    Minss,
982    Minsd,
983    Movaps,
984    Movapd,
985    Movd,
986    Movdqa,
987    Movdqu,
988    Movlhps,
989    Movmskps,
990    Movmskpd,
991    Movq,
992    Movss,
993    Movsd,
994    Movups,
995    Movupd,
996    Mulps,
997    Mulpd,
998    Mulss,
999    Mulsd,
1000    Orps,
1001    Orpd,
1002    Pabsb,
1003    Pabsw,
1004    Pabsd,
1005    Packssdw,
1006    Packsswb,
1007    Packusdw,
1008    Packuswb,
1009    Paddb,
1010    Paddd,
1011    Paddq,
1012    Paddw,
1013    Paddsb,
1014    Paddsw,
1015    Paddusb,
1016    Paddusw,
1017    Palignr,
1018    Pand,
1019    Pandn,
1020    Pavgb,
1021    Pavgw,
1022    Pblendvb,
1023    Pcmpeqb,
1024    Pcmpeqw,
1025    Pcmpeqd,
1026    Pcmpeqq,
1027    Pcmpgtb,
1028    Pcmpgtw,
1029    Pcmpgtd,
1030    Pcmpgtq,
1031    Pextrb,
1032    Pextrw,
1033    Pextrd,
1034    Pextrq,
1035    Pinsrb,
1036    Pinsrw,
1037    Pinsrd,
1038    Pmaddubsw,
1039    Pmaddwd,
1040    Pmaxsb,
1041    Pmaxsw,
1042    Pmaxsd,
1043    Pmaxub,
1044    Pmaxuw,
1045    Pmaxud,
1046    Pminsb,
1047    Pminsw,
1048    Pminsd,
1049    Pminub,
1050    Pminuw,
1051    Pminud,
1052    Pmovmskb,
1053    Pmovsxbd,
1054    Pmovsxbw,
1055    Pmovsxbq,
1056    Pmovsxwd,
1057    Pmovsxwq,
1058    Pmovsxdq,
1059    Pmovzxbd,
1060    Pmovzxbw,
1061    Pmovzxbq,
1062    Pmovzxwd,
1063    Pmovzxwq,
1064    Pmovzxdq,
1065    Pmuldq,
1066    Pmulhw,
1067    Pmulhuw,
1068    Pmulhrsw,
1069    Pmulld,
1070    Pmullw,
1071    Pmuludq,
1072    Por,
1073    Pshufb,
1074    Pshufd,
1075    Psllw,
1076    Pslld,
1077    Psllq,
1078    Psraw,
1079    Psrad,
1080    Psrlw,
1081    Psrld,
1082    Psrlq,
1083    Psubb,
1084    Psubd,
1085    Psubq,
1086    Psubw,
1087    Psubsb,
1088    Psubsw,
1089    Psubusb,
1090    Psubusw,
1091    Ptest,
1092    Punpckhbw,
1093    Punpckhwd,
1094    Punpcklbw,
1095    Punpcklwd,
1096    Pxor,
1097    Rcpss,
1098    Roundps,
1099    Roundpd,
1100    Roundss,
1101    Roundsd,
1102    Rsqrtss,
1103    Shufps,
1104    Sqrtps,
1105    Sqrtpd,
1106    Sqrtss,
1107    Sqrtsd,
1108    Ucomiss,
1109    Ucomisd,
1110    Unpcklps,
1111    Unpcklpd,
1112    Unpckhps,
1113    Xorps,
1114    Xorpd,
1115    Phaddw,
1116    Phaddd,
1117    Punpckhdq,
1118    Punpckldq,
1119    Punpckhqdq,
1120    Punpcklqdq,
1121    Pshuflw,
1122    Pshufhw,
1123    Pblendw,
1124    Movddup,
1125}
1126
1127impl SseOpcode {
1128    /// Which `InstructionSet` is the first supporting this opcode?
1129    pub(crate) fn available_from(&self) -> InstructionSet {
1130        use InstructionSet::*;
1131        match self {
1132            SseOpcode::Andps
1133            | SseOpcode::Andnps
1134            | SseOpcode::Comiss
1135            | SseOpcode::Cmpps
1136            | SseOpcode::Cmpss
1137            | SseOpcode::Cvtsi2ss
1138            | SseOpcode::Cvtss2si
1139            | SseOpcode::Cvttss2si
1140            | SseOpcode::Divps
1141            | SseOpcode::Divss
1142            | SseOpcode::Maxps
1143            | SseOpcode::Maxss
1144            | SseOpcode::Minps
1145            | SseOpcode::Minss
1146            | SseOpcode::Movaps
1147            | SseOpcode::Movlhps
1148            | SseOpcode::Movmskps
1149            | SseOpcode::Movss
1150            | SseOpcode::Movups
1151            | SseOpcode::Mulps
1152            | SseOpcode::Mulss
1153            | SseOpcode::Orps
1154            | SseOpcode::Rcpss
1155            | SseOpcode::Rsqrtss
1156            | SseOpcode::Shufps
1157            | SseOpcode::Sqrtps
1158            | SseOpcode::Sqrtss
1159            | SseOpcode::Ucomiss
1160            | SseOpcode::Unpcklps
1161            | SseOpcode::Unpckhps
1162            | SseOpcode::Xorps => SSE,
1163
1164            SseOpcode::Andpd
1165            | SseOpcode::Andnpd
1166            | SseOpcode::Cmppd
1167            | SseOpcode::Cmpsd
1168            | SseOpcode::Comisd
1169            | SseOpcode::Cvtdq2ps
1170            | SseOpcode::Cvtdq2pd
1171            | SseOpcode::Cvtpd2ps
1172            | SseOpcode::Cvtps2pd
1173            | SseOpcode::Cvtsd2ss
1174            | SseOpcode::Cvtsd2si
1175            | SseOpcode::Cvtsi2sd
1176            | SseOpcode::Cvtss2sd
1177            | SseOpcode::Cvttpd2dq
1178            | SseOpcode::Cvttps2dq
1179            | SseOpcode::Cvttsd2si
1180            | SseOpcode::Divpd
1181            | SseOpcode::Divsd
1182            | SseOpcode::Maxpd
1183            | SseOpcode::Maxsd
1184            | SseOpcode::Minpd
1185            | SseOpcode::Minsd
1186            | SseOpcode::Movapd
1187            | SseOpcode::Movd
1188            | SseOpcode::Movmskpd
1189            | SseOpcode::Movq
1190            | SseOpcode::Movsd
1191            | SseOpcode::Movupd
1192            | SseOpcode::Movdqa
1193            | SseOpcode::Movdqu
1194            | SseOpcode::Mulpd
1195            | SseOpcode::Mulsd
1196            | SseOpcode::Orpd
1197            | SseOpcode::Packssdw
1198            | SseOpcode::Packsswb
1199            | SseOpcode::Packuswb
1200            | SseOpcode::Paddb
1201            | SseOpcode::Paddd
1202            | SseOpcode::Paddq
1203            | SseOpcode::Paddw
1204            | SseOpcode::Paddsb
1205            | SseOpcode::Paddsw
1206            | SseOpcode::Paddusb
1207            | SseOpcode::Paddusw
1208            | SseOpcode::Pand
1209            | SseOpcode::Pandn
1210            | SseOpcode::Pavgb
1211            | SseOpcode::Pavgw
1212            | SseOpcode::Pcmpeqb
1213            | SseOpcode::Pcmpeqw
1214            | SseOpcode::Pcmpeqd
1215            | SseOpcode::Pcmpgtb
1216            | SseOpcode::Pcmpgtw
1217            | SseOpcode::Pcmpgtd
1218            | SseOpcode::Pextrw
1219            | SseOpcode::Pinsrw
1220            | SseOpcode::Pmaddwd
1221            | SseOpcode::Pmaxsw
1222            | SseOpcode::Pmaxub
1223            | SseOpcode::Pminsw
1224            | SseOpcode::Pminub
1225            | SseOpcode::Pmovmskb
1226            | SseOpcode::Pmulhw
1227            | SseOpcode::Pmulhuw
1228            | SseOpcode::Pmullw
1229            | SseOpcode::Pmuludq
1230            | SseOpcode::Por
1231            | SseOpcode::Pshufd
1232            | SseOpcode::Psllw
1233            | SseOpcode::Pslld
1234            | SseOpcode::Psllq
1235            | SseOpcode::Psraw
1236            | SseOpcode::Psrad
1237            | SseOpcode::Psrlw
1238            | SseOpcode::Psrld
1239            | SseOpcode::Psrlq
1240            | SseOpcode::Psubb
1241            | SseOpcode::Psubd
1242            | SseOpcode::Psubq
1243            | SseOpcode::Psubw
1244            | SseOpcode::Psubsb
1245            | SseOpcode::Psubsw
1246            | SseOpcode::Psubusb
1247            | SseOpcode::Psubusw
1248            | SseOpcode::Punpckhbw
1249            | SseOpcode::Punpckhwd
1250            | SseOpcode::Punpcklbw
1251            | SseOpcode::Punpcklwd
1252            | SseOpcode::Pxor
1253            | SseOpcode::Sqrtpd
1254            | SseOpcode::Sqrtsd
1255            | SseOpcode::Ucomisd
1256            | SseOpcode::Xorpd
1257            | SseOpcode::Punpckldq
1258            | SseOpcode::Punpckhdq
1259            | SseOpcode::Punpcklqdq
1260            | SseOpcode::Punpckhqdq
1261            | SseOpcode::Pshuflw
1262            | SseOpcode::Pshufhw
1263            | SseOpcode::Unpcklpd => SSE2,
1264
1265            SseOpcode::Pabsb
1266            | SseOpcode::Pabsw
1267            | SseOpcode::Pabsd
1268            | SseOpcode::Palignr
1269            | SseOpcode::Pmulhrsw
1270            | SseOpcode::Pshufb
1271            | SseOpcode::Phaddw
1272            | SseOpcode::Phaddd
1273            | SseOpcode::Pmaddubsw
1274            | SseOpcode::Movddup => SSSE3,
1275
1276            SseOpcode::Blendvpd
1277            | SseOpcode::Blendvps
1278            | SseOpcode::Insertps
1279            | SseOpcode::Packusdw
1280            | SseOpcode::Pblendvb
1281            | SseOpcode::Pcmpeqq
1282            | SseOpcode::Pextrb
1283            | SseOpcode::Pextrd
1284            | SseOpcode::Pextrq
1285            | SseOpcode::Pinsrb
1286            | SseOpcode::Pinsrd
1287            | SseOpcode::Pmaxsb
1288            | SseOpcode::Pmaxsd
1289            | SseOpcode::Pmaxuw
1290            | SseOpcode::Pmaxud
1291            | SseOpcode::Pminsb
1292            | SseOpcode::Pminsd
1293            | SseOpcode::Pminuw
1294            | SseOpcode::Pminud
1295            | SseOpcode::Pmovsxbd
1296            | SseOpcode::Pmovsxbw
1297            | SseOpcode::Pmovsxbq
1298            | SseOpcode::Pmovsxwd
1299            | SseOpcode::Pmovsxwq
1300            | SseOpcode::Pmovsxdq
1301            | SseOpcode::Pmovzxbd
1302            | SseOpcode::Pmovzxbw
1303            | SseOpcode::Pmovzxbq
1304            | SseOpcode::Pmovzxwd
1305            | SseOpcode::Pmovzxwq
1306            | SseOpcode::Pmovzxdq
1307            | SseOpcode::Pmuldq
1308            | SseOpcode::Pmulld
1309            | SseOpcode::Ptest
1310            | SseOpcode::Roundps
1311            | SseOpcode::Roundpd
1312            | SseOpcode::Roundss
1313            | SseOpcode::Roundsd
1314            | SseOpcode::Pblendw => SSE41,
1315
1316            SseOpcode::Pcmpgtq => SSE42,
1317        }
1318    }
1319
1320    /// Returns the src operand size for an instruction.
1321    pub(crate) fn src_size(&self) -> u8 {
1322        match self {
1323            SseOpcode::Movd => 4,
1324            _ => 8,
1325        }
1326    }
1327
1328    /// Is `src2` with this opcode a scalar, as for lane insertions?
1329    pub(crate) fn has_scalar_src2(self) -> bool {
1330        match self {
1331            SseOpcode::Pinsrb | SseOpcode::Pinsrw | SseOpcode::Pinsrd => true,
1332            SseOpcode::Pmovsxbw
1333            | SseOpcode::Pmovsxbd
1334            | SseOpcode::Pmovsxbq
1335            | SseOpcode::Pmovsxwd
1336            | SseOpcode::Pmovsxwq
1337            | SseOpcode::Pmovsxdq => true,
1338            SseOpcode::Pmovzxbw
1339            | SseOpcode::Pmovzxbd
1340            | SseOpcode::Pmovzxbq
1341            | SseOpcode::Pmovzxwd
1342            | SseOpcode::Pmovzxwq
1343            | SseOpcode::Pmovzxdq => true,
1344            _ => false,
1345        }
1346    }
1347}
1348
1349impl fmt::Debug for SseOpcode {
1350    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1351        let name = match self {
1352            SseOpcode::Andpd => "andpd",
1353            SseOpcode::Andps => "andps",
1354            SseOpcode::Andnps => "andnps",
1355            SseOpcode::Andnpd => "andnpd",
1356            SseOpcode::Blendvpd => "blendvpd",
1357            SseOpcode::Blendvps => "blendvps",
1358            SseOpcode::Cmpps => "cmpps",
1359            SseOpcode::Cmppd => "cmppd",
1360            SseOpcode::Cmpss => "cmpss",
1361            SseOpcode::Cmpsd => "cmpsd",
1362            SseOpcode::Comiss => "comiss",
1363            SseOpcode::Comisd => "comisd",
1364            SseOpcode::Cvtdq2ps => "cvtdq2ps",
1365            SseOpcode::Cvtdq2pd => "cvtdq2pd",
1366            SseOpcode::Cvtpd2ps => "cvtpd2ps",
1367            SseOpcode::Cvtps2pd => "cvtps2pd",
1368            SseOpcode::Cvtsd2ss => "cvtsd2ss",
1369            SseOpcode::Cvtsd2si => "cvtsd2si",
1370            SseOpcode::Cvtsi2ss => "cvtsi2ss",
1371            SseOpcode::Cvtsi2sd => "cvtsi2sd",
1372            SseOpcode::Cvtss2si => "cvtss2si",
1373            SseOpcode::Cvtss2sd => "cvtss2sd",
1374            SseOpcode::Cvttpd2dq => "cvttpd2dq",
1375            SseOpcode::Cvttps2dq => "cvttps2dq",
1376            SseOpcode::Cvttss2si => "cvttss2si",
1377            SseOpcode::Cvttsd2si => "cvttsd2si",
1378            SseOpcode::Divps => "divps",
1379            SseOpcode::Divpd => "divpd",
1380            SseOpcode::Divss => "divss",
1381            SseOpcode::Divsd => "divsd",
1382            SseOpcode::Insertps => "insertps",
1383            SseOpcode::Maxps => "maxps",
1384            SseOpcode::Maxpd => "maxpd",
1385            SseOpcode::Maxss => "maxss",
1386            SseOpcode::Maxsd => "maxsd",
1387            SseOpcode::Minps => "minps",
1388            SseOpcode::Minpd => "minpd",
1389            SseOpcode::Minss => "minss",
1390            SseOpcode::Minsd => "minsd",
1391            SseOpcode::Movaps => "movaps",
1392            SseOpcode::Movapd => "movapd",
1393            SseOpcode::Movd => "movd",
1394            SseOpcode::Movdqa => "movdqa",
1395            SseOpcode::Movdqu => "movdqu",
1396            SseOpcode::Movlhps => "movlhps",
1397            SseOpcode::Movmskps => "movmskps",
1398            SseOpcode::Movmskpd => "movmskpd",
1399            SseOpcode::Movq => "movq",
1400            SseOpcode::Movss => "movss",
1401            SseOpcode::Movsd => "movsd",
1402            SseOpcode::Movups => "movups",
1403            SseOpcode::Movupd => "movupd",
1404            SseOpcode::Mulps => "mulps",
1405            SseOpcode::Mulpd => "mulpd",
1406            SseOpcode::Mulss => "mulss",
1407            SseOpcode::Mulsd => "mulsd",
1408            SseOpcode::Orpd => "orpd",
1409            SseOpcode::Orps => "orps",
1410            SseOpcode::Pabsb => "pabsb",
1411            SseOpcode::Pabsw => "pabsw",
1412            SseOpcode::Pabsd => "pabsd",
1413            SseOpcode::Packssdw => "packssdw",
1414            SseOpcode::Packsswb => "packsswb",
1415            SseOpcode::Packusdw => "packusdw",
1416            SseOpcode::Packuswb => "packuswb",
1417            SseOpcode::Paddb => "paddb",
1418            SseOpcode::Paddd => "paddd",
1419            SseOpcode::Paddq => "paddq",
1420            SseOpcode::Paddw => "paddw",
1421            SseOpcode::Paddsb => "paddsb",
1422            SseOpcode::Paddsw => "paddsw",
1423            SseOpcode::Paddusb => "paddusb",
1424            SseOpcode::Paddusw => "paddusw",
1425            SseOpcode::Palignr => "palignr",
1426            SseOpcode::Pand => "pand",
1427            SseOpcode::Pandn => "pandn",
1428            SseOpcode::Pavgb => "pavgb",
1429            SseOpcode::Pavgw => "pavgw",
1430            SseOpcode::Pblendvb => "pblendvb",
1431            SseOpcode::Pcmpeqb => "pcmpeqb",
1432            SseOpcode::Pcmpeqw => "pcmpeqw",
1433            SseOpcode::Pcmpeqd => "pcmpeqd",
1434            SseOpcode::Pcmpeqq => "pcmpeqq",
1435            SseOpcode::Pcmpgtb => "pcmpgtb",
1436            SseOpcode::Pcmpgtw => "pcmpgtw",
1437            SseOpcode::Pcmpgtd => "pcmpgtd",
1438            SseOpcode::Pcmpgtq => "pcmpgtq",
1439            SseOpcode::Pextrb => "pextrb",
1440            SseOpcode::Pextrw => "pextrw",
1441            SseOpcode::Pextrd => "pextrd",
1442            SseOpcode::Pextrq => "pextrq",
1443            SseOpcode::Pinsrb => "pinsrb",
1444            SseOpcode::Pinsrw => "pinsrw",
1445            SseOpcode::Pinsrd => "pinsrd",
1446            SseOpcode::Pmaddubsw => "pmaddubsw",
1447            SseOpcode::Pmaddwd => "pmaddwd",
1448            SseOpcode::Pmaxsb => "pmaxsb",
1449            SseOpcode::Pmaxsw => "pmaxsw",
1450            SseOpcode::Pmaxsd => "pmaxsd",
1451            SseOpcode::Pmaxub => "pmaxub",
1452            SseOpcode::Pmaxuw => "pmaxuw",
1453            SseOpcode::Pmaxud => "pmaxud",
1454            SseOpcode::Pminsb => "pminsb",
1455            SseOpcode::Pminsw => "pminsw",
1456            SseOpcode::Pminsd => "pminsd",
1457            SseOpcode::Pminub => "pminub",
1458            SseOpcode::Pminuw => "pminuw",
1459            SseOpcode::Pminud => "pminud",
1460            SseOpcode::Pmovmskb => "pmovmskb",
1461            SseOpcode::Pmovsxbd => "pmovsxbd",
1462            SseOpcode::Pmovsxbw => "pmovsxbw",
1463            SseOpcode::Pmovsxbq => "pmovsxbq",
1464            SseOpcode::Pmovsxwd => "pmovsxwd",
1465            SseOpcode::Pmovsxwq => "pmovsxwq",
1466            SseOpcode::Pmovsxdq => "pmovsxdq",
1467            SseOpcode::Pmovzxbd => "pmovzxbd",
1468            SseOpcode::Pmovzxbw => "pmovzxbw",
1469            SseOpcode::Pmovzxbq => "pmovzxbq",
1470            SseOpcode::Pmovzxwd => "pmovzxwd",
1471            SseOpcode::Pmovzxwq => "pmovzxwq",
1472            SseOpcode::Pmovzxdq => "pmovzxdq",
1473            SseOpcode::Pmuldq => "pmuldq",
1474            SseOpcode::Pmulhw => "pmulhw",
1475            SseOpcode::Pmulhuw => "pmulhuw",
1476            SseOpcode::Pmulhrsw => "pmulhrsw",
1477            SseOpcode::Pmulld => "pmulld",
1478            SseOpcode::Pmullw => "pmullw",
1479            SseOpcode::Pmuludq => "pmuludq",
1480            SseOpcode::Por => "por",
1481            SseOpcode::Pshufb => "pshufb",
1482            SseOpcode::Pshufd => "pshufd",
1483            SseOpcode::Psllw => "psllw",
1484            SseOpcode::Pslld => "pslld",
1485            SseOpcode::Psllq => "psllq",
1486            SseOpcode::Psraw => "psraw",
1487            SseOpcode::Psrad => "psrad",
1488            SseOpcode::Psrlw => "psrlw",
1489            SseOpcode::Psrld => "psrld",
1490            SseOpcode::Psrlq => "psrlq",
1491            SseOpcode::Psubb => "psubb",
1492            SseOpcode::Psubd => "psubd",
1493            SseOpcode::Psubq => "psubq",
1494            SseOpcode::Psubw => "psubw",
1495            SseOpcode::Psubsb => "psubsb",
1496            SseOpcode::Psubsw => "psubsw",
1497            SseOpcode::Psubusb => "psubusb",
1498            SseOpcode::Psubusw => "psubusw",
1499            SseOpcode::Ptest => "ptest",
1500            SseOpcode::Punpckhbw => "punpckhbw",
1501            SseOpcode::Punpckhwd => "punpckhwd",
1502            SseOpcode::Punpcklbw => "punpcklbw",
1503            SseOpcode::Punpcklwd => "punpcklwd",
1504            SseOpcode::Pxor => "pxor",
1505            SseOpcode::Rcpss => "rcpss",
1506            SseOpcode::Roundps => "roundps",
1507            SseOpcode::Roundpd => "roundpd",
1508            SseOpcode::Roundss => "roundss",
1509            SseOpcode::Roundsd => "roundsd",
1510            SseOpcode::Rsqrtss => "rsqrtss",
1511            SseOpcode::Shufps => "shufps",
1512            SseOpcode::Sqrtps => "sqrtps",
1513            SseOpcode::Sqrtpd => "sqrtpd",
1514            SseOpcode::Sqrtss => "sqrtss",
1515            SseOpcode::Sqrtsd => "sqrtsd",
1516            SseOpcode::Ucomiss => "ucomiss",
1517            SseOpcode::Ucomisd => "ucomisd",
1518            SseOpcode::Unpcklps => "unpcklps",
1519            SseOpcode::Unpckhps => "unpckhps",
1520            SseOpcode::Xorps => "xorps",
1521            SseOpcode::Xorpd => "xorpd",
1522            SseOpcode::Phaddw => "phaddw",
1523            SseOpcode::Phaddd => "phaddd",
1524            SseOpcode::Punpckldq => "punpckldq",
1525            SseOpcode::Punpckhdq => "punpckhdq",
1526            SseOpcode::Punpcklqdq => "punpcklqdq",
1527            SseOpcode::Punpckhqdq => "punpckhqdq",
1528            SseOpcode::Pshuflw => "pshuflw",
1529            SseOpcode::Pshufhw => "pshufhw",
1530            SseOpcode::Pblendw => "pblendw",
1531            SseOpcode::Movddup => "movddup",
1532            SseOpcode::Unpcklpd => "unpcklpd",
1533        };
1534        write!(fmt, "{name}")
1535    }
1536}
1537
1538impl fmt::Display for SseOpcode {
1539    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1540        fmt::Debug::fmt(self, f)
1541    }
1542}
1543
1544pub use crate::isa::x64::lower::isle::generated_code::AvxOpcode;
1545
1546impl AvxOpcode {
1547    /// Which `InstructionSet`s support the opcode?
1548    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1549        match self {
1550            AvxOpcode::Vfmadd213ss
1551            | AvxOpcode::Vfmadd213sd
1552            | AvxOpcode::Vfmadd213ps
1553            | AvxOpcode::Vfmadd213pd
1554            | AvxOpcode::Vfmadd132ss
1555            | AvxOpcode::Vfmadd132sd
1556            | AvxOpcode::Vfmadd132ps
1557            | AvxOpcode::Vfmadd132pd
1558            | AvxOpcode::Vfnmadd213ss
1559            | AvxOpcode::Vfnmadd213sd
1560            | AvxOpcode::Vfnmadd213ps
1561            | AvxOpcode::Vfnmadd213pd
1562            | AvxOpcode::Vfnmadd132ss
1563            | AvxOpcode::Vfnmadd132sd
1564            | AvxOpcode::Vfnmadd132ps
1565            | AvxOpcode::Vfnmadd132pd
1566            | AvxOpcode::Vfmsub213ss
1567            | AvxOpcode::Vfmsub213sd
1568            | AvxOpcode::Vfmsub213ps
1569            | AvxOpcode::Vfmsub213pd
1570            | AvxOpcode::Vfmsub132ss
1571            | AvxOpcode::Vfmsub132sd
1572            | AvxOpcode::Vfmsub132ps
1573            | AvxOpcode::Vfmsub132pd
1574            | AvxOpcode::Vfnmsub213ss
1575            | AvxOpcode::Vfnmsub213sd
1576            | AvxOpcode::Vfnmsub213ps
1577            | AvxOpcode::Vfnmsub213pd
1578            | AvxOpcode::Vfnmsub132ss
1579            | AvxOpcode::Vfnmsub132sd
1580            | AvxOpcode::Vfnmsub132ps
1581            | AvxOpcode::Vfnmsub132pd => smallvec![InstructionSet::FMA],
1582            AvxOpcode::Vminps
1583            | AvxOpcode::Vminpd
1584            | AvxOpcode::Vmaxps
1585            | AvxOpcode::Vmaxpd
1586            | AvxOpcode::Vandnps
1587            | AvxOpcode::Vandnpd
1588            | AvxOpcode::Vpandn
1589            | AvxOpcode::Vcmpps
1590            | AvxOpcode::Vcmppd
1591            | AvxOpcode::Vpsrlw
1592            | AvxOpcode::Vpsrld
1593            | AvxOpcode::Vpsrlq
1594            | AvxOpcode::Vpaddb
1595            | AvxOpcode::Vpaddw
1596            | AvxOpcode::Vpaddd
1597            | AvxOpcode::Vpaddq
1598            | AvxOpcode::Vpaddsb
1599            | AvxOpcode::Vpaddsw
1600            | AvxOpcode::Vpaddusb
1601            | AvxOpcode::Vpaddusw
1602            | AvxOpcode::Vpsubb
1603            | AvxOpcode::Vpsubw
1604            | AvxOpcode::Vpsubd
1605            | AvxOpcode::Vpsubq
1606            | AvxOpcode::Vpsubsb
1607            | AvxOpcode::Vpsubsw
1608            | AvxOpcode::Vpsubusb
1609            | AvxOpcode::Vpsubusw
1610            | AvxOpcode::Vpavgb
1611            | AvxOpcode::Vpavgw
1612            | AvxOpcode::Vpand
1613            | AvxOpcode::Vandps
1614            | AvxOpcode::Vandpd
1615            | AvxOpcode::Vpor
1616            | AvxOpcode::Vorps
1617            | AvxOpcode::Vorpd
1618            | AvxOpcode::Vpxor
1619            | AvxOpcode::Vxorps
1620            | AvxOpcode::Vxorpd
1621            | AvxOpcode::Vpmullw
1622            | AvxOpcode::Vpmulld
1623            | AvxOpcode::Vpmulhw
1624            | AvxOpcode::Vpmulhd
1625            | AvxOpcode::Vpmulhrsw
1626            | AvxOpcode::Vpmulhuw
1627            | AvxOpcode::Vpmuldq
1628            | AvxOpcode::Vpmuludq
1629            | AvxOpcode::Vpunpckhwd
1630            | AvxOpcode::Vpunpcklwd
1631            | AvxOpcode::Vunpcklps
1632            | AvxOpcode::Vunpckhps
1633            | AvxOpcode::Vaddps
1634            | AvxOpcode::Vaddpd
1635            | AvxOpcode::Vsubps
1636            | AvxOpcode::Vsubpd
1637            | AvxOpcode::Vmulps
1638            | AvxOpcode::Vmulpd
1639            | AvxOpcode::Vdivps
1640            | AvxOpcode::Vdivpd
1641            | AvxOpcode::Vpcmpeqb
1642            | AvxOpcode::Vpcmpeqw
1643            | AvxOpcode::Vpcmpeqd
1644            | AvxOpcode::Vpcmpeqq
1645            | AvxOpcode::Vpcmpgtb
1646            | AvxOpcode::Vpcmpgtw
1647            | AvxOpcode::Vpcmpgtd
1648            | AvxOpcode::Vpcmpgtq
1649            | AvxOpcode::Vblendvps
1650            | AvxOpcode::Vblendvpd
1651            | AvxOpcode::Vpblendvb
1652            | AvxOpcode::Vmovlhps
1653            | AvxOpcode::Vpminsb
1654            | AvxOpcode::Vpminsw
1655            | AvxOpcode::Vpminsd
1656            | AvxOpcode::Vpminub
1657            | AvxOpcode::Vpminuw
1658            | AvxOpcode::Vpminud
1659            | AvxOpcode::Vpmaxsb
1660            | AvxOpcode::Vpmaxsw
1661            | AvxOpcode::Vpmaxsd
1662            | AvxOpcode::Vpmaxub
1663            | AvxOpcode::Vpmaxuw
1664            | AvxOpcode::Vpmaxud
1665            | AvxOpcode::Vpunpcklbw
1666            | AvxOpcode::Vpunpckhbw
1667            | AvxOpcode::Vpacksswb
1668            | AvxOpcode::Vpackssdw
1669            | AvxOpcode::Vpackuswb
1670            | AvxOpcode::Vpackusdw
1671            | AvxOpcode::Vpalignr
1672            | AvxOpcode::Vpinsrb
1673            | AvxOpcode::Vpinsrw
1674            | AvxOpcode::Vpinsrd
1675            | AvxOpcode::Vpinsrq
1676            | AvxOpcode::Vpmaddwd
1677            | AvxOpcode::Vpmaddubsw
1678            | AvxOpcode::Vinsertps
1679            | AvxOpcode::Vpshufb
1680            | AvxOpcode::Vshufps
1681            | AvxOpcode::Vpsllw
1682            | AvxOpcode::Vpslld
1683            | AvxOpcode::Vpsllq
1684            | AvxOpcode::Vpsraw
1685            | AvxOpcode::Vpsrad
1686            | AvxOpcode::Vpmovsxbw
1687            | AvxOpcode::Vpmovzxbw
1688            | AvxOpcode::Vpmovsxwd
1689            | AvxOpcode::Vpmovzxwd
1690            | AvxOpcode::Vpmovsxdq
1691            | AvxOpcode::Vpmovzxdq
1692            | AvxOpcode::Vaddss
1693            | AvxOpcode::Vaddsd
1694            | AvxOpcode::Vmulss
1695            | AvxOpcode::Vmulsd
1696            | AvxOpcode::Vsubss
1697            | AvxOpcode::Vsubsd
1698            | AvxOpcode::Vdivss
1699            | AvxOpcode::Vdivsd
1700            | AvxOpcode::Vpabsb
1701            | AvxOpcode::Vpabsw
1702            | AvxOpcode::Vpabsd
1703            | AvxOpcode::Vminss
1704            | AvxOpcode::Vminsd
1705            | AvxOpcode::Vmaxss
1706            | AvxOpcode::Vmaxsd
1707            | AvxOpcode::Vsqrtps
1708            | AvxOpcode::Vsqrtpd
1709            | AvxOpcode::Vroundpd
1710            | AvxOpcode::Vroundps
1711            | AvxOpcode::Vcvtdq2pd
1712            | AvxOpcode::Vcvtdq2ps
1713            | AvxOpcode::Vcvtpd2ps
1714            | AvxOpcode::Vcvtps2pd
1715            | AvxOpcode::Vcvttpd2dq
1716            | AvxOpcode::Vcvttps2dq
1717            | AvxOpcode::Vphaddw
1718            | AvxOpcode::Vphaddd
1719            | AvxOpcode::Vpunpckldq
1720            | AvxOpcode::Vpunpckhdq
1721            | AvxOpcode::Vpunpcklqdq
1722            | AvxOpcode::Vpunpckhqdq
1723            | AvxOpcode::Vpshuflw
1724            | AvxOpcode::Vpshufhw
1725            | AvxOpcode::Vpshufd
1726            | AvxOpcode::Vmovss
1727            | AvxOpcode::Vmovsd
1728            | AvxOpcode::Vmovups
1729            | AvxOpcode::Vmovupd
1730            | AvxOpcode::Vmovdqu
1731            | AvxOpcode::Vpextrb
1732            | AvxOpcode::Vpextrw
1733            | AvxOpcode::Vpextrd
1734            | AvxOpcode::Vpextrq
1735            | AvxOpcode::Vpblendw
1736            | AvxOpcode::Vmovddup
1737            | AvxOpcode::Vbroadcastss
1738            | AvxOpcode::Vmovd
1739            | AvxOpcode::Vmovq
1740            | AvxOpcode::Vmovmskps
1741            | AvxOpcode::Vmovmskpd
1742            | AvxOpcode::Vpmovmskb
1743            | AvxOpcode::Vcvtsi2ss
1744            | AvxOpcode::Vcvtsi2sd
1745            | AvxOpcode::Vcvtss2sd
1746            | AvxOpcode::Vcvtsd2ss
1747            | AvxOpcode::Vsqrtss
1748            | AvxOpcode::Vsqrtsd
1749            | AvxOpcode::Vroundss
1750            | AvxOpcode::Vroundsd
1751            | AvxOpcode::Vunpcklpd
1752            | AvxOpcode::Vptest
1753            | AvxOpcode::Vucomiss
1754            | AvxOpcode::Vucomisd => {
1755                smallvec![InstructionSet::AVX]
1756            }
1757
1758            AvxOpcode::Vpbroadcastb | AvxOpcode::Vpbroadcastw | AvxOpcode::Vpbroadcastd => {
1759                smallvec![InstructionSet::AVX2]
1760            }
1761        }
1762    }
1763
1764    /// Is the opcode known to be commutative?
1765    ///
1766    /// Note that this method is not exhaustive, and there may be commutative
1767    /// opcodes that we don't recognize as commutative.
1768    pub(crate) fn is_commutative(&self) -> bool {
1769        match *self {
1770            AvxOpcode::Vpaddb
1771            | AvxOpcode::Vpaddw
1772            | AvxOpcode::Vpaddd
1773            | AvxOpcode::Vpaddq
1774            | AvxOpcode::Vpaddsb
1775            | AvxOpcode::Vpaddsw
1776            | AvxOpcode::Vpaddusb
1777            | AvxOpcode::Vpaddusw
1778            | AvxOpcode::Vpand
1779            | AvxOpcode::Vandps
1780            | AvxOpcode::Vandpd
1781            | AvxOpcode::Vpor
1782            | AvxOpcode::Vorps
1783            | AvxOpcode::Vorpd
1784            | AvxOpcode::Vpxor
1785            | AvxOpcode::Vxorps
1786            | AvxOpcode::Vxorpd
1787            | AvxOpcode::Vpmuldq
1788            | AvxOpcode::Vpmuludq
1789            | AvxOpcode::Vaddps
1790            | AvxOpcode::Vaddpd
1791            | AvxOpcode::Vmulps
1792            | AvxOpcode::Vmulpd
1793            | AvxOpcode::Vpcmpeqb
1794            | AvxOpcode::Vpcmpeqw
1795            | AvxOpcode::Vpcmpeqd
1796            | AvxOpcode::Vpcmpeqq
1797            | AvxOpcode::Vaddss
1798            | AvxOpcode::Vaddsd
1799            | AvxOpcode::Vmulss
1800            | AvxOpcode::Vmulsd => true,
1801            _ => false,
1802        }
1803    }
1804}
1805
1806impl fmt::Display for AvxOpcode {
1807    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1808        format!("{self:?}").to_lowercase().fmt(f)
1809    }
1810}
1811
1812#[derive(Copy, Clone, PartialEq)]
1813#[allow(missing_docs)]
1814pub enum Avx512TupleType {
1815    Full,
1816    FullMem,
1817    Mem128,
1818}
1819
1820pub use crate::isa::x64::lower::isle::generated_code::Avx512Opcode;
1821
1822impl Avx512Opcode {
1823    /// Which `InstructionSet`s support the opcode?
1824    pub(crate) fn available_from(&self) -> SmallVec<[InstructionSet; 2]> {
1825        match self {
1826            Avx512Opcode::Vcvtudq2ps
1827            | Avx512Opcode::Vpabsq
1828            | Avx512Opcode::Vpsraq
1829            | Avx512Opcode::VpsraqImm => {
1830                smallvec![InstructionSet::AVX512F, InstructionSet::AVX512VL]
1831            }
1832            Avx512Opcode::Vpermi2b => {
1833                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512VBMI]
1834            }
1835            Avx512Opcode::Vpmullq => smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512DQ],
1836            Avx512Opcode::Vpopcntb => {
1837                smallvec![InstructionSet::AVX512VL, InstructionSet::AVX512BITALG]
1838            }
1839        }
1840    }
1841
1842    /// What is the "TupleType" of this opcode, which affects the scaling factor
1843    /// for 8-bit displacements when this instruction uses memory operands.
1844    ///
1845    /// This can be found in the encoding table for each instruction and is
1846    /// interpreted according to Table 2-34 and 2-35 in the Intel instruction
1847    /// manual.
1848    pub fn tuple_type(&self) -> Avx512TupleType {
1849        use Avx512Opcode::*;
1850        use Avx512TupleType::*;
1851
1852        match self {
1853            Vcvtudq2ps | Vpabsq | Vpmullq | VpsraqImm => Full,
1854            Vpermi2b | Vpopcntb => FullMem,
1855            Vpsraq => Mem128,
1856        }
1857    }
1858}
1859
1860impl fmt::Display for Avx512Opcode {
1861    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1862        let s = format!("{self:?}");
1863        f.write_str(&s.to_lowercase())
1864    }
1865}
1866
1867/// This defines the ways a value can be extended: either signed- or zero-extension, or none for
1868/// types that are not extended. Contrast with [ExtMode], which defines the widths from and to which
1869/// values can be extended.
1870#[allow(dead_code)]
1871#[derive(Clone, PartialEq)]
1872pub enum ExtKind {
1873    /// No extension.
1874    None,
1875    /// Sign-extend.
1876    SignExtend,
1877    /// Zero-extend.
1878    ZeroExtend,
1879}
1880
1881/// These indicate ways of extending (widening) a value, using the Intel
1882/// naming: B(yte) = u8, W(ord) = u16, L(ong)word = u32, Q(uad)word = u64
1883#[derive(Clone, PartialEq)]
1884pub enum ExtMode {
1885    /// Byte -> Longword.
1886    BL,
1887    /// Byte -> Quadword.
1888    BQ,
1889    /// Word -> Longword.
1890    WL,
1891    /// Word -> Quadword.
1892    WQ,
1893    /// Longword -> Quadword.
1894    LQ,
1895}
1896
1897impl ExtMode {
1898    /// Calculate the `ExtMode` from passed bit lengths of the from/to types.
1899    pub(crate) fn new(from_bits: u16, to_bits: u16) -> Option<ExtMode> {
1900        match (from_bits, to_bits) {
1901            (1, 8) | (1, 16) | (1, 32) | (8, 16) | (8, 32) => Some(ExtMode::BL),
1902            (1, 64) | (8, 64) => Some(ExtMode::BQ),
1903            (16, 32) => Some(ExtMode::WL),
1904            (16, 64) => Some(ExtMode::WQ),
1905            (32, 64) => Some(ExtMode::LQ),
1906            _ => None,
1907        }
1908    }
1909
1910    /// Return the source register size in bytes.
1911    pub(crate) fn src_size(&self) -> u8 {
1912        match self {
1913            ExtMode::BL | ExtMode::BQ => 1,
1914            ExtMode::WL | ExtMode::WQ => 2,
1915            ExtMode::LQ => 4,
1916        }
1917    }
1918
1919    /// Return the destination register size in bytes.
1920    pub(crate) fn dst_size(&self) -> u8 {
1921        match self {
1922            ExtMode::BL | ExtMode::WL => 4,
1923            ExtMode::BQ | ExtMode::WQ | ExtMode::LQ => 8,
1924        }
1925    }
1926
1927    /// Source size, as an integer type.
1928    pub(crate) fn src_type(&self) -> Type {
1929        match self {
1930            ExtMode::BL | ExtMode::BQ => I8,
1931            ExtMode::WL | ExtMode::WQ => I16,
1932            ExtMode::LQ => I32,
1933        }
1934    }
1935}
1936
1937impl fmt::Debug for ExtMode {
1938    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1939        let name = match self {
1940            ExtMode::BL => "bl",
1941            ExtMode::BQ => "bq",
1942            ExtMode::WL => "wl",
1943            ExtMode::WQ => "wq",
1944            ExtMode::LQ => "lq",
1945        };
1946        write!(fmt, "{name}")
1947    }
1948}
1949
1950impl fmt::Display for ExtMode {
1951    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1952        fmt::Debug::fmt(self, f)
1953    }
1954}
1955
1956/// These indicate the form of a scalar shift/rotate: left, signed right, unsigned right.
1957#[derive(Clone, Copy)]
1958pub enum ShiftKind {
1959    /// Left shift.
1960    ShiftLeft,
1961    /// Inserts zeros in the most significant bits.
1962    ShiftRightLogical,
1963    /// Replicates the sign bit in the most significant bits.
1964    ShiftRightArithmetic,
1965    /// Left rotation.
1966    RotateLeft,
1967    /// Right rotation.
1968    RotateRight,
1969}
1970
1971impl fmt::Debug for ShiftKind {
1972    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
1973        let name = match self {
1974            ShiftKind::ShiftLeft => "shl",
1975            ShiftKind::ShiftRightLogical => "shr",
1976            ShiftKind::ShiftRightArithmetic => "sar",
1977            ShiftKind::RotateLeft => "rol",
1978            ShiftKind::RotateRight => "ror",
1979        };
1980        write!(fmt, "{name}")
1981    }
1982}
1983
1984impl fmt::Display for ShiftKind {
1985    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
1986        fmt::Debug::fmt(self, f)
1987    }
1988}
1989
1990/// These indicate condition code tests.  Not all are represented since not all are useful in
1991/// compiler-generated code.
1992#[derive(Copy, Clone, PartialEq, Eq)]
1993#[repr(u8)]
1994pub enum CC {
1995    ///  overflow
1996    O = 0,
1997    /// no overflow
1998    NO = 1,
1999
2000    /// < unsigned
2001    B = 2,
2002    /// >= unsigned
2003    NB = 3,
2004
2005    /// zero
2006    Z = 4,
2007    /// not-zero
2008    NZ = 5,
2009
2010    /// <= unsigned
2011    BE = 6,
2012    /// > unsigned
2013    NBE = 7,
2014
2015    /// negative
2016    S = 8,
2017    /// not-negative
2018    NS = 9,
2019
2020    /// < signed
2021    L = 12,
2022    /// >= signed
2023    NL = 13,
2024
2025    /// <= signed
2026    LE = 14,
2027    /// > signed
2028    NLE = 15,
2029
2030    /// parity
2031    P = 10,
2032
2033    /// not parity
2034    NP = 11,
2035}
2036
2037impl CC {
2038    pub(crate) fn from_intcc(intcc: IntCC) -> Self {
2039        match intcc {
2040            IntCC::Equal => CC::Z,
2041            IntCC::NotEqual => CC::NZ,
2042            IntCC::SignedGreaterThanOrEqual => CC::NL,
2043            IntCC::SignedGreaterThan => CC::NLE,
2044            IntCC::SignedLessThanOrEqual => CC::LE,
2045            IntCC::SignedLessThan => CC::L,
2046            IntCC::UnsignedGreaterThanOrEqual => CC::NB,
2047            IntCC::UnsignedGreaterThan => CC::NBE,
2048            IntCC::UnsignedLessThanOrEqual => CC::BE,
2049            IntCC::UnsignedLessThan => CC::B,
2050        }
2051    }
2052
2053    pub(crate) fn invert(&self) -> Self {
2054        match self {
2055            CC::O => CC::NO,
2056            CC::NO => CC::O,
2057
2058            CC::B => CC::NB,
2059            CC::NB => CC::B,
2060
2061            CC::Z => CC::NZ,
2062            CC::NZ => CC::Z,
2063
2064            CC::BE => CC::NBE,
2065            CC::NBE => CC::BE,
2066
2067            CC::S => CC::NS,
2068            CC::NS => CC::S,
2069
2070            CC::L => CC::NL,
2071            CC::NL => CC::L,
2072
2073            CC::LE => CC::NLE,
2074            CC::NLE => CC::LE,
2075
2076            CC::P => CC::NP,
2077            CC::NP => CC::P,
2078        }
2079    }
2080
2081    pub(crate) fn get_enc(self) -> u8 {
2082        self as u8
2083    }
2084}
2085
2086impl fmt::Debug for CC {
2087    fn fmt(&self, fmt: &mut fmt::Formatter) -> fmt::Result {
2088        let name = match self {
2089            CC::O => "o",
2090            CC::NO => "no",
2091            CC::B => "b",
2092            CC::NB => "nb",
2093            CC::Z => "z",
2094            CC::NZ => "nz",
2095            CC::BE => "be",
2096            CC::NBE => "nbe",
2097            CC::S => "s",
2098            CC::NS => "ns",
2099            CC::L => "l",
2100            CC::NL => "nl",
2101            CC::LE => "le",
2102            CC::NLE => "nle",
2103            CC::P => "p",
2104            CC::NP => "np",
2105        };
2106        write!(fmt, "{name}")
2107    }
2108}
2109
2110impl fmt::Display for CC {
2111    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
2112        fmt::Debug::fmt(self, f)
2113    }
2114}
2115
2116/// Encode the ways that floats can be compared. This is used in float comparisons such as `cmpps`,
2117/// e.g.; it is distinguished from other float comparisons (e.g. `ucomiss`) in that those use EFLAGS
2118/// whereas [FcmpImm] is used as an immediate.
2119#[derive(Clone, Copy)]
2120pub enum FcmpImm {
2121    /// Equal comparison.
2122    Equal = 0x00,
2123    /// Less than comparison.
2124    LessThan = 0x01,
2125    /// Less than or equal comparison.
2126    LessThanOrEqual = 0x02,
2127    /// Unordered.
2128    Unordered = 0x03,
2129    /// Not equal comparison.
2130    NotEqual = 0x04,
2131    /// Unordered of greater than or equal comparison.
2132    UnorderedOrGreaterThanOrEqual = 0x05,
2133    /// Unordered or greater than comparison.
2134    UnorderedOrGreaterThan = 0x06,
2135    /// Ordered.
2136    Ordered = 0x07,
2137}
2138
2139impl FcmpImm {
2140    pub(crate) fn encode(self) -> u8 {
2141        self as u8
2142    }
2143}
2144
2145impl From<FloatCC> for FcmpImm {
2146    fn from(cond: FloatCC) -> Self {
2147        match cond {
2148            FloatCC::Equal => FcmpImm::Equal,
2149            FloatCC::LessThan => FcmpImm::LessThan,
2150            FloatCC::LessThanOrEqual => FcmpImm::LessThanOrEqual,
2151            FloatCC::Unordered => FcmpImm::Unordered,
2152            FloatCC::NotEqual => FcmpImm::NotEqual,
2153            FloatCC::UnorderedOrGreaterThanOrEqual => FcmpImm::UnorderedOrGreaterThanOrEqual,
2154            FloatCC::UnorderedOrGreaterThan => FcmpImm::UnorderedOrGreaterThan,
2155            FloatCC::Ordered => FcmpImm::Ordered,
2156            _ => panic!("unable to create comparison predicate for {cond}"),
2157        }
2158    }
2159}
2160
2161/// Encode the rounding modes used as part of the Rounding Control field.
2162/// Note, these rounding immediates only consider the rounding control field
2163/// (i.e. the rounding mode) which only take up the first two bits when encoded.
2164/// However the rounding immediate which this field helps make up, also includes
2165/// bits 3 and 4 which define the rounding select and precision mask respectively.
2166/// These two bits are not defined here and are implicitly set to zero when encoded.
2167#[derive(Clone, Copy)]
2168pub enum RoundImm {
2169    /// Round to nearest mode.
2170    RoundNearest = 0x00,
2171    /// Round down mode.
2172    RoundDown = 0x01,
2173    /// Round up mode.
2174    RoundUp = 0x02,
2175    /// Round to zero mode.
2176    RoundZero = 0x03,
2177}
2178
2179impl RoundImm {
2180    pub(crate) fn encode(self) -> u8 {
2181        self as u8
2182    }
2183}
2184
2185/// An operand's size in bits.
2186#[derive(Clone, Copy, PartialEq)]
2187pub enum OperandSize {
2188    /// 8-bit.
2189    Size8,
2190    /// 16-bit.
2191    Size16,
2192    /// 32-bit.
2193    Size32,
2194    /// 64-bit.
2195    Size64,
2196}
2197
2198impl OperandSize {
2199    pub(crate) fn from_bytes(num_bytes: u32) -> Self {
2200        match num_bytes {
2201            1 => OperandSize::Size8,
2202            2 => OperandSize::Size16,
2203            4 => OperandSize::Size32,
2204            8 => OperandSize::Size64,
2205            _ => unreachable!("Invalid OperandSize: {}", num_bytes),
2206        }
2207    }
2208
2209    // Computes the OperandSize for a given type.
2210    // For vectors, the OperandSize of the lanes is returned.
2211    pub(crate) fn from_ty(ty: Type) -> Self {
2212        Self::from_bytes(ty.lane_type().bytes())
2213    }
2214
2215    // Check that the value of self is one of the allowed sizes.
2216    pub(crate) fn is_one_of(&self, sizes: &[Self]) -> bool {
2217        sizes.iter().any(|val| *self == *val)
2218    }
2219
2220    pub(crate) fn to_bytes(&self) -> u8 {
2221        match self {
2222            Self::Size8 => 1,
2223            Self::Size16 => 2,
2224            Self::Size32 => 4,
2225            Self::Size64 => 8,
2226        }
2227    }
2228
2229    pub(crate) fn to_bits(&self) -> u8 {
2230        self.to_bytes() * 8
2231    }
2232
2233    pub(crate) fn to_type(&self) -> Type {
2234        match self {
2235            Self::Size8 => I8,
2236            Self::Size16 => I16,
2237            Self::Size32 => I32,
2238            Self::Size64 => I64,
2239        }
2240    }
2241}
2242
2243/// An x64 memory fence kind.
2244#[derive(Clone)]
2245#[allow(dead_code)]
2246pub enum FenceKind {
2247    /// `mfence` instruction ("Memory Fence")
2248    MFence,
2249    /// `lfence` instruction ("Load Fence")
2250    LFence,
2251    /// `sfence` instruction ("Store Fence")
2252    SFence,
2253}
cranelift_codegen/isa/x64/inst/args.rs

cranelift_codegen/isa/x64/inst/
args.rs
