Skip to main content

zydis_rs/encoder/
mod.rs

1//! x86/x64 指令编码器模块 / x86/x64 instruction encoder module
2//!
3//! 本模块提供将结构化表示转换为 x86/x64 指令机器码的功能。
4//!
5//! This module provides functionality for encoding x86/x64 instructions
6//! from a structured representation into raw bytes.
7//!
8//! ## 支持的指令编码 / Supported Instruction Encodings
9//!
10//! - Legacy encoding(传统编码)- 基础整数指令
11//! - VEX encoding(AVX/AVX2)- 128/256 位向量指令
12//! - EVEX encoding(AVX-512)- 512 位向量指令,掩码寄存器
13//! - XOP encoding(AMD XOP)- AMD 扩展操作指令
14//! - REX prefix(64 位扩展)
15//!
16//! ## 使用流程 / Usage Flow
17//!
18//! 1. 创建 `Encoder` 实例
19//! 2. 创建 `EncoderRequest` 并添加操作数
20//! 3. 调用 `encode()` 生成机器码
21//!
22//! ## 示例 / Example
23//!
24//! ```rust,ignore
25//! use zydis_rs::{Encoder, EncoderRequest, Mnemonic, Register, MachineMode};
26//!
27//! let encoder = Encoder::new(MachineMode::Long64, 64)?;
28//!
29//! // 编码 MOV RAX, RBX
30//! let request = EncoderRequest::new(Mnemonic::MOV)
31//!     .with_reg(Register::RAX)
32//!     .with_reg(Register::RBX);
33//! let bytes = encoder.encode(&request)?;
34//!
35//! // 编码 AVX-512 指令带掩码
36//! let request = EncoderRequest::new(Mnemonic::VADDPS)
37//!     .with_reg(Register::ZMM1)
38//!     .with_reg(Register::ZMM2)
39//!     .with_reg(Register::ZMM3)
40//!     .with_mask(Register::K1)
41//!     .with_zeroing_enabled();
42//! let bytes = encoder.encode(&request)?;
43//! # Ok::<(), zydis_rs::Error>(())
44//! ```
45
46pub mod evex;
47pub mod instruction_table;
48pub mod opcode;
49pub mod operand;
50pub mod request;
51pub mod types;
52pub mod utils;
53pub mod validation;
54pub mod vex;
55pub mod xop;
56
57// Re-exports
58pub use evex::{
59    EvexBroadcast, EvexInfo, EvexOpcodeMap, EvexPrefix, EvexVectorLength, EvexZeroingMode,
60};
61pub use instruction_table::{
62    EncodingPrefix, InstructionEncoding, OperandEncoding, OperandType, lookup_encoding,
63};
64pub use operand::{EncoderOperand, MemoryOperand};
65pub use request::EncoderRequest;
66pub use types::{
67    AddressSizeHint, BranchType, BranchWidth, BroadcastMode, EncodableEncoding, EncoderHints,
68    EncoderMode, OperandSizeHint, RoundingMode,
69};
70pub use validation::{EncoderValidator, ValidationError, ValidationErrorKind, ValidationResult};
71pub use vex::{VexInfo, VexOpcodeMap, VexPrefix, VexVectorLength};
72pub use xop::{XopEncoder, XopInfo, XopOpcodeMap, XopPrefix, XopVectorLength};
73
74// Legacy types for backward compatibility
75#[doc(hidden)]
76pub use types::BranchType as ZydisBranchType;
77#[doc(hidden)]
78pub use types::BranchWidth as ZydisBranchWidth;
79#[doc(hidden)]
80pub use types::EncodableEncoding as ZydisEncodableEncoding;
81
82use crate::error::{EncodeError, Error, Result};
83use crate::isa::{MachineMode, Mnemonic, Register};
84
85// Import utility functions
86use opcode::{AvxOpcodeMap, get_avx_opcode, get_mov_imm_opcode, get_mov_imm8_opcode};
87use utils::{
88    displacement_size, encode_modrm, encode_rex, encode_sib, needs_rex_b as check_rex_b,
89    needs_rex_for_8bit, needs_rex_r as check_rex_r, register_id, segment_override_prefix,
90};
91
92/// Get the ALU opcode extension for a mnemonic (used with 0x83 /n)
93fn get_alu_opcode_extension(mnemonic: Mnemonic) -> Option<u8> {
94    match mnemonic {
95        Mnemonic::ADD => Some(0),
96        Mnemonic::OR => Some(1),
97        Mnemonic::AND => Some(4),
98        Mnemonic::SUB => Some(5),
99        Mnemonic::XOR => Some(6),
100        Mnemonic::CMP => Some(7),
101        _ => None,
102    }
103}
104
105/// Get the ALU reg-reg opcode for "r/m, r" form
106fn get_alu_reg_reg_opcode(mnemonic: Mnemonic) -> Option<u8> {
107    match mnemonic {
108        Mnemonic::ADD => Some(0x01), // ADD r/m, r
109        Mnemonic::SUB => Some(0x29), // SUB r/m, r
110        Mnemonic::AND => Some(0x21), // AND r/m, r
111        Mnemonic::OR => Some(0x09),  // OR r/m, r
112        Mnemonic::XOR => Some(0x31), // XOR r/m, r
113        Mnemonic::CMP => Some(0x39), // CMP r/m, r
114        _ => None,
115    }
116}
117
118/// Check if a mnemonic is an AVX instruction
119fn is_avx_instruction(mnemonic: Mnemonic) -> bool {
120    matches!(
121        mnemonic,
122        Mnemonic::VMOVAPS
123            | Mnemonic::VMOVAPD
124            | Mnemonic::VMOVUPS
125            | Mnemonic::VMOVUPD
126            | Mnemonic::VADDPS
127            | Mnemonic::VADDPD
128            | Mnemonic::VSUBPS
129            | Mnemonic::VSUBPD
130            | Mnemonic::VMULPS
131            | Mnemonic::VMULPD
132            | Mnemonic::VDIVPS
133            | Mnemonic::VDIVPD
134            | Mnemonic::VXORPS
135            | Mnemonic::VXORPD
136            | Mnemonic::VSHUFPS
137            | Mnemonic::VSHUFPD
138            | Mnemonic::VPERM2I128
139            | Mnemonic::VPERM2F128
140            | Mnemonic::VPERMILPS
141            | Mnemonic::VPERMILPD
142            | Mnemonic::VPERMD
143            | Mnemonic::VPERMQ
144            | Mnemonic::VPERMPS
145            | Mnemonic::VPERMPD
146            | Mnemonic::VPSHUFB
147            | Mnemonic::VPSHUFD
148            | Mnemonic::VPSHUFHW
149            | Mnemonic::VPSHUFLW
150            | Mnemonic::VBROADCASTSS
151            | Mnemonic::VEXTRACTF128
152    )
153}
154
155/// Check if a mnemonic is an AVX instruction with imm8 operand
156fn is_avx_instruction_with_imm8(mnemonic: Mnemonic) -> bool {
157    matches!(
158        mnemonic,
159        Mnemonic::VSHUFPS
160            | Mnemonic::VSHUFPD
161            | Mnemonic::VPERM2I128
162            | Mnemonic::VPERM2F128
163            | Mnemonic::VPERMILPS
164            | Mnemonic::VPERMILPD
165            | Mnemonic::VPERMQ
166            | Mnemonic::VPERMPD
167            | Mnemonic::VPSHUFD
168            | Mnemonic::VPSHUFHW
169            | Mnemonic::VPSHUFLW
170            | Mnemonic::VEXTRACTF128
171    )
172}
173
174/// Check if a mnemonic is an AVX-512 capable instruction
175/// These instructions can use EVEX prefix with ZMM registers
176fn is_avx512_capable_instruction(mnemonic: Mnemonic) -> bool {
177    matches!(
178        mnemonic,
179        Mnemonic::VMOVAPS
180            | Mnemonic::VMOVAPD
181            | Mnemonic::VMOVUPS
182            | Mnemonic::VMOVUPD
183            | Mnemonic::VADDPS
184            | Mnemonic::VADDPD
185            | Mnemonic::VSUBPS
186            | Mnemonic::VSUBPD
187            | Mnemonic::VMULPS
188            | Mnemonic::VMULPD
189            | Mnemonic::VDIVPS
190            | Mnemonic::VDIVPD
191            | Mnemonic::VXORPS
192            | Mnemonic::VXORPD
193    )
194}
195
196/// x86/x64 指令编码器
197///
198/// The `Encoder` struct is responsible for converting instruction requests
199/// into raw bytes that can be executed by the CPU.
200///
201/// 编码器负责将指令请求转换为可执行的机器码。
202///
203/// # 支持的指令 / Supported Instructions
204///
205/// - 整数运算:ADD, SUB, AND, OR, XOR, CMP, TEST
206/// - 数据传送:MOV, LEA, MOVSX, MOVZX
207/// - 跳转指令:JMP, JE, JNE, JL, JG, JLE, JGE, CALL, RET
208/// - 栈操作:PUSH, POP
209/// - AVX 指令:VMOVAPS, VADDPS, VSUBPS, VMULPS, VDIVPS, VXORPS 等
210/// - AVX-512 指令:VPADDD, VPSUBD, VPANDD, VPORD 等
211///
212/// # 示例 / Example
213///
214/// ```rust,ignore
215/// use zydis_rs::{Encoder, EncoderRequest, Mnemonic, Register, MachineMode};
216///
217/// let encoder = Encoder::new(MachineMode::Long64, 64)?;
218///
219/// // 编码 MOV RAX, 0x123456789ABCDEF0
220/// let request = EncoderRequest::new(Mnemonic::MOV)
221///     .with_reg(Register::RAX)
222///     .with_imm(0x123456789ABCDEF0u64);
223/// let bytes = encoder.encode(&request)?;
224///
225/// // 编码 ADD RAX, RBX
226/// let request2 = EncoderRequest::new(Mnemonic::ADD)
227///     .with_reg(Register::RAX)
228///     .with_reg(Register::RBX);
229/// let bytes2 = encoder.encode(&request2)?;
230/// # Ok::<(), zydis_rs::Error>(())
231/// ```
232///
233/// # 注意事项 / Notes
234///
235/// - 单条指令最大长度为 15 字节
236/// - 64 位模式下某些指令需要 REX 前缀
237/// - AVX 指令使用 VEX 前缀
238/// - AVX-512 指令使用 EVEX 前缀
239#[derive(Debug, Clone, Copy)]
240pub struct Encoder {
241    /// The machine mode (16-bit, 32-bit, or 64-bit) / 机器模式
242    machine_mode: MachineMode,
243    /// The stack width in bits / 栈宽度(位)
244    stack_width: u8,
245}
246
247impl Encoder {
248    /// 创建指定机器模式的编码器
249    ///
250    /// Create a new encoder with the specified machine mode and stack width.
251    ///
252    /// # 参数 / Arguments
253    ///
254    /// * `mode` - 机器模式 / The machine mode (Long64, Protected32, etc.)
255    /// * `stack_width` - 栈宽度(16, 32 或 64 位)/ The stack width in bits (16, 32, or 64)
256    ///
257    /// # 返回 / Returns
258    ///
259    /// 返回编码器实例或错误
260    ///
261    /// # 错误 / Errors
262    ///
263    /// - `Error::InvalidStackWidth` - 如果栈宽度不是 16、32 或 64
264    ///
265    /// # 示例 / Example
266    ///
267    /// ```rust,ignore
268    /// use zydis_rs::{Encoder, MachineMode};
269    ///
270    /// let encoder = Encoder::new(MachineMode::Long64, 64)?;
271    /// # Ok::<(), zydis_rs::Error>(())
272    /// ```
273    pub fn new(mode: MachineMode, stack_width: u8) -> Result<Self> {
274        // Validate stack width
275        match stack_width {
276            16 | 32 | 64 => {}
277            _ => return Err(Error::InvalidStackWidth),
278        }
279
280        Ok(Self {
281            machine_mode: mode,
282            stack_width,
283        })
284    }
285
286    /// 获取机器模式 / Get the machine mode
287    #[must_use]
288    pub const fn machine_mode(&self) -> MachineMode {
289        self.machine_mode
290    }
291
292    /// 获取栈宽度 / Get the stack width
293    #[must_use]
294    pub const fn stack_width(&self) -> u8 {
295        self.stack_width
296    }
297
298    /// 编码指令请求为字节序列
299    ///
300    /// Encode an instruction request to bytes.
301    ///
302    /// # 参数 / Arguments
303    ///
304    /// * `request` - 要编码的指令请求 / The instruction request to encode
305    ///
306    /// # 返回 / Returns
307    ///
308    /// 包含编码后指令字节的向量
309    ///
310    /// # 错误 / Errors
311    ///
312    /// - `Error::UnsupportedInstruction` - 不支持的指令
313    /// - `Error::InvalidOperand` - 无效的操作数
314    /// - `Error::EncodeError` - 编码过程中的其他错误
315    ///
316    /// # 示例 / Example
317    ///
318    /// ```rust,ignore
319    /// use zydis_rs::{Encoder, MachineMode, EncoderRequest, Mnemonic, Register};
320    ///
321    /// let encoder = Encoder::new(MachineMode::Long64, 64)?;
322    /// let request = EncoderRequest::new(Mnemonic::MOV)
323    ///     .with_reg(Register::RAX)
324    ///     .with_reg(Register::RBX);
325    ///
326    /// let bytes = encoder.encode(&request)?;
327    /// # Ok::<(), zydis_rs::Error>(())
328    /// ```
329    pub fn encode(&self, request: &EncoderRequest) -> Result<alloc::vec::Vec<u8>> {
330        let mut buffer = [0u8; 15]; // Maximum instruction length is 15 bytes
331        let len = self.encode_to_buffer(request, &mut buffer)?;
332        Ok(buffer[..len].to_vec())
333    }
334
335    /// 编码指令到预分配的缓冲区
336    ///
337    /// Encode an instruction to a pre-allocated buffer.
338    ///
339    /// # 参数 / Arguments
340    ///
341    /// * `request` - 要编码的指令请求 / The instruction request to encode
342    /// * `buffer` - 写入编码字节的缓冲区(至少 15 字节)/ The buffer to write the encoded bytes to
343    ///
344    /// # 返回 / Returns
345    ///
346    /// 写入缓冲区的字节数
347    ///
348    /// # 错误 / Errors
349    ///
350    /// - `Error::BufferTooSmall` - 缓冲区太小(< 15 字节)
351    /// - `Error::UnsupportedInstruction` - 不支持的指令
352    /// - `Error::InvalidOperand` - 无效的操作数
353    ///
354    /// # 示例 / Example
355    ///
356    /// ```rust,ignore
357    /// use zydis_rs::{Encoder, EncoderRequest, Mnemonic, Register, MachineMode};
358    ///
359    /// let encoder = Encoder::new(MachineMode::Long64, 64)?;
360    /// let request = EncoderRequest::new(Mnemonic::NOP);
361    ///
362    /// let mut buffer = [0u8; 15];
363    /// let len = encoder.encode_to_buffer(&request, &mut buffer)?;
364    /// assert_eq!(len, 1);  // NOP is 0x90
365    /// assert_eq!(buffer[0], 0x90);
366    /// # Ok::<(), zydis_rs::Error>(())
367    /// ```
368    pub fn encode_to_buffer(&self, request: &EncoderRequest, buffer: &mut [u8]) -> Result<usize> {
369        // Validate buffer size (max instruction length is 15 bytes)
370        if buffer.len() < 15 {
371            return Err(Error::BufferTooSmall);
372        }
373
374        let offset = 0usize;
375
376        // Handle special cases first
377        match request.mnemonic {
378            Mnemonic::NOP => {
379                // NOP is simple: just 0x90
380                buffer[offset] = 0x90;
381                return Ok(1);
382            }
383            Mnemonic::RET => {
384                // Near return: 0xC3
385                buffer[offset] = 0xC3;
386                return Ok(1);
387            }
388            Mnemonic::INT3 => {
389                // INT3: 0xCC
390                buffer[offset] = 0xCC;
391                return Ok(1);
392            }
393            Mnemonic::HLT => {
394                // HLT: 0xF4
395                buffer[offset] = 0xF4;
396                return Ok(1);
397            }
398            Mnemonic::CLC => {
399                buffer[offset] = 0xF8;
400                return Ok(1);
401            }
402            Mnemonic::STC => {
403                buffer[offset] = 0xF9;
404                return Ok(1);
405            }
406            Mnemonic::CLI => {
407                buffer[offset] = 0xFA;
408                return Ok(1);
409            }
410            Mnemonic::STI => {
411                buffer[offset] = 0xFB;
412                return Ok(1);
413            }
414            Mnemonic::CLD => {
415                buffer[offset] = 0xFC;
416                return Ok(1);
417            }
418            Mnemonic::STD => {
419                buffer[offset] = 0xFD;
420                return Ok(1);
421            }
422            _ => {}
423        }
424
425        // Handle MOV reg, imm
426        if request.mnemonic == Mnemonic::MOV && request.operand_count == 2 {
427            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Imm(imm))) =
428                (request.operand(0), request.operand(1))
429            {
430                return self.encode_mov_reg_imm(*dst_reg, *imm, buffer);
431            }
432        }
433
434        // Handle reg-reg MOV
435        if request.mnemonic == Mnemonic::MOV && request.operand_count == 2 {
436            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Reg(src_reg))) =
437                (request.operand(0), request.operand(1))
438            {
439                return self.encode_mov_reg_reg(*dst_reg, *src_reg, buffer);
440            }
441        }
442
443        // Handle simple jumps
444        if request.mnemonic == Mnemonic::JMP && request.operand_count == 1 {
445            if let Some(EncoderOperand::Rel(offset)) = request.operand(0) {
446                return self.encode_jmp_rel(*offset, buffer);
447            }
448        }
449
450        // Handle conditional jumps
451        if is_conditional_jump(request.mnemonic) && request.operand_count == 1 {
452            if let Some(EncoderOperand::Rel(offset)) = request.operand(0) {
453                return self.encode_jcc_rel(request.mnemonic, *offset, buffer);
454            }
455        }
456
457        // Handle LEA
458        if request.mnemonic == Mnemonic::LEA && request.operand_count == 2 {
459            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Mem(mem))) =
460                (request.operand(0), request.operand(1))
461            {
462                return self.encode_lea(*dst_reg, mem, buffer);
463            }
464        }
465
466        // Handle ALU reg, imm8 (sign-extended): ADD/SUB/CMP/AND/OR/XOR reg, imm
467        if get_alu_opcode_extension(request.mnemonic).is_some() && request.operand_count == 2 {
468            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Imm(imm))) =
469                (request.operand(0), request.operand(1))
470            {
471                return self.encode_alu_reg_imm(*dst_reg, *imm, request.mnemonic, buffer);
472            }
473        }
474
475        // Handle ALU reg, reg: ADD/SUB/CMP/AND/OR/XOR reg, reg
476        if get_alu_reg_reg_opcode(request.mnemonic).is_some() && request.operand_count == 2 {
477            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Reg(src_reg))) =
478                (request.operand(0), request.operand(1))
479            {
480                return self.encode_alu_reg_reg(*dst_reg, *src_reg, request.mnemonic, buffer);
481            }
482        }
483
484        // Handle PUSH reg (50+rd)
485        if request.mnemonic == Mnemonic::PUSH && request.operand_count == 1 {
486            if let Some(EncoderOperand::Reg(reg)) = request.operand(0) {
487                return self.encode_push_pop_reg(*reg, true, buffer);
488            }
489        }
490
491        // Handle POP reg (58+rd)
492        if request.mnemonic == Mnemonic::POP && request.operand_count == 1 {
493            if let Some(EncoderOperand::Reg(reg)) = request.operand(0) {
494                return self.encode_push_pop_reg(*reg, false, buffer);
495            }
496        }
497
498        // Handle shift instructions with CL: SHL/SHR/SAR reg, CL
499        if matches!(
500            request.mnemonic,
501            Mnemonic::SHL | Mnemonic::SHR | Mnemonic::SAR | Mnemonic::SAL
502        ) && request.operand_count == 2
503        {
504            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Reg(src_reg))) =
505                (request.operand(0), request.operand(1))
506            {
507                // Check if src is CL
508                if *src_reg == Register::CL {
509                    return self.encode_shift_cl(*dst_reg, request.mnemonic, buffer);
510                }
511            }
512        }
513
514        // Handle AVX-512 instructions (EVEX prefix)
515        // Check if this is an AVX-512 capable instruction with ZMM registers or mask/zeroing
516        if is_avx512_capable_instruction(request.mnemonic) && request.operand_count == 3 {
517            let has_zmm = request.has_zmm_operand();
518            let has_mask = request.has_mask();
519            let is_zeroing = request.is_zeroing();
520
521            // Use EVEX if we have ZMM registers, mask register, or zeroing mode
522            if has_zmm || has_mask || is_zeroing {
523                if let (
524                    Some(EncoderOperand::Reg(dst_reg)),
525                    Some(EncoderOperand::Reg(src1_reg)),
526                    Some(EncoderOperand::Reg(src2_reg)),
527                ) = (request.operand(0), request.operand(1), request.operand(2))
528                {
529                    // AVX-512 reg, reg, reg form
530                    return self.encode_avx512_reg_reg(
531                        *dst_reg,
532                        *src1_reg,
533                        *src2_reg,
534                        request.mnemonic,
535                        request.mask_reg,
536                        request.zeroing_mode,
537                        buffer,
538                    );
539                }
540
541                if let (
542                    Some(EncoderOperand::Reg(dst_reg)),
543                    Some(EncoderOperand::Reg(src1_reg)),
544                    Some(EncoderOperand::Mem(mem)),
545                ) = (request.operand(0), request.operand(1), request.operand(2))
546                {
547                    // AVX-512 reg, reg, mem form
548                    return self.encode_avx512_reg_mem(
549                        *dst_reg,
550                        *src1_reg,
551                        mem,
552                        request.mnemonic,
553                        request.mask_reg,
554                        request.zeroing_mode,
555                        buffer,
556                    );
557                }
558            }
559        }
560
561        // Handle AVX instructions: VADDPS/VADDPD/VSUBPS/VSUBPD/VMULPS/VMULPD/VXORPS/VXORPD
562        // Format: VEX.128.0F.WIG opcode /r
563        // dst, src1, src2 (reg, reg, reg) or (reg, reg, mem)
564        if is_avx_instruction(request.mnemonic) && request.operand_count == 3 {
565            if let (
566                Some(EncoderOperand::Reg(dst_reg)),
567                Some(EncoderOperand::Reg(src1_reg)),
568                Some(EncoderOperand::Reg(src2_reg)),
569            ) = (request.operand(0), request.operand(1), request.operand(2))
570            {
571                // reg, reg, reg form
572                return self.encode_avx_reg_reg(
573                    *dst_reg,
574                    *src1_reg,
575                    *src2_reg,
576                    request.mnemonic,
577                    buffer,
578                );
579            }
580
581            if let (
582                Some(EncoderOperand::Reg(dst_reg)),
583                Some(EncoderOperand::Reg(src1_reg)),
584                Some(EncoderOperand::Mem(mem)),
585            ) = (request.operand(0), request.operand(1), request.operand(2))
586            {
587                // reg, reg, mem form
588                return self.encode_avx_reg_mem(*dst_reg, *src1_reg, mem, request.mnemonic, buffer);
589            }
590        }
591
592        // Handle AVX instructions with 4 operands (reg, reg, reg/mem, imm8):
593        // VSHUFPS, VSHUFPD, VPERM2I128
594        if is_avx_instruction_with_imm8(request.mnemonic) && request.operand_count == 4 {
595            if let (
596                Some(EncoderOperand::Reg(dst_reg)),
597                Some(EncoderOperand::Reg(src1_reg)),
598                Some(EncoderOperand::Reg(src2_reg)),
599                Some(EncoderOperand::Imm(imm8)),
600            ) = (
601                request.operand(0),
602                request.operand(1),
603                request.operand(2),
604                request.operand(3),
605            ) {
606                // reg, reg, reg, imm8 form
607                return self.encode_avx_reg_reg_imm8(
608                    *dst_reg,
609                    *src1_reg,
610                    *src2_reg,
611                    *imm8 as u8,
612                    request.mnemonic,
613                    buffer,
614                );
615            }
616
617            if let (
618                Some(EncoderOperand::Reg(dst_reg)),
619                Some(EncoderOperand::Reg(src1_reg)),
620                Some(EncoderOperand::Mem(mem)),
621                Some(EncoderOperand::Imm(imm8)),
622            ) = (
623                request.operand(0),
624                request.operand(1),
625                request.operand(2),
626                request.operand(3),
627            ) {
628                // reg, reg, mem, imm8 form
629                return self.encode_avx_reg_mem_imm8(
630                    *dst_reg,
631                    *src1_reg,
632                    mem,
633                    *imm8 as u8,
634                    request.mnemonic,
635                    buffer,
636                );
637            }
638        }
639
640        // Handle VBROADCASTSS: dst, mem (2 operands)
641        // VEX.128.0F38.WIG 18 /r
642        if request.mnemonic == Mnemonic::VBROADCASTSS && request.operand_count == 2 {
643            if let (Some(EncoderOperand::Reg(dst_reg)), Some(EncoderOperand::Mem(mem))) =
644                (request.operand(0), request.operand(1))
645            {
646                return self.encode_avx_broadcast_mem(*dst_reg, mem, request.mnemonic, buffer);
647            }
648        }
649
650        // Handle VEXTRACTF128: mem/xmm, ymm, imm8 (3 operands)
651        // VEX.256.66.0F3A.WIG 19 /r ib
652        if request.mnemonic == Mnemonic::VEXTRACTF128 && request.operand_count == 3 {
653            if let (
654                Some(EncoderOperand::Reg(dst_reg)),
655                Some(EncoderOperand::Reg(src_reg)),
656                Some(EncoderOperand::Imm(imm8)),
657            ) = (request.operand(0), request.operand(1), request.operand(2))
658            {
659                // xmm/mem, ymm, imm8 form (here: reg, reg, imm8)
660                return self.encode_avx_extractf128(*dst_reg, *src_reg, *imm8 as u8, buffer);
661            }
662        }
663
664        // Not yet implemented
665        Err(Error::NotYetImplemented(
666            alloc::format!("encoding for {:?}", request.mnemonic).leak(),
667        ))
668    }
669
670    /// Encode MOV reg, imm
671    fn encode_mov_reg_imm(&self, reg: Register, imm: u64, buffer: &mut [u8]) -> Result<usize> {
672        let mut offset = 0usize;
673        let reg_size = reg.size();
674
675        // Check for 8-bit register with 8-bit immediate
676        if reg_size == 8 {
677            let needs_rex = needs_rex_for_8bit(reg);
678
679            if needs_rex {
680                buffer[offset] = encode_rex(false, false, false, check_rex_b(reg));
681                offset += 1;
682            }
683
684            buffer[offset] = get_mov_imm8_opcode(register_id(reg));
685            offset += 1;
686            buffer[offset] = imm as u8;
687            offset += 1;
688
689            return Ok(offset);
690        }
691
692        // For 16/32/64-bit registers
693        // Determine if we need REX.W for 64-bit
694        let needs_rex_w = reg_size == 64 && self.machine_mode == MachineMode::Long64;
695
696        // Determine if we need 66h prefix for 16-bit
697        let needs_66 = reg_size == 16;
698
699        // Determine if we need REX.B for extended registers
700        let needs_rex_b = check_rex_b(reg);
701
702        // Emit prefix
703        if needs_66 {
704            buffer[offset] = 0x66;
705            offset += 1;
706        }
707
708        // Emit REX prefix if needed
709        if needs_rex_w || needs_rex_b {
710            buffer[offset] = encode_rex(needs_rex_w, false, false, needs_rex_b);
711            offset += 1;
712        }
713
714        // Emit opcode
715        buffer[offset] = get_mov_imm_opcode(register_id(reg), reg_size == 64);
716        offset += 1;
717
718        // Emit immediate
719        let imm_size = if reg_size == 64 {
720            8
721        } else {
722            (reg_size / 8) as usize
723        };
724        for i in 0..imm_size {
725            buffer[offset + i] = ((imm >> (i * 8)) & 0xFF) as u8;
726        }
727        offset += imm_size;
728
729        Ok(offset)
730    }
731
732    /// Encode MOV reg, reg
733    fn encode_mov_reg_reg(&self, dst: Register, src: Register, buffer: &mut [u8]) -> Result<usize> {
734        let mut offset = 0usize;
735        let op_size = dst.size().max(src.size());
736
737        // Validate operands have same size
738        if dst.size() != src.size() {
739            return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
740        }
741
742        // Prefixes
743        let needs_66 = op_size == 16;
744        let needs_rex_w = op_size == 64 && self.machine_mode == MachineMode::Long64;
745        let rex_r = check_rex_r(src);
746        let rex_b = check_rex_b(dst);
747
748        // For 8-bit registers that need REX (SPL, BPL, SIL, DIL, R8B-R15B)
749        let rex_for_8bit = op_size == 8 && (needs_rex_for_8bit(dst) || needs_rex_for_8bit(src));
750
751        // Emit 66h prefix for 16-bit
752        if needs_66 {
753            buffer[offset] = 0x66;
754            offset += 1;
755        }
756
757        // Emit REX prefix
758        if needs_rex_w || rex_r || rex_b || rex_for_8bit {
759            buffer[offset] = encode_rex(needs_rex_w, rex_r, false, rex_b);
760            offset += 1;
761        }
762
763        // Opcode: MOV r64, r/m64 is 8B /r
764        buffer[offset] = 0x8B;
765        offset += 1;
766
767        // ModRM: mod=11 (register), reg=src, rm=dst
768        buffer[offset] = encode_modrm(3, register_id(src), register_id(dst));
769        offset += 1;
770
771        Ok(offset)
772    }
773
774    /// Encode JMP rel
775    fn encode_jmp_rel(&self, offset: i32, buffer: &mut [u8]) -> Result<usize> {
776        let mut len = 0usize;
777
778        // Use short jump if possible (-128 to +127)
779        if (-128..=127).contains(&offset) {
780            buffer[len] = 0xEB; // JMP rel8
781            len += 1;
782            buffer[len] = offset as i8 as u8;
783            len += 1;
784        } else {
785            buffer[len] = 0xE9; // JMP rel32
786            len += 1;
787            buffer[len..len + 4].copy_from_slice(&offset.to_le_bytes());
788            len += 4;
789        }
790
791        Ok(len)
792    }
793
794    /// Encode conditional jump (Jcc)
795    fn encode_jcc_rel(&self, mnemonic: Mnemonic, offset: i32, buffer: &mut [u8]) -> Result<usize> {
796        let condition = get_jcc_condition(mnemonic);
797        let mut len = 0usize;
798
799        // Use short jump if possible (-128 to +127)
800        if (-128..=127).contains(&offset) {
801            buffer[len] = 0x70 + condition; // Jcc rel8
802            len += 1;
803            buffer[len] = offset as i8 as u8;
804            len += 1;
805        } else {
806            buffer[len] = 0x0F; // Two-byte Jcc
807            len += 1;
808            buffer[len] = 0x80 + condition;
809            len += 1;
810            buffer[len..len + 4].copy_from_slice(&offset.to_le_bytes());
811            len += 4;
812        }
813
814        Ok(len)
815    }
816
817    /// Encode ALU reg, imm8 (sign-extended): ADD/SUB/CMP/AND/OR/XOR reg, imm
818    /// Uses opcode 83 /n ib
819    fn encode_alu_reg_imm(
820        &self,
821        reg: Register,
822        imm: u64,
823        mnemonic: Mnemonic,
824        buffer: &mut [u8],
825    ) -> Result<usize> {
826        let mut offset = 0usize;
827        let reg_size = reg.size();
828
829        // Get the opcode extension for this ALU operation
830        let opcode_ext = get_alu_opcode_extension(mnemonic)
831            .ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
832
833        // Check if immediate fits in signed 8-bit
834        // First truncate to u8, then check if it can be sign-extended properly
835        // For the 83 /n ib form, the immediate is sign-extended to operand size
836        // So we need to check if the value fits in a signed byte when interpreted as such
837        let imm8 = imm as u8;
838        let signed_imm = imm8 as i8 as i64;
839        // The original value should be representable as a sign-extended imm8
840        // This means imm should equal the sign-extended value when truncated to operand size
841        // For simplicity, just check if imm's low byte gives the right sign extension
842        if imm > 255 && imm != (signed_imm as u64) {
843            // Value is too large and cannot be represented by sign-extending imm8
844            return Err(Error::EncodeFailed(EncodeError::ImmediateOutOfRange));
845        }
846
847        // For 8-bit registers, we would need a different encoding (80 /n ib)
848        if reg_size == 8 {
849            // 80 /n ib for 8-bit operands
850            let needs_rex = needs_rex_for_8bit(reg);
851
852            if needs_rex {
853                buffer[offset] = encode_rex(false, false, false, check_rex_b(reg));
854                offset += 1;
855            }
856
857            buffer[offset] = 0x80; // 8-bit operand, no sign extension needed
858            offset += 1;
859            buffer[offset] = encode_modrm(3, opcode_ext, register_id(reg));
860            offset += 1;
861            buffer[offset] = imm as u8;
862            offset += 1;
863
864            return Ok(offset);
865        }
866
867        // For 16/32/64-bit registers
868        let needs_rex_w = reg_size == 64 && self.machine_mode == MachineMode::Long64;
869        let needs_66 = reg_size == 16;
870        let needs_rex_b = check_rex_b(reg);
871
872        // Emit 66h prefix for 16-bit
873        if needs_66 {
874            buffer[offset] = 0x66;
875            offset += 1;
876        }
877
878        // Emit REX prefix
879        if needs_rex_w || needs_rex_b {
880            buffer[offset] = encode_rex(needs_rex_w, false, false, needs_rex_b);
881            offset += 1;
882        }
883
884        // Opcode: 83 /n ib (sign-extended imm8 to operand size)
885        buffer[offset] = 0x83;
886        offset += 1;
887
888        // ModRM: mod=11 (register), reg=opcode_ext, rm=dst
889        buffer[offset] = encode_modrm(3, opcode_ext, register_id(reg));
890        offset += 1;
891
892        // Immediate (imm8, sign-extended)
893        buffer[offset] = imm as u8;
894        offset += 1;
895
896        Ok(offset)
897    }
898
899    /// Encode ALU reg, reg: ADD/SUB/CMP/AND/OR/XOR reg, reg
900    /// Uses opcodes: ADD=01, SUB=29, AND=21, OR=09, XOR=31, CMP=39
901    fn encode_alu_reg_reg(
902        &self,
903        dst: Register,
904        src: Register,
905        mnemonic: Mnemonic,
906        buffer: &mut [u8],
907    ) -> Result<usize> {
908        let mut offset = 0usize;
909        let op_size = dst.size().max(src.size());
910
911        // Validate operands have same size
912        if dst.size() != src.size() {
913            return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
914        }
915
916        // Get the opcode for this ALU operation
917        let opcode = get_alu_reg_reg_opcode(mnemonic)
918            .ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
919
920        // Prefixes
921        let needs_66 = op_size == 16;
922        let needs_rex_w = op_size == 64 && self.machine_mode == MachineMode::Long64;
923        let rex_r = check_rex_r(src);
924        let rex_b = check_rex_b(dst);
925
926        // For 8-bit registers that need REX (SPL, BPL, SIL, DIL, R8B-R15B)
927        let rex_for_8bit = op_size == 8 && (needs_rex_for_8bit(dst) || needs_rex_for_8bit(src));
928
929        // Emit 66h prefix for 16-bit
930        if needs_66 {
931            buffer[offset] = 0x66;
932            offset += 1;
933        }
934
935        // Emit REX prefix
936        if needs_rex_w || rex_r || rex_b || rex_for_8bit {
937            buffer[offset] = encode_rex(needs_rex_w, rex_r, false, rex_b);
938            offset += 1;
939        }
940
941        // Emit opcode
942        buffer[offset] = opcode;
943        offset += 1;
944
945        // ModRM: mod=11 (register), reg=src, rm=dst
946        buffer[offset] = encode_modrm(3, register_id(src), register_id(dst));
947        offset += 1;
948
949        Ok(offset)
950    }
951
952    /// Encode PUSH reg (50+rd for 64-bit, FF /6 for others)
953    fn encode_push_pop_reg(
954        &self,
955        reg: Register,
956        is_push: bool,
957        buffer: &mut [u8],
958    ) -> Result<usize> {
959        let mut offset = 0usize;
960        let reg_size = reg.size();
961
962        // For 64-bit registers in 64-bit mode, use short form (50+rd for push, 58+rd for pop)
963        if reg_size == 64 && self.machine_mode == MachineMode::Long64 {
964            let needs_rex_b = check_rex_b(reg);
965
966            // Emit REX.B prefix if needed for R8-R15
967            if needs_rex_b {
968                buffer[offset] = encode_rex(false, false, false, true);
969                offset += 1;
970            }
971
972            // Opcode: 50+rd for PUSH, 58+rd for POP
973            buffer[offset] = if is_push {
974                0x50 + register_id(reg)
975            } else {
976                0x58 + register_id(reg)
977            };
978            offset += 1;
979
980            return Ok(offset);
981        }
982
983        // For other sizes, use FF /6 (PUSH) or 8F /0 (POP)
984        let needs_66 = reg_size == 16;
985        let needs_rex_w = reg_size == 64;
986        let needs_rex_b = check_rex_b(reg);
987
988        // Emit 66h prefix for 16-bit
989        if needs_66 {
990            buffer[offset] = 0x66;
991            offset += 1;
992        }
993
994        // Emit REX prefix
995        if needs_rex_w || needs_rex_b {
996            buffer[offset] = encode_rex(needs_rex_w, false, false, needs_rex_b);
997            offset += 1;
998        }
999
1000        // Opcode and ModRM
1001        if is_push {
1002            buffer[offset] = 0xFF; // PUSH r/m
1003            offset += 1;
1004            buffer[offset] = encode_modrm(3, 6, register_id(reg)); // /6
1005            offset += 1;
1006        } else {
1007            buffer[offset] = 0x8F; // POP r/m
1008            offset += 1;
1009            buffer[offset] = encode_modrm(3, 0, register_id(reg)); // /0
1010            offset += 1;
1011        }
1012
1013        Ok(offset)
1014    }
1015
1016    /// Encode shift instructions with CL: SHL/SHR/SAR reg, CL
1017    /// Uses opcode D3 /n (for 16/32/64-bit) or D2 /n (for 8-bit)
1018    fn encode_shift_cl(
1019        &self,
1020        reg: Register,
1021        mnemonic: Mnemonic,
1022        buffer: &mut [u8],
1023    ) -> Result<usize> {
1024        let mut offset = 0usize;
1025        let reg_size = reg.size();
1026
1027        // Get the opcode extension for shift operations
1028        let opcode_ext = match mnemonic {
1029            Mnemonic::SHL | Mnemonic::SAL => 4,
1030            Mnemonic::SHR => 5,
1031            Mnemonic::SAR => 7,
1032            _ => return Err(Error::EncodeFailed(EncodeError::InvalidMnemonic)),
1033        };
1034
1035        let needs_66 = reg_size == 16;
1036        let needs_rex_w = reg_size == 64 && self.machine_mode == MachineMode::Long64;
1037        let needs_rex_b = check_rex_b(reg);
1038
1039        // For 8-bit registers that need REX
1040        let rex_for_8bit = reg_size == 8 && needs_rex_for_8bit(reg);
1041
1042        // Emit 66h prefix for 16-bit
1043        if needs_66 {
1044            buffer[offset] = 0x66;
1045            offset += 1;
1046        }
1047
1048        // Emit REX prefix
1049        if needs_rex_w || needs_rex_b || rex_for_8bit {
1050            buffer[offset] = encode_rex(needs_rex_w, false, false, needs_rex_b);
1051            offset += 1;
1052        }
1053
1054        // Opcode: D0 (8-bit, 1), D1 (16/32/64-bit, 1), D2 (8-bit, CL), D3 (16/32/64-bit, CL)
1055        buffer[offset] = if reg_size == 8 { 0xD2 } else { 0xD3 };
1056        offset += 1;
1057
1058        // ModRM: mod=11 (register), reg=opcode_ext, rm=dst
1059        buffer[offset] = encode_modrm(3, opcode_ext, register_id(reg));
1060        offset += 1;
1061
1062        Ok(offset)
1063    }
1064
1065    /// Encode LEA
1066    fn encode_lea(&self, dst: Register, mem: &MemoryOperand, buffer: &mut [u8]) -> Result<usize> {
1067        let mut offset = 0usize;
1068
1069        // Determine address size
1070        let addr_size = if mem.base != Register::None {
1071            mem.base.size()
1072        } else if mem.index != Register::None {
1073            mem.index.size()
1074        } else {
1075            64 // Default to 64-bit addressing
1076        };
1077
1078        // Segment override prefix (must come before other prefixes)
1079        if mem.segment != Register::None && mem.segment != Register::DS {
1080            let seg_prefix = segment_override_prefix(mem.segment);
1081            if seg_prefix != 0 {
1082                buffer[offset] = seg_prefix;
1083                offset += 1;
1084            }
1085        }
1086
1087        // REX prefix calculation
1088        let needs_rex_w = dst.size() == 64;
1089        let rex_r = check_rex_r(dst);
1090        let rex_b = mem.base != Register::None && check_rex_b(mem.base);
1091        let rex_x = mem.index != Register::None && check_rex_r(mem.index);
1092
1093        // Address size override for 32-bit addressing in 64-bit mode
1094        if addr_size == 32 && self.machine_mode == MachineMode::Long64 {
1095            buffer[offset] = 0x67;
1096            offset += 1;
1097        }
1098
1099        // REX prefix
1100        if needs_rex_w || rex_r || rex_x || rex_b {
1101            buffer[offset] = encode_rex(needs_rex_w, rex_r, rex_x, rex_b);
1102            offset += 1;
1103        }
1104
1105        // Opcode: LEA r64, m
1106        buffer[offset] = 0x8D;
1107        offset += 1;
1108
1109        // Encode ModRM and SIB
1110        offset += self.encode_memory_operand(dst, mem, &mut buffer[offset..])?;
1111
1112        Ok(offset)
1113    }
1114
1115    /// Encode a memory operand (ModRM + SIB + displacement)
1116    ///
1117    /// Supports:
1118    /// - Base + disp8/disp32
1119    /// - Base + Index + Scale + disp
1120    /// - RIP-relative addressing
1121    /// - Absolute addressing via SIB
1122    /// - Segment override (must be added before calling this method)
1123    fn encode_memory_operand(
1124        &self,
1125        reg: Register,
1126        mem: &MemoryOperand,
1127        buffer: &mut [u8],
1128    ) -> Result<usize> {
1129        let mut offset = 0usize;
1130        let reg_id = register_id(reg);
1131        let base_id = if mem.base != Register::None {
1132            register_id(mem.base)
1133        } else {
1134            0
1135        };
1136        let index_id = if mem.index != Register::None {
1137            register_id(mem.index)
1138        } else {
1139            0
1140        };
1141
1142        // RIP-relative addressing: [RIP + disp32]
1143        if mem.is_rip_relative() {
1144            buffer[offset] = encode_modrm(0, reg_id, 5);
1145            offset += 1;
1146            buffer[offset..offset + 4].copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1147            offset += 4;
1148            return Ok(offset);
1149        }
1150
1151        // No base register - absolute addressing
1152        if mem.base == Register::None {
1153            if mem.index != Register::None {
1154                // [index*scale + disp32] - SIB with no base
1155                // Check for invalid index (RSP/ESP/SP/R12/R12D/R12W cannot be index)
1156                if is_stack_register(mem.index) {
1157                    return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1158                }
1159                buffer[offset] = encode_modrm(0, reg_id, 4); // rm=4 means SIB follows
1160                offset += 1;
1161                buffer[offset] = encode_sib(mem.scale, index_id, 5); // base=5 means disp32
1162                offset += 1;
1163                buffer[offset..offset + 4]
1164                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1165                offset += 4;
1166            } else {
1167                // Pure displacement [disp32] - use SIB with no index, base=5
1168                buffer[offset] = encode_modrm(0, reg_id, 4); // rm=4 means SIB follows
1169                offset += 1;
1170                buffer[offset] = encode_sib(1, 4, 5); // no index (index=4), base=5 for disp32
1171                offset += 1;
1172                buffer[offset..offset + 4]
1173                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1174                offset += 4;
1175            }
1176            return Ok(offset);
1177        }
1178
1179        // Has base register
1180        // Check for invalid index (RSP/ESP/SP/R12/R12D/R12W cannot be used as index)
1181        if mem.index != Register::None && is_stack_register(mem.index) {
1182            return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1183        }
1184
1185        let needs_sib = mem.requires_sib();
1186        let disp_size = displacement_size(mem.displacement);
1187
1188        // Check if base is RBP/R13 family (requires displacement)
1189        let base_is_rbp = is_rbp_family(mem.base);
1190
1191        // For RBP/R13 without displacement, we need to add a disp8 of 0
1192        let effective_disp_size = if base_is_rbp && disp_size == 0 {
1193            1 // Force disp8 with value 0
1194        } else {
1195            disp_size
1196        };
1197
1198        // Determine mod field
1199        let mod_val = match effective_disp_size {
1200            0 => 0,
1201            1 => 1,
1202            _ => 2,
1203        };
1204
1205        if needs_sib {
1206            // ModRM with SIB
1207            buffer[offset] = encode_modrm(mod_val, reg_id, 4); // rm=4 means SIB follows
1208            offset += 1;
1209
1210            // SIB byte
1211            // Special case: if base is RBP/R13 and no displacement, use base=5 in SIB
1212            // but the actual displacement will be encoded as disp8=0
1213            let sib_base = if base_is_rbp && disp_size == 0 {
1214                5 // Will use disp8 encoded below
1215            } else {
1216                base_id
1217            };
1218            buffer[offset] = encode_sib(mem.scale, index_id, sib_base);
1219            offset += 1;
1220        } else {
1221            // Simple ModRM without SIB
1222            // Special case: [RBP/R13] is encoded as mod=01, rm=101 with disp8
1223            if base_is_rbp && disp_size == 0 {
1224                buffer[offset] = encode_modrm(1, reg_id, 5); // mod=01, rm=101
1225            } else {
1226                buffer[offset] = encode_modrm(mod_val, reg_id, base_id);
1227            }
1228            offset += 1;
1229        }
1230
1231        // Displacement
1232        match effective_disp_size {
1233            1 => {
1234                buffer[offset] = mem.displacement as i8 as u8;
1235                offset += 1;
1236            }
1237            2 => {
1238                buffer[offset..offset + 2]
1239                    .copy_from_slice(&(mem.displacement as i16).to_le_bytes());
1240                offset += 2;
1241            }
1242            4 | 8 => {
1243                buffer[offset..offset + 4]
1244                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1245                offset += 4;
1246            }
1247            _ => {}
1248        }
1249
1250        Ok(offset)
1251    }
1252
1253    /// Encode AVX reg, reg instruction (VEX.128.0F.WIG opcode /r)
1254    ///
1255    /// Format: VEX + opcode + ModRM
1256    /// For instructions like VADDPS xmm1, xmm2, xmm3:
1257    /// - xmm1 is encoded in vvvv field (inverted)
1258    /// - xmm2 is encoded in ModRM.reg field
1259    /// - xmm3 is encoded in ModRM.rm field (mod=11)
1260    fn encode_avx_reg_reg(
1261        &self,
1262        dst: Register,
1263        src1: Register,
1264        src2: Register,
1265        mnemonic: Mnemonic,
1266        buffer: &mut [u8],
1267    ) -> Result<usize> {
1268        let mut offset = 0usize;
1269
1270        // Get AVX opcode info
1271        let avx_info =
1272            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1273
1274        // Convert opcode map
1275        let vex_map = match avx_info.opcode_map {
1276            AvxOpcodeMap::Map0F => vex::VexOpcodeMap::Map0F,
1277            AvxOpcodeMap::Map0F38 => vex::VexOpcodeMap::Map0F38,
1278            AvxOpcodeMap::Map0F3A => vex::VexOpcodeMap::Map0F3A,
1279        };
1280
1281        // Build VEX info
1282        let vex_info = VexInfo::new()
1283            .with_map(vex_map)
1284            .with_prefix(avx_info.vex_prefix)
1285            .with_128bit()
1286            .with_vvvv(dst); // dst goes into vvvv field
1287
1288        // Encode VEX prefix
1289        let vex_bytes = vex::encode_vex(&vex_info, Some(src1), Some(src2), None);
1290        buffer[offset..offset + vex_bytes.len()].copy_from_slice(&vex_bytes);
1291        offset += vex_bytes.len();
1292
1293        // Emit opcode
1294        buffer[offset] = avx_info.opcode;
1295        offset += 1;
1296
1297        // Emit ModRM: mod=11 (register), reg=src1, rm=src2
1298        buffer[offset] = encode_modrm(3, register_id(src1), register_id(src2));
1299        offset += 1;
1300
1301        // Emit imm8 if required
1302        if avx_info.has_imm8 {
1303            // For 3-operand AVX instructions with imm8, we don't emit here
1304            // This is handled by encode_avx_reg_reg_imm8
1305            return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1306        }
1307
1308        Ok(offset)
1309    }
1310
1311    /// Encode AVX reg, mem instruction (VEX.128.0F.WIG opcode /r)
1312    ///
1313    /// Format: VEX + opcode + ModRM [+ SIB + displacement]
1314    fn encode_avx_reg_mem(
1315        &self,
1316        dst: Register,
1317        src1: Register,
1318        mem: &MemoryOperand,
1319        mnemonic: Mnemonic,
1320        buffer: &mut [u8],
1321    ) -> Result<usize> {
1322        let mut offset = 0usize;
1323
1324        // Get AVX opcode info
1325        let avx_info =
1326            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1327
1328        // Convert opcode map
1329        let vex_map = match avx_info.opcode_map {
1330            AvxOpcodeMap::Map0F => vex::VexOpcodeMap::Map0F,
1331            AvxOpcodeMap::Map0F38 => vex::VexOpcodeMap::Map0F38,
1332            AvxOpcodeMap::Map0F3A => vex::VexOpcodeMap::Map0F3A,
1333        };
1334
1335        // Build VEX info
1336        let vex_info = VexInfo::new()
1337            .with_map(vex_map)
1338            .with_prefix(avx_info.vex_prefix)
1339            .with_128bit()
1340            .with_vvvv(dst); // dst goes into vvvv field
1341
1342        // Encode VEX prefix
1343        let vex_bytes = vex::encode_vex(&vex_info, Some(src1), Some(mem.base), Some(mem.index));
1344        buffer[offset..offset + vex_bytes.len()].copy_from_slice(&vex_bytes);
1345        offset += vex_bytes.len();
1346
1347        // Emit opcode
1348        buffer[offset] = avx_info.opcode;
1349        offset += 1;
1350
1351        // Encode ModRM and memory operand
1352        // src1 goes into ModRM.reg field, mem operand into rm field
1353        offset += self.encode_avx_memory_operand(src1, mem, &mut buffer[offset..])?;
1354
1355        Ok(offset)
1356    }
1357
1358    /// Encode AVX reg, reg, reg, imm8 instruction
1359    /// For instructions like VSHUFPS xmm1, xmm2, xmm3, imm8
1360    fn encode_avx_reg_reg_imm8(
1361        &self,
1362        dst: Register,
1363        src1: Register,
1364        src2: Register,
1365        imm8: u8,
1366        mnemonic: Mnemonic,
1367        buffer: &mut [u8],
1368    ) -> Result<usize> {
1369        let mut offset = 0usize;
1370
1371        // Get AVX opcode info
1372        let avx_info =
1373            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1374
1375        // Convert opcode map
1376        let vex_map = match avx_info.opcode_map {
1377            AvxOpcodeMap::Map0F => vex::VexOpcodeMap::Map0F,
1378            AvxOpcodeMap::Map0F38 => vex::VexOpcodeMap::Map0F38,
1379            AvxOpcodeMap::Map0F3A => vex::VexOpcodeMap::Map0F3A,
1380        };
1381
1382        // Determine vector length based on register type and instruction
1383        let vector_length = if matches!(mnemonic, Mnemonic::VPERM2I128) {
1384            // VPERM2I128 requires 256-bit (YMM) registers
1385            vex::VexVectorLength::L256
1386        } else {
1387            vex::VexVectorLength::L128
1388        };
1389
1390        // Build VEX info
1391        let vex_info = VexInfo::new()
1392            .with_map(vex_map)
1393            .with_prefix(avx_info.vex_prefix)
1394            .with_vector_length(vector_length)
1395            .with_vvvv(dst);
1396
1397        // Encode VEX prefix
1398        let vex_bytes = vex::encode_vex(&vex_info, Some(src1), Some(src2), None);
1399        buffer[offset..offset + vex_bytes.len()].copy_from_slice(&vex_bytes);
1400        offset += vex_bytes.len();
1401
1402        // Emit opcode
1403        buffer[offset] = avx_info.opcode;
1404        offset += 1;
1405
1406        // Emit ModRM: mod=11 (register), reg=src1, rm=src2
1407        buffer[offset] = encode_modrm(3, register_id(src1), register_id(src2));
1408        offset += 1;
1409
1410        // Emit imm8
1411        buffer[offset] = imm8;
1412        offset += 1;
1413
1414        Ok(offset)
1415    }
1416
1417    /// Encode AVX reg, reg, mem, imm8 instruction
1418    /// For instructions like VSHUFPS xmm1, xmm2, [mem], imm8
1419    fn encode_avx_reg_mem_imm8(
1420        &self,
1421        dst: Register,
1422        src1: Register,
1423        mem: &MemoryOperand,
1424        imm8: u8,
1425        mnemonic: Mnemonic,
1426        buffer: &mut [u8],
1427    ) -> Result<usize> {
1428        let mut offset = 0usize;
1429
1430        // Get AVX opcode info
1431        let avx_info =
1432            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1433
1434        // Convert opcode map
1435        let vex_map = match avx_info.opcode_map {
1436            AvxOpcodeMap::Map0F => vex::VexOpcodeMap::Map0F,
1437            AvxOpcodeMap::Map0F38 => vex::VexOpcodeMap::Map0F38,
1438            AvxOpcodeMap::Map0F3A => vex::VexOpcodeMap::Map0F3A,
1439        };
1440
1441        // Build VEX info
1442        let vex_info = VexInfo::new()
1443            .with_map(vex_map)
1444            .with_prefix(avx_info.vex_prefix)
1445            .with_128bit()
1446            .with_vvvv(dst);
1447
1448        // Encode VEX prefix
1449        let vex_bytes = vex::encode_vex(&vex_info, Some(src1), Some(mem.base), Some(mem.index));
1450        buffer[offset..offset + vex_bytes.len()].copy_from_slice(&vex_bytes);
1451        offset += vex_bytes.len();
1452
1453        // Emit opcode
1454        buffer[offset] = avx_info.opcode;
1455        offset += 1;
1456
1457        // Encode ModRM and memory operand
1458        offset += self.encode_avx_memory_operand(src1, mem, &mut buffer[offset..])?;
1459
1460        // Emit imm8
1461        buffer[offset] = imm8;
1462        offset += 1;
1463
1464        Ok(offset)
1465    }
1466
1467    /// Encode VBROADCASTSS instruction
1468    /// VEX.128.0F38.WIG 18 /r
1469    fn encode_avx_broadcast_mem(
1470        &self,
1471        dst: Register,
1472        mem: &MemoryOperand,
1473        mnemonic: Mnemonic,
1474        buffer: &mut [u8],
1475    ) -> Result<usize> {
1476        let mut offset = 0usize;
1477
1478        // Get AVX opcode info
1479        let avx_info =
1480            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1481
1482        // Build VEX info - VBROADCASTSS uses Map0F38
1483        let vex_info = VexInfo::new()
1484            .with_map(vex::VexOpcodeMap::Map0F38)
1485            .with_prefix(avx_info.vex_prefix)
1486            .with_128bit();
1487        // No vvvv field for VBROADCASTSS
1488
1489        // Encode VEX prefix
1490        let vex_bytes = vex::encode_vex(&vex_info, Some(dst), Some(mem.base), Some(mem.index));
1491        buffer[offset..offset + vex_bytes.len()].copy_from_slice(&vex_bytes);
1492        offset += vex_bytes.len();
1493
1494        // Emit opcode
1495        buffer[offset] = avx_info.opcode;
1496        offset += 1;
1497
1498        // Encode ModRM and memory operand
1499        // dst goes into ModRM.reg field
1500        offset += self.encode_avx_memory_operand(dst, mem, &mut buffer[offset..])?;
1501
1502        Ok(offset)
1503    }
1504
1505    /// Encode VEXTRACTF128 instruction
1506    /// VEX.256.66.0F3A.WIG 19 /r ib
1507    fn encode_avx_extractf128(
1508        &self,
1509        dst: Register,
1510        src: Register,
1511        imm8: u8,
1512        buffer: &mut [u8],
1513    ) -> Result<usize> {
1514        let mut offset = 0usize;
1515
1516        // Get AVX opcode info
1517        let avx_info = get_avx_opcode(Mnemonic::VEXTRACTF128)
1518            .ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1519
1520        // Build VEX info - VEXTRACTF128 uses Map0F3A with 256-bit source
1521        let vex_info = VexInfo::new()
1522            .with_map(vex::VexOpcodeMap::Map0F3A)
1523            .with_prefix(avx_info.vex_prefix)
1524            .with_256bit();
1525        // No vvvv field for VEXTRACTF128
1526
1527        // Encode VEX prefix - src (YMM) goes in reg field
1528        let vex_bytes = vex::encode_vex(&vex_info, Some(src), Some(dst), None);
1529        buffer[offset..offset + vex_bytes.len()].copy_from_slice(&vex_bytes);
1530        offset += vex_bytes.len();
1531
1532        // Emit opcode
1533        buffer[offset] = avx_info.opcode;
1534        offset += 1;
1535
1536        // Emit ModRM: mod=11 (register), reg=src (YMM), rm=dst (XMM)
1537        buffer[offset] = encode_modrm(3, register_id(src), register_id(dst));
1538        offset += 1;
1539
1540        // Emit imm8
1541        buffer[offset] = imm8;
1542        offset += 1;
1543
1544        Ok(offset)
1545    }
1546
1547    /// Encode AVX memory operand (ModRM + SIB + displacement)
1548    /// Similar to encode_memory_operand but for AVX instructions
1549    fn encode_avx_memory_operand(
1550        &self,
1551        reg: Register,
1552        mem: &MemoryOperand,
1553        buffer: &mut [u8],
1554    ) -> Result<usize> {
1555        let mut offset = 0usize;
1556        let reg_id = register_id(reg);
1557        let base_id = if mem.base != Register::None {
1558            register_id(mem.base)
1559        } else {
1560            0
1561        };
1562        let index_id = if mem.index != Register::None {
1563            register_id(mem.index)
1564        } else {
1565            0
1566        };
1567
1568        // RIP-relative addressing: [RIP + disp32]
1569        if mem.is_rip_relative() {
1570            buffer[offset] = encode_modrm(0, reg_id, 5);
1571            offset += 1;
1572            buffer[offset..offset + 4].copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1573            offset += 4;
1574            return Ok(offset);
1575        }
1576
1577        // No base register - absolute addressing
1578        if mem.base == Register::None {
1579            if mem.index != Register::None {
1580                // [index*scale + disp32] - SIB with no base
1581                if is_stack_register(mem.index) {
1582                    return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1583                }
1584                buffer[offset] = encode_modrm(0, reg_id, 4); // rm=4 means SIB follows
1585                offset += 1;
1586                buffer[offset] = encode_sib(mem.scale, index_id, 5); // base=5 means disp32
1587                offset += 1;
1588                buffer[offset..offset + 4]
1589                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1590                offset += 4;
1591            } else {
1592                // Pure displacement [disp32] - use SIB with no index, base=5
1593                buffer[offset] = encode_modrm(0, reg_id, 4); // rm=4 means SIB follows
1594                offset += 1;
1595                buffer[offset] = encode_sib(1, 4, 5); // no index (index=4), base=5 for disp32
1596                offset += 1;
1597                buffer[offset..offset + 4]
1598                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1599                offset += 4;
1600            }
1601            return Ok(offset);
1602        }
1603
1604        // Has base register
1605        if mem.index != Register::None && is_stack_register(mem.index) {
1606            return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1607        }
1608
1609        let needs_sib = mem.requires_sib();
1610        let disp_size = displacement_size(mem.displacement);
1611
1612        // Check if base is RBP/R13 family (requires displacement)
1613        let base_is_rbp = is_rbp_family(mem.base);
1614
1615        // For RBP/R13 without displacement, we need to add a disp8 of 0
1616        let effective_disp_size = if base_is_rbp && disp_size == 0 {
1617            1 // Force disp8 with value 0
1618        } else {
1619            disp_size
1620        };
1621
1622        // Determine mod field
1623        let mod_val = match effective_disp_size {
1624            0 => 0,
1625            1 => 1,
1626            _ => 2,
1627        };
1628
1629        if needs_sib {
1630            // ModRM with SIB
1631            buffer[offset] = encode_modrm(mod_val, reg_id, 4); // rm=4 means SIB follows
1632            offset += 1;
1633
1634            let sib_base = if base_is_rbp && disp_size == 0 {
1635                5 // Will use disp8 encoded below
1636            } else {
1637                base_id
1638            };
1639            buffer[offset] = encode_sib(mem.scale, index_id, sib_base);
1640            offset += 1;
1641        } else {
1642            // Simple ModRM without SIB
1643            if base_is_rbp && disp_size == 0 {
1644                buffer[offset] = encode_modrm(1, reg_id, 5); // mod=01, rm=101
1645            } else {
1646                buffer[offset] = encode_modrm(mod_val, reg_id, base_id);
1647            }
1648            offset += 1;
1649        }
1650
1651        // Displacement
1652        match effective_disp_size {
1653            1 => {
1654                buffer[offset] = mem.displacement as i8 as u8;
1655                offset += 1;
1656            }
1657            2 => {
1658                buffer[offset..offset + 2]
1659                    .copy_from_slice(&(mem.displacement as i16).to_le_bytes());
1660                offset += 2;
1661            }
1662            4 | 8 => {
1663                buffer[offset..offset + 4]
1664                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1665                offset += 4;
1666            }
1667            _ => {}
1668        }
1669
1670        Ok(offset)
1671    }
1672
1673    /// Encode AVX-512 reg, reg instruction (EVEX-encoded)
1674    ///
1675    /// Format: EVEX + opcode + ModRM
1676    /// For instructions like VADDPS.Z zmm1, zmm2, zmm3:
1677    /// - zmm1 is encoded in vvvv field
1678    /// - zmm2 is encoded in ModRM.reg field
1679    /// - zmm3 is encoded in ModRM.rm field (mod=11)
1680    #[allow(clippy::too_many_arguments)]
1681    fn encode_avx512_reg_reg(
1682        &self,
1683        dst: Register,
1684        src1: Register,
1685        src2: Register,
1686        mnemonic: Mnemonic,
1687        mask_reg: Option<Register>,
1688        zeroing: evex::EvexZeroingMode,
1689        buffer: &mut [u8],
1690    ) -> Result<usize> {
1691        let mut offset = 0usize;
1692
1693        // Get AVX opcode info (EVEX uses same opcodes as VEX for these instructions)
1694        let avx_info =
1695            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1696
1697        // Determine vector length based on register type
1698        let vector_length = if dst.is_zmm() || src1.is_zmm() || src2.is_zmm() {
1699            EvexVectorLength::L512
1700        } else if dst.is_ymm() || src1.is_ymm() || src2.is_ymm() {
1701            EvexVectorLength::L256
1702        } else {
1703            EvexVectorLength::L128
1704        };
1705
1706        // Convert VEX prefix to EVEX prefix
1707        let evex_prefix = match avx_info.vex_prefix {
1708            vex::VexPrefix::None => EvexPrefix::None,
1709            vex::VexPrefix::Prefix66 => EvexPrefix::Prefix66,
1710            vex::VexPrefix::PrefixF3 => EvexPrefix::PrefixF3,
1711            vex::VexPrefix::PrefixF2 => EvexPrefix::PrefixF2,
1712        };
1713
1714        // Build EVEX info
1715        let evex_info = EvexInfo::new()
1716            .with_map(EvexOpcodeMap::Map0F)
1717            .with_prefix(evex_prefix)
1718            .with_vvvv(dst)
1719            .with_mask_opt(mask_reg)
1720            .with_zeroing(zeroing);
1721
1722        // Set vector length
1723        let evex_info = match vector_length {
1724            EvexVectorLength::L128 => evex_info.with_128bit(),
1725            EvexVectorLength::L256 => evex_info.with_256bit(),
1726            EvexVectorLength::L512 => evex_info.with_512bit(),
1727        };
1728
1729        // Encode EVEX prefix
1730        let evex_bytes = evex::encode_evex(&evex_info, Some(src1), None, Some(src2));
1731        buffer[offset..offset + evex_bytes.len()].copy_from_slice(&evex_bytes);
1732        offset += evex_bytes.len();
1733
1734        // Emit opcode
1735        buffer[offset] = avx_info.opcode;
1736        offset += 1;
1737
1738        // Emit ModRM: mod=11 (register), reg=src1, rm=src2
1739        // For EVEX, we use evex_register_id to handle extended registers
1740        buffer[offset] = encode_modrm(
1741            3,
1742            evex::evex_register_id(src1),
1743            evex::evex_register_id(src2),
1744        );
1745        offset += 1;
1746
1747        Ok(offset)
1748    }
1749
1750    /// Encode AVX-512 reg, mem instruction (EVEX-encoded)
1751    ///
1752    /// Format: EVEX + opcode + ModRM [+ SIB + displacement]
1753    #[allow(clippy::too_many_arguments)]
1754    fn encode_avx512_reg_mem(
1755        &self,
1756        dst: Register,
1757        src1: Register,
1758        mem: &MemoryOperand,
1759        mnemonic: Mnemonic,
1760        mask_reg: Option<Register>,
1761        zeroing: evex::EvexZeroingMode,
1762        buffer: &mut [u8],
1763    ) -> Result<usize> {
1764        let mut offset = 0usize;
1765
1766        // Get AVX opcode info
1767        let avx_info =
1768            get_avx_opcode(mnemonic).ok_or(Error::EncodeFailed(EncodeError::InvalidMnemonic))?;
1769
1770        // Determine vector length based on register type
1771        let vector_length = if dst.is_zmm() || src1.is_zmm() {
1772            EvexVectorLength::L512
1773        } else if dst.is_ymm() || src1.is_ymm() {
1774            EvexVectorLength::L256
1775        } else {
1776            EvexVectorLength::L128
1777        };
1778
1779        // Convert VEX prefix to EVEX prefix
1780        let evex_prefix = match avx_info.vex_prefix {
1781            vex::VexPrefix::None => EvexPrefix::None,
1782            vex::VexPrefix::Prefix66 => EvexPrefix::Prefix66,
1783            vex::VexPrefix::PrefixF3 => EvexPrefix::PrefixF3,
1784            vex::VexPrefix::PrefixF2 => EvexPrefix::PrefixF2,
1785        };
1786
1787        // Build EVEX info
1788        let evex_info = EvexInfo::new()
1789            .with_map(EvexOpcodeMap::Map0F)
1790            .with_prefix(evex_prefix)
1791            .with_vvvv(dst)
1792            .with_mask_opt(mask_reg)
1793            .with_zeroing(zeroing);
1794
1795        // Set vector length
1796        let evex_info = match vector_length {
1797            EvexVectorLength::L128 => evex_info.with_128bit(),
1798            EvexVectorLength::L256 => evex_info.with_256bit(),
1799            EvexVectorLength::L512 => evex_info.with_512bit(),
1800        };
1801
1802        // Encode EVEX prefix
1803        let evex_bytes = evex::encode_evex(&evex_info, Some(src1), Some(mem.index), Some(mem.base));
1804        buffer[offset..offset + evex_bytes.len()].copy_from_slice(&evex_bytes);
1805        offset += evex_bytes.len();
1806
1807        // Emit opcode
1808        buffer[offset] = avx_info.opcode;
1809        offset += 1;
1810
1811        // Encode ModRM and memory operand
1812        offset += self.encode_avx512_memory_operand(src1, mem, &mut buffer[offset..])?;
1813
1814        Ok(offset)
1815    }
1816
1817    /// Encode AVX-512 memory operand (ModRM + SIB + displacement)
1818    /// Similar to encode_avx_memory_operand but uses EVEX register encoding
1819    fn encode_avx512_memory_operand(
1820        &self,
1821        reg: Register,
1822        mem: &MemoryOperand,
1823        buffer: &mut [u8],
1824    ) -> Result<usize> {
1825        let mut offset = 0usize;
1826        let reg_id = evex::evex_register_id(reg);
1827        let base_id = if mem.base != Register::None {
1828            evex::evex_register_id(mem.base)
1829        } else {
1830            0
1831        };
1832        let index_id = if mem.index != Register::None {
1833            evex::evex_register_id(mem.index)
1834        } else {
1835            0
1836        };
1837
1838        // RIP-relative addressing: [RIP + disp32]
1839        if mem.is_rip_relative() {
1840            buffer[offset] = encode_modrm(0, reg_id, 5);
1841            offset += 1;
1842            buffer[offset..offset + 4].copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1843            offset += 4;
1844            return Ok(offset);
1845        }
1846
1847        // No base register - absolute addressing
1848        if mem.base == Register::None {
1849            if mem.index != Register::None {
1850                // [index*scale + disp32] - SIB with no base
1851                if is_stack_register(mem.index) {
1852                    return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1853                }
1854                buffer[offset] = encode_modrm(0, reg_id, 4); // rm=4 means SIB follows
1855                offset += 1;
1856                buffer[offset] = encode_sib(mem.scale, index_id, 5); // base=5 means disp32
1857                offset += 1;
1858                buffer[offset..offset + 4]
1859                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1860                offset += 4;
1861            } else {
1862                // Pure displacement [disp32] - use SIB with no index, base=5
1863                buffer[offset] = encode_modrm(0, reg_id, 4); // rm=4 means SIB follows
1864                offset += 1;
1865                buffer[offset] = encode_sib(1, 4, 5); // no index (index=4), base=5 for disp32
1866                offset += 1;
1867                buffer[offset..offset + 4]
1868                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1869                offset += 4;
1870            }
1871            return Ok(offset);
1872        }
1873
1874        // Has base register
1875        if mem.index != Register::None && is_stack_register(mem.index) {
1876            return Err(Error::EncodeFailed(EncodeError::InvalidOperandCombination));
1877        }
1878
1879        let needs_sib = mem.requires_sib();
1880        let disp_size = displacement_size(mem.displacement);
1881
1882        // Check if base is RBP/R13 family (requires displacement)
1883        let base_is_rbp = is_rbp_family(mem.base);
1884
1885        // For RBP/R13 without displacement, we need to add a disp8 of 0
1886        let effective_disp_size = if base_is_rbp && disp_size == 0 {
1887            1 // Force disp8 with value 0
1888        } else {
1889            disp_size
1890        };
1891
1892        // Determine mod field
1893        let mod_val = match effective_disp_size {
1894            0 => 0,
1895            1 => 1,
1896            _ => 2,
1897        };
1898
1899        if needs_sib {
1900            // ModRM with SIB
1901            buffer[offset] = encode_modrm(mod_val, reg_id, 4); // rm=4 means SIB follows
1902            offset += 1;
1903
1904            let sib_base = if base_is_rbp && disp_size == 0 {
1905                5 // Will use disp8 encoded below
1906            } else {
1907                base_id
1908            };
1909            buffer[offset] = encode_sib(mem.scale, index_id, sib_base);
1910            offset += 1;
1911        } else {
1912            // Simple ModRM without SIB
1913            if base_is_rbp && disp_size == 0 {
1914                buffer[offset] = encode_modrm(1, reg_id, 5); // mod=01, rm=101
1915            } else {
1916                buffer[offset] = encode_modrm(mod_val, reg_id, base_id);
1917            }
1918            offset += 1;
1919        }
1920
1921        // Displacement
1922        match effective_disp_size {
1923            1 => {
1924                buffer[offset] = mem.displacement as i8 as u8;
1925                offset += 1;
1926            }
1927            2 => {
1928                buffer[offset..offset + 2]
1929                    .copy_from_slice(&(mem.displacement as i16).to_le_bytes());
1930                offset += 2;
1931            }
1932            4 | 8 => {
1933                buffer[offset..offset + 4]
1934                    .copy_from_slice(&(mem.displacement as i32).to_le_bytes());
1935                offset += 4;
1936            }
1937            _ => {}
1938        }
1939
1940        Ok(offset)
1941    }
1942}
1943
1944/// Check if a register is a stack pointer register (cannot be used as index)
1945fn is_stack_register(reg: Register) -> bool {
1946    matches!(
1947        reg,
1948        Register::RSP
1949            | Register::ESP
1950            | Register::SP
1951            | Register::R12
1952            | Register::R12D
1953            | Register::R12W
1954    )
1955}
1956
1957/// Check if a register is RBP/R13 family (requires displacement)
1958fn is_rbp_family(reg: Register) -> bool {
1959    matches!(
1960        reg,
1961        Register::RBP
1962            | Register::R13
1963            | Register::EBP
1964            | Register::R13D
1965            | Register::BP
1966            | Register::R13W
1967    )
1968}
1969
1970/// Check if a mnemonic is a conditional jump
1971fn is_conditional_jump(mnemonic: Mnemonic) -> bool {
1972    matches!(
1973        mnemonic,
1974        Mnemonic::JO
1975            | Mnemonic::JNO
1976            | Mnemonic::JB
1977            | Mnemonic::JNB
1978            | Mnemonic::JAE
1979            | Mnemonic::JZ
1980            | Mnemonic::JE
1981            | Mnemonic::JNZ
1982            | Mnemonic::JNE
1983            | Mnemonic::JBE
1984            | Mnemonic::JNG
1985            | Mnemonic::JS
1986            | Mnemonic::JNS
1987            | Mnemonic::JP
1988            | Mnemonic::JNP
1989            | Mnemonic::JPE
1990            | Mnemonic::JPO
1991            | Mnemonic::JC
1992            | Mnemonic::JNC
1993            | Mnemonic::JL
1994            | Mnemonic::JGE
1995            | Mnemonic::JLE
1996            | Mnemonic::JG
1997            | Mnemonic::JNL
1998    )
1999}
2000
2001/// Get the condition code for a conditional jump
2002fn get_jcc_condition(mnemonic: Mnemonic) -> u8 {
2003    match mnemonic {
2004        Mnemonic::JO => 0,
2005        Mnemonic::JNO => 1,
2006        Mnemonic::JB | Mnemonic::JC => 2, // JB = JC = JNAE
2007        Mnemonic::JNB | Mnemonic::JAE | Mnemonic::JNC | Mnemonic::JNL => 3, // JNB = JAE = JNC
2008        Mnemonic::JZ | Mnemonic::JE => 4,
2009        Mnemonic::JNZ | Mnemonic::JNE => 5,
2010        Mnemonic::JBE | Mnemonic::JNG => 6, // JBE = JNG
2011        // JA not in enum, use 7 for "above" condition
2012        Mnemonic::JS => 8,
2013        Mnemonic::JNS => 9,
2014        Mnemonic::JP | Mnemonic::JPE => 10,
2015        Mnemonic::JNP | Mnemonic::JPO => 11,
2016        Mnemonic::JL => 12,
2017        Mnemonic::JGE => 13,
2018        Mnemonic::JLE => 14,
2019        Mnemonic::JG => 15,
2020        _ => 0,
2021    }
2022}
2023
2024impl Default for Encoder {
2025    fn default() -> Self {
2026        Self::new(MachineMode::Long64, 64).unwrap()
2027    }
2028}
2029
2030#[cfg(test)]
2031mod tests {
2032    use super::*;
2033    use alloc::vec;
2034
2035    #[test]
2036    fn test_encoder_new() {
2037        let encoder = Encoder::new(MachineMode::Long64, 64);
2038        assert!(encoder.is_ok());
2039
2040        let encoder = encoder.unwrap();
2041        assert_eq!(encoder.machine_mode(), MachineMode::Long64);
2042        assert_eq!(encoder.stack_width(), 64);
2043    }
2044
2045    #[test]
2046    fn test_encoder_invalid_stack_width() {
2047        let encoder = Encoder::new(MachineMode::Long64, 128);
2048        assert!(encoder.is_err());
2049    }
2050
2051    #[test]
2052    fn test_encoder_default() {
2053        let encoder = Encoder::default();
2054        assert_eq!(encoder.machine_mode(), MachineMode::Long64);
2055        assert_eq!(encoder.stack_width(), 64);
2056    }
2057
2058    // ========================================
2059    // Memory operand encoding tests
2060    // ========================================
2061
2062    #[test]
2063    fn test_encode_lea_rip_relative() {
2064        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2065
2066        // LEA RAX, [RIP + 0x1000]
2067        let mem = MemoryOperand::base_disp(Register::RIP, 0x1000);
2068        let request = EncoderRequest::new(Mnemonic::LEA)
2069            .with_reg(Register::RAX)
2070            .with_mem(mem);
2071        let bytes = encoder.encode(&request).unwrap();
2072
2073        // Should be: 48 8D 05 00 10 00 00
2074        assert_eq!(bytes[0], 0x48); // REX.W
2075        assert_eq!(bytes[1], 0x8D); // LEA
2076        // ModRM: mod=00, reg=000 (RAX), rm=101 (RIP-relative)
2077        assert_eq!(bytes[2] & 0xC7, 0x05); // mod=00, reg=000, rm=101
2078        // Check displacement (little-endian)
2079        assert_eq!(bytes[3], 0x00);
2080        assert_eq!(bytes[4], 0x10);
2081        assert_eq!(bytes[5], 0x00);
2082        assert_eq!(bytes[6], 0x00);
2083    }
2084
2085    #[test]
2086    fn test_encode_lea_sib() {
2087        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2088
2089        // LEA RAX, [RAX + RCX*4 + 0x10]
2090        let mem = MemoryOperand::full(Register::DS, Register::RAX, Register::RCX, 4, 0x10);
2091        let request = EncoderRequest::new(Mnemonic::LEA)
2092            .with_reg(Register::RAX)
2093            .with_mem(mem);
2094        let bytes = encoder.encode(&request).unwrap();
2095
2096        // Verify encoding contains SIB byte
2097        assert!(bytes.len() >= 5); // REX + opcode + ModRM + SIB + disp8
2098        assert_eq!(bytes[0], 0x48); // REX.W
2099        assert_eq!(bytes[1], 0x8D); // LEA
2100        // ModRM should indicate SIB follows
2101        assert_eq!(bytes[2] & 0x07, 0x04); // rm=4 means SIB
2102    }
2103
2104    #[test]
2105    fn test_encode_lea_base_disp() {
2106        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2107
2108        // LEA RAX, [RBP - 16]
2109        let mem = MemoryOperand::base_disp(Register::RBP, -16);
2110        let request = EncoderRequest::new(Mnemonic::LEA)
2111            .with_reg(Register::RAX)
2112            .with_mem(mem);
2113        let bytes = encoder.encode(&request).unwrap();
2114
2115        // RBP requires SIB in 64-bit mode
2116        assert!(bytes.len() >= 4); // REX + opcode + ModRM + SIB + disp8
2117        assert_eq!(bytes[0], 0x48); // REX.W
2118        assert_eq!(bytes[1], 0x8D); // LEA
2119    }
2120
2121    #[test]
2122    fn test_encode_lea_rsp_base() {
2123        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2124
2125        // LEA RAX, [RSP + 8]
2126        let mem = MemoryOperand::base_disp(Register::RSP, 8);
2127        let request = EncoderRequest::new(Mnemonic::LEA)
2128            .with_reg(Register::RAX)
2129            .with_mem(mem);
2130        let bytes = encoder.encode(&request).unwrap();
2131
2132        // RSP requires SIB
2133        assert!(bytes.len() >= 5); // REX + opcode + ModRM + SIB + disp8
2134        assert_eq!(bytes[0], 0x48); // REX.W
2135        assert_eq!(bytes[1], 0x8D); // LEA
2136        // ModRM should indicate SIB follows
2137        assert_eq!(bytes[2] & 0x07, 0x04); // rm=4 means SIB
2138    }
2139
2140    #[test]
2141    fn test_encode_segment_override() {
2142        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2143
2144        // LEA RAX, FS:[0x1000] - using segment override
2145        let mem = MemoryOperand::base_disp(Register::None, 0x1000).with_segment(Register::FS);
2146        let request = EncoderRequest::new(Mnemonic::LEA)
2147            .with_reg(Register::RAX)
2148            .with_mem(mem);
2149        let bytes = encoder.encode(&request).unwrap();
2150
2151        // Should start with FS segment override prefix (0x64)
2152        assert_eq!(bytes[0], 0x64); // FS segment override
2153    }
2154
2155    #[test]
2156    fn test_encode_lea_gs_segment() {
2157        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2158
2159        // LEA RAX, GS:[RCX]
2160        let mem = MemoryOperand::base(Register::RCX).with_segment(Register::GS);
2161        let request = EncoderRequest::new(Mnemonic::LEA)
2162            .with_reg(Register::RAX)
2163            .with_mem(mem);
2164        let bytes = encoder.encode(&request).unwrap();
2165
2166        // Should start with GS segment override prefix (0x65)
2167        assert_eq!(bytes[0], 0x65); // GS segment override
2168    }
2169
2170    #[test]
2171    fn test_encode_absolute_addressing() {
2172        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2173
2174        // LEA RAX, [0x12345678] - absolute address
2175        let mem = MemoryOperand::base_disp(Register::None, 0x12345678);
2176        let request = EncoderRequest::new(Mnemonic::LEA)
2177            .with_reg(Register::RAX)
2178            .with_mem(mem);
2179        let bytes = encoder.encode(&request).unwrap();
2180
2181        // Should use SIB with base=5 for absolute addressing
2182        assert!(bytes.len() >= 8); // REX + opcode + ModRM + SIB + disp32
2183        assert_eq!(bytes[2] & 0x07, 0x04); // rm=4 means SIB follows
2184    }
2185
2186    #[test]
2187    fn test_encode_index_scale_no_base() {
2188        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2189
2190        // LEA RAX, [RCX*4 + 0x1000] - index with scale, no base
2191        let mem = MemoryOperand::full(Register::DS, Register::None, Register::RCX, 4, 0x1000);
2192        let request = EncoderRequest::new(Mnemonic::LEA)
2193            .with_reg(Register::RAX)
2194            .with_mem(mem);
2195        let bytes = encoder.encode(&request).unwrap();
2196
2197        // Should use SIB with base=5 for disp32
2198        assert!(bytes.len() >= 8); // REX + opcode + ModRM + SIB + disp32
2199    }
2200
2201    #[test]
2202    fn test_encode_invalid_rsp_index() {
2203        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2204
2205        // LEA RAX, [RAX + RSP*4] - RSP cannot be index, should fail
2206        let mem = MemoryOperand::full(Register::DS, Register::RAX, Register::RSP, 4, 0);
2207        let request = EncoderRequest::new(Mnemonic::LEA)
2208            .with_reg(Register::RAX)
2209            .with_mem(mem);
2210        let result = encoder.encode(&request);
2211
2212        // Should return error for invalid operand combination
2213        assert!(result.is_err());
2214    }
2215
2216    #[test]
2217    fn test_is_stack_register() {
2218        assert!(is_stack_register(Register::RSP));
2219        assert!(is_stack_register(Register::ESP));
2220        assert!(is_stack_register(Register::SP));
2221        assert!(is_stack_register(Register::R12));
2222        assert!(is_stack_register(Register::R12D));
2223        assert!(is_stack_register(Register::R12W));
2224        assert!(!is_stack_register(Register::RAX));
2225        assert!(!is_stack_register(Register::RCX));
2226    }
2227
2228    #[test]
2229    fn test_is_rbp_family() {
2230        assert!(is_rbp_family(Register::RBP));
2231        assert!(is_rbp_family(Register::R13));
2232        assert!(is_rbp_family(Register::EBP));
2233        assert!(is_rbp_family(Register::R13D));
2234        assert!(is_rbp_family(Register::BP));
2235        assert!(is_rbp_family(Register::R13W));
2236        assert!(!is_rbp_family(Register::RAX));
2237        assert!(!is_rbp_family(Register::RSP));
2238    }
2239
2240    // ========================================
2241    // ALU instruction encoding tests
2242    // ========================================
2243
2244    #[test]
2245    fn test_encode_add_reg_imm() {
2246        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2247
2248        // ADD RAX, 1
2249        let request = EncoderRequest::new(Mnemonic::ADD)
2250            .with_reg(Register::RAX)
2251            .with_imm(1);
2252        let bytes = encoder.encode(&request).unwrap();
2253        // REX.W + 83 /0 ib => 48 83 C0 01
2254        assert_eq!(bytes, vec![0x48, 0x83, 0xC0, 0x01]);
2255    }
2256
2257    #[test]
2258    fn test_encode_sub_reg_imm() {
2259        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2260
2261        // SUB RAX, 5
2262        let request = EncoderRequest::new(Mnemonic::SUB)
2263            .with_reg(Register::RAX)
2264            .with_imm(5);
2265        let bytes = encoder.encode(&request).unwrap();
2266        // REX.W + 83 /5 ib => 48 83 E8 05
2267        assert_eq!(bytes, vec![0x48, 0x83, 0xE8, 0x05]);
2268    }
2269
2270    #[test]
2271    fn test_encode_cmp_reg_imm() {
2272        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2273
2274        // CMP RBX, 0
2275        let request = EncoderRequest::new(Mnemonic::CMP)
2276            .with_reg(Register::RBX)
2277            .with_imm(0);
2278        let bytes = encoder.encode(&request).unwrap();
2279        // REX.W + 83 /7 ib => 48 83 FB 00
2280        assert_eq!(bytes, vec![0x48, 0x83, 0xFB, 0x00]);
2281    }
2282
2283    #[test]
2284    fn test_encode_xor_reg_imm() {
2285        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2286
2287        // XOR RCX, 0xFF
2288        let request = EncoderRequest::new(Mnemonic::XOR)
2289            .with_reg(Register::RCX)
2290            .with_imm(0xFF);
2291        let bytes = encoder.encode(&request).unwrap();
2292        // REX.W + 83 /6 ib => 48 83 F1 FF
2293        assert_eq!(bytes, vec![0x48, 0x83, 0xF1, 0xFF]);
2294    }
2295
2296    #[test]
2297    fn test_encode_and_reg_imm() {
2298        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2299
2300        // AND RDX, 0x0F
2301        let request = EncoderRequest::new(Mnemonic::AND)
2302            .with_reg(Register::RDX)
2303            .with_imm(0x0F);
2304        let bytes = encoder.encode(&request).unwrap();
2305        // REX.W + 83 /4 ib => 48 83 E2 0F
2306        assert_eq!(bytes, vec![0x48, 0x83, 0xE2, 0x0F]);
2307    }
2308
2309    #[test]
2310    fn test_encode_or_reg_imm() {
2311        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2312
2313        // OR RSI, 0x80
2314        let request = EncoderRequest::new(Mnemonic::OR)
2315            .with_reg(Register::RSI)
2316            .with_imm(0x80);
2317        let bytes = encoder.encode(&request).unwrap();
2318        // REX.W + 83 /1 ib => 48 83 CE 80
2319        assert_eq!(bytes, vec![0x48, 0x83, 0xCE, 0x80]);
2320    }
2321
2322    #[test]
2323    fn test_encode_add_reg_reg() {
2324        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2325
2326        // ADD RAX, RBX
2327        let request = EncoderRequest::new(Mnemonic::ADD)
2328            .with_reg(Register::RAX)
2329            .with_reg(Register::RBX);
2330        let bytes = encoder.encode(&request).unwrap();
2331        // REX.W + 01 /r => 48 01 D8
2332        assert_eq!(bytes, vec![0x48, 0x01, 0xD8]);
2333    }
2334
2335    #[test]
2336    fn test_encode_sub_reg_reg() {
2337        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2338
2339        // SUB RAX, RBX
2340        let request = EncoderRequest::new(Mnemonic::SUB)
2341            .with_reg(Register::RAX)
2342            .with_reg(Register::RBX);
2343        let bytes = encoder.encode(&request).unwrap();
2344        // REX.W + 29 /r => 48 29 D8
2345        assert_eq!(bytes, vec![0x48, 0x29, 0xD8]);
2346    }
2347
2348    #[test]
2349    fn test_encode_xor_reg_reg() {
2350        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2351
2352        // XOR RAX, RCX
2353        let request = EncoderRequest::new(Mnemonic::XOR)
2354            .with_reg(Register::RAX)
2355            .with_reg(Register::RCX);
2356        let bytes = encoder.encode(&request).unwrap();
2357        // REX.W + 31 /r => 48 31 C8
2358        assert_eq!(bytes, vec![0x48, 0x31, 0xC8]);
2359    }
2360
2361    #[test]
2362    fn test_encode_and_reg_reg() {
2363        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2364
2365        // AND RAX, RDX
2366        let request = EncoderRequest::new(Mnemonic::AND)
2367            .with_reg(Register::RAX)
2368            .with_reg(Register::RDX);
2369        let bytes = encoder.encode(&request).unwrap();
2370        // REX.W + 21 /r => 48 21 D0
2371        assert_eq!(bytes, vec![0x48, 0x21, 0xD0]);
2372    }
2373
2374    #[test]
2375    fn test_encode_or_reg_reg() {
2376        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2377
2378        // OR RAX, RSI
2379        let request = EncoderRequest::new(Mnemonic::OR)
2380            .with_reg(Register::RAX)
2381            .with_reg(Register::RSI);
2382        let bytes = encoder.encode(&request).unwrap();
2383        // REX.W + 09 /r => 48 09 F0
2384        assert_eq!(bytes, vec![0x48, 0x09, 0xF0]);
2385    }
2386
2387    #[test]
2388    fn test_encode_cmp_reg_reg() {
2389        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2390
2391        // CMP RAX, RBX
2392        let request = EncoderRequest::new(Mnemonic::CMP)
2393            .with_reg(Register::RAX)
2394            .with_reg(Register::RBX);
2395        let bytes = encoder.encode(&request).unwrap();
2396        // REX.W + 39 /r => 48 39 D8
2397        assert_eq!(bytes, vec![0x48, 0x39, 0xD8]);
2398    }
2399
2400    #[test]
2401    fn test_encode_alu_extended_reg() {
2402        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2403
2404        // ADD R8, 1
2405        let request = EncoderRequest::new(Mnemonic::ADD)
2406            .with_reg(Register::R8)
2407            .with_imm(1);
2408        let bytes = encoder.encode(&request).unwrap();
2409        // REX.WB + 83 /0 ib => 49 83 C0 01
2410        assert_eq!(bytes, vec![0x49, 0x83, 0xC0, 0x01]);
2411
2412        // ADD RAX, R8
2413        let request = EncoderRequest::new(Mnemonic::ADD)
2414            .with_reg(Register::RAX)
2415            .with_reg(Register::R8);
2416        let bytes = encoder.encode(&request).unwrap();
2417        // REX.WR + 01 /r => 4C 01 C0
2418        assert_eq!(bytes, vec![0x4C, 0x01, 0xC0]);
2419    }
2420
2421    // ========================================
2422    // PUSH/POP instruction encoding tests
2423    // ========================================
2424
2425    #[test]
2426    fn test_encode_push_reg() {
2427        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2428
2429        // PUSH RAX
2430        let request = EncoderRequest::new(Mnemonic::PUSH).with_reg(Register::RAX);
2431        let bytes = encoder.encode(&request).unwrap();
2432        // 50+rd => 50
2433        assert_eq!(bytes, vec![0x50]);
2434
2435        // PUSH RBX
2436        let request = EncoderRequest::new(Mnemonic::PUSH).with_reg(Register::RBX);
2437        let bytes = encoder.encode(&request).unwrap();
2438        assert_eq!(bytes, vec![0x53]);
2439    }
2440
2441    #[test]
2442    fn test_encode_push_extended_reg() {
2443        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2444
2445        // PUSH R8
2446        let request = EncoderRequest::new(Mnemonic::PUSH).with_reg(Register::R8);
2447        let bytes = encoder.encode(&request).unwrap();
2448        // REX.B + 50+rd => 41 50
2449        assert_eq!(bytes, vec![0x41, 0x50]);
2450
2451        // PUSH R15
2452        let request = EncoderRequest::new(Mnemonic::PUSH).with_reg(Register::R15);
2453        let bytes = encoder.encode(&request).unwrap();
2454        // REX.B + 50+rd => 41 57
2455        assert_eq!(bytes, vec![0x41, 0x57]);
2456    }
2457
2458    #[test]
2459    fn test_encode_pop_reg() {
2460        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2461
2462        // POP RAX
2463        let request = EncoderRequest::new(Mnemonic::POP).with_reg(Register::RAX);
2464        let bytes = encoder.encode(&request).unwrap();
2465        // 58+rd => 58
2466        assert_eq!(bytes, vec![0x58]);
2467
2468        // POP RBX
2469        let request = EncoderRequest::new(Mnemonic::POP).with_reg(Register::RBX);
2470        let bytes = encoder.encode(&request).unwrap();
2471        assert_eq!(bytes, vec![0x5B]);
2472    }
2473
2474    #[test]
2475    fn test_encode_pop_extended_reg() {
2476        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2477
2478        // POP R8
2479        let request = EncoderRequest::new(Mnemonic::POP).with_reg(Register::R8);
2480        let bytes = encoder.encode(&request).unwrap();
2481        // REX.B + 58+rd => 41 58
2482        assert_eq!(bytes, vec![0x41, 0x58]);
2483
2484        // POP R15
2485        let request = EncoderRequest::new(Mnemonic::POP).with_reg(Register::R15);
2486        let bytes = encoder.encode(&request).unwrap();
2487        // REX.B + 58+rd => 41 5F
2488        assert_eq!(bytes, vec![0x41, 0x5F]);
2489    }
2490
2491    // ========================================
2492    // Shift instruction encoding tests
2493    // ========================================
2494
2495    #[test]
2496    fn test_encode_shl_cl() {
2497        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2498
2499        // SHL RAX, CL
2500        let request = EncoderRequest::new(Mnemonic::SHL)
2501            .with_reg(Register::RAX)
2502            .with_reg(Register::CL);
2503        let bytes = encoder.encode(&request).unwrap();
2504        // REX.W + D3 /4 => 48 D3 E0
2505        assert_eq!(bytes, vec![0x48, 0xD3, 0xE0]);
2506    }
2507
2508    #[test]
2509    fn test_encode_shr_cl() {
2510        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2511
2512        // SHR RBX, CL
2513        let request = EncoderRequest::new(Mnemonic::SHR)
2514            .with_reg(Register::RBX)
2515            .with_reg(Register::CL);
2516        let bytes = encoder.encode(&request).unwrap();
2517        // REX.W + D3 /5 => 48 D3 EB
2518        assert_eq!(bytes, vec![0x48, 0xD3, 0xEB]);
2519    }
2520
2521    #[test]
2522    fn test_encode_sar_cl() {
2523        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2524
2525        // SAR RCX, CL
2526        let request = EncoderRequest::new(Mnemonic::SAR)
2527            .with_reg(Register::RCX)
2528            .with_reg(Register::CL);
2529        let bytes = encoder.encode(&request).unwrap();
2530        // REX.W + D3 /7 => 48 D3 F9
2531        assert_eq!(bytes, vec![0x48, 0xD3, 0xF9]);
2532    }
2533
2534    #[test]
2535    fn test_encode_shl_extended_reg() {
2536        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2537
2538        // SHL R8, CL
2539        let request = EncoderRequest::new(Mnemonic::SHL)
2540            .with_reg(Register::R8)
2541            .with_reg(Register::CL);
2542        let bytes = encoder.encode(&request).unwrap();
2543        // REX.WB + D3 /4 => 49 D3 E0
2544        assert_eq!(bytes, vec![0x49, 0xD3, 0xE0]);
2545    }
2546
2547    // ========================================
2548    // AVX instruction encoding tests
2549    // ========================================
2550
2551    #[test]
2552    fn test_encode_vaddps_reg_reg() {
2553        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2554
2555        // VADDPS xmm1, xmm2, xmm3
2556        // VEX.128.0F.WIG 58 /r
2557        // xmm1 = vvvv (inverted), xmm2 = reg, xmm3 = rm
2558        let request = EncoderRequest::new(Mnemonic::VADDPS)
2559            .with_reg(Register::XMM1)
2560            .with_reg(Register::XMM2)
2561            .with_reg(Register::XMM3);
2562        let bytes = encoder.encode(&request).unwrap();
2563
2564        // VEX prefix: C5 (2-byte) since no extended registers
2565        // R~=1 (XMM2 not extended), vvvv~=~1=0xE, L=0, pp=00
2566        // Byte 2: 1 1110 0 00 = 0xF0
2567        // Opcode: 58
2568        // ModRM: mod=11, reg=2 (XMM2), rm=3 (XMM3) = 0xD3
2569        assert_eq!(bytes.len(), 4);
2570        assert_eq!(bytes[0], 0xC5); // VEX 2-byte prefix
2571        // R~=1 -> 0x80, vvvv~=~1=0xE << 3 = 0x70, total 0xF0
2572        assert_eq!(bytes[1], 0xF0);
2573        assert_eq!(bytes[2], 0x58); // opcode
2574        assert_eq!(bytes[3], 0xD3); // ModRM: 11 010 011
2575    }
2576
2577    #[test]
2578    fn test_encode_vaddpd_reg_reg() {
2579        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2580
2581        // VADDPD xmm1, xmm2, xmm3
2582        // VEX.128.66.0F.WIG 58 /r
2583        let request = EncoderRequest::new(Mnemonic::VADDPD)
2584            .with_reg(Register::XMM1)
2585            .with_reg(Register::XMM2)
2586            .with_reg(Register::XMM3);
2587        let bytes = encoder.encode(&request).unwrap();
2588
2589        // VEX prefix with 66 prefix (pp=01)
2590        assert_eq!(bytes.len(), 4);
2591        assert_eq!(bytes[0], 0xC5); // VEX 2-byte prefix
2592        // R~=1, vvvv~=~1=0xE, L=0, pp=01 => 1 1110 0 01 = 0xF1
2593        assert_eq!(bytes[1], 0xF1);
2594        assert_eq!(bytes[2], 0x58); // opcode
2595        assert_eq!(bytes[3], 0xD3); // ModRM
2596    }
2597
2598    #[test]
2599    fn test_encode_vsubps_reg_reg() {
2600        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2601
2602        // VSUBPS xmm0, xmm1, xmm2
2603        // VEX.128.0F.WIG 5C /r
2604        let request = EncoderRequest::new(Mnemonic::VSUBPS)
2605            .with_reg(Register::XMM0)
2606            .with_reg(Register::XMM1)
2607            .with_reg(Register::XMM2);
2608        let bytes = encoder.encode(&request).unwrap();
2609
2610        assert_eq!(bytes.len(), 4);
2611        assert_eq!(bytes[0], 0xC5); // VEX prefix
2612        // R~=1, vvvv~=~0=0xF, L=0, pp=00 => 1 1111 0 00 = 0xF8
2613        assert_eq!(bytes[1], 0xF8);
2614        assert_eq!(bytes[2], 0x5C); // opcode
2615        // ModRM: mod=11, reg=1, rm=2 = 0xCA
2616        assert_eq!(bytes[3], 0xCA);
2617    }
2618
2619    #[test]
2620    fn test_encode_vsubpd_reg_reg() {
2621        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2622
2623        // VSUBPD xmm0, xmm1, xmm2
2624        // VEX.128.66.0F.WIG 5C /r
2625        let request = EncoderRequest::new(Mnemonic::VSUBPD)
2626            .with_reg(Register::XMM0)
2627            .with_reg(Register::XMM1)
2628            .with_reg(Register::XMM2);
2629        let bytes = encoder.encode(&request).unwrap();
2630
2631        assert_eq!(bytes.len(), 4);
2632        assert_eq!(bytes[0], 0xC5);
2633        // R~=1, vvvv~=~0=0xF, L=0, pp=01 => 0xF9
2634        assert_eq!(bytes[1], 0xF9);
2635        assert_eq!(bytes[2], 0x5C);
2636        assert_eq!(bytes[3], 0xCA);
2637    }
2638
2639    #[test]
2640    fn test_encode_vmulps_reg_reg() {
2641        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2642
2643        // VMULPS xmm1, xmm2, xmm3
2644        // VEX.128.0F.WIG 59 /r
2645        let request = EncoderRequest::new(Mnemonic::VMULPS)
2646            .with_reg(Register::XMM1)
2647            .with_reg(Register::XMM2)
2648            .with_reg(Register::XMM3);
2649        let bytes = encoder.encode(&request).unwrap();
2650
2651        assert_eq!(bytes.len(), 4);
2652        assert_eq!(bytes[0], 0xC5);
2653        assert_eq!(bytes[1], 0xF0); // vvvv~=~1=0xE
2654        assert_eq!(bytes[2], 0x59); // opcode
2655        assert_eq!(bytes[3], 0xD3);
2656    }
2657
2658    #[test]
2659    fn test_encode_vmulpd_reg_reg() {
2660        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2661
2662        // VMULPD xmm1, xmm2, xmm3
2663        // VEX.128.66.0F.WIG 59 /r
2664        let request = EncoderRequest::new(Mnemonic::VMULPD)
2665            .with_reg(Register::XMM1)
2666            .with_reg(Register::XMM2)
2667            .with_reg(Register::XMM3);
2668        let bytes = encoder.encode(&request).unwrap();
2669
2670        assert_eq!(bytes.len(), 4);
2671        assert_eq!(bytes[0], 0xC5);
2672        assert_eq!(bytes[1], 0xF1); // pp=01
2673        assert_eq!(bytes[2], 0x59);
2674        assert_eq!(bytes[3], 0xD3);
2675    }
2676
2677    #[test]
2678    fn test_encode_vxorps_reg_reg() {
2679        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2680
2681        // VXORPS xmm1, xmm2, xmm3
2682        // VEX.128.0F.WIG 57 /r
2683        let request = EncoderRequest::new(Mnemonic::VXORPS)
2684            .with_reg(Register::XMM1)
2685            .with_reg(Register::XMM2)
2686            .with_reg(Register::XMM3);
2687        let bytes = encoder.encode(&request).unwrap();
2688
2689        assert_eq!(bytes.len(), 4);
2690        assert_eq!(bytes[0], 0xC5);
2691        assert_eq!(bytes[1], 0xF0);
2692        assert_eq!(bytes[2], 0x57); // opcode
2693        assert_eq!(bytes[3], 0xD3);
2694    }
2695
2696    #[test]
2697    fn test_encode_vxorpd_reg_reg() {
2698        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2699
2700        // VXORPD xmm1, xmm2, xmm3
2701        // VEX.128.66.0F.WIG 57 /r
2702        let request = EncoderRequest::new(Mnemonic::VXORPD)
2703            .with_reg(Register::XMM1)
2704            .with_reg(Register::XMM2)
2705            .with_reg(Register::XMM3);
2706        let bytes = encoder.encode(&request).unwrap();
2707
2708        assert_eq!(bytes.len(), 4);
2709        assert_eq!(bytes[0], 0xC5);
2710        assert_eq!(bytes[1], 0xF1); // pp=01
2711        assert_eq!(bytes[2], 0x57);
2712        assert_eq!(bytes[3], 0xD3);
2713    }
2714
2715    #[test]
2716    fn test_encode_vaddps_extended_reg() {
2717        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2718
2719        // VADDPS xmm8, xmm9, xmm10
2720        // This uses 3-byte VEX due to extended registers
2721        let request = EncoderRequest::new(Mnemonic::VADDPS)
2722            .with_reg(Register::XMM8)
2723            .with_reg(Register::XMM9)
2724            .with_reg(Register::XMM10);
2725        let bytes = encoder.encode(&request).unwrap();
2726
2727        // Should use 3-byte VEX since we have extended registers
2728        assert_eq!(bytes.len(), 5);
2729        assert_eq!(bytes[0], 0xC4); // 3-byte VEX prefix
2730        // R~=0 (XMM9 extended), X~=1 (no index), B~=0 (XMM10 extended), mmmmm=00001
2731        // Byte 2: 0 1 0 00001 = 0x41
2732        assert_eq!(bytes[1], 0x41);
2733        // W=0, vvvv~=~8=0x7, L=0, pp=00 => 0 0111 0 00 = 0x38
2734        assert_eq!(bytes[2], 0x38);
2735        assert_eq!(bytes[3], 0x58); // opcode
2736        // ModRM: mod=11, reg=1 (XMM9=9&7=1), rm=2 (XMM10=10&7=2) = 0xCA
2737        assert_eq!(bytes[4], 0xCA);
2738    }
2739
2740    #[test]
2741    fn test_encode_vaddps_reg_mem() {
2742        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2743
2744        // VADDPS xmm1, xmm2, [rax]
2745        let mem = MemoryOperand::base(Register::RAX);
2746        let request = EncoderRequest::new(Mnemonic::VADDPS)
2747            .with_reg(Register::XMM1)
2748            .with_reg(Register::XMM2)
2749            .with_mem(mem);
2750        let bytes = encoder.encode(&request).unwrap();
2751
2752        assert_eq!(bytes.len(), 4);
2753        assert_eq!(bytes[0], 0xC5); // 2-byte VEX
2754        assert_eq!(bytes[1], 0xF0); // R~=1, vvvv~=~1=0xE
2755        assert_eq!(bytes[2], 0x58); // opcode
2756        // ModRM: mod=00, reg=2, rm=0 (RAX) = 0x10
2757        assert_eq!(bytes[3], 0x10);
2758    }
2759
2760    #[test]
2761    fn test_encode_vaddps_reg_mem_disp() {
2762        let encoder = Encoder::new(MachineMode::Long64, 64).unwrap();
2763
2764        // VADDPS xmm1, xmm2, [rax+0x10]
2765        let mem = MemoryOperand::base_disp(Register::RAX, 0x10);
2766        let request = EncoderRequest::new(Mnemonic::VADDPS)
2767            .with_reg(Register::XMM1)
2768            .with_reg(Register::XMM2)
2769            .with_mem(mem);
2770        let bytes = encoder.encode(&request).unwrap();
2771
2772        assert_eq!(bytes.len(), 5);
2773        assert_eq!(bytes[0], 0xC5);
2774        assert_eq!(bytes[1], 0xF0);
2775        assert_eq!(bytes[2], 0x58);
2776        // ModRM: mod=01, reg=2, rm=0 = 0x50
2777        assert_eq!(bytes[3], 0x50);
2778        assert_eq!(bytes[4], 0x10); // disp8
2779    }
2780}