#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CodecBits {
Bits32,
Bits64,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct StructuredPrologue {
pub saves: Vec<String>,
pub saves_after: Vec<String>,
pub frame: bool,
pub sub_esp: u32,
pub cf_protect: bool,
pub frame_alt_encoding: bool,
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
pub struct StructuredEpilogue {
pub saves: Vec<String>,
pub leave: bool,
pub pop_frame: bool,
pub add_esp: u32,
pub ret_imm: u16,
}
#[must_use]
pub fn decode_prologue(bytes: &[u8], bits: CodecBits) -> Option<StructuredPrologue> {
let mut p = StructuredPrologue::default();
let mut i = 0;
let endbr = match bits {
CodecBits::Bits32 => &[0xf3u8, 0x0f, 0x1e, 0xfb][..],
CodecBits::Bits64 => &[0xf3u8, 0x0f, 0x1e, 0xfa][..],
};
if bytes.get(i..i + endbr.len()) == Some(endbr) {
p.cf_protect = true;
i += endbr.len();
}
while let Some(&b) = bytes.get(i) {
if (0x50..=0x57).contains(&b) {
p.saves.push(push_reg_name(b, false)?);
i += 1;
} else if matches!(bits, CodecBits::Bits64)
&& bytes
.get(i..i + 2)
.is_some_and(|s| s[0] == 0x41 && (0x50..=0x57).contains(&s[1]))
{
p.saves.push(push_reg_name(bytes[i + 1], true)?);
i += 2;
} else {
break;
}
}
if matches!(p.saves.last().map(String::as_str), Some("ebp" | "rbp")) {
if let Some(mov_b) = bytes.get(i..i + mov_bp_sp_len(bits)) {
if let Some(alt) = mov_bp_sp_form(mov_b, bits) {
p.frame = true;
p.frame_alt_encoding = alt;
p.saves.pop(); i += mov_bp_sp_len(bits);
}
}
}
if let Some(consumed) = parse_sub_esp(&bytes[i..], bits) {
p.sub_esp = consumed.imm;
i += consumed.len;
}
while let Some(&b) = bytes.get(i) {
if (0x50..=0x57).contains(&b) {
p.saves_after.push(push_reg_name(b, false)?);
i += 1;
} else if matches!(bits, CodecBits::Bits64)
&& bytes
.get(i..i + 2)
.is_some_and(|s| s[0] == 0x41 && (0x50..=0x57).contains(&s[1]))
{
p.saves_after.push(push_reg_name(bytes[i + 1], true)?);
i += 2;
} else {
break;
}
}
if i != bytes.len() {
return None;
}
Some(p)
}
#[must_use]
pub fn encode_prologue(p: &StructuredPrologue, bits: CodecBits) -> Vec<u8> {
let mut out: Vec<u8> = Vec::new();
if p.cf_protect {
let endbr = match bits {
CodecBits::Bits32 => &[0xf3u8, 0x0f, 0x1e, 0xfb][..],
CodecBits::Bits64 => &[0xf3u8, 0x0f, 0x1e, 0xfa][..],
};
out.extend_from_slice(endbr);
}
for r in &p.saves {
push_reg_encoded(r, bits, &mut out);
}
if p.frame {
out.push(0x55);
let mov_bytes: &[u8] = match (bits, p.frame_alt_encoding) {
(CodecBits::Bits32, false) => &[0x8b, 0xec],
(CodecBits::Bits32, true) => &[0x89, 0xe5],
(CodecBits::Bits64, false) => &[0x48, 0x8b, 0xec],
(CodecBits::Bits64, true) => &[0x48, 0x89, 0xe5],
};
out.extend_from_slice(mov_bytes);
}
if p.sub_esp > 0 {
encode_sub_esp(p.sub_esp, bits, &mut out);
}
for r in &p.saves_after {
push_reg_encoded(r, bits, &mut out);
}
out
}
#[must_use]
pub fn decode_epilogue(bytes: &[u8], bits: CodecBits) -> Option<StructuredEpilogue> {
let mut e = StructuredEpilogue::default();
let mut i = 0;
while let Some(&b) = bytes.get(i) {
if (0x58..=0x5f).contains(&b) {
e.saves.push(pop_reg_name(b, false)?);
i += 1;
} else if matches!(bits, CodecBits::Bits64)
&& bytes
.get(i..i + 2)
.is_some_and(|s| s[0] == 0x41 && (0x58..=0x5f).contains(&s[1]))
{
e.saves.push(pop_reg_name(bytes[i + 1], true)?);
i += 2;
} else {
break;
}
}
if let Some(consumed) = parse_add_esp(&bytes[i..], bits) {
e.add_esp = consumed.imm;
i += consumed.len;
}
if bytes.get(i) == Some(&0xc9) {
e.leave = true;
i += 1;
} else if matches!(e.saves.last().map(String::as_str), Some("ebp" | "rbp")) {
e.pop_frame = true;
e.saves.pop();
}
match bytes.get(i) {
Some(&0xc3) => {
i += 1;
}
Some(&0xc2) => {
let lo = *bytes.get(i + 1)?;
let hi = *bytes.get(i + 2)?;
e.ret_imm = u16::from(lo) | (u16::from(hi) << 8);
i += 3;
}
_ => return None,
}
if i != bytes.len() {
return None;
}
Some(e)
}
#[must_use]
pub fn encode_epilogue(e: &StructuredEpilogue, bits: CodecBits) -> Vec<u8> {
let mut out: Vec<u8> = Vec::new();
for r in &e.saves {
pop_reg_encoded(r, bits, &mut out);
}
if e.add_esp > 0 {
encode_add_esp(e.add_esp, bits, &mut out);
}
if e.leave {
out.push(0xc9);
} else if e.pop_frame {
out.push(0x5d);
}
if e.ret_imm == 0 {
out.push(0xc3);
} else {
out.push(0xc2);
out.push((e.ret_imm & 0xff) as u8);
out.push(((e.ret_imm >> 8) & 0xff) as u8);
}
out
}
#[derive(Debug, Clone, Default, PartialEq, Eq)]
#[allow(clippy::struct_excessive_bools)]
pub struct ProfileInputs {
pub saves_used: Vec<String>,
pub frame_required: bool,
pub sub_esp: u32,
pub cf_protect: bool,
pub stack_arg_count: u32,
pub abi: String,
pub bits: u32,
pub frame_alt: bool,
pub body_has_call: bool,
}
fn compute_aligned_sub_esp(inputs: &ProfileInputs) -> u32 {
let max_var_off = inputs.sub_esp;
if inputs.bits == 64 {
if inputs.frame_required {
(max_var_off + 15) & !15u32
} else if max_var_off == 0 && !inputs.body_has_call {
0
} else if max_var_off <= 8 {
8
} else {
(((max_var_off - 8) + 15) & !15u32) + 8
}
} else {
max_var_off
}
}
#[must_use]
pub fn default_prologue(inputs: &ProfileInputs) -> StructuredPrologue {
let (saves, saves_after) = if inputs.frame_required {
(Vec::new(), inputs.saves_used.clone())
} else {
(inputs.saves_used.clone(), Vec::new())
};
let sub_esp = compute_aligned_sub_esp(inputs);
StructuredPrologue {
saves,
saves_after,
frame: inputs.frame_required,
sub_esp,
cf_protect: inputs.cf_protect,
frame_alt_encoding: inputs.frame_alt,
}
}
#[must_use]
pub fn default_epilogue(inputs: &ProfileInputs) -> StructuredEpilogue {
let saves: Vec<String> = inputs.saves_used.iter().rev().cloned().collect();
let ret_imm = match inputs.abi.as_str() {
"stdcall" | "thiscall" | "fastcall" => {
u16::try_from(inputs.stack_arg_count * 4).unwrap_or(0)
}
_ => 0,
};
let sub_esp = compute_aligned_sub_esp(inputs);
let leave = inputs.frame_required && sub_esp > 0;
let pop_frame = inputs.frame_required && !leave;
let add_esp = if inputs.frame_required { 0 } else { sub_esp };
StructuredEpilogue {
saves,
leave,
pop_frame,
add_esp,
ret_imm,
}
}
#[must_use]
pub fn prologue_roundtrips(bytes: &[u8], bits: CodecBits) -> Option<StructuredPrologue> {
let p = decode_prologue(bytes, bits)?;
if encode_prologue(&p, bits) == bytes {
Some(p)
} else {
None
}
}
#[must_use]
pub fn epilogue_roundtrips(bytes: &[u8], bits: CodecBits) -> Option<StructuredEpilogue> {
let e = decode_epilogue(bytes, bits)?;
if encode_epilogue(&e, bits) == bytes {
Some(e)
} else {
None
}
}
fn push_reg_name(opcode: u8, rex_b: bool) -> Option<String> {
let n = (opcode - 0x50) | (u8::from(rex_b) << 3);
Some(reg_name_for_opcode_index(n)?.to_string())
}
fn pop_reg_name(opcode: u8, rex_b: bool) -> Option<String> {
let n = (opcode - 0x58) | (u8::from(rex_b) << 3);
Some(reg_name_for_opcode_index(n)?.to_string())
}
fn reg_name_for_opcode_index(n: u8) -> Option<&'static str> {
Some(match n {
0 => "eax",
1 => "ecx",
2 => "edx",
3 => "ebx",
4 => "esp",
5 => "ebp",
6 => "esi",
7 => "edi",
8 => "r8",
9 => "r9",
10 => "r10",
11 => "r11",
12 => "r12",
13 => "r13",
14 => "r14",
15 => "r15",
_ => return None,
})
}
fn push_reg_encoded(name: &str, bits: CodecBits, out: &mut Vec<u8>) {
if let Some((idx, rex_b)) = encode_index_of(name) {
if rex_b {
assert!(matches!(bits, CodecBits::Bits64));
out.push(0x41);
}
out.push(0x50 + idx);
}
}
fn pop_reg_encoded(name: &str, bits: CodecBits, out: &mut Vec<u8>) {
if let Some((idx, rex_b)) = encode_index_of(name) {
if rex_b {
assert!(matches!(bits, CodecBits::Bits64));
out.push(0x41);
}
out.push(0x58 + idx);
}
}
fn encode_index_of(name: &str) -> Option<(u8, bool)> {
Some(match name {
"eax" | "rax" => (0, false),
"ecx" | "rcx" => (1, false),
"edx" | "rdx" => (2, false),
"ebx" | "rbx" => (3, false),
"esp" | "rsp" => (4, false),
"ebp" | "rbp" => (5, false),
"esi" | "rsi" => (6, false),
"edi" | "rdi" => (7, false),
"r8" => (0, true),
"r9" => (1, true),
"r10" => (2, true),
"r11" => (3, true),
"r12" => (4, true),
"r13" => (5, true),
"r14" => (6, true),
"r15" => (7, true),
_ => return None,
})
}
fn mov_bp_sp_len(bits: CodecBits) -> usize {
match bits {
CodecBits::Bits32 => 2,
CodecBits::Bits64 => 3,
}
}
fn mov_bp_sp_form(b: &[u8], bits: CodecBits) -> Option<bool> {
match bits {
CodecBits::Bits32 => match b {
[0x8b, 0xec] => Some(false),
[0x89, 0xe5] => Some(true),
_ => None,
},
CodecBits::Bits64 => match b {
[0x48, 0x8b, 0xec] => Some(false),
[0x48, 0x89, 0xe5] => Some(true),
_ => None,
},
}
}
struct ParsedImm {
imm: u32,
len: usize,
}
fn parse_sub_esp(bytes: &[u8], bits: CodecBits) -> Option<ParsedImm> {
let rex = matches!(bits, CodecBits::Bits64);
let offset = usize::from(rex);
if rex && bytes.first() != Some(&0x48) {
return None;
}
match bytes.get(offset)? {
0x83 if bytes.get(offset + 1) == Some(&0xec) => {
let imm = bytes.get(offset + 2).copied()?;
Some(ParsedImm {
imm: u32::from(imm),
len: offset + 3,
})
}
0x81 if bytes.get(offset + 1) == Some(&0xec) => {
let b0 = bytes.get(offset + 2).copied()?;
let b1 = bytes.get(offset + 3).copied()?;
let b2 = bytes.get(offset + 4).copied()?;
let b3 = bytes.get(offset + 5).copied()?;
let imm = u32::from_le_bytes([b0, b1, b2, b3]);
Some(ParsedImm {
imm,
len: offset + 6,
})
}
_ => None,
}
}
fn parse_add_esp(bytes: &[u8], bits: CodecBits) -> Option<ParsedImm> {
let rex = matches!(bits, CodecBits::Bits64);
let offset = usize::from(rex);
if rex && bytes.first() != Some(&0x48) {
return None;
}
match bytes.get(offset)? {
0x83 if bytes.get(offset + 1) == Some(&0xc4) => {
let imm = bytes.get(offset + 2).copied()?;
Some(ParsedImm {
imm: u32::from(imm),
len: offset + 3,
})
}
0x81 if bytes.get(offset + 1) == Some(&0xc4) => {
let b0 = bytes.get(offset + 2).copied()?;
let b1 = bytes.get(offset + 3).copied()?;
let b2 = bytes.get(offset + 4).copied()?;
let b3 = bytes.get(offset + 5).copied()?;
let imm = u32::from_le_bytes([b0, b1, b2, b3]);
Some(ParsedImm {
imm,
len: offset + 6,
})
}
_ => None,
}
}
fn encode_sub_esp(imm: u32, bits: CodecBits, out: &mut Vec<u8>) {
if matches!(bits, CodecBits::Bits64) {
out.push(0x48);
}
if imm < 0x80 {
out.extend_from_slice(&[0x83, 0xec, imm as u8]);
} else {
out.push(0x81);
out.push(0xec);
out.extend_from_slice(&imm.to_le_bytes());
}
}
fn encode_add_esp(imm: u32, bits: CodecBits, out: &mut Vec<u8>) {
if matches!(bits, CodecBits::Bits64) {
out.push(0x48);
}
if imm < 0x80 {
out.extend_from_slice(&[0x83, 0xc4, imm as u8]);
} else {
out.push(0x81);
out.push(0xc4);
out.extend_from_slice(&imm.to_le_bytes());
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn decode_msvc_x86_full_prologue() {
let bytes = [0x55, 0x8b, 0xec, 0x83, 0xec, 0x40, 0x53, 0x56, 0x57];
let p = decode_prologue(&bytes, CodecBits::Bits32).expect("decode");
assert!(p.frame);
assert_eq!(p.sub_esp, 0x40);
assert_eq!(p.saves_after, vec!["ebx", "esi", "edi"]);
assert_eq!(p.saves, Vec::<String>::new());
assert!(!p.cf_protect);
assert_eq!(encode_prologue(&p, CodecBits::Bits32), bytes);
}
#[test]
fn decode_pure_saves() {
let bytes = [0x53, 0x56, 0x57];
let p = decode_prologue(&bytes, CodecBits::Bits32).expect("decode");
assert_eq!(p.saves, vec!["ebx", "esi", "edi"]);
assert!(!p.frame);
assert_eq!(p.sub_esp, 0);
assert_eq!(encode_prologue(&p, CodecBits::Bits32), bytes);
}
#[test]
fn decode_endbr_then_frame() {
let bytes = [0xf3, 0x0f, 0x1e, 0xfa, 0x55, 0x48, 0x8b, 0xec];
let p = decode_prologue(&bytes, CodecBits::Bits64).expect("decode");
assert!(p.cf_protect);
assert!(p.frame);
assert!(p.saves.is_empty());
assert_eq!(encode_prologue(&p, CodecBits::Bits64), bytes);
}
#[test]
fn decode_msvc_epilogue_saves_imm() {
let bytes = [0x5f, 0x5e, 0x5b, 0x5d, 0xc2, 0x0c, 0x00];
let e = decode_epilogue(&bytes, CodecBits::Bits32).expect("decode");
assert_eq!(e.saves, vec!["edi", "esi", "ebx"]);
assert!(e.pop_frame);
assert_eq!(e.ret_imm, 0x0c);
assert_eq!(encode_epilogue(&e, CodecBits::Bits32), bytes);
}
#[test]
fn decode_leave_ret() {
let bytes = [0xc9, 0xc3];
let e = decode_epilogue(&bytes, CodecBits::Bits32).expect("decode");
assert!(e.leave);
assert!(!e.pop_frame);
assert_eq!(e.ret_imm, 0);
assert_eq!(encode_epilogue(&e, CodecBits::Bits32), bytes);
}
#[test]
fn decode_bare_ret() {
let bytes = [0xc3];
let e = decode_epilogue(&bytes, CodecBits::Bits32).expect("decode");
assert!(e.saves.is_empty());
assert!(!e.leave && !e.pop_frame);
assert_eq!(e.ret_imm, 0);
assert_eq!(encode_epilogue(&e, CodecBits::Bits32), bytes);
}
}