ghostscope_platform/
calling_convention.rs

1use crate::types::{CodeReader, PlatformError};
2use tracing::{debug, warn};
3
4/// Platform-specific calling convention and prologue analysis
5pub trait CallingConvention {
6    /// Get the register number for a parameter at the given index
7    fn get_parameter_register(param_index: usize) -> Option<u16>;
8
9    /// Maximum number of parameters passed in registers
10    fn max_register_parameters() -> usize;
11
12    /// Skip function prologue and return the address where the function body starts
13    fn skip_prologue<R: CodeReader>(
14        function_start: u64,
15        code_reader: &R,
16    ) -> Result<u64, PlatformError>;
17
18    /// Check if we're currently in the function prologue
19    fn is_in_prologue<R: CodeReader>(pc: u64, function_start: u64, code_reader: &R) -> bool {
20        match Self::skip_prologue(function_start, code_reader) {
21            Ok(prologue_end) => pc < prologue_end,
22            Err(_) => false, // Conservative: assume not in prologue if we can't determine
23        }
24    }
25}
26
27/// x86-64 System V ABI calling convention
28pub struct X86_64SystemV;
29
30impl CallingConvention for X86_64SystemV {
31    fn get_parameter_register(param_index: usize) -> Option<u16> {
32        // System V ABI parameter registers (DWARF register numbers)
33        const PARAMETER_REGISTERS: [u16; 6] = [
34            5, // RDI (DWARF register 5) - 1st parameter
35            4, // RSI (DWARF register 4) - 2nd parameter
36            1, // RDX (DWARF register 1) - 3rd parameter
37            2, // RCX (DWARF register 2) - 4th parameter
38            8, // R8  (DWARF register 8) - 5th parameter
39            9, // R9  (DWARF register 9) - 6th parameter
40        ];
41
42        PARAMETER_REGISTERS.get(param_index).copied()
43    }
44
45    fn max_register_parameters() -> usize {
46        6
47    }
48
49    fn skip_prologue<R: CodeReader>(
50        function_start: u64,
51        code_reader: &R,
52    ) -> Result<u64, PlatformError> {
53        debug!(
54            "Analyzing prologue for function starting at 0x{:x}",
55            function_start
56        );
57
58        // 1. Try DWARF line information first
59        if let Some(prologue_end) = find_prologue_end_from_line_info(function_start, code_reader) {
60            debug!(
61                "Found prologue end from DWARF line info: 0x{:x}",
62                prologue_end
63            );
64            return Ok(prologue_end);
65        }
66
67        // 2. Fall back to instruction pattern analysis
68        if let Some(prologue_end) =
69            analyze_x86_64_prologue_instructions(function_start, code_reader)
70        {
71            debug!(
72                "Found prologue end from instruction analysis: 0x{:x}",
73                prologue_end
74            );
75            return Ok(prologue_end);
76        }
77
78        // 3. Cannot determine prologue end
79        warn!(
80            "Cannot determine prologue end for function at 0x{:x}",
81            function_start
82        );
83        Err(PlatformError::PrologueAnalysisFailed(
84            "Cannot determine prologue end - parameters may be optimized".to_string(),
85        ))
86    }
87}
88
89/// Find prologue end using DWARF line number information
90fn find_prologue_end_from_line_info<R: CodeReader>(
91    function_start: u64,
92    code_reader: &R,
93) -> Option<u64> {
94    debug!("Trying to find prologue end from DWARF line info");
95
96    // Try to find the next is_stmt=true instruction after function start
97    // This follows GDB's approach for prologue detection
98    if let Some(prologue_end) = code_reader.find_next_stmt_address(function_start) {
99        debug!(
100            "Found next is_stmt=true instruction at 0x{:x} (offset +{} from function start 0x{:x})",
101            prologue_end,
102            prologue_end - function_start,
103            function_start
104        );
105        Some(prologue_end)
106    } else {
107        debug!(
108            "No next is_stmt=true instruction found after function start 0x{:x}",
109            function_start
110        );
111        None
112    }
113}
114
115/// Analyze x86-64 instruction patterns to find prologue end
116fn analyze_x86_64_prologue_instructions<R: CodeReader>(
117    function_start: u64,
118    code_reader: &R,
119) -> Option<u64> {
120    debug!(
121        "Analyzing x86-64 prologue instructions starting at 0x{:x}",
122        function_start
123    );
124
125    let mut pc = function_start;
126
127    // Skip endbr64 instruction if present (0xf3 0x0f 0x1e 0xfa)
128    if let Some(bytes) = code_reader.read_code_bytes(pc, 4) {
129        if bytes.len() >= 4 && bytes == [0xf3, 0x0f, 0x1e, 0xfa] {
130            debug!("Found endbr64 at 0x{:x}, skipping", pc);
131            pc += 4;
132        }
133    }
134
135    // Look for push %rbp (0x55)
136    if let Some(bytes) = code_reader.read_code_bytes(pc, 1) {
137        if !bytes.is_empty() && bytes[0] == 0x55 {
138            debug!("Found push %%rbp at 0x{:x}", pc);
139            pc += 1;
140
141            // Look for mov %rsp,%rbp patterns
142            if let Some(mov_bytes) = code_reader.read_code_bytes(pc, 3) {
143                if mov_bytes.len() >= 3 {
144                    // Pattern 1: 0x48 0x89 0xe5 (mov %rsp,%rbp)
145                    // Pattern 2: 0x48 0x8b 0xec (mov %rsp,%rbp - alternative encoding)
146                    if mov_bytes == [0x48, 0x89, 0xe5] || mov_bytes == [0x48, 0x8b, 0xec] {
147                        debug!("Found mov %%rsp,%%rbp at 0x{:x}", pc);
148                        pc += 3;
149
150                        // Skip optional stack allocation: sub $imm,%rsp
151                        pc = skip_stack_allocation(pc, code_reader);
152
153                        debug!("Prologue analysis complete, body starts at 0x{:x}", pc);
154                        return Some(pc);
155                    }
156                }
157            }
158
159            // Check for 32-bit mov %esp,%ebp patterns
160            if let Some(mov_bytes) = code_reader.read_code_bytes(pc, 2) {
161                if mov_bytes.len() >= 2 {
162                    // Pattern: 0x89 0xe5 (mov %esp,%ebp)
163                    // Pattern: 0x8b 0xec (mov %esp,%ebp - alternative)
164                    if mov_bytes == [0x89, 0xe5] || mov_bytes == [0x8b, 0xec] {
165                        debug!("Found 32-bit mov %%esp,%%ebp at 0x{:x}", pc);
166                        pc += 2;
167                        pc = skip_stack_allocation(pc, code_reader);
168                        return Some(pc);
169                    }
170                }
171            }
172        }
173    }
174
175    // Check for frameless function (no %rbp setup)
176    // Look for immediate stack allocation: sub $imm,%rsp
177    let allocation_end = skip_stack_allocation(function_start, code_reader);
178    if allocation_end > function_start {
179        debug!("Found frameless function with stack allocation");
180        return Some(allocation_end);
181    }
182
183    // If no prologue pattern found, assume function starts immediately
184    debug!("No standard prologue pattern found, assuming function body starts immediately");
185    Some(function_start)
186}
187
188/// Skip stack allocation instructions and return the address after them
189fn skip_stack_allocation<R: CodeReader>(pc: u64, code_reader: &R) -> u64 {
190    let mut current_pc = pc;
191
192    // Look for sub $imm,%rsp patterns
193    if let Some(bytes) = code_reader.read_code_bytes(current_pc, 4) {
194        if bytes.len() >= 4 {
195            // Pattern 1: 0x48 0x83 0xec 0xXX (sub $imm8,%rsp)
196            if bytes[0] == 0x48 && bytes[1] == 0x83 && bytes[2] == 0xec {
197                debug!("Found sub $0x{:x},%%rsp at 0x{:x}", bytes[3], current_pc);
198                current_pc += 4;
199            }
200            // Pattern 2: 0x48 0x81 0xec (sub $imm32,%rsp) - need to read more bytes
201            else if bytes[0] == 0x48 && bytes[1] == 0x81 && bytes[2] == 0xec {
202                if let Some(extended_bytes) = code_reader.read_code_bytes(current_pc, 7) {
203                    if extended_bytes.len() >= 7 {
204                        let imm32 = u32::from_le_bytes([
205                            extended_bytes[3],
206                            extended_bytes[4],
207                            extended_bytes[5],
208                            extended_bytes[6],
209                        ]);
210                        debug!("Found sub $0x{:x},%%rsp at 0x{:x}", imm32, current_pc);
211                        current_pc += 7;
212                    }
213                }
214            }
215        }
216    }
217
218    current_pc
219}
220
221/// Check if a parameter should be accessed via register in the current context
222/// Returns the register number if parameter should be in register, None otherwise
223pub fn get_parameter_register_in_context<R: CodeReader>(
224    param_index: usize,
225    param_name: &str,
226    pc: u64,
227    function_start: u64,
228    code_reader: &R,
229) -> Result<Option<u16>, PlatformError> {
230    debug!(
231        "Checking parameter '{}' (index={}) register access at PC 0x{:x}, function_start=0x{:x}",
232        param_name, param_index, pc, function_start
233    );
234
235    // Use x86-64 System V ABI for now
236    // TODO: Detect platform and use appropriate calling convention
237
238    if X86_64SystemV::is_in_prologue(pc, function_start, code_reader) {
239        debug!("PC 0x{:x} is in prologue, checking register parameter", pc);
240
241        // In prologue, parameter is in register
242        if let Some(reg) = X86_64SystemV::get_parameter_register(param_index) {
243            debug!("Parameter '{}' is in register {}", param_name, reg);
244            Ok(Some(reg))
245        } else {
246            // Parameter index >= 6, likely optimized away
247            Err(PlatformError::ParameterOptimized(format!(
248                "Parameter '{param_name}' (index {param_index}) likely optimized away (>6 parameters)"
249            )))
250        }
251    } else {
252        debug!(
253            "PC 0x{:x} is after prologue, parameter should be on stack",
254            pc
255        );
256
257        // After prologue, parameter should be on stack
258        Ok(None)
259    }
260}
261
262#[cfg(test)]
263mod tests {
264    use super::*;
265
266    #[test]
267    fn test_x86_64_parameter_registers() {
268        assert_eq!(X86_64SystemV::get_parameter_register(0), Some(5)); // RDI
269        assert_eq!(X86_64SystemV::get_parameter_register(1), Some(4)); // RSI
270        assert_eq!(X86_64SystemV::get_parameter_register(2), Some(1)); // RDX
271        assert_eq!(X86_64SystemV::get_parameter_register(3), Some(2)); // RCX
272        assert_eq!(X86_64SystemV::get_parameter_register(4), Some(8)); // R8
273        assert_eq!(X86_64SystemV::get_parameter_register(5), Some(9)); // R9
274        assert_eq!(X86_64SystemV::get_parameter_register(6), None); // Beyond registers
275
276        assert_eq!(X86_64SystemV::max_register_parameters(), 6);
277    }
278}
ghostscope_platform/calling_convention.rs

ghostscope_platform/
calling_convention.rs
