memscope_rs/export/binary/
format.rs

1//! Binary file format definitions using simple TLV (Type-Length-Value) structure
2
3use bincode::{Decode, Encode};
4
5/// File magic bytes for format identification
6pub const MAGIC_BYTES: &[u8; 8] = b"MEMSCOPE";
7
8pub const FORMAT_VERSION: u32 = 3; // Updated for Task 6.2: Enhanced safety analysis and passport tracking
9pub const HEADER_SIZE: usize = 32; // Enhanced header: 8+4+4+1+2+2+1+4+4+2 = 32 bytes
10pub const ALLOCATION_RECORD_TYPE: u8 = 1;
11
12// Task 6.2: New segment types for enhanced safety analysis
13pub const UNSAFE_REPORT_SEGMENT_TYPE: u8 = 2;
14pub const MEMORY_PASSPORT_SEGMENT_TYPE: u8 = 3;
15pub const CALL_STACK_SEGMENT_TYPE: u8 = 4;
16pub const FFI_FUNCTION_SEGMENT_TYPE: u8 = 5;
17pub const ADVANCED_METRICS_SEGMENT_TYPE: u8 = 6;
18
19// Segment magic identifiers
20pub const UNSAFE_REPORT_MAGIC: &[u8; 4] = b"USAF"; // Unsafe Analysis segment
21pub const MEMORY_PASSPORT_MAGIC: &[u8; 4] = b"MPPT"; // Memory Passport segment
22pub const CALL_STACK_MAGIC: &[u8; 4] = b"CSTK"; // Call Stack segment
23pub const FFI_FUNCTION_MAGIC: &[u8; 4] = b"FFIR"; // FFI Function Resolution segment
24pub const ADVANCED_METRICS_MAGIC: &[u8; 4] = b"ADVD"; // Advanced Data segment identifier
25
26/// Binary export mode for header identification
27#[repr(u8)]
28#[derive(Debug, Clone, Copy, PartialEq, serde::Serialize, serde::Deserialize, Encode, Decode)]
29pub enum BinaryExportMode {
30    /// User-only export mode (strict filtering)
31    UserOnly = 0,
32    /// Full export mode (loose filtering, all data)
33    Full = 1,
34}
35
36impl From<u8> for BinaryExportMode {
37    fn from(value: u8) -> Self {
38        match value {
39            0 => BinaryExportMode::UserOnly,
40            1 => BinaryExportMode::Full,
41            _ => BinaryExportMode::UserOnly, // Default fallback
42        }
43    }
44}
45
46/// Feature flags for enhanced binary format (bit field)
47pub mod feature_flags {
48    /// Call stack normalization enabled
49    pub const CALL_STACK_NORMALIZATION: u8 = 0b00000001;
50    /// FFI function resolution enabled
51    pub const FFI_FUNCTION_RESOLUTION: u8 = 0b00000010;
52    /// Safety analysis enabled
53    pub const SAFETY_ANALYSIS: u8 = 0b00000100;
54    /// Memory passport tracking enabled
55    pub const MEMORY_PASSPORT_TRACKING: u8 = 0b00001000;
56    /// Enhanced lifetime analysis enabled
57    pub const ENHANCED_LIFETIME_ANALYSIS: u8 = 0b00010000;
58    /// Reserved for future use
59    pub const RESERVED_1: u8 = 0b00100000;
60    pub const RESERVED_2: u8 = 0b01000000;
61    pub const RESERVED_3: u8 = 0b10000000;
62}
63
64/// Enhanced file header structure (32 bytes fixed size)
65/// Extended to include safety analysis and passport tracking information
66#[repr(C)]
67#[derive(Debug, Clone, PartialEq, serde::Serialize, serde::Deserialize, Encode, Decode)]
68pub struct FileHeader {
69    pub magic: [u8; 8],           // 8 bytes: File magic identifier
70    pub version: u32,             // 4 bytes: Format version
71    pub total_count: u32,         // 4 bytes: Total allocation count (user + system)
72    pub export_mode: u8,          // 1 byte: Export mode (user_only vs full)
73    pub user_count: u16,          // 2 bytes: User allocation count (var_name.is_some())
74    pub system_count: u16,        // 2 bytes: System allocation count (var_name.is_none())
75    pub features_enabled: u8,     // 1 byte: Feature flags (bit field)
76    pub unsafe_report_count: u32, // 4 bytes: Number of unsafe reports
77    pub passport_count: u32,      // 4 bytes: Number of memory passports
78    pub reserved: u16,            // 2 bytes: Reserved for future use
79}
80
81impl FileHeader {
82    /// Create a new file header with enhanced information
83    pub fn new(
84        total_count: u32,
85        export_mode: BinaryExportMode,
86        user_count: u16,
87        system_count: u16,
88    ) -> Self {
89        Self {
90            magic: *MAGIC_BYTES,
91            version: FORMAT_VERSION,
92            total_count,
93            export_mode: export_mode as u8,
94            user_count,
95            system_count,
96            features_enabled: feature_flags::CALL_STACK_NORMALIZATION
97                | feature_flags::FFI_FUNCTION_RESOLUTION,
98            unsafe_report_count: 0,
99            passport_count: 0,
100            reserved: 0,
101        }
102    }
103
104    /// Create a legacy header for backward compatibility
105    pub fn new_legacy(count: u32) -> Self {
106        Self {
107            magic: *MAGIC_BYTES,
108            version: FORMAT_VERSION,
109            total_count: count,
110            export_mode: BinaryExportMode::UserOnly as u8,
111            user_count: count as u16,
112            system_count: 0,
113            features_enabled: 0,
114            unsafe_report_count: 0,
115            passport_count: 0,
116            reserved: 0,
117        }
118    }
119
120    pub fn is_valid_magic(&self) -> bool {
121        self.magic == *MAGIC_BYTES
122    }
123
124    pub fn is_compatible_version(&self) -> bool {
125        // Support backward compatibility: can read older versions
126        self.version <= FORMAT_VERSION && self.version >= 1
127    }
128
129    pub fn get_version(&self) -> u32 {
130        self.version
131    }
132
133    pub fn is_legacy_version(&self) -> bool {
134        self.version < FORMAT_VERSION
135    }
136
137    /// Get the export mode from the header
138    pub fn get_export_mode(&self) -> BinaryExportMode {
139        BinaryExportMode::from(self.export_mode)
140    }
141
142    /// Check if this is a user-only binary
143    pub fn is_user_only(&self) -> bool {
144        self.get_export_mode() == BinaryExportMode::UserOnly
145    }
146
147    /// Check if this is a full binary
148    pub fn is_full_binary(&self) -> bool {
149        self.get_export_mode() == BinaryExportMode::Full
150    }
151
152    /// Get allocation count information
153    pub fn get_allocation_counts(&self) -> (u32, u16, u16) {
154        (self.total_count, self.user_count, self.system_count)
155    }
156
157    /// Validate allocation count consistency
158    pub fn is_count_consistent(&self) -> bool {
159        self.total_count == (self.user_count as u32 + self.system_count as u32)
160    }
161
162    /// Convert header to bytes using Little Endian format
163    pub fn to_bytes(&self) -> [u8; HEADER_SIZE] {
164        let mut bytes = [0u8; HEADER_SIZE];
165
166        bytes[0..8].copy_from_slice(&self.magic); // 8 bytes: magic
167        bytes[8..12].copy_from_slice(&self.version.to_le_bytes()); // 4 bytes: version
168        bytes[12..16].copy_from_slice(&self.total_count.to_le_bytes()); // 4 bytes: total_count
169        bytes[16] = self.export_mode; // 1 byte: export_mode
170        bytes[17..19].copy_from_slice(&self.user_count.to_le_bytes()); // 2 bytes: user_count
171        bytes[19..21].copy_from_slice(&self.system_count.to_le_bytes()); // 2 bytes: system_count
172        bytes[21] = self.features_enabled; // 1 byte: features_enabled
173        bytes[22..24].copy_from_slice(&self.unsafe_report_count.to_le_bytes()[0..2]); // 2 bytes: unsafe_report_count (lower 16 bits)
174        bytes[24..28].copy_from_slice(&self.passport_count.to_le_bytes()); // 4 bytes: passport_count
175        bytes[28..30].copy_from_slice(&self.reserved.to_le_bytes()); // 2 bytes: reserved
176
177        bytes
178    }
179
180    /// Create header from bytes using Little Endian format
181    pub fn from_bytes(bytes: &[u8; HEADER_SIZE]) -> Self {
182        let mut magic = [0u8; 8];
183        magic.copy_from_slice(&bytes[0..8]);
184
185        let version = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]);
186        let total_count = u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]);
187        let export_mode = bytes[16];
188        let user_count = u16::from_le_bytes([bytes[17], bytes[18]]);
189        let system_count = u16::from_le_bytes([bytes[19], bytes[20]]);
190        let features_enabled = bytes[21];
191        let unsafe_report_count = u16::from_le_bytes([bytes[22], bytes[23]]) as u32;
192        let passport_count = u32::from_le_bytes([bytes[24], bytes[25], bytes[26], bytes[27]]);
193        let reserved = u16::from_le_bytes([bytes[28], bytes[29]]);
194
195        Self {
196            magic,
197            version,
198            total_count,
199            export_mode,
200            user_count,
201            system_count,
202            features_enabled,
203            unsafe_report_count,
204            passport_count,
205            reserved,
206        }
207    }
208}
209
210/// Allocation record structure for binary serialization with improve.md extensions
211#[derive(Debug, Clone, PartialEq)]
212pub struct AllocationRecord {
213    pub ptr: u64,
214    pub size: u64,
215    pub timestamp: u64,
216    pub var_name: Option<String>,
217    pub type_name: Option<String>,
218    pub thread_id: String,
219    // improve.md extensions
220    pub lifetime_ms: Option<u64>,
221    pub borrow_info: Option<crate::core::types::BorrowInfo>,
222    pub clone_info: Option<crate::core::types::CloneInfo>,
223    pub ownership_history_available: bool,
224}
225
226impl AllocationRecord {
227    /// Calculate serialized size in bytes with improve.md extensions
228    #[allow(dead_code)]
229    pub fn serialized_size(&self) -> usize {
230        let mut size = 1 + 4; // Type + Length
231        size += 8 + 8 + 8; // ptr + size + timestamp
232
233        size += 4; // var_name_len
234        if let Some(ref name) = self.var_name {
235            size += name.len();
236        }
237
238        size += 4; // type_name_len
239        if let Some(ref name) = self.type_name {
240            size += name.len();
241        }
242
243        size += 4; // thread_id_len
244        size += self.thread_id.len();
245
246        // improve.md extensions
247        size += 8; // lifetime_ms (Option<u64> as u64, 0 for None)
248        size += 1; // borrow_info presence flag
249        if self.borrow_info.is_some() {
250            size += 4 + 4 + 4 + 8; // immutable_borrows + mutable_borrows + max_concurrent + timestamp
251        }
252        size += 1; // clone_info presence flag
253        if self.clone_info.is_some() {
254            size += 4 + 1 + 8; // clone_count + is_clone + original_ptr (Option<usize> as u64)
255        }
256        size += 1; // ownership_history_available
257
258        size
259    }
260}
261
262/// Task 6: Advanced metrics segment header
263#[repr(C)]
264#[derive(Debug, Clone, PartialEq)]
265pub struct AdvancedMetricsHeader {
266    pub magic: [u8; 4],      // "ADVD"
267    pub segment_size: u32,   // Size of the entire segment including header
268    pub metrics_bitmap: u32, // Bitmap indicating which metrics are present
269    pub reserved: u32,       // Reserved for future use
270}
271
272impl AdvancedMetricsHeader {
273    pub fn new(segment_size: u32, metrics_bitmap: u32) -> Self {
274        Self {
275            magic: *ADVANCED_METRICS_MAGIC,
276            segment_size,
277            metrics_bitmap,
278            reserved: 0,
279        }
280    }
281
282    pub fn is_valid_magic(&self) -> bool {
283        self.magic == *ADVANCED_METRICS_MAGIC
284    }
285
286    /// Convert header to bytes using Little Endian format
287    pub fn to_bytes(&self) -> [u8; 16] {
288        let mut bytes = [0u8; 16];
289
290        bytes[0..4].copy_from_slice(&self.magic);
291        bytes[4..8].copy_from_slice(&self.segment_size.to_le_bytes());
292        bytes[8..12].copy_from_slice(&self.metrics_bitmap.to_le_bytes());
293        bytes[12..16].copy_from_slice(&self.reserved.to_le_bytes());
294
295        bytes
296    }
297
298    /// Create header from bytes using Little Endian format
299    pub fn from_bytes(bytes: &[u8; 16]) -> Self {
300        let mut magic = [0u8; 4];
301        magic.copy_from_slice(&bytes[0..4]);
302
303        let segment_size = u32::from_le_bytes([bytes[4], bytes[5], bytes[6], bytes[7]]);
304        let metrics_bitmap = u32::from_le_bytes([bytes[8], bytes[9], bytes[10], bytes[11]]);
305        let reserved = u32::from_le_bytes([bytes[12], bytes[13], bytes[14], bytes[15]]);
306
307        Self {
308            magic,
309            segment_size,
310            metrics_bitmap,
311            reserved,
312        }
313    }
314}
315
316/// Task 6: Metrics bitmap flags for identifying which advanced metrics are present
317#[repr(u32)]
318#[derive(Debug, Clone, Copy, PartialEq)]
319pub enum MetricsBitmapFlags {
320    LifecycleAnalysis = 1 << 0,     // Task 4 lifecycle metrics
321    ContainerAnalysis = 1 << 1,     // Task 3 container analysis
322    TypeUsageStats = 1 << 2,        // Task 2 type usage statistics
323    SourceAnalysis = 1 << 3,        // Source code analysis
324    FragmentationAnalysis = 1 << 4, // Memory fragmentation analysis
325    ThreadContext = 1 << 5,         // Thread context information
326    DropChainAnalysis = 1 << 6,     // Drop chain analysis
327    ZstAnalysis = 1 << 7,           // Zero-sized type analysis
328    HealthScoring = 1 << 8,         // Memory health scoring
329    PerformanceBenchmarks = 1 << 9, // Performance benchmark data
330                                    // Bits 10-31 reserved for future metrics
331}
332
333impl MetricsBitmapFlags {
334    /// Check if a specific metric is enabled in the bitmap
335    pub fn is_enabled(bitmap: u32, flag: MetricsBitmapFlags) -> bool {
336        (bitmap & flag as u32) != 0
337    }
338
339    /// Enable a specific metric in the bitmap
340    pub fn enable(bitmap: u32, flag: MetricsBitmapFlags) -> u32 {
341        bitmap | (flag as u32)
342    }
343
344    /// Disable a specific metric in the bitmap
345    #[allow(dead_code)]
346    pub fn disable(bitmap: u32, flag: MetricsBitmapFlags) -> u32 {
347        bitmap & !(flag as u32)
348    }
349}
350
351/// Endian conversion utilities
352pub mod endian {
353    #[allow(dead_code)]
354    pub fn u32_to_le_bytes(value: u32) -> [u8; 4] {
355        value.to_le_bytes()
356    }
357
358    #[allow(dead_code)]
359    pub fn u32_from_le_bytes(bytes: [u8; 4]) -> u32 {
360        u32::from_le_bytes(bytes)
361    }
362
363    #[allow(dead_code)]
364    pub fn u64_to_le_bytes(value: u64) -> [u8; 8] {
365        value.to_le_bytes()
366    }
367
368    #[allow(dead_code)]
369    pub fn u64_from_le_bytes(bytes: [u8; 8]) -> u64 {
370        u64::from_le_bytes(bytes)
371    }
372}
373
374#[cfg(test)]
375mod tests {
376    use super::*;
377
378    #[test]
379    fn test_file_header_creation() {
380        let header = FileHeader::new(100, BinaryExportMode::Full, 60, 40);
381        assert_eq!(header.magic, *MAGIC_BYTES);
382        assert_eq!(header.version, FORMAT_VERSION);
383        assert_eq!(header.total_count, 100);
384        assert_eq!(header.user_count, 60);
385        assert_eq!(header.system_count, 40);
386        assert_eq!(header.get_export_mode(), BinaryExportMode::Full);
387        assert!(header.is_valid_magic());
388        assert!(header.is_compatible_version());
389        assert!(header.is_count_consistent());
390        assert!(header.is_full_binary());
391        assert!(!header.is_user_only());
392    }
393
394    #[test]
395    fn test_file_header_serialization() {
396        let header = FileHeader::new(42, BinaryExportMode::UserOnly, 42, 0);
397        let bytes = header.to_bytes();
398        let deserialized = FileHeader::from_bytes(&bytes);
399
400        assert_eq!(header, deserialized);
401    }
402
403    #[test]
404    fn test_legacy_header_creation() {
405        let header = FileHeader::new_legacy(50);
406        assert_eq!(header.total_count, 50);
407        assert_eq!(header.user_count, 50);
408        assert_eq!(header.system_count, 0);
409        assert_eq!(header.get_export_mode(), BinaryExportMode::UserOnly);
410        assert!(header.is_user_only());
411        assert!(!header.is_full_binary());
412    }
413
414    #[test]
415    fn test_binary_export_mode_conversion() {
416        assert_eq!(BinaryExportMode::from(0), BinaryExportMode::UserOnly);
417        assert_eq!(BinaryExportMode::from(1), BinaryExportMode::Full);
418        assert_eq!(BinaryExportMode::from(255), BinaryExportMode::UserOnly); // Default fallback
419    }
420
421    #[test]
422    fn test_allocation_record_size_calculation() {
423        let record = AllocationRecord {
424            ptr: 0x1000,
425            size: 1024,
426            timestamp: 1234567890,
427            var_name: Some("test_var".to_string()),
428            type_name: Some("i32".to_string()),
429            thread_id: "main".to_string(),
430            // improve.md extensions
431            lifetime_ms: None,
432            borrow_info: None,
433            clone_info: None,
434            ownership_history_available: false,
435        };
436
437        let expected_size = 1 + 4 + // Type + Length
438                           8 + 8 + 8 + // ptr + size + timestamp
439                           4 + 8 + // var_name_len + var_name ("test_var" = 8 chars)
440                           4 + 3 + // type_name_len + type_name ("i32" = 3 chars)
441                           4 + 4 + // thread_id_len + thread_id ("main" = 4 chars)
442                           11; // Additional fields for improve.md extensions (8 + 1 + 1 + 1)
443
444        assert_eq!(record.serialized_size(), expected_size);
445    }
446
447    #[test]
448    fn test_endian_conversion() {
449        let value = 0x12345678u32;
450        let bytes = endian::u32_to_le_bytes(value);
451        let converted = endian::u32_from_le_bytes(bytes);
452        assert_eq!(value, converted);
453
454        let value64 = 0x123456789ABCDEFu64;
455        let bytes64 = endian::u64_to_le_bytes(value64);
456        let converted64 = endian::u64_from_le_bytes(bytes64);
457        assert_eq!(value64, converted64);
458    }
459
460    #[test]
461    fn test_advanced_metrics_header_creation() {
462        let header = AdvancedMetricsHeader::new(1024, 0x12345678);
463        assert_eq!(header.magic, *ADVANCED_METRICS_MAGIC);
464        assert_eq!(header.segment_size, 1024);
465        assert_eq!(header.metrics_bitmap, 0x12345678);
466        assert_eq!(header.reserved, 0);
467        assert!(header.is_valid_magic());
468    }
469
470    #[test]
471    fn test_advanced_metrics_header_serialization() {
472        let header = AdvancedMetricsHeader::new(2048, 0xABCDEF00);
473        let bytes = header.to_bytes();
474        let deserialized = AdvancedMetricsHeader::from_bytes(&bytes);
475
476        assert_eq!(header, deserialized);
477    }
478
479    #[test]
480    fn test_metrics_bitmap_flags() {
481        let mut bitmap = 0u32;
482
483        // Test enabling flags
484        bitmap = MetricsBitmapFlags::enable(bitmap, MetricsBitmapFlags::LifecycleAnalysis);
485        assert!(MetricsBitmapFlags::is_enabled(
486            bitmap,
487            MetricsBitmapFlags::LifecycleAnalysis
488        ));
489        assert!(!MetricsBitmapFlags::is_enabled(
490            bitmap,
491            MetricsBitmapFlags::ContainerAnalysis
492        ));
493
494        bitmap = MetricsBitmapFlags::enable(bitmap, MetricsBitmapFlags::ContainerAnalysis);
495        assert!(MetricsBitmapFlags::is_enabled(
496            bitmap,
497            MetricsBitmapFlags::LifecycleAnalysis
498        ));
499        assert!(MetricsBitmapFlags::is_enabled(
500            bitmap,
501            MetricsBitmapFlags::ContainerAnalysis
502        ));
503
504        // Test disabling flags
505        bitmap = MetricsBitmapFlags::disable(bitmap, MetricsBitmapFlags::LifecycleAnalysis);
506        assert!(!MetricsBitmapFlags::is_enabled(
507            bitmap,
508            MetricsBitmapFlags::LifecycleAnalysis
509        ));
510        assert!(MetricsBitmapFlags::is_enabled(
511            bitmap,
512            MetricsBitmapFlags::ContainerAnalysis
513        ));
514    }
515}
memscope_rs/export/binary/format.rs

memscope_rs/export/binary/
format.rs
