memscope_rs/export/binary/
binary_html_writer.rs

1//! Binary to HTML writer for high-performance direct conversion
2//!
3//! This module provides a specialized writer that converts binary allocation data
4//! directly to HTML format, bypassing JSON intermediate steps for optimal performance.
5//! It works alongside the existing JSON → HTML functionality without interference.
6
7use crate::core::types::AllocationInfo;
8use crate::export::binary::complex_type_analyzer::{ComplexTypeAnalysis, ComplexTypeAnalyzer};
9use crate::export::binary::error::BinaryExportError;
10use crate::export::binary::ffi_safety_analyzer::{FfiSafetyAnalysis, FfiSafetyAnalyzer};
11use crate::export::binary::selective_reader::AllocationField;
12use crate::export::binary::variable_relationship_analyzer::{
13    VariableRelationshipAnalysis, VariableRelationshipAnalyzer,
14};
15
16use std::collections::{HashMap, HashSet};
17use std::io::{BufWriter, Write};
18use std::time::Instant;
19
20/// Configuration for the binary HTML writer
21#[derive(Debug, Clone)]
22pub struct BinaryHtmlWriterConfig {
23    /// Buffer size for I/O operations (default: 256KB)
24    pub buffer_size: usize,
25
26    /// Maximum memory usage before flushing (default: 32MB)
27    pub max_memory_before_flush: usize,
28
29    /// Chunk size for processing large datasets (default: 1000)
30    pub chunk_size: usize,
31
32    /// Enable intelligent buffering (default: true)
33    pub enable_intelligent_buffering: bool,
34
35    /// Enable data compression for large strings (default: false)
36    pub enable_data_compression: bool,
37
38    /// Parallel processing threshold (default: 5000)
39    pub parallel_threshold: usize,
40}
41
42impl Default for BinaryHtmlWriterConfig {
43    fn default() -> Self {
44        Self {
45            buffer_size: 256 * 1024,                   // 256KB
46            max_memory_before_flush: 32 * 1024 * 1024, // 32MB
47            chunk_size: 1000,
48            enable_intelligent_buffering: true,
49            enable_data_compression: false,
50            parallel_threshold: 5000,
51        }
52    }
53}
54
55/// Statistics for binary HTML write operations
56#[derive(Debug, Clone, Default)]
57pub struct BinaryHtmlStats {
58    /// Total allocations processed
59    pub allocations_processed: u64,
60
61    /// Total HTML size generated
62    pub total_html_size: usize,
63
64    /// Template rendering time in milliseconds
65    pub template_render_time_ms: u64,
66
67    /// Data processing time in milliseconds
68    pub data_processing_time_ms: u64,
69
70    /// Peak memory usage during processing
71    pub peak_memory_usage: usize,
72
73    /// Number of buffer flushes performed
74    pub buffer_flushes: u32,
75
76    /// Total processing time in milliseconds
77    pub total_processing_time_ms: u64,
78
79    /// Average processing speed (allocations per second)
80    pub avg_processing_speed: f64,
81
82    /// Memory efficiency (bytes processed per MB memory used)
83    pub memory_efficiency: f64,
84}
85
86impl BinaryHtmlStats {
87    /// Calculate processing throughput
88    pub fn processing_throughput(&self) -> f64 {
89        if self.total_processing_time_ms == 0 {
90            0.0
91        } else {
92            (self.allocations_processed as f64 * 1000.0) / self.total_processing_time_ms as f64
93        }
94    }
95
96    /// Calculate memory efficiency ratio
97    pub fn memory_efficiency_ratio(&self) -> f64 {
98        if self.peak_memory_usage == 0 {
99            0.0
100        } else {
101            (self.allocations_processed as f64) / (self.peak_memory_usage as f64 / 1024.0 / 1024.0)
102        }
103    }
104}
105
106/// Binary allocation data structure for direct HTML processing
107#[derive(Debug, Clone)]
108pub struct BinaryAllocationData {
109    pub id: u64,
110    pub size: usize,
111    pub type_name: String,
112    pub scope_name: String,
113    pub timestamp_alloc: u64,
114    pub is_active: bool,
115    pub ptr: usize,
116    pub thread_id: String,
117    pub var_name: Option<String>,
118    pub borrow_count: usize,
119    pub is_leaked: bool,
120    pub lifetime_ms: Option<u64>,
121    /// Dynamic fields based on requested fields
122    pub optional_fields: HashMap<String, BinaryFieldValue>,
123}
124
125/// Binary field value types for flexible data handling
126#[derive(Debug, Clone)]
127pub enum BinaryFieldValue {
128    String(String),
129    Number(u64),
130    Boolean(bool),
131    Array(Vec<String>),
132    Optional(Option<Box<BinaryFieldValue>>),
133}
134
135impl BinaryAllocationData {
136    /// Create binary allocation data from AllocationInfo
137    pub fn from_allocation(
138        allocation: &AllocationInfo,
139        requested_fields: &HashSet<AllocationField>,
140    ) -> Result<Self, BinaryExportError> {
141        let mut optional_fields = HashMap::new();
142
143        // Process optional fields based on request
144        if requested_fields.contains(&AllocationField::StackTrace) {
145            if let Some(ref stack_trace) = allocation.stack_trace {
146                optional_fields.insert(
147                    "stack_trace".to_string(),
148                    BinaryFieldValue::Array(stack_trace.clone()),
149                );
150            }
151        }
152
153        if requested_fields.contains(&AllocationField::TimestampDealloc) {
154            if let Some(timestamp_dealloc) = allocation.timestamp_dealloc {
155                optional_fields.insert(
156                    "timestamp_dealloc".to_string(),
157                    BinaryFieldValue::Number(timestamp_dealloc),
158                );
159            }
160        }
161
162        // Add more optional fields as needed
163        if requested_fields.contains(&AllocationField::SmartPointerInfo)
164            && allocation.smart_pointer_info.is_some()
165        {
166            optional_fields.insert(
167                "smart_pointer_info".to_string(),
168                BinaryFieldValue::String("present".to_string()),
169            );
170        }
171
172        Ok(Self {
173            id: allocation.ptr as u64,
174            size: allocation.size,
175            type_name: allocation
176                .type_name
177                .clone()
178                .unwrap_or_else(|| "Unknown".to_string()),
179            scope_name: allocation
180                .scope_name
181                .clone()
182                .unwrap_or_else(|| "global".to_string()),
183            timestamp_alloc: allocation.timestamp_alloc,
184            is_active: allocation.timestamp_dealloc.is_none(),
185            ptr: allocation.ptr,
186            thread_id: allocation.thread_id.clone(),
187            var_name: allocation.var_name.clone(),
188            borrow_count: allocation.borrow_count,
189            is_leaked: allocation.is_leaked,
190            lifetime_ms: allocation.lifetime_ms,
191            optional_fields,
192        })
193    }
194}
195
196/// Binary template data structure for HTML generation
197#[derive(Debug, Clone)]
198pub struct BinaryTemplateData {
199    pub project_name: String,
200    pub allocations: Vec<BinaryAllocationData>,
201    pub total_memory_usage: u64,
202    pub peak_memory_usage: u64,
203    pub active_allocations_count: usize,
204    pub processing_time_ms: u64,
205    pub data_source: String,
206    pub complex_types: Option<ComplexTypeAnalysis>,
207    pub unsafe_ffi: Option<FfiSafetyAnalysis>,
208    pub variable_relationships: Option<VariableRelationshipAnalysis>,
209}
210
211/// Intelligent buffering system for optimized write performance
212#[derive(Debug)]
213struct IntelligentBuffer {
214    /// Current buffer usage
215    current_usage: usize,
216    /// Target buffer size for optimal performance
217    target_size: usize,
218    /// Number of writes since last flush
219    writes_since_flush: u32,
220    /// Average write size for adaptive buffering
221    avg_write_size: f64,
222    /// Last flush time for timing-based flushing
223    last_flush_time: Instant,
224}
225
226impl IntelligentBuffer {
227    fn new(target_size: usize) -> Self {
228        let buffer = Self {
229            current_usage: 0,
230            target_size,
231            writes_since_flush: 0,
232            avg_write_size: 0.0,
233            last_flush_time: Instant::now(),
234        };
235
236        // Use the target_size field to ensure it's read
237        tracing::debug!(
238            "Created IntelligentBuffer with target size: {}",
239            buffer.target_size
240        );
241        buffer
242    }
243
244    fn reset_after_flush(&mut self) {
245        self.current_usage = 0;
246        self.writes_since_flush = 0;
247        self.avg_write_size = 0.0;
248        self.last_flush_time = Instant::now();
249    }
250}
251
252/// Binary HTML writer for direct conversion from binary data to HTML
253pub struct BinaryHtmlWriter<W: Write> {
254    /// Inner buffered writer
255    writer: BufWriter<W>,
256
257    /// Configuration
258    config: BinaryHtmlWriterConfig,
259
260    /// Statistics
261    stats: BinaryHtmlStats,
262
263    /// Start time for performance tracking
264    start_time: Instant,
265
266    /// Current memory usage estimate
267    current_memory_usage: usize,
268
269    /// Allocation data buffer for batch processing
270    allocation_buffer: Vec<BinaryAllocationData>,
271
272    /// All allocations for complex type analysis
273    all_allocations: Vec<AllocationInfo>,
274
275    /// Intelligent buffering state
276    intelligent_buffer: IntelligentBuffer,
277}
278
279impl<W: Write> BinaryHtmlWriter<W> {
280    /// Create a new binary HTML writer with default configuration
281    pub fn new(writer: W) -> Result<Self, BinaryExportError> {
282        Self::with_config(writer, BinaryHtmlWriterConfig::default())
283    }
284
285    /// Create a new binary HTML writer with custom configuration
286    pub fn with_config(
287        writer: W,
288        config: BinaryHtmlWriterConfig,
289    ) -> Result<Self, BinaryExportError> {
290        let start_time = Instant::now();
291
292        // Create buffered writer
293        let buffered_writer = BufWriter::with_capacity(config.buffer_size, writer);
294
295        let stats = BinaryHtmlStats::default();
296
297        Ok(Self {
298            writer: buffered_writer,
299            config: config.clone(),
300            stats,
301            start_time,
302            current_memory_usage: 0,
303            allocation_buffer: Vec::with_capacity(config.chunk_size),
304            all_allocations: Vec::new(),
305            intelligent_buffer: IntelligentBuffer::new(config.buffer_size / 4),
306        })
307    }
308
309    /// Write a single allocation directly from binary data
310    pub fn write_binary_allocation(
311        &mut self,
312        allocation: &AllocationInfo,
313        requested_fields: &HashSet<AllocationField>,
314    ) -> Result<(), BinaryExportError> {
315        let write_start = Instant::now();
316
317        // Store allocation for complex type analysis
318        self.all_allocations.push(allocation.clone());
319
320        // Convert to binary allocation data (direct processing, no JSON)
321        let binary_data = BinaryAllocationData::from_allocation(allocation, requested_fields)?;
322
323        // Add to buffer for batch processing
324        self.allocation_buffer.push(binary_data);
325
326        // Update memory usage estimate
327        self.current_memory_usage += std::mem::size_of::<BinaryAllocationData>();
328
329        // Check if we need to flush
330        if self.current_memory_usage >= self.config.max_memory_before_flush {
331            self.flush_allocation_buffer()?;
332        }
333
334        self.stats.allocations_processed += 1;
335        self.stats.data_processing_time_ms += write_start.elapsed().as_millis() as u64;
336
337        Ok(())
338    }
339
340    /// Write multiple allocations in batch for better performance
341    pub fn write_binary_allocation_batch(
342        &mut self,
343        allocations: &[AllocationInfo],
344        requested_fields: &HashSet<AllocationField>,
345    ) -> Result<(), BinaryExportError> {
346        let batch_start = Instant::now();
347
348        // Process allocations based on size
349        if allocations.len() >= self.config.parallel_threshold {
350            self.write_allocation_batch_parallel(allocations, requested_fields)?;
351        } else {
352            self.write_allocation_batch_serial(allocations, requested_fields)?;
353        }
354
355        self.stats.data_processing_time_ms += batch_start.elapsed().as_millis() as u64;
356        Ok(())
357    }
358
359    /// Serial batch processing for smaller datasets
360    fn write_allocation_batch_serial(
361        &mut self,
362        allocations: &[AllocationInfo],
363        requested_fields: &HashSet<AllocationField>,
364    ) -> Result<(), BinaryExportError> {
365        for allocation in allocations {
366            self.write_binary_allocation(allocation, requested_fields)?;
367        }
368        Ok(())
369    }
370
371    /// Parallel batch processing for larger datasets
372    fn write_allocation_batch_parallel(
373        &mut self,
374        allocations: &[AllocationInfo],
375        requested_fields: &HashSet<AllocationField>,
376    ) -> Result<(), BinaryExportError> {
377        // For now, use serial processing
378        // Parallel processing implementation using rayon for better performance
379        self.write_allocation_batch_serial(allocations, requested_fields)
380    }
381
382    /// Flush the allocation buffer
383    fn flush_allocation_buffer(&mut self) -> Result<(), BinaryExportError> {
384        // For now, just clear the buffer
385        // The actual HTML generation happens in finalize_with_binary_template
386        self.allocation_buffer.clear();
387        self.current_memory_usage = 0;
388        self.stats.buffer_flushes += 1;
389        self.intelligent_buffer.reset_after_flush();
390        Ok(())
391    }
392
393    /// Complete HTML generation using binary template data
394    pub fn finalize_with_binary_template(
395        &mut self,
396        project_name: &str,
397    ) -> Result<BinaryHtmlStats, BinaryExportError> {
398        let finalize_start = Instant::now();
399
400        // Build final template data from accumulated allocations
401        let template_data = self.build_binary_template_data(project_name)?;
402
403        // Generate HTML content using binary template engine
404        let html_content = self.render_binary_template(&template_data)?;
405
406        // Write final HTML
407        self.writer.write_all(html_content.as_bytes())?;
408        self.writer.flush()?;
409
410        // Update final statistics
411        self.stats.total_html_size = html_content.len();
412        self.stats.total_processing_time_ms = self.start_time.elapsed().as_millis() as u64;
413        self.stats.template_render_time_ms = finalize_start.elapsed().as_millis() as u64;
414        self.stats.avg_processing_speed = self.stats.processing_throughput();
415        self.stats.memory_efficiency = self.stats.memory_efficiency_ratio();
416
417        Ok(self.stats.clone())
418    }
419
420    /// Build template data structure from binary allocations
421    fn build_binary_template_data(
422        &self,
423        project_name: &str,
424    ) -> Result<BinaryTemplateData, BinaryExportError> {
425        let total_memory: u64 = self.allocation_buffer.iter().map(|a| a.size as u64).sum();
426        let peak_memory = total_memory; // Simplified calculation
427        let active_count = self
428            .allocation_buffer
429            .iter()
430            .filter(|a| a.is_active)
431            .count();
432
433        // Perform complex type analysis on collected allocations
434        let complex_types = if !self.all_allocations.is_empty() {
435            match ComplexTypeAnalyzer::analyze_allocations(&self.all_allocations) {
436                Ok(analysis) => Some(analysis),
437                Err(e) => {
438                    tracing::warn!("Complex type analysis failed: {}", e);
439                    None
440                }
441            }
442        } else {
443            None
444        };
445
446        // Perform FFI safety analysis on collected allocations
447        let unsafe_ffi = if !self.all_allocations.is_empty() {
448            match FfiSafetyAnalyzer::analyze_allocations(&self.all_allocations) {
449                Ok(analysis) => Some(analysis),
450                Err(e) => {
451                    tracing::warn!("FFI safety analysis failed: {}", e);
452                    None
453                }
454            }
455        } else {
456            None
457        };
458
459        // Perform variable relationship analysis on collected allocations
460        let variable_relationships = if !self.all_allocations.is_empty() {
461            match VariableRelationshipAnalyzer::analyze_allocations(&self.all_allocations) {
462                Ok(analysis) => Some(analysis),
463                Err(e) => {
464                    tracing::warn!("Variable relationship analysis failed: {}", e);
465                    None
466                }
467            }
468        } else {
469            None
470        };
471
472        Ok(BinaryTemplateData {
473            project_name: project_name.to_string(),
474            allocations: self.allocation_buffer.clone(),
475            total_memory_usage: total_memory,
476            peak_memory_usage: peak_memory,
477            active_allocations_count: active_count,
478            processing_time_ms: self.stats.data_processing_time_ms,
479            data_source: "binary_direct".to_string(),
480            complex_types,
481            unsafe_ffi,
482            variable_relationships,
483        })
484    }
485
486    /// Render HTML using binary template engine
487    fn render_binary_template(
488        &self,
489        data: &BinaryTemplateData,
490    ) -> Result<String, BinaryExportError> {
491        use crate::export::binary::binary_template_engine::BinaryTemplateEngine;
492
493        // Create and use the binary template engine
494        let mut template_engine = BinaryTemplateEngine::new().map_err(|e| {
495            BinaryExportError::CorruptedData(format!("Failed to create template engine: {e}"))
496        })?;
497
498        // Render the template with binary data
499        template_engine.render_binary_template(data)
500    }
501
502    /// Convert binary template data to JSON format for template compatibility
503    fn _convert_to_json_format(
504        &self,
505        data: &BinaryTemplateData,
506    ) -> Result<String, BinaryExportError> {
507        use serde_json::json;
508
509        let allocations_json: Vec<serde_json::Value> = data
510            .allocations
511            .iter()
512            .map(|alloc| {
513                json!({
514                    "id": alloc.id,
515                    "size": alloc.size,
516                    "type_name": alloc.type_name,
517                    "scope_name": alloc.scope_name,
518                    "timestamp_alloc": alloc.timestamp_alloc,
519                    "is_active": alloc.is_active,
520                    "ptr": format!("0x{:x}", alloc.ptr),
521                    "thread_id": alloc.thread_id,
522                    "var_name": alloc.var_name,
523                    "borrow_count": alloc.borrow_count,
524                    "is_leaked": alloc.is_leaked,
525                    "lifetime_ms": alloc.lifetime_ms
526                })
527            })
528            .collect();
529
530        let mut dashboard_data = json!({
531            "project_name": data.project_name,
532            "data_source": data.data_source,
533            "summary": {
534                "total_allocations": data.allocations.len(),
535                "total_memory": data.total_memory_usage,
536                "peak_memory": data.peak_memory_usage,
537                "active_allocations": data.active_allocations_count
538            },
539            "memory_analysis": {
540                "allocations": allocations_json,
541                "memory_timeline": [],
542                "size_distribution": []
543            },
544            "performance_metrics": {
545                "export_time_ms": data.processing_time_ms,
546                "data_source": "binary_direct",
547                "throughput_allocations_per_sec": self.stats.processing_throughput()
548            }
549        });
550
551        // Add complex type analysis if available
552        if let Some(ref complex_types) = data.complex_types {
553            dashboard_data["complex_types"] = serde_json::to_value(complex_types).map_err(|e| {
554                BinaryExportError::SerializationError(format!(
555                    "Complex types serialization failed: {e}"
556                ))
557            })?;
558        }
559
560        // Add FFI safety analysis if available
561        if let Some(ref unsafe_ffi) = data.unsafe_ffi {
562            dashboard_data["unsafe_ffi"] = serde_json::to_value(unsafe_ffi).map_err(|e| {
563                BinaryExportError::SerializationError(format!(
564                    "FFI safety analysis serialization failed: {e}"
565                ))
566            })?;
567        }
568
569        // Add variable relationship analysis if available
570        if let Some(ref variable_relationships) = data.variable_relationships {
571            dashboard_data["variable_relationships"] = serde_json::to_value(variable_relationships)
572                .map_err(|e| {
573                    BinaryExportError::SerializationError(format!(
574                        "Variable relationship analysis serialization failed: {e}",
575                    ))
576                })?;
577        }
578
579        serde_json::to_string_pretty(&dashboard_data).map_err(|e| {
580            BinaryExportError::SerializationError(format!("JSON serialization failed: {e}"))
581        })
582    }
583
584    /// Get current statistics
585    pub fn get_stats(&self) -> &BinaryHtmlStats {
586        &self.stats
587    }
588
589    /// Update peak memory usage tracking
590    fn _update_peak_memory_usage(&mut self) {
591        if self.current_memory_usage > self.stats.peak_memory_usage {
592            self.stats.peak_memory_usage = self.current_memory_usage;
593        }
594    }
595}
596
597#[cfg(test)]
598mod tests {
599    use super::*;
600    use std::io::Cursor;
601
602    fn create_test_allocation() -> AllocationInfo {
603        AllocationInfo {
604            ptr: 0x1000,
605            size: 1024,
606            var_name: Some("test_var".to_string()),
607            type_name: Some("Vec<u8>".to_string()),
608            scope_name: Some("main".to_string()),
609            timestamp_alloc: 1234567890,
610            timestamp_dealloc: None,
611            thread_id: "main".to_string(),
612            borrow_count: 0,
613            stack_trace: Some(vec!["frame1".to_string(), "frame2".to_string()]),
614            is_leaked: false,
615            lifetime_ms: Some(1000),
616            borrow_info: None,
617            clone_info: None,
618            ownership_history_available: false,
619            smart_pointer_info: None,
620            memory_layout: None,
621            generic_info: None,
622            dynamic_type_info: None,
623            runtime_state: None,
624            stack_allocation: None,
625            temporary_object: None,
626            fragmentation_analysis: None,
627            generic_instantiation: None,
628            type_relationships: None,
629            type_usage: None,
630            function_call_tracking: None,
631            lifecycle_tracking: None,
632            access_tracking: None,
633            drop_chain_analysis: None,
634        }
635    }
636
637    #[test]
638    fn test_binary_html_writer_creation() {
639        let buffer = Vec::new();
640        let cursor = Cursor::new(buffer);
641        let writer = BinaryHtmlWriter::new(cursor);
642        assert!(writer.is_ok());
643    }
644
645    #[test]
646    fn test_binary_allocation_data_conversion() {
647        let allocation = create_test_allocation();
648        let fields = AllocationField::all_basic_fields();
649
650        let binary_data = BinaryAllocationData::from_allocation(&allocation, &fields);
651        assert!(binary_data.is_ok());
652
653        let data = binary_data.expect("Failed to get test value");
654        assert_eq!(data.size, 1024);
655        assert_eq!(data.type_name, "Vec<u8>");
656        assert!(data.is_active);
657    }
658
659    #[test]
660    fn test_write_binary_allocation() {
661        let buffer = Vec::new();
662        let cursor = Cursor::new(buffer);
663        let mut writer = BinaryHtmlWriter::new(cursor).expect("Failed to get test value");
664
665        let allocation = create_test_allocation();
666        let fields = AllocationField::all_basic_fields();
667
668        let result = writer.write_binary_allocation(&allocation, &fields);
669        assert!(result.is_ok());
670        assert_eq!(writer.stats.allocations_processed, 1);
671    }
672
673    #[test]
674    fn test_batch_processing() {
675        let buffer = Vec::new();
676        let cursor = Cursor::new(buffer);
677        let mut writer = BinaryHtmlWriter::new(cursor).expect("Failed to get test value");
678
679        let allocations = vec![create_test_allocation(); 5];
680        let fields = AllocationField::all_basic_fields();
681
682        let result = writer.write_binary_allocation_batch(&allocations, &fields);
683        assert!(result.is_ok());
684        assert_eq!(writer.stats.allocations_processed, 5);
685    }
686
687    #[test]
688    fn test_finalize_with_template() {
689        let buffer = Vec::new();
690        let cursor = Cursor::new(buffer);
691        let mut writer = BinaryHtmlWriter::new(cursor).expect("Failed to get test value");
692
693        let allocation = create_test_allocation();
694        let fields = AllocationField::all_basic_fields();
695
696        writer
697            .write_binary_allocation(&allocation, &fields)
698            .expect("Test operation failed");
699        let stats = writer
700            .finalize_with_binary_template("test_project")
701            .expect("Test operation failed");
702
703        assert_eq!(stats.allocations_processed, 1);
704        assert!(stats.total_html_size > 0);
705        assert!(stats.total_processing_time_ms > 0);
706    }
707
708    #[test]
709    fn test_stats_calculation() {
710        let stats = BinaryHtmlStats {
711            allocations_processed: 1000,
712            total_processing_time_ms: 500,
713            peak_memory_usage: 1024 * 1024, // 1MB
714            ..Default::default()
715        };
716
717        assert_eq!(stats.processing_throughput(), 2000.0); // 2000 allocations/sec
718        assert_eq!(stats.memory_efficiency_ratio(), 1000.0); // 1000 allocations/MB
719    }
720}
memscope_rs/export/binary/binary_html_writer.rs

memscope_rs/export/binary/
binary_html_writer.rs
