trustformers_debug/
data_export.rs

1//! Data export capabilities for debugging tools
2//!
3//! This module provides comprehensive data export functionality supporting
4//! multiple formats including CSV, Excel, JSON, HDF5, and more.
5
6use anyhow::Result;
7use chrono::{DateTime, Utc};
8use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10use uuid::Uuid;
11
12/// Data export manager for debugging tools
13#[derive(Debug, Clone)]
14pub struct DataExportManager {
15    /// Export configuration
16    config: ExportConfig,
17    /// Active export jobs
18    active_jobs: HashMap<Uuid, ExportJob>,
19    /// Export history
20    export_history: Vec<ExportRecord>,
21    /// Supported formats
22    supported_formats: Vec<ExportFormat>,
23}
24
25/// Export configuration
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct ExportConfig {
28    /// Default export directory
29    pub default_directory: String,
30    /// Maximum file size (bytes)
31    pub max_file_size: u64,
32    /// Enable compression
33    pub enable_compression: bool,
34    /// Default format
35    pub default_format: ExportFormat,
36    /// Include metadata
37    pub include_metadata: bool,
38    /// Export templates
39    pub templates: Vec<ExportTemplate>,
40}
41
42/// Export job tracking
43#[derive(Debug, Clone, Serialize, Deserialize)]
44pub struct ExportJob {
45    /// Job identifier
46    pub id: Uuid,
47    /// Job name
48    pub name: String,
49    /// Export format
50    pub format: ExportFormat,
51    /// Output path
52    pub output_path: String,
53    /// Job status
54    pub status: ExportStatus,
55    /// Progress percentage
56    pub progress: f64,
57    /// Start time
58    pub started_at: DateTime<Utc>,
59    /// Completion time
60    pub completed_at: Option<DateTime<Utc>>,
61    /// Data size (bytes)
62    pub data_size: u64,
63    /// Error message (if failed)
64    pub error_message: Option<String>,
65    /// Export options
66    pub options: ExportOptions,
67}
68
69/// Export record for history
70#[derive(Debug, Clone, Serialize, Deserialize)]
71pub struct ExportRecord {
72    /// Record identifier
73    pub id: Uuid,
74    /// Export job ID
75    pub job_id: Uuid,
76    /// Export timestamp
77    pub timestamp: DateTime<Utc>,
78    /// File path
79    pub file_path: String,
80    /// File size
81    pub file_size: u64,
82    /// Export format
83    pub format: ExportFormat,
84    /// Success status
85    pub success: bool,
86    /// Duration (seconds)
87    pub duration: f64,
88}
89
90/// Export template for common configurations
91#[derive(Debug, Clone, Serialize, Deserialize)]
92pub struct ExportTemplate {
93    /// Template identifier
94    pub id: String,
95    /// Template name
96    pub name: String,
97    /// Description
98    pub description: String,
99    /// Export format
100    pub format: ExportFormat,
101    /// Export options
102    pub options: ExportOptions,
103    /// Data filters
104    pub filters: DataFilters,
105    /// Template tags
106    pub tags: Vec<String>,
107}
108
109/// Export options
110#[derive(Debug, Clone, Serialize, Deserialize)]
111pub struct ExportOptions {
112    /// Include headers (for CSV/Excel)
113    pub include_headers: bool,
114    /// Date format
115    pub date_format: String,
116    /// Precision for floats
117    pub float_precision: u32,
118    /// Field separator (for CSV)
119    pub separator: String,
120    /// Compression level (0-9)
121    pub compression_level: u32,
122    /// Include metadata
123    pub include_metadata: bool,
124    /// Custom formatting options
125    pub custom_options: HashMap<String, serde_json::Value>,
126}
127
128/// Data filters for selective export
129#[derive(Debug, Clone, Serialize, Deserialize)]
130pub struct DataFilters {
131    /// Date range filter
132    pub date_range: Option<DateRange>,
133    /// Include specific data types
134    pub data_types: Vec<DataType>,
135    /// Exclude fields
136    pub exclude_fields: Vec<String>,
137    /// Include only fields
138    pub include_fields: Option<Vec<String>>,
139    /// Custom filters
140    pub custom_filters: HashMap<String, serde_json::Value>,
141}
142
143/// Date range for filtering
144#[derive(Debug, Clone, Serialize, Deserialize)]
145pub struct DateRange {
146    /// Start date
147    pub start: DateTime<Utc>,
148    /// End date
149    pub end: DateTime<Utc>,
150}
151
152/// Export formats supported
153#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Hash, Eq)]
154pub enum ExportFormat {
155    /// Comma-separated values
156    Csv,
157    /// Excel workbook
158    Excel,
159    /// JSON format
160    Json,
161    /// Pretty-printed JSON
162    JsonPretty,
163    /// HDF5 format
164    Hdf5,
165    /// Parquet format
166    Parquet,
167    /// XML format
168    Xml,
169    /// YAML format
170    Yaml,
171    /// SQLite database
172    Sqlite,
173    /// MessagePack
174    MessagePack,
175    /// Apache Arrow
176    Arrow,
177    /// Custom format
178    Custom(String),
179}
180
181/// Export status
182#[derive(Debug, Clone, Serialize, Deserialize)]
183pub enum ExportStatus {
184    Pending,
185    InProgress,
186    Completed,
187    Failed,
188    Cancelled,
189}
190
191/// Data types that can be exported
192#[derive(Debug, Clone, Serialize, Deserialize)]
193pub enum DataType {
194    TensorData,
195    GradientData,
196    PerformanceMetrics,
197    MemoryProfiles,
198    ActivityLogs,
199    AnnotationData,
200    CommentData,
201    ModelDiagnostics,
202    TrainingDynamics,
203    ArchitectureAnalysis,
204    Custom(String),
205}
206
207/// Exportable data container
208#[derive(Debug, Clone, Serialize, Deserialize)]
209pub struct ExportableData {
210    /// Data identifier
211    pub id: Uuid,
212    /// Data name
213    pub name: String,
214    /// Data type
215    pub data_type: DataType,
216    /// Creation timestamp
217    pub timestamp: DateTime<Utc>,
218    /// Data content
219    pub content: ExportDataContent,
220    /// Metadata
221    pub metadata: HashMap<String, serde_json::Value>,
222    /// Data size
223    pub size: u64,
224}
225
226/// Content of exportable data
227#[derive(Debug, Clone, Serialize, Deserialize)]
228pub enum ExportDataContent {
229    /// Tabular data
230    Table(TableData),
231    /// Time series data
232    TimeSeries(TimeSeriesData),
233    /// Key-value pairs
234    KeyValue(HashMap<String, serde_json::Value>),
235    /// Structured data
236    Structured(serde_json::Value),
237    /// Binary data
238    Binary(Vec<u8>),
239    /// Text data
240    Text(String),
241}
242
243/// Tabular data structure
244#[derive(Debug, Clone, Serialize, Deserialize)]
245pub struct TableData {
246    /// Column headers
247    pub headers: Vec<String>,
248    /// Data rows
249    pub rows: Vec<Vec<serde_json::Value>>,
250    /// Column types
251    pub column_types: HashMap<String, ColumnType>,
252}
253
254/// Time series data structure
255#[derive(Debug, Clone, Serialize, Deserialize)]
256pub struct TimeSeriesData {
257    /// Timestamps
258    pub timestamps: Vec<DateTime<Utc>>,
259    /// Data series
260    pub series: HashMap<String, Vec<f64>>,
261    /// Series metadata
262    pub metadata: HashMap<String, String>,
263}
264
265/// Column data types
266#[derive(Debug, Clone, Serialize, Deserialize)]
267pub enum ColumnType {
268    Integer,
269    Float,
270    String,
271    Boolean,
272    DateTime,
273    Binary,
274}
275
276impl DataExportManager {
277    /// Create a new data export manager
278    pub fn new(config: ExportConfig) -> Self {
279        let supported_formats = vec![
280            ExportFormat::Csv,
281            ExportFormat::Excel,
282            ExportFormat::Json,
283            ExportFormat::JsonPretty,
284            ExportFormat::Xml,
285            ExportFormat::Yaml,
286            ExportFormat::Sqlite,
287        ];
288
289        Self {
290            config,
291            active_jobs: HashMap::new(),
292            export_history: Vec::new(),
293            supported_formats,
294        }
295    }
296
297    /// Start an export job
298    pub fn start_export(
299        &mut self,
300        name: String,
301        data: Vec<ExportableData>,
302        format: ExportFormat,
303        output_path: String,
304        options: ExportOptions,
305    ) -> Result<Uuid> {
306        let job_id = Uuid::new_v4();
307
308        // Calculate total data size
309        let data_size: u64 = data.iter().map(|d| d.size).sum();
310
311        // Check file size limit
312        if data_size > self.config.max_file_size {
313            return Err(anyhow::anyhow!("Data size exceeds maximum file size limit"));
314        }
315
316        let job = ExportJob {
317            id: job_id,
318            name: name.clone(),
319            format: format.clone(),
320            output_path: output_path.clone(),
321            status: ExportStatus::Pending,
322            progress: 0.0,
323            started_at: Utc::now(),
324            completed_at: None,
325            data_size,
326            error_message: None,
327            options: options.clone(),
328        };
329
330        self.active_jobs.insert(job_id, job);
331
332        // Start the actual export process
333        self.execute_export(job_id, data, options)?;
334
335        Ok(job_id)
336    }
337
338    /// Execute the export process
339    fn execute_export(
340        &mut self,
341        job_id: Uuid,
342        data: Vec<ExportableData>,
343        options: ExportOptions,
344    ) -> Result<()> {
345        // Extract job info to avoid multiple mutable borrows
346        let (format, output_path) = {
347            if let Some(job) = self.active_jobs.get_mut(&job_id) {
348                job.status = ExportStatus::InProgress;
349                (job.format.clone(), job.output_path.clone())
350            } else {
351                return Err(anyhow::anyhow!("Export job not found"));
352            }
353        };
354
355        let result = match format {
356            ExportFormat::Csv => self.export_csv(&data, &output_path, &options),
357            ExportFormat::Json => self.export_json(&data, &output_path, &options),
358            ExportFormat::JsonPretty => self.export_json_pretty(&data, &output_path, &options),
359            ExportFormat::Excel => self.export_excel(&data, &output_path, &options),
360            ExportFormat::Xml => self.export_xml(&data, &output_path, &options),
361            ExportFormat::Yaml => self.export_yaml(&data, &output_path, &options),
362            ExportFormat::Sqlite => self.export_sqlite(&data, &output_path, &options),
363            // Binary/columnar formats that require optional external crates or services.
364            // These are intentionally not implemented to keep the core dependency set
365            // minimal.  Callers should use JSON or CSV as a universal fallback.
366            ExportFormat::Hdf5 => Err(anyhow::anyhow!(
367                "HDF5 export is not supported in this build. Use JSON or CSV instead."
368            )),
369            ExportFormat::Parquet => Err(anyhow::anyhow!(
370                "Parquet export is not supported in this build. Use JSON or CSV instead."
371            )),
372            ExportFormat::MessagePack => Err(anyhow::anyhow!(
373                "MessagePack export is not supported in this build. Use JSON instead."
374            )),
375            ExportFormat::Arrow => Err(anyhow::anyhow!(
376                "Apache Arrow export is not supported in this build. Use JSON or CSV instead."
377            )),
378            ExportFormat::Custom(ref name) => Err(anyhow::anyhow!(
379                "Custom export format '{}' is not registered. \
380                 Register a handler or use one of the built-in formats.",
381                name
382            )),
383        };
384
385        // Update job status
386        if let Some(job) = self.active_jobs.get_mut(&job_id) {
387            match result {
388                Ok(_) => {
389                    job.status = ExportStatus::Completed;
390                    job.progress = 100.0;
391                    job.completed_at = Some(Utc::now());
392
393                    // Add to history by cloning the job
394                    let job_copy = job.clone();
395                    self.add_export_record(&job_copy);
396                },
397                Err(e) => {
398                    job.status = ExportStatus::Failed;
399                    job.error_message = Some(e.to_string());
400                },
401            }
402        }
403
404        Ok(())
405    }
406
407    /// Export to CSV format
408    fn export_csv(
409        &mut self,
410        data: &[ExportableData],
411        output_path: &str,
412        options: &ExportOptions,
413    ) -> Result<()> {
414        use std::fs::File;
415        use std::io::Write;
416
417        let mut file = File::create(output_path)?;
418
419        for item in data {
420            match &item.content {
421                ExportDataContent::Table(table_data) => {
422                    // Write headers
423                    if options.include_headers {
424                        let header_line = table_data.headers.join(&options.separator);
425                        writeln!(file, "{}", header_line)?;
426                    }
427
428                    // Write data rows
429                    for row in &table_data.rows {
430                        let row_values: Vec<String> =
431                            row.iter().map(|v| self.format_value_for_csv(v, options)).collect();
432                        let row_line = row_values.join(&options.separator);
433                        writeln!(file, "{}", row_line)?;
434                    }
435                },
436                ExportDataContent::TimeSeries(ts_data) => {
437                    // Write time series data
438                    if options.include_headers {
439                        let mut headers = vec!["timestamp".to_string()];
440                        headers.extend(ts_data.series.keys().cloned());
441                        let header_line = headers.join(&options.separator);
442                        writeln!(file, "{}", header_line)?;
443                    }
444
445                    for (i, timestamp) in ts_data.timestamps.iter().enumerate() {
446                        let mut row = vec![timestamp.format(&options.date_format).to_string()];
447                        for series_name in ts_data.series.keys() {
448                            if let Some(series) = ts_data.series.get(series_name) {
449                                if let Some(value) = series.get(i) {
450                                    row.push(format!(
451                                        "{:.precision$}",
452                                        value,
453                                        precision = options.float_precision as usize
454                                    ));
455                                } else {
456                                    row.push("".to_string());
457                                }
458                            }
459                        }
460                        let row_line = row.join(&options.separator);
461                        writeln!(file, "{}", row_line)?;
462                    }
463                },
464                _ => {
465                    // Convert other formats to JSON and then to CSV-like representation
466                    let json_str = serde_json::to_string(&item.content)?;
467                    writeln!(file, "{}", json_str)?;
468                },
469            }
470        }
471
472        Ok(())
473    }
474
475    /// Export to JSON format
476    fn export_json(
477        &mut self,
478        data: &[ExportableData],
479        output_path: &str,
480        _options: &ExportOptions,
481    ) -> Result<()> {
482        use std::fs::File;
483
484        let file = File::create(output_path)?;
485        serde_json::to_writer(file, data)?;
486        Ok(())
487    }
488
489    /// Export to pretty JSON format
490    fn export_json_pretty(
491        &mut self,
492        data: &[ExportableData],
493        output_path: &str,
494        _options: &ExportOptions,
495    ) -> Result<()> {
496        use std::fs::File;
497
498        let file = File::create(output_path)?;
499        serde_json::to_writer_pretty(file, data)?;
500        Ok(())
501    }
502
503    /// Export to Excel (.xlsx) format as a real Office Open XML workbook
504    fn export_excel(
505        &mut self,
506        data: &[ExportableData],
507        output_path: &str,
508        options: &ExportOptions,
509    ) -> Result<()> {
510        use oxiarc_archive::zip::ZipWriter;
511
512        let rows = build_xlsx_rows(data, options)?;
513        let sheet_xml = build_xlsx_sheet(&rows);
514
515        let mut buffer: Vec<u8> = Vec::new();
516        {
517            let mut writer = ZipWriter::new(&mut buffer);
518            writer
519                .add_file("[Content_Types].xml", XLSX_CONTENT_TYPES.as_bytes())
520                .map_err(|e| anyhow::anyhow!("xlsx: failed to write [Content_Types].xml: {e}"))?;
521            writer
522                .add_file("_rels/.rels", XLSX_ROOT_RELS.as_bytes())
523                .map_err(|e| anyhow::anyhow!("xlsx: failed to write _rels/.rels: {e}"))?;
524            writer
525                .add_file("xl/workbook.xml", XLSX_WORKBOOK.as_bytes())
526                .map_err(|e| anyhow::anyhow!("xlsx: failed to write xl/workbook.xml: {e}"))?;
527            writer
528                .add_file("xl/_rels/workbook.xml.rels", XLSX_WORKBOOK_RELS.as_bytes())
529                .map_err(|e| {
530                    anyhow::anyhow!("xlsx: failed to write xl/_rels/workbook.xml.rels: {e}")
531                })?;
532            writer.add_file("xl/worksheets/sheet1.xml", sheet_xml.as_bytes()).map_err(|e| {
533                anyhow::anyhow!("xlsx: failed to write xl/worksheets/sheet1.xml: {e}")
534            })?;
535            writer
536                .finish()
537                .map_err(|e| anyhow::anyhow!("xlsx: failed to finalize workbook package: {e}"))?;
538        }
539
540        std::fs::write(output_path, &buffer)?;
541        Ok(())
542    }
543
544    /// Export to XML format
545    fn export_xml(
546        &mut self,
547        data: &[ExportableData],
548        output_path: &str,
549        _options: &ExportOptions,
550    ) -> Result<()> {
551        use std::fs::File;
552        use std::io::Write;
553
554        let mut file = File::create(output_path)?;
555
556        writeln!(file, "<?xml version=\"1.0\" encoding=\"UTF-8\"?>")?;
557        writeln!(file, "<export_data>")?;
558
559        for item in data {
560            writeln!(
561                file,
562                "  <data_item id=\"{}\" type=\"{:?}\">",
563                item.id, item.data_type
564            )?;
565            writeln!(file, "    <name>{}</name>", item.name)?;
566            writeln!(
567                file,
568                "    <timestamp>{}</timestamp>",
569                item.timestamp.to_rfc3339()
570            )?;
571            writeln!(file, "    <size>{}</size>", item.size)?;
572
573            // Convert content to XML (simplified)
574            let content_json = serde_json::to_string(&item.content)?;
575            writeln!(file, "    <content><![CDATA[{}]]></content>", content_json)?;
576
577            writeln!(file, "  </data_item>")?;
578        }
579
580        writeln!(file, "</export_data>")?;
581        Ok(())
582    }
583
584    /// Export to YAML format
585    fn export_yaml(
586        &mut self,
587        data: &[ExportableData],
588        output_path: &str,
589        _options: &ExportOptions,
590    ) -> Result<()> {
591        use std::fs::File;
592
593        let file = File::create(output_path)?;
594        serde_json::to_writer_pretty(file, data)?;
595        Ok(())
596    }
597
598    /// Export to SQLite database
599    fn export_sqlite(
600        &mut self,
601        data: &[ExportableData],
602        output_path: &str,
603        _options: &ExportOptions,
604    ) -> Result<()> {
605        // This would use rusqlite or similar library
606        // For now, we'll create a JSON file as a placeholder
607        self.export_json(data, output_path, _options)
608    }
609
610    /// Helper function to format values for CSV
611    fn format_value_for_csv(&self, value: &serde_json::Value, options: &ExportOptions) -> String {
612        match value {
613            serde_json::Value::Number(n) => {
614                if let Some(f) = n.as_f64() {
615                    format!(
616                        "{:.precision$}",
617                        f,
618                        precision = options.float_precision as usize
619                    )
620                } else {
621                    n.to_string()
622                }
623            },
624            serde_json::Value::String(s) => {
625                // Escape quotes and commas
626                if s.contains(',') || s.contains('"') || s.contains('\n') {
627                    format!("\"{}\"", s.replace('"', "\"\""))
628                } else {
629                    s.clone()
630                }
631            },
632            _ => value.to_string(),
633        }
634    }
635
636    /// Add export record to history
637    fn add_export_record(&mut self, job: &ExportJob) {
638        let record = ExportRecord {
639            id: Uuid::new_v4(),
640            job_id: job.id,
641            timestamp: Utc::now(),
642            file_path: job.output_path.clone(),
643            file_size: job.data_size,
644            format: job.format.clone(),
645            success: matches!(job.status, ExportStatus::Completed),
646            duration: job
647                .completed_at
648                .map(|end| (end - job.started_at).num_milliseconds() as f64 / 1000.0)
649                .unwrap_or(0.0),
650        };
651
652        self.export_history.push(record);
653    }
654
655    /// Get export job status
656    pub fn get_job_status(&self, job_id: Uuid) -> Option<&ExportJob> {
657        self.active_jobs.get(&job_id)
658    }
659
660    /// Get export history
661    pub fn get_export_history(&self) -> &[ExportRecord] {
662        &self.export_history
663    }
664
665    /// Create export template
666    pub fn create_template(
667        &mut self,
668        name: String,
669        description: String,
670        format: ExportFormat,
671        options: ExportOptions,
672        filters: DataFilters,
673        tags: Vec<String>,
674    ) -> String {
675        let template_id = Uuid::new_v4().to_string();
676
677        let template = ExportTemplate {
678            id: template_id.clone(),
679            name,
680            description,
681            format,
682            options,
683            filters,
684            tags,
685        };
686
687        self.config.templates.push(template);
688        template_id
689    }
690
691    /// Apply export template
692    pub fn apply_template(
693        &self,
694        template_id: &str,
695    ) -> Option<(&ExportFormat, &ExportOptions, &DataFilters)> {
696        self.config
697            .templates
698            .iter()
699            .find(|t| t.id == template_id)
700            .map(|t| (&t.format, &t.options, &t.filters))
701    }
702
703    /// Get supported formats
704    pub fn get_supported_formats(&self) -> &[ExportFormat] {
705        &self.supported_formats
706    }
707
708    /// Cancel export job
709    pub fn cancel_job(&mut self, job_id: Uuid) -> Result<()> {
710        if let Some(job) = self.active_jobs.get_mut(&job_id) {
711            if matches!(job.status, ExportStatus::Pending | ExportStatus::InProgress) {
712                job.status = ExportStatus::Cancelled;
713                Ok(())
714            } else {
715                Err(anyhow::anyhow!("Job cannot be cancelled in current status"))
716            }
717        } else {
718            Err(anyhow::anyhow!("Job not found"))
719        }
720    }
721
722    /// Get export statistics
723    pub fn get_export_statistics(&self) -> ExportStatistics {
724        let total_exports = self.export_history.len();
725        let successful_exports = self.export_history.iter().filter(|r| r.success).count();
726        let total_size: u64 = self.export_history.iter().map(|r| r.file_size).sum();
727        let avg_duration = if total_exports > 0 {
728            self.export_history.iter().map(|r| r.duration).sum::<f64>() / total_exports as f64
729        } else {
730            0.0
731        };
732
733        let format_stats: HashMap<ExportFormat, usize> =
734            self.export_history.iter().fold(HashMap::new(), |mut acc, record| {
735                *acc.entry(record.format.clone()).or_insert(0) += 1;
736                acc
737            });
738
739        ExportStatistics {
740            total_exports,
741            successful_exports,
742            failed_exports: total_exports - successful_exports,
743            total_size_bytes: total_size,
744            average_duration_seconds: avg_duration,
745            format_statistics: format_stats,
746            active_jobs: self.active_jobs.len(),
747        }
748    }
749}
750
751/// OOXML `[Content_Types].xml` part declaring the package content types.
752const XLSX_CONTENT_TYPES: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
753<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>"#;
754
755/// OOXML `_rels/.rels` package-level relationships part.
756const XLSX_ROOT_RELS: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
757<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#;
758
759/// OOXML `xl/workbook.xml` part defining a single worksheet.
760const XLSX_WORKBOOK: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
761<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Sheet1" sheetId="1" r:id="rId1"/></sheets></workbook>"#;
762
763/// OOXML `xl/_rels/workbook.xml.rels` part linking the workbook to its worksheet.
764const XLSX_WORKBOOK_RELS: &str = r#"<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
765<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#;
766
767/// A single spreadsheet cell value to emit into a worksheet.
768enum XlsxCell {
769    /// Inline (shared-string-free) text cell, emitted as `<c t="inlineStr">`.
770    Inline(String),
771    /// Numeric cell, emitted as `<c><v>..</v></c>` with the pre-formatted literal.
772    Number(String),
773    /// Empty placeholder cell.
774    Empty,
775}
776
777/// XML-escape `&`, `<`, `>`, `"`, `'` for safe inclusion in OOXML parts.
778fn xlsx_xml_escape(input: &str) -> String {
779    let mut out = String::with_capacity(input.len());
780    for ch in input.chars() {
781        match ch {
782            '&' => out.push_str("&amp;"),
783            '<' => out.push_str("&lt;"),
784            '>' => out.push_str("&gt;"),
785            '"' => out.push_str("&quot;"),
786            '\'' => out.push_str("&apos;"),
787            _ => out.push(ch),
788        }
789    }
790    out
791}
792
793/// Convert a zero-based column index to an Excel column name (0 -> "A", 26 -> "AA").
794fn xlsx_column_name(index: usize) -> String {
795    let mut n = index + 1;
796    let mut name = String::new();
797    while n > 0 {
798        let rem = (n - 1) % 26;
799        name.insert(0, (b'A' + rem as u8) as char);
800        n = (n - 1) / 26;
801    }
802    name
803}
804
805/// Map a JSON value to a worksheet cell, mirroring `format_value_for_csv` numeric rules.
806fn xlsx_value_to_cell(value: &serde_json::Value, options: &ExportOptions) -> XlsxCell {
807    match value {
808        serde_json::Value::Number(n) => {
809            if let Some(f) = n.as_f64() {
810                XlsxCell::Number(format!(
811                    "{:.precision$}",
812                    f,
813                    precision = options.float_precision as usize
814                ))
815            } else {
816                XlsxCell::Number(n.to_string())
817            }
818        },
819        serde_json::Value::String(s) => XlsxCell::Inline(s.clone()),
820        _ => XlsxCell::Inline(value.to_string()),
821    }
822}
823
824/// Build the worksheet rows from the export data, mirroring `export_csv` layout.
825fn build_xlsx_rows(data: &[ExportableData], options: &ExportOptions) -> Result<Vec<Vec<XlsxCell>>> {
826    let mut rows: Vec<Vec<XlsxCell>> = Vec::new();
827    for item in data {
828        match &item.content {
829            ExportDataContent::Table(table_data) => {
830                if options.include_headers {
831                    rows.push(
832                        table_data.headers.iter().map(|h| XlsxCell::Inline(h.clone())).collect(),
833                    );
834                }
835                for row in &table_data.rows {
836                    rows.push(row.iter().map(|v| xlsx_value_to_cell(v, options)).collect());
837                }
838            },
839            ExportDataContent::TimeSeries(ts_data) => {
840                if options.include_headers {
841                    let mut header = vec![XlsxCell::Inline("timestamp".to_string())];
842                    header.extend(ts_data.series.keys().map(|k| XlsxCell::Inline(k.clone())));
843                    rows.push(header);
844                }
845                for (i, timestamp) in ts_data.timestamps.iter().enumerate() {
846                    let mut cells = vec![XlsxCell::Inline(
847                        timestamp.format(&options.date_format).to_string(),
848                    )];
849                    for series_name in ts_data.series.keys() {
850                        if let Some(series) = ts_data.series.get(series_name) {
851                            if let Some(value) = series.get(i) {
852                                cells.push(XlsxCell::Number(format!(
853                                    "{:.precision$}",
854                                    value,
855                                    precision = options.float_precision as usize
856                                )));
857                            } else {
858                                cells.push(XlsxCell::Empty);
859                            }
860                        }
861                    }
862                    rows.push(cells);
863                }
864            },
865            _ => {
866                let json_str = serde_json::to_string(&item.content)?;
867                rows.push(vec![XlsxCell::Inline(json_str)]);
868            },
869        }
870    }
871    Ok(rows)
872}
873
874/// Render worksheet rows into the `xl/worksheets/sheet1.xml` OOXML part.
875fn build_xlsx_sheet(rows: &[Vec<XlsxCell>]) -> String {
876    let mut sheet = String::new();
877    sheet.push_str("<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"yes\"?>");
878    sheet.push_str(
879        "<worksheet xmlns=\"http://schemas.openxmlformats.org/spreadsheetml/2006/main\">",
880    );
881    sheet.push_str("<sheetData>");
882    for (row_idx, row) in rows.iter().enumerate() {
883        let row_num = row_idx + 1;
884        sheet.push_str(&format!("<row r=\"{row_num}\">"));
885        for (col_idx, cell) in row.iter().enumerate() {
886            let cell_ref = format!("{}{row_num}", xlsx_column_name(col_idx));
887            match cell {
888                XlsxCell::Inline(text) => {
889                    sheet.push_str(&format!(
890                        "<c r=\"{cell_ref}\" t=\"inlineStr\"><is><t xml:space=\"preserve\">{}</t></is></c>",
891                        xlsx_xml_escape(text)
892                    ));
893                },
894                XlsxCell::Number(num) => {
895                    sheet.push_str(&format!("<c r=\"{cell_ref}\"><v>{num}</v></c>"));
896                },
897                XlsxCell::Empty => {
898                    sheet.push_str(&format!("<c r=\"{cell_ref}\"/>"));
899                },
900            }
901        }
902        sheet.push_str("</row>");
903    }
904    sheet.push_str("</sheetData></worksheet>");
905    sheet
906}
907
908/// Export statistics
909#[derive(Debug, Clone, Serialize, Deserialize)]
910pub struct ExportStatistics {
911    pub total_exports: usize,
912    pub successful_exports: usize,
913    pub failed_exports: usize,
914    pub total_size_bytes: u64,
915    pub average_duration_seconds: f64,
916    pub format_statistics: HashMap<ExportFormat, usize>,
917    pub active_jobs: usize,
918}
919
920impl Default for ExportConfig {
921    fn default() -> Self {
922        Self {
923            default_directory: "./exports".to_string(),
924            max_file_size: 1024 * 1024 * 1024, // 1GB
925            enable_compression: true,
926            default_format: ExportFormat::Json,
927            include_metadata: true,
928            templates: Vec::new(),
929        }
930    }
931}
932
933impl Default for ExportOptions {
934    fn default() -> Self {
935        Self {
936            include_headers: true,
937            date_format: "%Y-%m-%d %H:%M:%S UTC".to_string(),
938            float_precision: 6,
939            separator: ",".to_string(),
940            compression_level: 6,
941            include_metadata: true,
942            custom_options: HashMap::new(),
943        }
944    }
945}
946
947impl Default for DataFilters {
948    fn default() -> Self {
949        Self {
950            date_range: None,
951            data_types: vec![
952                DataType::TensorData,
953                DataType::GradientData,
954                DataType::PerformanceMetrics,
955            ],
956            exclude_fields: Vec::new(),
957            include_fields: None,
958            custom_filters: HashMap::new(),
959        }
960    }
961}
962
963#[cfg(test)]
964mod tests {
965    use super::*;
966    use tempfile::tempdir;
967
968    // These values are test data, not approximations of mathematical constants
969    #[allow(clippy::approx_constant)]
970    fn create_test_data() -> Vec<ExportableData> {
971        let table_data = TableData {
972            headers: vec![
973                "id".to_string(),
974                "value".to_string(),
975                "timestamp".to_string(),
976            ],
977            rows: vec![
978                vec![
979                    serde_json::Value::Number(serde_json::Number::from(1)),
980                    serde_json::Value::Number(
981                        serde_json::Number::from_f64(3.14).expect("operation failed in test"),
982                    ),
983                    serde_json::Value::String("2023-01-01T12:00:00Z".to_string()),
984                ],
985                vec![
986                    serde_json::Value::Number(serde_json::Number::from(2)),
987                    serde_json::Value::Number(
988                        serde_json::Number::from_f64(2.71).expect("operation failed in test"),
989                    ),
990                    serde_json::Value::String("2023-01-01T12:01:00Z".to_string()),
991                ],
992            ],
993            column_types: HashMap::new(),
994        };
995
996        vec![ExportableData {
997            id: Uuid::new_v4(),
998            name: "Test Data".to_string(),
999            data_type: DataType::TensorData,
1000            timestamp: Utc::now(),
1001            content: ExportDataContent::Table(table_data),
1002            metadata: HashMap::new(),
1003            size: 1024,
1004        }]
1005    }
1006
1007    #[test]
1008    fn test_export_manager_creation() {
1009        let config = ExportConfig::default();
1010        let manager = DataExportManager::new(config);
1011
1012        assert!(manager.get_supported_formats().contains(&ExportFormat::Json));
1013        assert!(manager.get_supported_formats().contains(&ExportFormat::Csv));
1014    }
1015
1016    #[test]
1017    fn test_csv_export() {
1018        let config = ExportConfig::default();
1019        let mut manager = DataExportManager::new(config);
1020        let test_data = create_test_data();
1021
1022        let temp_dir = tempdir().expect("temp file creation failed");
1023        let output_path = temp_dir.path().join("test.csv").to_string_lossy().to_string();
1024
1025        let job_id = manager
1026            .start_export(
1027                "Test CSV Export".to_string(),
1028                test_data,
1029                ExportFormat::Csv,
1030                output_path.clone(),
1031                ExportOptions::default(),
1032            )
1033            .expect("operation failed in test");
1034
1035        // Check job was created
1036        assert!(manager.active_jobs.contains_key(&job_id));
1037
1038        // Check file was created
1039        assert!(std::path::Path::new(&output_path).exists());
1040    }
1041
1042    #[test]
1043    fn test_json_export() {
1044        let config = ExportConfig::default();
1045        let mut manager = DataExportManager::new(config);
1046        let test_data = create_test_data();
1047
1048        let temp_dir = tempdir().expect("temp file creation failed");
1049        let output_path = temp_dir.path().join("test.json").to_string_lossy().to_string();
1050
1051        let job_id = manager
1052            .start_export(
1053                "Test JSON Export".to_string(),
1054                test_data,
1055                ExportFormat::Json,
1056                output_path.clone(),
1057                ExportOptions::default(),
1058            )
1059            .expect("operation failed in test");
1060
1061        assert!(manager.active_jobs.contains_key(&job_id));
1062        assert!(std::path::Path::new(&output_path).exists());
1063    }
1064
1065    #[test]
1066    fn test_export_template() {
1067        let config = ExportConfig::default();
1068        let mut manager = DataExportManager::new(config);
1069
1070        let template_id = manager.create_template(
1071            "CSV Template".to_string(),
1072            "Standard CSV export".to_string(),
1073            ExportFormat::Csv,
1074            ExportOptions::default(),
1075            DataFilters::default(),
1076            vec!["csv".to_string(), "standard".to_string()],
1077        );
1078
1079        let (format, options, _filters) =
1080            manager.apply_template(&template_id).expect("temp file creation failed");
1081        assert_eq!(*format, ExportFormat::Csv);
1082        assert!(options.include_headers);
1083    }
1084
1085    #[test]
1086    fn test_export_statistics() {
1087        let config = ExportConfig::default();
1088        let mut manager = DataExportManager::new(config);
1089
1090        // Add some mock export records
1091        manager.export_history.push(ExportRecord {
1092            id: Uuid::new_v4(),
1093            job_id: Uuid::new_v4(),
1094            timestamp: Utc::now(),
1095            file_path: "test1.csv".to_string(),
1096            file_size: 1024,
1097            format: ExportFormat::Csv,
1098            success: true,
1099            duration: 2.5,
1100        });
1101
1102        manager.export_history.push(ExportRecord {
1103            id: Uuid::new_v4(),
1104            job_id: Uuid::new_v4(),
1105            timestamp: Utc::now(),
1106            file_path: "test2.json".to_string(),
1107            file_size: 2048,
1108            format: ExportFormat::Json,
1109            success: true,
1110            duration: 1.8,
1111        });
1112
1113        let stats = manager.get_export_statistics();
1114        assert_eq!(stats.total_exports, 2);
1115        assert_eq!(stats.successful_exports, 2);
1116        assert_eq!(stats.total_size_bytes, 3072);
1117    }
1118
1119    #[test]
1120    fn test_excel_export_roundtrip() {
1121        use oxiarc_archive::zip::ZipReader;
1122        use std::io::Cursor;
1123
1124        let config = ExportConfig::default();
1125        let mut manager = DataExportManager::new(config);
1126        let test_data = create_test_data();
1127
1128        let file_name = format!("trustformers_xlsx_test_{}.xlsx", Uuid::new_v4());
1129        let output_path = std::env::temp_dir().join(file_name).to_string_lossy().to_string();
1130
1131        manager
1132            .start_export(
1133                "Test Excel Export".to_string(),
1134                test_data,
1135                ExportFormat::Excel,
1136                output_path.clone(),
1137                ExportOptions::default(),
1138            )
1139            .expect("excel export should succeed");
1140
1141        assert!(std::path::Path::new(&output_path).exists());
1142
1143        let bytes = std::fs::read(&output_path).expect("read xlsx bytes");
1144        let mut reader = ZipReader::new(Cursor::new(bytes)).expect("open xlsx as zip");
1145        let entries = reader.entries().to_vec();
1146        let names: Vec<String> = entries.iter().map(|e| e.name.clone()).collect();
1147
1148        assert!(
1149            names.iter().any(|n| n == "[Content_Types].xml"),
1150            "missing [Content_Types].xml; entries = {names:?}"
1151        );
1152        assert!(
1153            names.iter().any(|n| n == "xl/workbook.xml"),
1154            "missing xl/workbook.xml; entries = {names:?}"
1155        );
1156        assert!(
1157            names.iter().any(|n| n == "xl/worksheets/sheet1.xml"),
1158            "missing xl/worksheets/sheet1.xml; entries = {names:?}"
1159        );
1160
1161        let sheet_entry = entries
1162            .iter()
1163            .find(|e| e.name == "xl/worksheets/sheet1.xml")
1164            .expect("sheet1.xml entry");
1165        let sheet_bytes = reader.extract(sheet_entry).expect("extract sheet1.xml");
1166        let sheet_xml = String::from_utf8(sheet_bytes).expect("sheet1.xml is utf8");
1167
1168        assert!(sheet_xml.contains("<worksheet"));
1169        // Header text from create_test_data(): "id", "value", "timestamp".
1170        assert!(sheet_xml.contains("id"), "sheet missing header text");
1171        assert!(sheet_xml.contains("value"), "sheet missing header text");
1172        assert!(sheet_xml.contains("timestamp"), "sheet missing header text");
1173
1174        let _ = std::fs::remove_file(&output_path);
1175    }
1176}
trustformers_debug/data_export.rs

trustformers_debug/
data_export.rs
