scirs2_cluster/serialization/
export.rs

1//! Advanced export functionality for clustering models
2//!
3//! This module provides sophisticated export capabilities including
4//! multiple formats, metadata enrichment, and cross-platform compatibility.
5
6use crate::error::{ClusteringError, Result};
7use scirs2_core::ndarray::{Array1, Array2};
8use serde::{Deserialize, Serialize};
9use serde_json::Value;
10use std::collections::HashMap;
11use std::path::Path;
12
13#[cfg(feature = "yaml")]
14use serde_yaml;
15
16use super::core::{EnhancedModelMetadata, PlatformInfo, SerializableModel};
17use super::models::*;
18
19/// Export formats supported by the serialization system
20#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
21pub enum ExportFormat {
22    /// JSON format with full metadata
23    Json,
24    /// Compressed JSON format
25    JsonGz,
26    /// Binary format (MessagePack)
27    Binary,
28    /// CSV format (for simple models)
29    Csv,
30    /// YAML format
31    Yaml,
32    /// XML format
33    Xml,
34    /// HDF5 format (for large datasets)
35    Hdf5,
36    /// Custom format with user-defined structure
37    Custom(String),
38}
39
40/// Trait for advanced export capabilities
41pub trait AdvancedExport {
42    /// Export model to specified format with metadata
43    fn export_with_metadata(
44        &self,
45        format: ExportFormat,
46        metadata: Option<ModelMetadata>,
47    ) -> Result<Vec<u8>>;
48
49    /// Export to file with automatic format detection
50    fn export_to_file<P: AsRef<Path>>(&self, path: P) -> Result<()>;
51
52    /// Export model summary for quick inspection
53    fn export_summary(&self) -> Result<String>;
54
55    /// Export model in a format compatible with other libraries
56    fn export_compatible(&self, target_library: &str) -> Result<Value>;
57
58    /// Validate model before export
59    fn validate_for_export(&self) -> Result<()>;
60}
61
62/// Comprehensive model metadata for exports
63#[derive(Serialize, Deserialize, Debug, Clone)]
64pub struct ModelMetadata {
65    /// Model name and version
66    pub model_info: ModelInfo,
67    /// Algorithm configuration
68    pub algorithm_config: AlgorithmConfig,
69    /// Performance metrics
70    pub performance_metrics: PerformanceMetrics,
71    /// Data characteristics
72    pub data_characteristics: ModelDataCharacteristics,
73    /// Export settings
74    pub export_settings: ExportSettings,
75}
76
77/// Model information
78#[derive(Serialize, Deserialize, Debug, Clone)]
79pub struct ModelInfo {
80    /// Model name
81    pub name: String,
82    /// Model version
83    pub version: String,
84    /// Creation timestamp
85    pub created_at: String,
86    /// Author/creator
87    pub author: Option<String>,
88    /// Description
89    pub description: Option<String>,
90}
91
92/// Algorithm configuration details
93#[derive(Serialize, Deserialize, Debug, Clone)]
94pub struct AlgorithmConfig {
95    /// Algorithm name
96    pub algorithm: String,
97    /// Hyperparameters used
98    pub hyperparameters: HashMap<String, Value>,
99    /// Preprocessing steps applied
100    pub preprocessing: Vec<String>,
101    /// Random seed used
102    pub random_seed: Option<u64>,
103    /// Convergence criteria
104    pub convergence_criteria: Option<HashMap<String, f64>>,
105}
106
107/// Performance metrics collected during training
108#[derive(Serialize, Deserialize, Debug, Clone)]
109pub struct PerformanceMetrics {
110    /// Training time in seconds
111    pub training_time_seconds: f64,
112    /// Memory usage in MB
113    pub peak_memory_mb: f64,
114    /// CPU utilization percentage
115    pub cpu_utilization: f64,
116    /// Model quality metrics
117    pub quality_metrics: HashMap<String, f64>,
118    /// Convergence information
119    pub convergence_info: Option<ConvergenceInfo>,
120}
121
122/// Convergence information
123#[derive(Serialize, Deserialize, Debug, Clone)]
124pub struct ConvergenceInfo {
125    /// Whether algorithm converged
126    pub converged: bool,
127    /// Number of iterations to convergence
128    pub iterations: usize,
129    /// Final objective value
130    pub final_objective: f64,
131    /// Convergence tolerance achieved
132    pub tolerance_achieved: f64,
133}
134
135/// Data characteristics for model validation
136#[derive(Serialize, Deserialize, Debug, Clone)]
137pub struct ModelDataCharacteristics {
138    /// Number of samples
139    pub n_samples: usize,
140    /// Number of features
141    pub n_features: usize,
142    /// Feature names (if available)
143    pub feature_names: Option<Vec<String>>,
144    /// Data types for each feature
145    pub feature_types: Option<Vec<String>>,
146    /// Statistical summaries
147    pub feature_statistics: Option<HashMap<String, FeatureStats>>,
148}
149
150/// Statistical summary for a feature
151#[derive(Serialize, Deserialize, Debug, Clone)]
152pub struct FeatureStats {
153    /// Mean value
154    pub mean: f64,
155    /// Standard deviation
156    pub std: f64,
157    /// Minimum value
158    pub min: f64,
159    /// Maximum value
160    pub max: f64,
161    /// Missing value count
162    pub missing_count: usize,
163}
164
165/// Export settings and options
166#[derive(Serialize, Deserialize, Debug, Clone)]
167pub struct ExportSettings {
168    /// Include raw model data
169    pub include_raw_data: bool,
170    /// Include training data
171    pub include_training_data: bool,
172    /// Compression level (0-9)
173    pub compression_level: Option<u8>,
174    /// Precision for floating point values
175    pub float_precision: Option<usize>,
176    /// Custom export options
177    pub custom_options: HashMap<String, Value>,
178}
179
180impl Default for ExportSettings {
181    fn default() -> Self {
182        Self {
183            include_raw_data: true,
184            include_training_data: false,
185            compression_level: None,
186            float_precision: Some(6),
187            custom_options: HashMap::new(),
188        }
189    }
190}
191
192/// Implementation of AdvancedExport for KMeansModel
193impl AdvancedExport for KMeansModel {
194    fn export_with_metadata(
195        &self,
196        format: ExportFormat,
197        metadata: Option<ModelMetadata>,
198    ) -> Result<Vec<u8>> {
199        let export_data = KMeansExportData {
200            model: self.clone(),
201            metadata,
202            format_version: "1.0".to_string(),
203            export_timestamp: chrono::Utc::now().to_rfc3339(),
204        };
205
206        match format {
207            ExportFormat::Json => {
208                let json = serde_json::to_string_pretty(&export_data)
209                    .map_err(|e| ClusteringError::InvalidInput(e.to_string()))?;
210                Ok(json.into_bytes())
211            }
212            ExportFormat::JsonGz => {
213                let json = serde_json::to_string(&export_data)
214                    .map_err(|e| ClusteringError::InvalidInput(e.to_string()))?;
215
216                oxiarc_deflate::gzip_compress(json.as_bytes(), 6)
217                    .map_err(|e| ClusteringError::InvalidInput(e.to_string()))
218            }
219            #[cfg(feature = "yaml")]
220            ExportFormat::Yaml => {
221                let yaml = serde_yaml::to_string(&export_data)
222                    .map_err(|e| ClusteringError::InvalidInput(e.to_string()))?;
223                Ok(yaml.into_bytes())
224            }
225            #[cfg(not(feature = "yaml"))]
226            ExportFormat::Yaml => Err(ClusteringError::InvalidInput(
227                "YAML support not enabled. Enable the 'yaml' feature".to_string(),
228            )),
229            ExportFormat::Csv => self.export_csv(),
230            _ => Err(ClusteringError::InvalidInput(format!(
231                "Unsupported export format: {:?}",
232                format
233            ))),
234        }
235    }
236
237    fn export_to_file<P: AsRef<Path>>(&self, path: P) -> Result<()> {
238        let path = path.as_ref();
239        let format = detect_format_from_extension(path)?;
240        let data = self.export_with_metadata(format, None)?;
241
242        std::fs::write(path, data)
243            .map_err(|e| ClusteringError::InvalidInput(format!("Failed to write file: {}", e)))
244    }
245
246    fn export_summary(&self) -> Result<String> {
247        let summary = KMeansSummary {
248            algorithm: "K-Means".to_string(),
249            n_clusters: self.n_clusters,
250            n_features: self.centroids.ncols(),
251            n_iterations: self.n_iter,
252            inertia: self.inertia,
253            has_labels: self.labels.is_some(),
254        };
255
256        serde_json::to_string_pretty(&summary)
257            .map_err(|e| ClusteringError::InvalidInput(e.to_string()))
258    }
259
260    fn export_compatible(&self, target_library: &str) -> Result<Value> {
261        match target_library.to_lowercase().as_str() {
262            "sklearn" | "scikit-learn" => self.to_sklearn_format(),
263            "tensorflow" | "tf" => self.to_tensorflow_format(),
264            "pytorch" => self.to_pytorch_format(),
265            _ => Err(ClusteringError::InvalidInput(format!(
266                "Unsupported target library: {}",
267                target_library
268            ))),
269        }
270    }
271
272    fn validate_for_export(&self) -> Result<()> {
273        if self.centroids.is_empty() {
274            return Err(ClusteringError::InvalidInput(
275                "Cannot export model with empty centroids".to_string(),
276            ));
277        }
278
279        if self.n_clusters == 0 {
280            return Err(ClusteringError::InvalidInput(
281                "Cannot export model with zero clusters".to_string(),
282            ));
283        }
284
285        if self.centroids.nrows() != self.n_clusters {
286            return Err(ClusteringError::InvalidInput(
287                "Centroids shape inconsistent with n_clusters".to_string(),
288            ));
289        }
290
291        Ok(())
292    }
293}
294
295/// Export data structure for K-Means
296#[derive(Serialize, Deserialize, Debug, Clone)]
297struct KMeansExportData {
298    model: KMeansModel,
299    metadata: Option<ModelMetadata>,
300    format_version: String,
301    export_timestamp: String,
302}
303
304/// Summary structure for K-Means
305#[derive(Serialize, Deserialize, Debug, Clone)]
306struct KMeansSummary {
307    algorithm: String,
308    n_clusters: usize,
309    n_features: usize,
310    n_iterations: usize,
311    inertia: f64,
312    has_labels: bool,
313}
314
315impl KMeansModel {
316    /// Export as CSV format
317    fn export_csv(&self) -> Result<Vec<u8>> {
318        let mut csv_content = String::new();
319
320        // Header
321        csv_content.push_str("cluster_id");
322        for i in 0..self.centroids.ncols() {
323            csv_content.push_str(&format!(",feature_{}", i));
324        }
325        csv_content.push('\n');
326
327        // Centroids data
328        for (cluster_id, centroid) in self.centroids.rows().into_iter().enumerate() {
329            csv_content.push_str(&cluster_id.to_string());
330            for value in centroid {
331                csv_content.push_str(&format!(",{:.6}", value));
332            }
333            csv_content.push('\n');
334        }
335
336        Ok(csv_content.into_bytes())
337    }
338
339    /// Convert to scikit-learn compatible format
340    fn to_sklearn_format(&self) -> Result<Value> {
341        use serde_json::json;
342
343        Ok(json!({
344            "cluster_centers_": self.centroids.as_slice().expect("Operation failed"),
345            "labels_": self.labels.as_ref().map(|l| l.as_slice().expect("Operation failed")),
346            "inertia_": self.inertia,
347            "n_iter_": self.n_iter,
348            "n_clusters": self.n_clusters,
349            "_sklearn_version": "1.0.0"
350        }))
351    }
352
353    /// Convert to TensorFlow compatible format
354    fn to_tensorflow_format(&self) -> Result<Value> {
355        use serde_json::json;
356
357        Ok(json!({
358            "centroids": {
359                "data": self.centroids.as_slice().expect("Operation failed"),
360                "shape": [self.centroids.nrows(), self.centroids.ncols()],
361                "dtype": "float64"
362            },
363            "metadata": {
364                "n_clusters": self.n_clusters,
365                "inertia": self.inertia,
366                "iterations": self.n_iter
367            }
368        }))
369    }
370
371    /// Convert to PyTorch compatible format
372    fn to_pytorch_format(&self) -> Result<Value> {
373        use serde_json::json;
374
375        Ok(json!({
376            "state_dict": {
377                "centroids": self.centroids.as_slice().expect("Operation failed")
378            },
379            "hyperparameters": {
380                "n_clusters": self.n_clusters
381            },
382            "metrics": {
383                "inertia": self.inertia,
384                "n_iter": self.n_iter
385            }
386        }))
387    }
388}
389
390/// Detect export format from file extension
391fn detect_format_from_extension<P: AsRef<Path>>(path: P) -> Result<ExportFormat> {
392    let path = path.as_ref();
393    let extension = path
394        .extension()
395        .and_then(|ext| ext.to_str())
396        .unwrap_or("")
397        .to_lowercase();
398
399    match extension.as_str() {
400        "json" => Ok(ExportFormat::Json),
401        "gz" | "json.gz" => Ok(ExportFormat::JsonGz),
402        "yaml" | "yml" => Ok(ExportFormat::Yaml),
403        "csv" => Ok(ExportFormat::Csv),
404        "xml" => Ok(ExportFormat::Xml),
405        "h5" | "hdf5" => Ok(ExportFormat::Hdf5),
406        _ => Err(ClusteringError::InvalidInput(format!(
407            "Unknown file extension: {}",
408            extension
409        ))),
410    }
411}
412
413/// Export utility functions
414pub mod utils {
415    use super::*;
416
417    /// Create default metadata for a model
418    pub fn create_default_metadata(algorithm_name: &str) -> ModelMetadata {
419        ModelMetadata {
420            model_info: ModelInfo {
421                name: format!("{}_model", algorithm_name),
422                version: "1.0.0".to_string(),
423                created_at: chrono::Utc::now().to_rfc3339(),
424                author: None,
425                description: None,
426            },
427            algorithm_config: AlgorithmConfig {
428                algorithm: algorithm_name.to_string(),
429                hyperparameters: HashMap::new(),
430                preprocessing: Vec::new(),
431                random_seed: None,
432                convergence_criteria: None,
433            },
434            performance_metrics: PerformanceMetrics {
435                training_time_seconds: 0.0,
436                peak_memory_mb: 0.0,
437                cpu_utilization: 0.0,
438                quality_metrics: HashMap::new(),
439                convergence_info: None,
440            },
441            data_characteristics: ModelDataCharacteristics {
442                n_samples: 0,
443                n_features: 0,
444                feature_names: None,
445                feature_types: None,
446                feature_statistics: None,
447            },
448            export_settings: ExportSettings::default(),
449        }
450    }
451
452    /// Compress data using gzip
453    pub fn compress_data(data: &[u8]) -> Result<Vec<u8>> {
454        oxiarc_deflate::gzip_compress(data, 6)
455            .map_err(|e| ClusteringError::InvalidInput(e.to_string()))
456    }
457
458    /// Decompress gzip data
459    pub fn decompress_data(compressed: &[u8]) -> Result<Vec<u8>> {
460        oxiarc_deflate::gzip_decompress(compressed)
461            .map_err(|e| ClusteringError::InvalidInput(e.to_string()))
462    }
463}
464
465#[cfg(test)]
466mod tests {
467    use super::*;
468    use scirs2_core::ndarray::Array2;
469
470    #[test]
471    fn test_kmeans_export_summary() {
472        let centroids =
473            Array2::from_shape_vec((2, 2), vec![0.0, 0.0, 1.0, 1.0]).expect("Operation failed");
474        let model = KMeansModel::new(centroids, 2, 10, 0.5, None);
475
476        let summary = model.export_summary().expect("Operation failed");
477        assert!(summary.contains("K-Means"));
478        assert!(summary.contains("\"n_clusters\": 2"));
479    }
480
481    #[test]
482    fn test_format_detection() {
483        assert_eq!(
484            detect_format_from_extension("model.json").expect("Operation failed"),
485            ExportFormat::Json
486        );
487        assert_eq!(
488            detect_format_from_extension("model.yaml").expect("Operation failed"),
489            ExportFormat::Yaml
490        );
491        assert_eq!(
492            detect_format_from_extension("model.csv").expect("Operation failed"),
493            ExportFormat::Csv
494        );
495    }
496
497    #[test]
498    fn test_sklearn_compatibility() {
499        let centroids =
500            Array2::from_shape_vec((2, 2), vec![0.0, 0.0, 1.0, 1.0]).expect("Operation failed");
501        let model = KMeansModel::new(centroids, 2, 10, 0.5, None);
502
503        let sklearn_format = model
504            .export_compatible("sklearn")
505            .expect("Operation failed");
506        assert!(sklearn_format.get("cluster_centers_").is_some());
507        assert!(sklearn_format.get("_sklearn_version").is_some());
508    }
509}
scirs2_cluster/serialization/export.rs

scirs2_cluster/serialization/
export.rs
