optirs_core/research/
reproducibility.rs

1// Reproducibility tools for research experiments
2//
3// This module provides tools to ensure research experiments can be reproduced
4// exactly, including environment capture, dependency tracking, and result verification.
5
6use crate::error::{OptimError, Result};
7use chrono::{DateTime, Utc};
8use serde::{Deserialize, Serialize};
9use std::collections::HashMap;
10use std::path::PathBuf;
11
12/// Reproducibility manager for tracking experiment reproducibility
13#[derive(Debug, Clone, Serialize, Deserialize)]
14pub struct ReproducibilityManager {
15    /// Environment snapshots
16    pub environments: HashMap<String, EnvironmentSnapshot>,
17    /// Reproducibility reports
18    pub reports: Vec<ReproducibilityReport>,
19    /// Verification results
20    pub verifications: Vec<VerificationResult>,
21    /// Configuration
22    pub config: ReproducibilityConfig,
23}
24
25/// Complete environment snapshot for reproducibility
26#[derive(Debug, Clone, Serialize, Deserialize)]
27pub struct EnvironmentSnapshot {
28    /// Snapshot identifier
29    pub id: String,
30    /// Snapshot timestamp
31    pub timestamp: DateTime<Utc>,
32    /// System information
33    pub system_info: SystemInfo,
34    /// Software dependencies
35    pub dependencies: Vec<Dependency>,
36    /// Environment variables
37    pub environment_variables: HashMap<String, String>,
38    /// Hardware configuration
39    pub hardware_config: HardwareConfig,
40    /// Random seeds
41    pub random_seeds: Vec<u64>,
42    /// Data checksums
43    pub data_checksums: HashMap<String, String>,
44    /// Configuration hashes
45    pub config_hashes: HashMap<String, String>,
46}
47
48/// System information for reproducibility
49#[derive(Debug, Clone, Serialize, Deserialize)]
50pub struct SystemInfo {
51    /// Operating system
52    pub os: String,
53    /// OS version
54    pub os_version: String,
55    /// Kernel version
56    pub kernel_version: Option<String>,
57    /// Architecture
58    pub architecture: String,
59    /// Hostname
60    pub hostname: String,
61    /// Timezone
62    pub timezone: String,
63    /// Locale settings
64    pub locale: HashMap<String, String>,
65}
66
67/// Software dependency information
68#[derive(Debug, Clone, Serialize, Deserialize)]
69pub struct Dependency {
70    /// Package name
71    pub name: String,
72    /// Version
73    pub version: String,
74    /// Source/registry
75    pub source: String,
76    /// Checksum
77    pub checksum: Option<String>,
78    /// Installation path
79    pub install_path: Option<String>,
80}
81
82/// Hardware configuration for reproducibility
83#[derive(Debug, Clone, Serialize, Deserialize)]
84pub struct HardwareConfig {
85    /// CPU information
86    pub cpu: CpuSpec,
87    /// Memory information
88    pub memory: MemorySpec,
89    /// GPU information
90    pub gpu: Option<GpuSpec>,
91    /// Storage information
92    pub storage: Vec<StorageSpec>,
93}
94
95/// CPU specification
96#[derive(Debug, Clone, Serialize, Deserialize)]
97pub struct CpuSpec {
98    /// CPU model
99    pub model: String,
100    /// Number of cores
101    pub cores: usize,
102    /// Number of threads
103    pub threads: usize,
104    /// Base frequency (MHz)
105    pub base_frequency: u32,
106    /// Max frequency (MHz)
107    pub max_frequency: u32,
108    /// Cache information
109    pub cache: HashMap<String, String>,
110    /// CPU flags/features
111    pub flags: Vec<String>,
112}
113
114/// Memory specification
115#[derive(Debug, Clone, Serialize, Deserialize)]
116pub struct MemorySpec {
117    /// Total memory (bytes)
118    pub total_bytes: u64,
119    /// Available memory (bytes)
120    pub available_bytes: u64,
121    /// Memory type (DDR4, etc.)
122    pub memory_type: String,
123    /// Memory speed (MHz)
124    pub speed_mhz: u32,
125}
126
127/// GPU specification
128#[derive(Debug, Clone, Serialize, Deserialize)]
129pub struct GpuSpec {
130    /// GPU model
131    pub model: String,
132    /// GPU memory (bytes)
133    pub memory_bytes: u64,
134    /// Driver version
135    pub driver_version: String,
136    /// CUDA version (if applicable)
137    pub cuda_version: Option<String>,
138    /// Compute capability
139    pub compute_capability: Option<String>,
140}
141
142/// Storage specification
143#[derive(Debug, Clone, Serialize, Deserialize)]
144pub struct StorageSpec {
145    /// Device name
146    pub device: String,
147    /// Storage type (SSD, HDD, etc.)
148    pub storage_type: String,
149    /// Total size (bytes)
150    pub size_bytes: u64,
151    /// Available space (bytes)
152    pub available_bytes: u64,
153    /// File system
154    pub filesystem: String,
155}
156
157/// Reproducibility report
158#[derive(Debug, Clone, Serialize, Deserialize)]
159pub struct ReproducibilityReport {
160    /// Report ID
161    pub id: String,
162    /// Experiment ID
163    pub experiment_id: String,
164    /// Environment snapshot ID
165    pub environment_id: String,
166    /// Reproducibility score
167    pub reproducibility_score: f64,
168    /// Checklist results
169    pub checklist: ReproducibilityChecklist,
170    /// Issues found
171    pub issues: Vec<ReproducibilityIssue>,
172    /// Recommendations
173    pub recommendations: Vec<String>,
174    /// Generation timestamp
175    pub generated_at: DateTime<Utc>,
176}
177
178/// Reproducibility checklist
179#[derive(Debug, Clone, Serialize, Deserialize)]
180pub struct ReproducibilityChecklist {
181    /// Random seed documented
182    pub random_seed_documented: bool,
183    /// Dependencies pinned
184    pub dependencies_pinned: bool,
185    /// Environment captured
186    pub environment_captured: bool,
187    /// Data versioned
188    pub data_versioned: bool,
189    /// Code versioned
190    pub code_versioned: bool,
191    /// Hardware documented
192    pub hardware_documented: bool,
193    /// Configuration hashed
194    pub configuration_hashed: bool,
195    /// Results verified
196    pub results_verified: bool,
197}
198
199/// Reproducibility issue
200#[derive(Debug, Clone, Serialize, Deserialize)]
201pub struct ReproducibilityIssue {
202    /// Issue type
203    pub issue_type: IssueType,
204    /// Severity level
205    pub severity: IssueSeverity,
206    /// Description
207    pub description: String,
208    /// Affected component
209    pub component: String,
210    /// Suggested fix
211    pub suggested_fix: Option<String>,
212}
213
214/// Types of reproducibility issues
215#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
216pub enum IssueType {
217    /// Missing random seed
218    MissingRandomSeed,
219    /// Unpinned dependencies
220    UnpinnedDependencies,
221    /// Missing environment info
222    MissingEnvironment,
223    /// Data not versioned
224    DataNotVersioned,
225    /// Code not versioned
226    CodeNotVersioned,
227    /// Hardware not documented
228    HardwareNotDocumented,
229    /// Configuration not hashed
230    ConfigurationNotHashed,
231    /// Non-deterministic behavior
232    NonDeterministic,
233    /// Platform-specific code
234    PlatformSpecific,
235    /// External dependencies
236    ExternalDependencies,
237}
238
239/// Issue severity levels
240#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Ord)]
241pub enum IssueSeverity {
242    /// Critical issue - prevents reproducibility
243    Critical,
244    /// High severity - likely to affect reproducibility
245    High,
246    /// Medium severity - may affect reproducibility
247    Medium,
248    /// Low severity - minor impact on reproducibility
249    Low,
250    /// Info only
251    Info,
252}
253
254/// Verification result for reproducibility
255#[derive(Debug, Clone, Serialize, Deserialize)]
256pub struct VerificationResult {
257    /// Verification ID
258    pub id: String,
259    /// Original experiment ID
260    pub original_experiment_id: String,
261    /// Reproduction experiment ID
262    pub reproduction_experiment_id: String,
263    /// Verification status
264    pub status: VerificationStatus,
265    /// Similarity metrics
266    pub similarity_metrics: SimilarityMetrics,
267    /// Differences found
268    pub differences: Vec<Difference>,
269    /// Verification timestamp
270    pub verified_at: DateTime<Utc>,
271}
272
273/// Verification status
274#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
275pub enum VerificationStatus {
276    /// Exact reproduction
277    ExactMatch,
278    /// Close reproduction (within tolerance)
279    CloseMatch,
280    /// Partial reproduction
281    PartialMatch,
282    /// No match
283    NoMatch,
284    /// Verification failed
285    VerificationFailed,
286}
287
288/// Similarity metrics between original and reproduction
289#[derive(Debug, Clone, Serialize, Deserialize)]
290pub struct SimilarityMetrics {
291    /// Overall similarity score (0.0 to 1.0)
292    pub overall_similarity: f64,
293    /// Result similarity
294    pub result_similarity: f64,
295    /// Performance similarity
296    pub performance_similarity: f64,
297    /// Configuration similarity
298    pub configuration_similarity: f64,
299    /// Environment similarity
300    pub environment_similarity: f64,
301}
302
303/// Difference between original and reproduction
304#[derive(Debug, Clone, Serialize, Deserialize)]
305pub struct Difference {
306    /// Category of difference
307    pub category: DifferenceCategory,
308    /// Field or metric name
309    pub field: String,
310    /// Original value
311    pub original_value: String,
312    /// Reproduction value
313    pub reproduction_value: String,
314    /// Difference magnitude
315    pub magnitude: f64,
316    /// Significance
317    pub significant: bool,
318}
319
320/// Categories of differences
321#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
322pub enum DifferenceCategory {
323    /// Difference in results
324    Results,
325    /// Difference in performance
326    Performance,
327    /// Difference in configuration
328    Configuration,
329    /// Difference in environment
330    Environment,
331    /// Difference in dependencies
332    Dependencies,
333    /// Difference in hardware
334    Hardware,
335}
336
337/// Reproducibility configuration
338#[derive(Debug, Clone, Serialize, Deserialize)]
339pub struct ReproducibilityConfig {
340    /// Tolerance for numerical comparisons
341    pub numerical_tolerance: f64,
342    /// Tolerance for performance comparisons
343    pub performance_tolerance: f64,
344    /// Minimum reproducibility score
345    pub min_reproducibility_score: f64,
346    /// Auto-capture environment
347    pub auto_capture_environment: bool,
348    /// Auto-verify results
349    pub auto_verify_results: bool,
350    /// Storage settings
351    pub storage: ReproducibilityStorage,
352}
353
354/// Storage settings for reproducibility data
355#[derive(Debug, Clone, Serialize, Deserialize)]
356pub struct ReproducibilityStorage {
357    /// Base storage directory
358    pub base_directory: PathBuf,
359    /// Compress snapshots
360    pub compress_snapshots: bool,
361    /// Retention period (days)
362    pub retention_days: u32,
363    /// Maximum storage size (bytes)
364    pub max_storage_bytes: u64,
365}
366
367impl ReproducibilityManager {
368    /// Create a new reproducibility manager
369    pub fn new(config: ReproducibilityConfig) -> Self {
370        Self {
371            environments: HashMap::new(),
372            reports: Vec::new(),
373            verifications: Vec::new(),
374            config,
375        }
376    }
377
378    /// Capture current environment snapshot
379    pub fn capture_environment(&mut self) -> Result<String> {
380        let snapshot_id = uuid::Uuid::new_v4().to_string();
381        let snapshot = EnvironmentSnapshot {
382            id: snapshot_id.clone(),
383            timestamp: Utc::now(),
384            system_info: self.capture_system_info()?,
385            dependencies: self.capture_dependencies()?,
386            environment_variables: self.capture_environment_variables(),
387            hardware_config: self.capture_hardware_config()?,
388            random_seeds: vec![42], // Default seed
389            data_checksums: HashMap::new(),
390            config_hashes: HashMap::new(),
391        };
392
393        self.environments.insert(snapshot_id.clone(), snapshot);
394        Ok(snapshot_id)
395    }
396
397    /// Generate reproducibility report for experiment
398    pub fn generate_report(&mut self, experiment_id: &str, environment_id: &str) -> Result<String> {
399        let environment = self.environments.get(environment_id).ok_or_else(|| {
400            OptimError::InvalidConfig("Environment snapshot not found".to_string())
401        })?;
402
403        let checklist = self.evaluate_checklist(environment);
404        let (score, issues) = self.calculate_reproducibility_score(&checklist, environment);
405        let recommendations = self.generate_recommendations(&issues);
406
407        let report_id = uuid::Uuid::new_v4().to_string();
408        let report = ReproducibilityReport {
409            id: report_id.clone(),
410            experiment_id: experiment_id.to_string(),
411            environment_id: environment_id.to_string(),
412            reproducibility_score: score,
413            checklist,
414            issues,
415            recommendations,
416            generated_at: Utc::now(),
417        };
418
419        self.reports.push(report);
420        Ok(report_id)
421    }
422
423    /// Verify reproducibility between two experiments
424    pub fn verify_reproducibility(
425        &mut self,
426        original_experiment_id: &str,
427        reproduction_experiment_id: &str,
428    ) -> Result<String> {
429        // This would compare the actual experiment results
430        // For now, we'll create a placeholder verification
431
432        let verification_id = uuid::Uuid::new_v4().to_string();
433        let verification = VerificationResult {
434            id: verification_id.clone(),
435            original_experiment_id: original_experiment_id.to_string(),
436            reproduction_experiment_id: reproduction_experiment_id.to_string(),
437            status: VerificationStatus::CloseMatch, // Placeholder
438            similarity_metrics: SimilarityMetrics {
439                overall_similarity: 0.95,
440                result_similarity: 0.98,
441                performance_similarity: 0.92,
442                configuration_similarity: 1.0,
443                environment_similarity: 0.90,
444            },
445            differences: Vec::new(),
446            verified_at: Utc::now(),
447        };
448
449        self.verifications.push(verification);
450        Ok(verification_id)
451    }
452
453    fn capture_system_info(&self) -> Result<SystemInfo> {
454        Ok(SystemInfo {
455            os: std::env::consts::OS.to_string(),
456            os_version: "Unknown".to_string(), // Would use system APIs
457            kernel_version: None,
458            architecture: std::env::consts::ARCH.to_string(),
459            hostname: std::env::var("HOSTNAME").unwrap_or_else(|_| "unknown".to_string()),
460            timezone: "UTC".to_string(), // Would detect actual timezone
461            locale: HashMap::new(),
462        })
463    }
464
465    fn capture_dependencies(&self) -> Result<Vec<Dependency>> {
466        // In a real implementation, this would parse Cargo.lock, requirements.txt, etc.
467        Ok(vec![Dependency {
468            name: "scirs2-optim".to_string(),
469            version: "0.1.0".to_string(),
470            source: "local".to_string(),
471            checksum: None,
472            install_path: None,
473        }])
474    }
475
476    fn capture_environment_variables(&self) -> HashMap<String, String> {
477        std::env::vars().collect()
478    }
479
480    fn capture_hardware_config(&self) -> Result<HardwareConfig> {
481        Ok(HardwareConfig {
482            cpu: CpuSpec {
483                model: "Unknown CPU".to_string(),
484                cores: std::thread::available_parallelism()
485                    .map(|p| p.get())
486                    .unwrap_or(1),
487                threads: std::thread::available_parallelism()
488                    .map(|p| p.get())
489                    .unwrap_or(1),
490                base_frequency: 0,
491                max_frequency: 0,
492                cache: HashMap::new(),
493                flags: Vec::new(),
494            },
495            memory: MemorySpec {
496                total_bytes: 8 * 1024 * 1024 * 1024,     // 8GB default
497                available_bytes: 6 * 1024 * 1024 * 1024, // 6GB default
498                memory_type: "Unknown".to_string(),
499                speed_mhz: 0,
500            },
501            gpu: None,
502            storage: Vec::new(),
503        })
504    }
505
506    fn evaluate_checklist(&self, environment: &EnvironmentSnapshot) -> ReproducibilityChecklist {
507        ReproducibilityChecklist {
508            random_seed_documented: !environment.random_seeds.is_empty(),
509            dependencies_pinned: !environment.dependencies.is_empty(),
510            environment_captured: true, // We have the snapshot
511            data_versioned: !environment.data_checksums.is_empty(),
512            code_versioned: false,     // Would check git info
513            hardware_documented: true, // We captured hardware info
514            configuration_hashed: !environment.config_hashes.is_empty(),
515            results_verified: false, // Would check verification status
516        }
517    }
518
519    fn calculate_reproducibility_score(
520        &self,
521        checklist: &ReproducibilityChecklist,
522        environment: &EnvironmentSnapshot,
523    ) -> (f64, Vec<ReproducibilityIssue>) {
524        let mut score = 0.0;
525        let mut issues = Vec::new();
526        let total_checks = 8.0;
527
528        if checklist.random_seed_documented {
529            score += 1.0;
530        } else {
531            issues.push(ReproducibilityIssue {
532                issue_type: IssueType::MissingRandomSeed,
533                severity: IssueSeverity::High,
534                description: "Random seed not documented".to_string(),
535                component: "Random Number Generation".to_string(),
536                suggested_fix: Some("Set and document random seeds for all RNGs".to_string()),
537            });
538        }
539
540        if checklist.dependencies_pinned {
541            score += 1.0;
542        } else {
543            issues.push(ReproducibilityIssue {
544                issue_type: IssueType::UnpinnedDependencies,
545                severity: IssueSeverity::Critical,
546                description: "Dependencies not pinned to specific versions".to_string(),
547                component: "Dependencies".to_string(),
548                suggested_fix: Some("Pin all dependencies to exact versions".to_string()),
549            });
550        }
551
552        if checklist.environment_captured {
553            score += 1.0;
554        }
555
556        if checklist.data_versioned {
557            score += 1.0;
558        } else {
559            issues.push(ReproducibilityIssue {
560                issue_type: IssueType::DataNotVersioned,
561                severity: IssueSeverity::High,
562                description: "Data not versioned or checksummed".to_string(),
563                component: "Data Management".to_string(),
564                suggested_fix: Some("Version control data or provide checksums".to_string()),
565            });
566        }
567
568        if checklist.code_versioned {
569            score += 1.0;
570        } else {
571            issues.push(ReproducibilityIssue {
572                issue_type: IssueType::CodeNotVersioned,
573                severity: IssueSeverity::Critical,
574                description: "Code not under version control".to_string(),
575                component: "Source Code".to_string(),
576                suggested_fix: Some("Use Git or other version control system".to_string()),
577            });
578        }
579
580        if checklist.hardware_documented {
581            score += 1.0;
582        }
583
584        if checklist.configuration_hashed {
585            score += 1.0;
586        } else {
587            issues.push(ReproducibilityIssue {
588                issue_type: IssueType::ConfigurationNotHashed,
589                severity: IssueSeverity::Medium,
590                description: "Configuration not hashed for integrity".to_string(),
591                component: "Configuration".to_string(),
592                suggested_fix: Some("Generate and store configuration hashes".to_string()),
593            });
594        }
595
596        if checklist.results_verified {
597            score += 1.0;
598        }
599
600        (score / total_checks, issues)
601    }
602
603    fn generate_recommendations(&self, issues: &[ReproducibilityIssue]) -> Vec<String> {
604        let mut recommendations = Vec::new();
605
606        for issue in issues {
607            if let Some(fix) = &issue.suggested_fix {
608                recommendations.push(format!("{}: {}", issue.component, fix));
609            }
610        }
611
612        if issues
613            .iter()
614            .any(|i| i.issue_type == IssueType::MissingRandomSeed)
615        {
616            recommendations.push("Use consistent random seeds across all components".to_string());
617        }
618
619        if issues
620            .iter()
621            .any(|i| i.issue_type == IssueType::UnpinnedDependencies)
622        {
623            recommendations.push("Create a lockfile with exact dependency versions".to_string());
624        }
625
626        recommendations.push("Document the complete experimental procedure".to_string());
627        recommendations.push("Provide clear instructions for reproduction".to_string());
628
629        recommendations
630    }
631}
632
633impl Default for ReproducibilityConfig {
634    fn default() -> Self {
635        Self {
636            numerical_tolerance: 1e-6,
637            performance_tolerance: 0.1,     // 10%
638            min_reproducibility_score: 0.8, // 80%
639            auto_capture_environment: true,
640            auto_verify_results: false,
641            storage: ReproducibilityStorage {
642                base_directory: PathBuf::from("./reproducibility"),
643                compress_snapshots: true,
644                retention_days: 365,
645                max_storage_bytes: 10 * 1024 * 1024 * 1024, // 10GB
646            },
647        }
648    }
649}
650
651#[cfg(test)]
652mod tests {
653    use super::*;
654
655    #[test]
656    fn test_reproducibility_manager_creation() {
657        let config = ReproducibilityConfig::default();
658        let manager = ReproducibilityManager::new(config);
659
660        assert!(manager.environments.is_empty());
661        assert!(manager.reports.is_empty());
662        assert!(manager.verifications.is_empty());
663    }
664
665    #[test]
666    fn test_environment_capture() {
667        let config = ReproducibilityConfig::default();
668        let mut manager = ReproducibilityManager::new(config);
669
670        let snapshot_id = manager.capture_environment().unwrap();
671
672        assert!(manager.environments.contains_key(&snapshot_id));
673        let snapshot = &manager.environments[&snapshot_id];
674        assert_eq!(snapshot.system_info.os, std::env::consts::OS);
675    }
676
677    #[test]
678    fn test_reproducibility_report() {
679        let config = ReproducibilityConfig::default();
680        let mut manager = ReproducibilityManager::new(config);
681
682        let env_id = manager.capture_environment().unwrap();
683        let report_id = manager.generate_report("test_experiment", &env_id).unwrap();
684
685        assert!(!manager.reports.is_empty());
686        let report = &manager.reports[0];
687        assert_eq!(report.id, report_id);
688        assert_eq!(report.experiment_id, "test_experiment");
689    }
690}
optirs_core/research/reproducibility.rs

optirs_core/research/
reproducibility.rs
