Struct BatchOperations

pub struct BatchOperations { /* private fields */ }

Batch operations manager for dataset caching

Implementations

impl BatchOperations

pub fn new(cache: CacheManager) -> Self

Create a new batch operations manager

Examples found in repository

examples/batch_operations_demo.rs (line 17)

fn main() {
    println!("=== Batch Operations Demonstration ===\n");

    // Create a cache manager for demonstration
    let cache_dir = scirs2_datasets::get_cache_dir().unwrap();
    let cache_manager = CacheManager::new(cache_dir, 100, 3600);

    println!("=== Setting up Batch Operations Manager =====");
    let batch_ops = BatchOperations::new(cache_manager)
        .with_parallel(false) // Use sequential for deterministic demo output
        .with_retry_config(2, Duration::from_millis(500));

    println!("Batch operations manager configured:");
    println!("  - Parallel processing: disabled (for demo)");
    println!("  - Max retries: 2");
    println!("  - Retry delay: 500ms");

    // Demonstrate cache setup with sample data
    println!("\n=== Sample Data Setup ========================");
    setup_sample_cache_data(&batch_ops);

    // Demonstrate batch statistics
    println!("\n=== Cache Statistics ==========================");
    demonstrate_cache_statistics(&batch_ops);

    // Demonstrate batch processing
    println!("\n=== Batch Processing =========================");
    demonstrate_batch_processing(&batch_ops);

    // Demonstrate selective cleanup
    println!("\n=== Selective Cache Cleanup ==================");
    demonstrate_selective_cleanup(&batch_ops);

    // Show final cache state
    println!("\n=== Final Cache State =========================");
    show_final_cache_state(&batch_ops);

    // Performance considerations
    println!("\n=== Performance Considerations ================");
    demonstrate_performance_features();

    println!("\n=== Batch Operations Demo Complete ===========");
}

pub fn with_parallel(self, parallel: bool) -> Self

Configure parallel processing

Examples found in repository

examples/batch_operations_demo.rs (line 18)

fn main() {
    println!("=== Batch Operations Demonstration ===\n");

    // Create a cache manager for demonstration
    let cache_dir = scirs2_datasets::get_cache_dir().unwrap();
    let cache_manager = CacheManager::new(cache_dir, 100, 3600);

    println!("=== Setting up Batch Operations Manager =====");
    let batch_ops = BatchOperations::new(cache_manager)
        .with_parallel(false) // Use sequential for deterministic demo output
        .with_retry_config(2, Duration::from_millis(500));

    println!("Batch operations manager configured:");
    println!("  - Parallel processing: disabled (for demo)");
    println!("  - Max retries: 2");
    println!("  - Retry delay: 500ms");

    // Demonstrate cache setup with sample data
    println!("\n=== Sample Data Setup ========================");
    setup_sample_cache_data(&batch_ops);

    // Demonstrate batch statistics
    println!("\n=== Cache Statistics ==========================");
    demonstrate_cache_statistics(&batch_ops);

    // Demonstrate batch processing
    println!("\n=== Batch Processing =========================");
    demonstrate_batch_processing(&batch_ops);

    // Demonstrate selective cleanup
    println!("\n=== Selective Cache Cleanup ==================");
    demonstrate_selective_cleanup(&batch_ops);

    // Show final cache state
    println!("\n=== Final Cache State =========================");
    show_final_cache_state(&batch_ops);

    // Performance considerations
    println!("\n=== Performance Considerations ================");
    demonstrate_performance_features();

    println!("\n=== Batch Operations Demo Complete ===========");
}

pub fn with_retry_config( self, max_retries: usize, retry_delay: Duration, ) -> Self

Configure retry settings

Examples found in repository

examples/batch_operations_demo.rs (line 19)

fn main() {
    println!("=== Batch Operations Demonstration ===\n");

    // Create a cache manager for demonstration
    let cache_dir = scirs2_datasets::get_cache_dir().unwrap();
    let cache_manager = CacheManager::new(cache_dir, 100, 3600);

    println!("=== Setting up Batch Operations Manager =====");
    let batch_ops = BatchOperations::new(cache_manager)
        .with_parallel(false) // Use sequential for deterministic demo output
        .with_retry_config(2, Duration::from_millis(500));

    println!("Batch operations manager configured:");
    println!("  - Parallel processing: disabled (for demo)");
    println!("  - Max retries: 2");
    println!("  - Retry delay: 500ms");

    // Demonstrate cache setup with sample data
    println!("\n=== Sample Data Setup ========================");
    setup_sample_cache_data(&batch_ops);

    // Demonstrate batch statistics
    println!("\n=== Cache Statistics ==========================");
    demonstrate_cache_statistics(&batch_ops);

    // Demonstrate batch processing
    println!("\n=== Batch Processing =========================");
    demonstrate_batch_processing(&batch_ops);

    // Demonstrate selective cleanup
    println!("\n=== Selective Cache Cleanup ==================");
    demonstrate_selective_cleanup(&batch_ops);

    // Show final cache state
    println!("\n=== Final Cache State =========================");
    show_final_cache_state(&batch_ops);

    // Performance considerations
    println!("\n=== Performance Considerations ================");
    demonstrate_performance_features();

    println!("\n=== Batch Operations Demo Complete ===========");
}

pub fn batch_verify_integrity( &self, files_and_hashes: &[(&str, &str)], ) -> BatchResult

Verify integrity of multiple cached files

pub fn selective_cleanup( &self, patterns: &[&str], max_age_days: Option<u32>, ) -> Result<BatchResult>

Clean up cache selectively based on patterns

Examples found in repository

examples/batch_operations_demo.rs (line 169)

fn demonstrate_selective_cleanup(batch_ops: &BatchOperations) {
    println!("Demonstrating selective cache cleanup...");

    // Show current cache state
    let initial_stats = batch_ops.get_cache_statistics().unwrap();
    println!(
        "Before cleanup: {} files, {}",
        initial_stats.success_count,
        format_bytes(initial_stats.total_bytes)
    );

    // Example 1: Clean up temporary files
    println!("\n1. Cleaning up temporary files (*.tmp):");
    match batch_ops.selective_cleanup(&["*.tmp"], None) {
        Ok(result) => {
            println!("   {}", result.summary());
            if result.success_count > 0 {
                println!("   Removed {} temporary files", result.success_count);
            }
        }
        Err(e) => println!("   Failed: {}", e),
    }

    // Example 2: Clean up old files (demo with 0 days to show functionality)
    println!("\n2. Age-based cleanup (files older than 0 days - for demo):");
    match batch_ops.selective_cleanup(&["*"], Some(0)) {
        Ok(result) => {
            println!("   {}", result.summary());
            println!("   (Note: Using 0 days for demonstration - all files are 'old')");
        }
        Err(e) => println!("   Failed: {}", e),
    }

    // Show final cache state
    let final_stats = batch_ops.get_cache_statistics().unwrap_or_default();
    println!(
        "\nAfter cleanup: {} files, {}",
        final_stats.success_count,
        format_bytes(final_stats.total_bytes)
    );

    let freed_space = initial_stats
        .total_bytes
        .saturating_sub(final_stats.total_bytes);
    if freed_space > 0 {
        println!("Space freed: {}", format_bytes(freed_space));
    }
}

pub fn batch_process<F, T, E>( &self, names: &[String], processor: F, ) -> BatchResult

where F: Fn(&str, &[u8]) -> Result<T, E> + Sync + Send + 'static, E: Display, T: Send,

Process multiple datasets with a given function

Examples found in repository

examples/batch_operations_demo.rs (lines 114-120)

fn demonstrate_batch_processing(batch_ops: &BatchOperations) {
    println!("Processing multiple cached files in batch...");

    // Get list of cached files
    let cached_files = batch_ops.list_cached_files().unwrap_or_default();

    if cached_files.is_empty() {
        println!("No cached files found for processing");
        return;
    }

    println!("Found {} files to process", cached_files.len());

    // Example 1: Validate file sizes
    println!("\n1. File Size Validation:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        if data.len() < 10 {
            Err(format!("File {} too small ({} bytes)", name, data.len()))
        } else {
            Ok(data.len())
        }
    });

    println!("   {}", result.summary());
    if result.failure_count > 0 {
        for (file, error) in &result.failures {
            println!("   ⚠ {}: {}", file, error);
        }
    }

    // Example 2: Content type detection
    println!("\n2. Content Type Detection:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        let content_type = detect_content_type(name, data);
        println!("   {} -> {}", name, content_type);
        Ok::<String, String>(content_type)
    });

    println!("   {}", result.summary());

    // Example 3: Data integrity check
    println!("\n3. Data Integrity Check:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        // Simple check: ensure data is not all zeros
        let all_zeros = data.iter().all(|&b| b == 0);
        if all_zeros && data.len() > 100 {
            Err("Suspicious: large file with all zeros".to_string())
        } else {
            let checksum = data.iter().map(|&b| b as u32).sum::<u32>();
            println!("   {} checksum: {}", name, checksum);
            Ok(checksum)
        }
    });

    println!("   {}", result.summary());
}

pub fn cache_manager(&self) -> &CacheManager

Get access to the underlying cache manager

pub fn write_cached(&self, name: &str, data: &[u8]) -> Result<()>

Write data to cache

Examples found in repository

examples/batch_operations_demo.rs (line 68)

fn setup_sample_cache_data(batch_ops: &BatchOperations) {
    println!("Creating sample cached datasets...");

    // Create various types of sample data
    let sample_datasets = [
        ("iris_processed.csv", create_csv_data()),
        ("experiment_001.json", create_json_data()),
        ("temp_file_001.tmp", create_binary_data(100)),
        ("temp_file_002.tmp", create_binary_data(200)),
        ("large_dataset.dat", create_binary_data(1024)),
        ("model_weights.bin", create_binary_data(512)),
        ("results_summary.txt", create_text_data()),
    ];

    for (name, data) in sample_datasets {
        if let Err(e) = batch_ops.write_cached(name, &data) {
            println!("  Warning: Failed to cache {}: {}", name, e);
        } else {
            println!("  ✓ Cached {} ({} bytes)", name, data.len());
        }
    }
}

pub fn read_cached(&self, name: &str) -> Result<Vec<u8>>

Read data from cache

Examples found in repository

examples/batch_operations_demo.rs (line 214)

fn show_final_cache_state(batch_ops: &BatchOperations) {
    println!("Final cache contents:");

    match batch_ops.list_cached_files() {
        Ok(files) => {
            if files.is_empty() {
                println!("  Cache is empty");
            } else {
                for file in files {
                    if let Ok(data) = batch_ops.read_cached(&file) {
                        println!("  {} ({} bytes)", file, data.len());
                    }
                }
            }
        }
        Err(e) => println!("  Failed to list files: {}", e),
    }

    // Print detailed cache report
    if let Err(e) = batch_ops.print_cache_report() {
        println!("Failed to generate cache report: {}", e);
    }
}

pub fn list_cached_files(&self) -> Result<Vec<String>>

List cached files

Examples found in repository

examples/batch_operations_demo.rs (line 103)

fn demonstrate_batch_processing(batch_ops: &BatchOperations) {
    println!("Processing multiple cached files in batch...");

    // Get list of cached files
    let cached_files = batch_ops.list_cached_files().unwrap_or_default();

    if cached_files.is_empty() {
        println!("No cached files found for processing");
        return;
    }

    println!("Found {} files to process", cached_files.len());

    // Example 1: Validate file sizes
    println!("\n1. File Size Validation:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        if data.len() < 10 {
            Err(format!("File {} too small ({} bytes)", name, data.len()))
        } else {
            Ok(data.len())
        }
    });

    println!("   {}", result.summary());
    if result.failure_count > 0 {
        for (file, error) in &result.failures {
            println!("   ⚠ {}: {}", file, error);
        }
    }

    // Example 2: Content type detection
    println!("\n2. Content Type Detection:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        let content_type = detect_content_type(name, data);
        println!("   {} -> {}", name, content_type);
        Ok::<String, String>(content_type)
    });

    println!("   {}", result.summary());

    // Example 3: Data integrity check
    println!("\n3. Data Integrity Check:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        // Simple check: ensure data is not all zeros
        let all_zeros = data.iter().all(|&b| b == 0);
        if all_zeros && data.len() > 100 {
            Err("Suspicious: large file with all zeros".to_string())
        } else {
            let checksum = data.iter().map(|&b| b as u32).sum::<u32>();
            println!("   {} checksum: {}", name, checksum);
            Ok(checksum)
        }
    });

    println!("   {}", result.summary());
}

fn demonstrate_selective_cleanup(batch_ops: &BatchOperations) {
    println!("Demonstrating selective cache cleanup...");

    // Show current cache state
    let initial_stats = batch_ops.get_cache_statistics().unwrap();
    println!(
        "Before cleanup: {} files, {}",
        initial_stats.success_count,
        format_bytes(initial_stats.total_bytes)
    );

    // Example 1: Clean up temporary files
    println!("\n1. Cleaning up temporary files (*.tmp):");
    match batch_ops.selective_cleanup(&["*.tmp"], None) {
        Ok(result) => {
            println!("   {}", result.summary());
            if result.success_count > 0 {
                println!("   Removed {} temporary files", result.success_count);
            }
        }
        Err(e) => println!("   Failed: {}", e),
    }

    // Example 2: Clean up old files (demo with 0 days to show functionality)
    println!("\n2. Age-based cleanup (files older than 0 days - for demo):");
    match batch_ops.selective_cleanup(&["*"], Some(0)) {
        Ok(result) => {
            println!("   {}", result.summary());
            println!("   (Note: Using 0 days for demonstration - all files are 'old')");
        }
        Err(e) => println!("   Failed: {}", e),
    }

    // Show final cache state
    let final_stats = batch_ops.get_cache_statistics().unwrap_or_default();
    println!(
        "\nAfter cleanup: {} files, {}",
        final_stats.success_count,
        format_bytes(final_stats.total_bytes)
    );

    let freed_space = initial_stats
        .total_bytes
        .saturating_sub(final_stats.total_bytes);
    if freed_space > 0 {
        println!("Space freed: {}", format_bytes(freed_space));
    }
}

fn show_final_cache_state(batch_ops: &BatchOperations) {
    println!("Final cache contents:");

    match batch_ops.list_cached_files() {
        Ok(files) => {
            if files.is_empty() {
                println!("  Cache is empty");
            } else {
                for file in files {
                    if let Ok(data) = batch_ops.read_cached(&file) {
                        println!("  {} ({} bytes)", file, data.len());
                    }
                }
            }
        }
        Err(e) => println!("  Failed to list files: {}", e),
    }

    // Print detailed cache report
    if let Err(e) = batch_ops.print_cache_report() {
        println!("Failed to generate cache report: {}", e);
    }
}

pub fn print_cache_report(&self) -> Result<()>

Print cache report

Examples found in repository

examples/batch_operations_demo.rs (line 224)

fn show_final_cache_state(batch_ops: &BatchOperations) {
    println!("Final cache contents:");

    match batch_ops.list_cached_files() {
        Ok(files) => {
            if files.is_empty() {
                println!("  Cache is empty");
            } else {
                for file in files {
                    if let Ok(data) = batch_ops.read_cached(&file) {
                        println!("  {} ({} bytes)", file, data.len());
                    }
                }
            }
        }
        Err(e) => println!("  Failed to list files: {}", e),
    }

    // Print detailed cache report
    if let Err(e) = batch_ops.print_cache_report() {
        println!("Failed to generate cache report: {}", e);
    }
}

pub fn get_cache_statistics(&self) -> Result<BatchResult>

Get statistics about cached datasets

Examples found in repository

examples/batch_operations_demo.rs (line 77)

fn demonstrate_cache_statistics(batch_ops: &BatchOperations) {
    match batch_ops.get_cache_statistics() {
        Ok(result) => {
            println!("{}", result.summary());
            println!("Cache analysis:");
            println!("  - Files processed: {}", result.success_count);
            println!("  - Total cache size: {}", format_bytes(result.total_bytes));
            println!(
                "  - Analysis time: {:.2}ms",
                result.elapsed_time.as_millis()
            );

            if result.failure_count > 0 {
                println!("  - Failed files: {}", result.failure_count);
                for (file, error) in &result.failures {
                    println!("    • {}: {}", file, error);
                }
            }
        }
        Err(e) => println!("Failed to get cache statistics: {}", e),
    }
}

fn demonstrate_batch_processing(batch_ops: &BatchOperations) {
    println!("Processing multiple cached files in batch...");

    // Get list of cached files
    let cached_files = batch_ops.list_cached_files().unwrap_or_default();

    if cached_files.is_empty() {
        println!("No cached files found for processing");
        return;
    }

    println!("Found {} files to process", cached_files.len());

    // Example 1: Validate file sizes
    println!("\n1. File Size Validation:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        if data.len() < 10 {
            Err(format!("File {} too small ({} bytes)", name, data.len()))
        } else {
            Ok(data.len())
        }
    });

    println!("   {}", result.summary());
    if result.failure_count > 0 {
        for (file, error) in &result.failures {
            println!("   ⚠ {}: {}", file, error);
        }
    }

    // Example 2: Content type detection
    println!("\n2. Content Type Detection:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        let content_type = detect_content_type(name, data);
        println!("   {} -> {}", name, content_type);
        Ok::<String, String>(content_type)
    });

    println!("   {}", result.summary());

    // Example 3: Data integrity check
    println!("\n3. Data Integrity Check:");
    let result = batch_ops.batch_process(&cached_files, |name, data| {
        // Simple check: ensure data is not all zeros
        let all_zeros = data.iter().all(|&b| b == 0);
        if all_zeros && data.len() > 100 {
            Err("Suspicious: large file with all zeros".to_string())
        } else {
            let checksum = data.iter().map(|&b| b as u32).sum::<u32>();
            println!("   {} checksum: {}", name, checksum);
            Ok(checksum)
        }
    });

    println!("   {}", result.summary());
}

fn demonstrate_selective_cleanup(batch_ops: &BatchOperations) {
    println!("Demonstrating selective cache cleanup...");

    // Show current cache state
    let initial_stats = batch_ops.get_cache_statistics().unwrap();
    println!(
        "Before cleanup: {} files, {}",
        initial_stats.success_count,
        format_bytes(initial_stats.total_bytes)
    );

    // Example 1: Clean up temporary files
    println!("\n1. Cleaning up temporary files (*.tmp):");
    match batch_ops.selective_cleanup(&["*.tmp"], None) {
        Ok(result) => {
            println!("   {}", result.summary());
            if result.success_count > 0 {
                println!("   Removed {} temporary files", result.success_count);
            }
        }
        Err(e) => println!("   Failed: {}", e),
    }

    // Example 2: Clean up old files (demo with 0 days to show functionality)
    println!("\n2. Age-based cleanup (files older than 0 days - for demo):");
    match batch_ops.selective_cleanup(&["*"], Some(0)) {
        Ok(result) => {
            println!("   {}", result.summary());
            println!("   (Note: Using 0 days for demonstration - all files are 'old')");
        }
        Err(e) => println!("   Failed: {}", e),
    }

    // Show final cache state
    let final_stats = batch_ops.get_cache_statistics().unwrap_or_default();
    println!(
        "\nAfter cleanup: {} files, {}",
        final_stats.success_count,
        format_bytes(final_stats.total_bytes)
    );

    let freed_space = initial_stats
        .total_bytes
        .saturating_sub(final_stats.total_bytes);
    if freed_space > 0 {
        println!("Space freed: {}", format_bytes(freed_space));
    }
}