Function train_test_split 

pub fn train_test_split(
    dataset: &Dataset,
    test_size: f64,
    random_seed: Option<u64>,
) -> Result<(Dataset, Dataset)>

Split a dataset into training and test sets

This function creates a random split of the dataset while preserving the metadata and feature information in both resulting datasets.

Arguments

• dataset - The dataset to split
• test_size - Fraction of samples to include in test set (0.0 to 1.0)
• random_seed - Optional random seed for reproducible splits

Returns

A tuple of (train_dataset, test_dataset)

Examples

use ndarray::Array2;
use scirs2__datasets::utils::{Dataset, train_test_split};

let data = Array2::from_shape_vec((10, 3), (0..30).map(|x| x as f64).collect()).unwrap();
let dataset = Dataset::new(data, None);

let (train, test) = train_test_split(&dataset, 0.3, Some(42)).unwrap();
assert_eq!(train.n_samples() + test.n_samples(), 10);

Examples found in repository

examples/dataset_loaders.rs (line 37)

fn main() {
    // Check if a CSV file is provided as a command-line argument
    let args: Vec<String> = env::args().collect();
    if args.len() < 2 {
        println!("Usage: {} <path_to_csv_file>", args[0]);
        println!("Example: {} examples/sampledata.csv", args[0]);
        return;
    }

    let filepath = &args[1];

    // Verify the file exists
    if !Path::new(filepath).exists() {
        println!("Error: File '{filepath}' does not exist");
        return;
    }

    // Load CSV file
    println!("Loading CSV file: {filepath}");
    let csv_config = loaders::CsvConfig {
        has_header: true,
        target_column: None,
        ..Default::default()
    };
    match loaders::load_csv(filepath, csv_config) {
        Ok(dataset) => {
            print_dataset_info(&dataset, "Loaded CSV");

            // Split the dataset for demonstration
            println!("\nDemonstrating train-test split...");
            match train_test_split(&dataset, 0.2, Some(42)) {
                Ok((train, test)) => {
                    println!("Training set: {} samples", train.n_samples());
                    println!("Test set: {} samples", test.n_samples());

                    // Save as JSON for demonstration
                    let jsonpath = format!("{filepath}.json");
                    println!("\nSaving training dataset to JSON: {jsonpath}");
                    if let Err(e) = loaders::save_json(&train, &jsonpath) {
                        println!("Error saving JSON: {e}");
                    } else {
                        println!("Successfully saved JSON file");

                        // Load back the JSON file
                        println!("\nLoading back from JSON file...");
                        match loaders::load_json(&jsonpath) {
                            Ok(loaded) => {
                                print_dataset_info(&loaded, "Loaded JSON");
                            }
                            Err(e) => println!("Error loading JSON: {e}"),
                        }
                    }
                }
                Err(e) => println!("Error splitting dataset: {e}"),
            }
        }
        Err(e) => println!("Error loading CSV: {e}"),
    }
}

examples/real_world_datasets.rs (line 147)

fn demonstrate_classification_datasets() -> Result<(), Box<dyn std::error::Error>> {
    println!("🎯 CLASSIFICATION DATASETS");
    println!("{}", "-".repeat(40));

    // Titanic dataset
    println!("Loading Titanic dataset...");
    let titanic = load_titanic()?;

    println!("Titanic Dataset:");
    println!(
        "  Description: {}",
        titanic
            .metadata
            .get("description")
            .unwrap_or(&"Unknown".to_string())
    );
    println!("  Samples: {}", titanic.n_samples());
    println!("  Features: {}", titanic.n_features());

    if let Some(featurenames) = titanic.featurenames() {
        println!("  Features: {featurenames:?}");
    }

    if let Some(targetnames) = titanic.targetnames() {
        println!("  Classes: {targetnames:?}");
    }

    // Analyze class distribution
    if let Some(target) = &titanic.target {
        let mut class_counts = HashMap::new();
        for &class in target.iter() {
            *class_counts.entry(class as i32).or_insert(0) += 1;
        }
        println!("  Class distribution: {class_counts:?}");

        // Calculate survival rate
        let survived = class_counts.get(&1).unwrap_or(&0);
        let total = titanic.n_samples();
        println!(
            "  Survival rate: {:.1}%",
            (*survived as f64 / total as f64) * 100.0
        );
    }

    // Demonstrate train/test split
    let (train, test) = train_test_split(&titanic, 0.2, Some(42))?;
    println!(
        "  Train/test split: {} train, {} test",
        train.n_samples(),
        test.n_samples()
    );

    // Adult (Census Income) dataset
    println!("\nLoading Adult (Census Income) dataset...");
    match load_adult() {
        Ok(adult) => {
            println!("Adult Dataset:");
            println!(
                "  Description: {}",
                adult
                    .metadata
                    .get("description")
                    .unwrap_or(&"Unknown".to_string())
            );
            println!("  Samples: {}", adult.n_samples());
            println!("  Features: {}", adult.n_features());
            println!("  Task: Predict income >$50K based on census data");
        }
        Err(e) => {
            println!("  Note: Adult dataset requires download: {e}");
            println!("  This is expected for the demonstration");
        }
    }

    println!();
    Ok(())
}

#[allow(dead_code)]
fn demonstrate_regression_datasets() -> Result<(), Box<dyn std::error::Error>> {
    println!("📈 REGRESSION DATASETS");
    println!("{}", "-".repeat(40));

    // California Housing dataset
    println!("Loading California Housing dataset...");
    let housing = load_california_housing()?;

    println!("California Housing Dataset:");
    println!(
        "  Description: {}",
        housing
            .metadata
            .get("description")
            .unwrap_or(&"Unknown".to_string())
    );
    println!("  Samples: {}", housing.n_samples());
    println!("  Features: {}", housing.n_features());

    if let Some(featurenames) = housing.featurenames() {
        println!("  Features: {featurenames:?}");
    }

    // Analyze target distribution
    if let Some(target) = &housing.target {
        let mean = target.mean().unwrap_or(0.0);
        let std = target.std(0.0);
        let min = target.iter().fold(f64::INFINITY, |a, &b| a.min(b));
        let max = target.iter().fold(f64::NEG_INFINITY, |a, &b| a.max(b));

        println!("  Target (house value) statistics:");
        println!("    Mean: {mean:.2} (hundreds of thousands)");
        println!("    Std:  {std:.2}");
        println!("    Range: [{min:.2}, {max:.2}]");
    }

    // Red Wine Quality dataset
    println!("\nLoading Red Wine Quality dataset...");
    let wine = load_red_wine_quality()?;

    println!("Red Wine Quality Dataset:");
    println!(
        "  Description: {}",
        wine.metadata
            .get("description")
            .unwrap_or(&"Unknown".to_string())
    );
    println!("  Samples: {}", wine.n_samples());
    println!("  Features: {}", wine.n_features());

    if let Some(target) = &wine.target {
        let mean_quality = target.mean().unwrap_or(0.0);
        println!("  Average wine quality: {mean_quality:.1}/10");

        // Quality distribution
        let mut quality_counts = HashMap::new();
        for &quality in target.iter() {
            let q = quality.round() as i32;
            *quality_counts.entry(q).or_insert(0) += 1;
        }
        println!("  Quality distribution: {quality_counts:?}");
    }

    println!();
    Ok(())
}

#[allow(dead_code)]
fn demonstrate_healthcare_datasets() -> Result<(), Box<dyn std::error::Error>> {
    println!("🏥 HEALTHCARE DATASETS");
    println!("{}", "-".repeat(40));

    // Heart Disease dataset
    println!("Loading Heart Disease dataset...");
    let heart = load_heart_disease()?;

    println!("Heart Disease Dataset:");
    println!(
        "  Description: {}",
        heart
            .metadata
            .get("description")
            .unwrap_or(&"Unknown".to_string())
    );
    println!("  Samples: {}", heart.n_samples());
    println!("  Features: {}", heart.n_features());

    if let Some(featurenames) = heart.featurenames() {
        println!("  Clinical features: {:?}", &featurenames[..5]); // Show first 5
        println!("  ... and {} more features", featurenames.len() - 5);
    }

    // Analyze risk factors
    if let Some(target) = &heart.target {
        let mut disease_counts = HashMap::new();
        for &disease in target.iter() {
            *disease_counts.entry(disease as i32).or_insert(0) += 1;
        }

        let with_disease = disease_counts.get(&1).unwrap_or(&0);
        let total = heart.n_samples();
        println!(
            "  Disease prevalence: {:.1}% ({}/{})",
            (*with_disease as f64 / total as f64) * 100.0,
            with_disease,
            total
        );
    }

    // Demonstrate feature analysis
    println!("  Sample clinical parameter ranges:");
    let age_col = heart.data.column(0);
    let age_mean = age_col.mean();
    let age_std = age_col.std(0.0);
    println!("    Age: {age_mean:.1} ± {age_std:.1} years");

    println!();
    Ok(())
}

#[allow(dead_code)]
fn demonstrate_advanced_operations() -> Result<(), Box<dyn std::error::Error>> {
    println!("🔧 ADVANCED DATASET OPERATIONS");
    println!("{}", "-".repeat(40));

    let housing = load_california_housing()?;

    // Data preprocessing pipeline
    println!("Preprocessing pipeline for California Housing:");

    // 1. Train/test split
    let (mut train, test) = train_test_split(&housing, 0.2, Some(42))?;
    println!(
        "  1. Split: {} train, {} test",
        train.n_samples(),
        test.n_samples()
    );

    // 2. Feature scaling
    let mut pipeline = MLPipeline::default();
    train = pipeline.prepare_dataset(&train)?;
    println!("  2. Standardized features");

    // 3. Cross-validation setup
    let cv_folds = k_fold_split(train.n_samples(), 5, true, Some(42))?;
    println!("  3. Created {} CV folds", cv_folds.len());

    // Feature correlation analysis (simplified)
    println!("  4. Feature analysis:");
    println!("     • {} numerical features", train.n_features());
    println!("     • Ready for machine learning models");

    // Custom dataset configuration
    println!("\nCustom dataset loading configuration:");
    let config = RealWorldConfig {
        use_cache: true,
        download_if_missing: false, // Don't download in demo
        return_preprocessed: true,
        subset: Some("small".to_string()),
        random_state: Some(42),
        ..Default::default()
    };

    println!("  • Caching: {}", config.use_cache);
    println!("  • Download missing: {}", config.download_if_missing);
    println!("  • Preprocessed: {}", config.return_preprocessed);
    println!("  • Subset: {:?}", config.subset);

    println!();
    Ok(())
}

examples/data_generators.rs (line 24)

fn main() -> Result<(), Box<dyn std::error::Error>> {
    println!("Creating synthetic datasets...\n");

    // Generate classification dataset
    let n_samples = 100;
    let n_features = 5;

    let classificationdata = make_classification(
        n_samples,
        n_features,
        3,        // 3 classes
        2,        // 2 clusters per class
        3,        // 3 informative features
        Some(42), // random seed
    )?;

    // Train-test split
    let (train, test) = train_test_split(&classificationdata, 0.2, Some(42))?;

    println!("Classification dataset:");
    println!("  Total samples: {}", classificationdata.n_samples());
    println!("  Features: {}", classificationdata.n_features());
    println!("  Training samples: {}", train.n_samples());
    println!("  Test samples: {}", test.n_samples());

    // Generate regression dataset
    let regressiondata = make_regression(
        n_samples,
        n_features,
        3,   // 3 informative features
        0.5, // noise level
        Some(42),
    )?;

    println!("\nRegression dataset:");
    println!("  Samples: {}", regressiondata.n_samples());
    println!("  Features: {}", regressiondata.n_features());

    // Normalize the data (in-place)
    let mut data_copy = regressiondata.data.clone();
    normalize(&mut data_copy);
    println!("  Data normalized successfully");

    // Generate clustering data (blobs)
    let clusteringdata = make_blobs(
        n_samples,
        2,   // 2 features for easy visualization
        4,   // 4 clusters
        0.8, // cluster standard deviation
        Some(42),
    )?;

    println!("\nClustering dataset (blobs):");
    println!("  Samples: {}", clusteringdata.n_samples());
    println!("  Features: {}", clusteringdata.n_features());

    // Find the number of clusters by finding the max value of target
    let num_clusters = clusteringdata.target.as_ref().map_or(0, |t| {
        let mut max_val = -1.0;
        for &val in t.iter() {
            if val > max_val {
                max_val = val;
            }
        }
        (max_val as usize) + 1
    });

    println!("  Clusters: {num_clusters}");

    // Generate time series data
    let time_series = make_time_series(
        100,  // 100 time steps
        3,    // 3 features/variables
        true, // with trend
        true, // with seasonality
        0.2,  // noise level
        Some(42),
    )?;

    println!("\nTime series dataset:");
    println!("  Time steps: {}", time_series.n_samples());
    println!("  Features: {}", time_series.n_features());

    Ok(())
}

examples/datasets_streaming_demo.rs (line 148)

fn demonstrate_memory_efficient_processing() -> Result<(), Box<dyn std::error::Error>> {
    println!("💾 MEMORY-EFFICIENT PROCESSING");
    println!("{}", "-".repeat(40));

    // Compare memory usage: streaming vs. in-memory
    let datasetsize = 50_000;
    let n_features = 50;

    println!("Comparing memory usage for {datasetsize} samples with {n_features} features");

    // In-memory approach (for comparison)
    println!("\n1. In-memory approach:");
    let start_mem = get_memory_usage();
    let start_time = Instant::now();

    let in_memorydataset = make_classification(datasetsize, n_features, 5, 2, 25, Some(42))?;
    let (train, test) = train_test_split(&in_memorydataset, 0.2, Some(42))?;

    let in_memory_time = start_time.elapsed();
    let in_memory_mem = get_memory_usage() - start_mem;

    println!("  Time: {:.2}s", in_memory_time.as_secs_f64());
    println!("  Memory usage: ~{in_memory_mem:.1} MB");
    println!("  Train samples: {}", train.n_samples());
    println!("  Test samples: {}", test.n_samples());

    // Streaming approach
    println!("\n2. Streaming approach:");
    let stream_start_time = Instant::now();
    let stream_start_mem = get_memory_usage();

    let config = StreamConfig {
        chunk_size: 5_000, // Smaller chunks for memory efficiency
        buffer_size: 2,    // Smaller buffer
        num_workers: 2,
        memory_limit_mb: Some(50),
        ..Default::default()
    };

    let mut stream = stream_classification(datasetsize, n_features, 5, config)?;

    let mut total_processed = 0;
    let mut train_samples = 0;
    let mut test_samples = 0;

    while let Some(chunk) = stream.next_chunk()? {
        total_processed += chunk.n_samples();

        // Simulate train/test split on chunk level
        let chunk_trainsize = (chunk.n_samples() as f64 * 0.8) as usize;
        train_samples += chunk_trainsize;
        test_samples += chunk.n_samples() - chunk_trainsize;

        // Process chunk (simulate some computation)
        let _mean = chunk.data.mean_axis(ndarray::Axis(0));
        let _std = chunk.data.std_axis(ndarray::Axis(0), 0.0);

        if chunk.is_last {
            break;
        }
    }

    let stream_time = stream_start_time.elapsed();
    let stream_mem = get_memory_usage() - stream_start_mem;

    println!("  Time: {:.2}s", stream_time.as_secs_f64());
    println!("  Memory usage: ~{stream_mem:.1} MB");
    println!("  Train samples: {train_samples}");
    println!("  Test samples: {test_samples}");
    println!("  Total processed: {total_processed}");

    // Comparison
    println!("\n3. Comparison:");
    println!(
        "  Memory savings: {:.1}x less memory",
        in_memory_mem / stream_mem.max(1.0)
    );
    println!(
        "  Time overhead: {:.1}x",
        stream_time.as_secs_f64() / in_memory_time.as_secs_f64()
    );
    println!("  Streaming is beneficial for large datasets that don't fit in memory");

    println!();
    Ok(())
}