dendritic-datasets 1.5.0

Prebuilt datasets that can be imported for ML model training
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57

use std::fs::File; 
use dendritic_ndarray::ndarray::NDArray;
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::errors::Result; 
use crate::utils::*;

/// Load schema for airfoil noise data
pub fn load_airfoil_schema() -> Schema {
    Schema::new(vec![
        Field::new("x0", DataType::Float64, false),
        Field::new("x1", DataType::Float64, false),
        Field::new("x2", DataType::Float64, false),
        Field::new("x3", DataType::Float64, false),
        Field::new("x4", DataType::Float64, false),
        Field::new("y", DataType::Float64, false),
    ])
}

/// Utility method to convert airfoil csv to parquet
pub fn convert_airfoil_csv_to_parquet() {

    let airfoil_schema = load_airfoil_schema();

    csv_to_parquet(
        airfoil_schema,
        "data/airfoil_noise_data.csv",
        "data/airfoil_noise_data.parquet"
    ); 
}

/// Load airfoil noise data from path
pub fn load_airfoil_data(path: &str) -> Result<(NDArray<f64>, NDArray<f64>)> {
    
    let file = File::open(path).unwrap();
    let mut reader = ParquetRecordBatchReaderBuilder::try_new(file)?
        .build()?;

    let batch = reader.next().unwrap().unwrap();
    let (input, y_train) = select_features(
        batch.clone(),
        vec![
            "x0",
            "x1",
            "x2",
            "x3",
            "x4",
            "y"
        ],
        "y"
    );


    //let x_train = min_max_scalar(input).unwrap();
    Ok((input, y_train))

}