dendritic-datasets 1.5.0

Prebuilt datasets that can be imported for ML model training
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92

use std::fs::File; 
use dendritic_preprocessing::standard_scalar::*;
use dendritic_ndarray::ndarray::NDArray;
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::errors::Result; 
use crate::utils::*;


/// loading the schema for the breast cancer data
pub fn load_breast_cancer_schema() -> Schema {
    Schema::new(vec![
        Field::new("id", DataType::Utf8, false),
        Field::new("diagnosis", DataType::Utf8, false),
        Field::new("radius_mean", DataType::Float64, false),
        Field::new("texture_mean", DataType::Float64, false),
        Field::new("perimiter_mean", DataType::Float64, false),
        Field::new("area_mean", DataType::Float64, false),
        Field::new("smoothness_mean", DataType::Float64, false),
        Field::new("compactness_mean", DataType::Float64, false),
        Field::new("concavity_mean", DataType::Float64, false),
        Field::new("concave_points_mean", DataType::Float64, false),
        Field::new("symmetry_mean", DataType::Float64, false),
        Field::new("fractal_dimension_mean", DataType::Float64, false),
        Field::new("radius_se", DataType::Float64, false),
        Field::new("texture_se", DataType::Float64, false),
        Field::new("perimeter_se", DataType::Float64, false),
        Field::new("area_se", DataType::Float64, false),
        Field::new("smoothness_se", DataType::Float64, false),
        Field::new("compactness_se", DataType::Float64, false),
        Field::new("concavity_se", DataType::Float64, false),
        Field::new("concave_points_se", DataType::Float64, false),
        Field::new("symmetry_se", DataType::Float64, false),
        Field::new("fractal_dimensions_se", DataType::Float64, false),
        Field::new("radius_worst", DataType::Float64, false),
        Field::new("texture_worst", DataType::Float64, false),
        Field::new("perimeter_worst", DataType::Float64, false),
        Field::new("area_worst", DataType::Float64, false),
        Field::new("smoothness_worst", DataType::Float64, false),
        Field::new("compactness_worst", DataType::Float64, false),
        Field::new("concavity_worst", DataType::Float64, false),
        Field::new("concave_points_worst", DataType::Float64, false), 
        Field::new("symmetry_worst", DataType::Float64, false),
        Field::new("fractal_dimension_worst", DataType::Float64, false),
        Field::new("diagnosis_code", DataType::Float64, false)
    ])
}

/// Utility to convert breast cancer data to parquet
pub fn convert_breast_cancer_csv_to_parquet() {

    let breast_cancer_schema = load_breast_cancer_schema();

    csv_to_parquet(
        breast_cancer_schema,
        "data/breast_cancer.csv",
        "data/breast_cancer.parquet"
    );
}

/// Load breast cancer data from parquet file
pub fn load_breast_cancer(path: &str) -> Result<(NDArray<f64>, NDArray<f64>)> {
    
    /* switch to datasets/data directory */
    let file = File::open(path).unwrap();
    let mut reader = ParquetRecordBatchReaderBuilder::try_new(file)?
        .build()?;

    let batch = reader.next().unwrap().unwrap();
    let (input, y_train) = select_features(
        batch.clone(),
        vec![
            "radius_mean",
            "texture_mean",
            "smoothness_mean",
            "compactness_mean",
            "symmetry_mean", 
            "fractal_dimension_mean",
            "radius_se",
            "texture_se",
            "smoothness_se",
            "compactness_se",
            "symmetry_se",
            "fractal_dimensions_se"
        ],
        "diagnosis_code"
    );

    let x_train = min_max_scalar(input).unwrap();
    Ok((x_train, y_train))

}