dendritic-datasets 1.5.0

Prebuilt datasets that can be imported for ML model training
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113

use std::fs::File; 
use dendritic_preprocessing::standard_scalar::*;
use dendritic_ndarray::ndarray::NDArray;
use arrow_schema::{DataType, Field, Schema};
use parquet::arrow::arrow_reader::ParquetRecordBatchReaderBuilder;
use parquet::errors::Result; 
use crate::utils::*;



/// loading the schema for alzheimers disease data
pub fn load_alzheimers_schema() -> Schema {
    Schema::new(vec![
        Field::new("patient_id", DataType::Utf8, false),
        Field::new("age", DataType::Float64, false),
        Field::new("gender", DataType::Float64, false),
        Field::new("ethnicity", DataType::Float64, false),
        Field::new("education_level", DataType::Float64, false),
        Field::new("bmi", DataType::Float64, false),
        Field::new("smoking", DataType::Float64, false),
        Field::new("alchohol_consumption", DataType::Float64, false),
        Field::new("physical_activity", DataType::Float64, false),
        Field::new("diet_quality", DataType::Float64, false),
        Field::new("sleep_quality", DataType::Float64, false),
        Field::new("family_history", DataType::Float64, false),
        Field::new("cardiovascular_disease", DataType::Float64, false),
        Field::new("diabetes", DataType::Float64, false),
        Field::new("depression", DataType::Float64, false),
        Field::new("head_injury", DataType::Float64, false),
        Field::new("hyptertension", DataType::Float64, false),
        Field::new("systolic_bp", DataType::Float64, false),
        Field::new("distolic_dp", DataType::Float64, false),
        Field::new("cholesterol_total", DataType::Float64, false),
        Field::new("cholesterol_ldl", DataType::Float64, false),
        Field::new("cholesterol_hdl", DataType::Float64, false),
        Field::new("cholesterol_tryglicerides", DataType::Float64, false),
        Field::new("mmse", DataType::Float64, false),
        Field::new("functional_assesment", DataType::Float64, false),
        Field::new("memory_complaints", DataType::Float64, false),
        Field::new("behavorial_problems", DataType::Float64, false),
        Field::new("adl", DataType::Float64, false),
        Field::new("confusion", DataType::Float64, false),
        Field::new("disorientation", DataType::Float64, false), 
        Field::new("personality_changes", DataType::Float64, false),
        Field::new("difficulty_w_tasks", DataType::Float64, false),
        Field::new("forgetfullness", DataType::Float64, false),
        Field::new("diagnosis", DataType::Float64, false),
        Field::new("doctor", DataType::Utf8, false),
    ])
}

/// Utility method for converting alzheimers data to parquet
pub fn convert_alzhiemers_to_parquet() {

    let alz_schema = load_alzheimers_schema();

    csv_to_parquet(
        alz_schema,
        "data/alzheimers_disease_data.csv",
        "data/alzheimers.parquet"
    );
}

/// Load alzhiemers data from path
pub fn load_alzhiemers(path: &str) -> Result<(NDArray<f64>, NDArray<f64>)> {
    
    let file = File::open(path).unwrap();
    let mut reader = ParquetRecordBatchReaderBuilder::try_new(file)?
        .build()?;

    let batch = reader.next().unwrap().unwrap();
    let (input, y_train) = select_features(
        batch.clone(),
        vec![
            "age",
            "gender",
            "ethnicity",
            "education_level",
            "alchohol_consumption",
            "bmi",
            "smoking",
            "physical_activity",
            "diet_quality",
            "sleep_quality",
            "family_history",
            "cardiovascular_disease",
            "diabetes",
            "depression",
            "head_injury",
            "hyptertension",
            "systolic_bp", 
            "distolic_dp",
            "cholesterol_total",
            "cholesterol_ldl",
            "cholesterol_hdl", 
            "cholesterol_tryglicerides",
            "confusion",
            "disorientation",
            "personality_changes",
            "difficulty_w_tasks",
            "forgetfullness",
        ],
        "diagnosis"
    );

    let x_train = min_max_scalar(input).unwrap();
    Ok((x_train, y_train))

}