mnist 0.6.0

MNIST data set parser.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215

#![cfg(test)]

use super::*;
use ndarray::prelude::*;

#[test]
fn test_example() {
    // Deconstruct the returned Mnist struct.
    let Mnist {
        trn_img,
        trn_lbl,
        tst_img,
        tst_lbl,
        ..
    } = MnistBuilder::new()
        .label_format_digit()
        .training_set_length(50_000)
        .validation_set_length(10_000)
        .test_set_length(10_000)
        .finalize();

    let image_num = 0;
    // Can use an Array2 or Array3 here (Array3 for visualization)
    let train_data = Array3::from_shape_vec((50_000, 28, 28), trn_img)
        .expect("Error converting images to Array3 struct")
        .map(|x| *x as f32 / 256.0);
    println!("{:#.1?}\n", train_data.slice(s![image_num, .., ..]));

    // Convert the returned Mnist struct to Array2 format
    let train_labels: Array2<f32> = Array2::from_shape_vec((50_000, 1), trn_lbl)
        .expect("Error converting training labels to Array2 struct")
        .map(|x| *x as f32);
    println!(
        "The first digit is a {:?}",
        train_labels.slice(s![image_num, ..])
    );

    let _test_data = Array3::from_shape_vec((10_000, 28, 28), tst_img)
        .expect("Error converting images to Array3 struct")
        .map(|x| *x as f32 / 256.);

    let _test_labels: Array2<f32> = Array2::from_shape_vec((10_000, 1), tst_lbl)
        .expect("Error converting testing labels to Array2 struct")
        .map(|x| *x as f32);
}

#[test]
fn test_00() {
    let mnist = MnistBuilder::new().finalize();
    assert!(mnist.trn_img.len() == 60_000 * 28 * 28);
    assert!(mnist.trn_lbl.len() == 60_000);
    assert!(mnist.val_img.len() == 0);
    assert!(mnist.val_lbl.len() == 0);
    assert!(mnist.tst_img.len() == 10_000 * 28 * 28);
    assert!(mnist.tst_lbl.len() == 10_000);
    assert!(mnist.trn_lbl[0] == 5);
    assert!(mnist.tst_lbl[0] == 7);
}

#[test]
fn test_01() {
    let mnist = MnistBuilder::new()
        .label_format_one_hot()
        .training_set_length(50_000)
        .validation_set_length(10_000)
        .test_set_length(10_000)
        .finalize();
    assert!(mnist.trn_img.len() == 50_000 * 28 * 28);
    assert!(mnist.trn_lbl.len() == 50_000 * 10);
    assert!(mnist.val_img.len() == 10_000 * 28 * 28);
    assert!(mnist.val_lbl.len() == 10_000 * 10);
    assert!(mnist.tst_img.len() == 10_000 * 28 * 28);
    assert!(mnist.tst_lbl.len() == 10_000 * 10);
    assert!(mnist.trn_lbl[0] == 0);
    assert!(mnist.trn_lbl[9] == 0);
    assert!(mnist.trn_lbl[5] == 1);
    assert!(mnist.tst_lbl[0] == 0);
    assert!(mnist.tst_lbl[9] == 0);
    assert!(mnist.tst_lbl[7] == 1);
}

#[test]
#[should_panic(
    expected = "Total data set length (70001) greater than maximum possible length (70000)."
)]
fn test_02() {
    let _ = MnistBuilder::new()
        .training_set_length(50_001)
        .validation_set_length(10_000)
        .test_set_length(10_000)
        .finalize();
}

#[test]
#[should_panic(
    expected = "Total data set length (70001) greater than maximum possible length (70000)."
)]
fn test_03() {
    let _ = MnistBuilder::new()
        .training_set_length(50_000)
        .validation_set_length(10_001)
        .test_set_length(10_000)
        .finalize();
}

#[test]
#[should_panic(
    expected = "Total data set length (70001) greater than maximum possible length (70000)."
)]
fn test_04() {
    let _ = MnistBuilder::new()
        .training_set_length(50_000)
        .validation_set_length(10_000)
        .test_set_length(10_001)
        .finalize();
}

#[test]
#[should_panic(
    expected = "Total data set length (70300) greater than maximum possible length (70000)."
)]
fn test_05() {
    let _ = MnistBuilder::new()
        .training_set_length(50_100)
        .validation_set_length(10_100)
        .test_set_length(10_100)
        .finalize();
}

#[test]
fn test_06() {
    let mnist = MnistBuilder::new()
        .label_format_one_hot()
        .training_set_length(50_000)
        .validation_set_length(10_000)
        .test_set_length(10_000)
        .base_path("data_sets/mnist")
        .training_images_filename("training_images")
        .training_labels_filename("training_labels")
        .test_images_filename("test_images")
        .test_labels_filename("test_labels")
        .finalize();
    assert!(mnist.trn_img.len() == 50_000 * 28 * 28);
    assert!(mnist.trn_lbl.len() == 50_000 * 10);
    assert!(mnist.val_img.len() == 10_000 * 28 * 28);
    assert!(mnist.val_lbl.len() == 10_000 * 10);
    assert!(mnist.tst_img.len() == 10_000 * 28 * 28);
    assert!(mnist.tst_lbl.len() == 10_000 * 10);
    assert!(mnist.trn_lbl[5] == 1);
    assert!(mnist.tst_lbl[7] == 1);
}

#[test]
#[should_panic(
    expected = "Total data set length (70001) greater than maximum possible length (70000)."
)]
fn test_07() {
    let _ = MnistBuilder::new()
        .training_set_length(50_000)
        .validation_set_length(10_000)
        .test_set_length(10_001)
        .finalize();
}

#[test]
#[should_panic(
    expected = "Unable to find path to images at \"wrong/path/train-images-idx3-ubyte\"."
)]
fn test_08() {
    let _ = MnistBuilder::new().base_path("wrong/path").finalize();
}

#[test]
#[should_panic(expected = "Unable to find path to images at \"data/test\".")]
fn test_09() {
    let _ = MnistBuilder::new()
        .training_images_filename("test")
        .finalize();
}

#[test]
#[should_panic(expected = "Unable to find path to labels at \"data/test\".")]
fn test_10() {
    let _ = MnistBuilder::new()
        .training_labels_filename("test")
        .finalize();
}

#[test]
#[should_panic(expected = "Unable to find path to images at \"data/test\".")]
fn test_11() {
    let _ = MnistBuilder::new().test_images_filename("test").finalize();
}

#[test]
#[should_panic(expected = "Unable to find path to labels at \"data/test\".")]
fn test_12() {
    let _ = MnistBuilder::new().test_labels_filename("test").finalize();
}

#[test]
fn normalize_vector() {
    use super::normalize_vector;

    let v: Vec<u8> = vec![0, 1, 2, 127, 128, 129, 254, 255];
    let normalized_v: Vec<f32> = normalize_vector(&v);
    let expected: Vec<f32> = vec![
        0.0, 0.00392157, 0.00784314, 0.49803922, 0.50196078, 0.50588235, 0.99607843, 1.0,
    ];

    expected
        .iter()
        .zip(normalized_v.iter())
        .for_each(|(value, expected)| assert!((value - expected).abs() < 1.0e-6));
}