sarek 0.1.0

A work-in-progress, experimental neural network library utilizing TensorFlow Keras
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

use {
    byteorder::{
        BigEndian,
        ReadBytesExt
    },
    flate2::{
        read::{
            GzDecoder
        }
    },
    log::{
        info
    },
    sarek::{
        DataSet,
        SliceSource
    },
    std::{
        io::{
            Cursor,
            Read
        }
    }
};

static MNIST_TRAINING_IMAGES: &[u8] = include_bytes!( "data/train-images-idx3-ubyte.gz" );
static MNIST_TRAINING_LABELS: &[u8] = include_bytes!( "data/train-labels-idx1-ubyte.gz" );
static MNIST_TEST_IMAGES: &[u8] = include_bytes!( "data/t10k-images-idx3-ubyte.gz" );
static MNIST_TEST_LABELS: &[u8] = include_bytes!( "data/t10k-labels-idx1-ubyte.gz" );

fn create_reader( data: &'static [u8] ) -> Cursor< Vec< u8 > > {
    let mut decoder = GzDecoder::new( data );
    let mut data = Vec::new();
    decoder.read_to_end( &mut data ).unwrap();

    Cursor::new( data )
}

fn load_images( images: &'static [u8] ) -> (u32, u32, u32, Vec< u8 >) {
    let mut reader = create_reader( images );
    let magic = reader.read_u32::< BigEndian >().unwrap();
    assert_eq!( magic, 0x00000803 );

    let count = reader.read_u32::< BigEndian >().unwrap();
    let height = reader.read_u32::< BigEndian >().unwrap();
    let width = reader.read_u32::< BigEndian >().unwrap();

    let position = reader.position() as usize;
    let data = reader.into_inner()[ position.. ].to_vec();

    (count, width, height, data)
}

fn load_labels( labels: &'static [u8] ) -> (u32, Vec< u8 >) {
    let mut reader = create_reader( labels );
    let magic = reader.read_u32::< BigEndian >().unwrap();
    assert_eq!( magic, 0x00000801 );

    let count = reader.read_u32::< BigEndian >().unwrap();
    let position = reader.position() as usize;
    let data = reader.into_inner()[ position.. ].to_vec();

    (count, data)
}

type VecSource< T > = SliceSource< T, Vec< T > >;

fn load( raw_labels: &'static [u8], raw_images: &'static [u8] ) -> DataSet< VecSource< f32 >, VecSource< u32 > > {
    let (label_count, labels) = load_labels( raw_labels );
    let (image_count, width, height, image_data) = load_images( raw_images );

    assert_eq!( label_count, image_count );

    let images: Vec< f32 > = image_data.into_iter().map( |value| value as f32 / 255.0 ).collect();
    let categories: Vec< u32 > = labels.iter().map( |&category| category as u32 ).collect();

    let images = SliceSource::from( (width as usize, height as usize).into(), images );
    let categories = SliceSource::from( 1.into(), categories );

    DataSet::new( images, categories )
}

pub fn load_training_data_set() -> DataSet< VecSource< f32 >, VecSource< u32 > > {
    info!( "Starting to load the training data set..." );
    let data = load( MNIST_TRAINING_LABELS, MNIST_TRAINING_IMAGES );
    info!( "Finished loading the training data set!" );
    data
}

pub fn load_test_data_set() -> DataSet< VecSource< f32 >, VecSource< u32 > > {
    info!( "Starting to load the test data set..." );
    let data = load( MNIST_TEST_LABELS, MNIST_TEST_IMAGES );
    info!( "Finished loading the test data set!" );
    data
}