flashlight_tensor 0.4.5

gpu/cpu tensor library focused around matrix and neural network operations
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164

use flashlight_tensor::prelude::*;
use std::time::Instant;

#[tokio::main]
async fn main(){
    full_test(1, 1, 800).await;
    full_test(100, 50, 20).await;
    full_test(1000, 500, 100).await;
}

async fn full_test(iterations: u32, samples: u32, neurons: u32){
    backward_bias(iterations, samples, neurons).await;
    backward_weights(iterations, samples, neurons).await;
    backward_gradient(iterations, samples, neurons).await;
    forward_propagation(iterations, samples, neurons).await;
}

async fn backward_bias(iterations: u32, samples: u32, neurons: u32){
    println!("Backward bias\nIterations: {}\nSamples: {}\nneurons: {}\n", iterations, samples, neurons);

    let start_init = Instant::now();

    let grad_output: Tensor<f32> = Tensor::fill(0.69, &[neurons, samples]);

    let linear_cache: Tensor<f32> = Tensor::fill(0.2137, &[neurons, samples]);

    let bias: Tensor<f32> = Tensor::fill(0.12412, &[neurons, 1]);

    let learning_rate = 0.01;

    let mut runner = GpuRunner::init(2, MemoryMetric::GB);

    let prep_init = Instant::now();

    for _i in 0..iterations{
        let sample = Sample::from_data(vec!{bias.clone(), grad_output.clone(), linear_cache.clone()}, vec!{learning_rate}, &[]);

        runner.append(sample);
    }
    let prep_duration = prep_init.elapsed();

    let first_runtime = Instant::now();
    runner.backward_bias().await;
    let first_duration = first_runtime.elapsed();

    let second_runtime = Instant::now();
    let _full_gpu_output = runner.backward_bias().await;
    let second_duration = second_runtime.elapsed();

    let start_duration = start_init.elapsed();

    println!("Cpu prep time: {:?}\nFirst operaiton runtime(init): {:?}\nSecond operation runtime(update): {:?}\nFull first gpu runtime: {:?}\nFull second gpu runtime: {:?}", prep_duration, first_duration, second_duration, start_duration - second_duration, start_duration - first_duration);
    println!("_____________________________________________________________");
}

async fn backward_weights(iterations: u32, samples: u32, neurons: u32){
    println!("Backward weights\nIterations: {}\nSamples: {}\nneurons: {}\n", iterations, samples, neurons);

    let start_init = Instant::now();

    let grad_output: Tensor<f32> = Tensor::fill(0.69, &[neurons, samples]);

    let linear_cache: Tensor<f32> = Tensor::fill(0.2137, &[neurons, samples]);

    let weights: Tensor<f32> = Tensor::fill(0.12412, &[neurons, neurons]);

    let learning_rate = 0.01;

    let mut runner = GpuRunner::init(2, MemoryMetric::GB);

    let prep_init = Instant::now();

    for _i in 0..iterations{
        let sample = Sample::from_data(vec!{weights.clone(), grad_output.clone(), linear_cache.clone()}, vec!{learning_rate}, &[]);

        runner.append(sample);
    }
    let prep_duration = prep_init.elapsed();

    let first_runtime = Instant::now();
    runner.backward_weight().await;
    let first_duration = first_runtime.elapsed();

    let second_runtime = Instant::now();
    let _full_gpu_output = runner.backward_weight().await;
    let second_duration = second_runtime.elapsed();

    let start_duration = start_init.elapsed();

    println!("Cpu prep time: {:?}\nFirst operaiton runtime(init): {:?}\nSecond operation runtime(update): {:?}\nFull first gpu runtime: {:?}\nFull second gpu runtime: {:?}", prep_duration, first_duration, second_duration, start_duration - second_duration, start_duration - first_duration);
    println!("_____________________________________________________________");
}

async fn backward_gradient(iterations: u32, samples: u32, neurons: u32){
    println!("Backward gradient\nIterations: {}\nSamples: {}\nneurons: {}\n", iterations, samples, neurons);

    let start_init = Instant::now();

    let grad_output: Tensor<f32> = Tensor::fill(0.69, &[neurons, samples]);

    let weights: Tensor<f32> = Tensor::fill(0.12412, &[neurons, neurons]);


    let mut runner = GpuRunner::init(2, MemoryMetric::GB);

    let prep_init = Instant::now();

    for _i in 0..iterations{
        let sample = Sample::from_data(vec!{weights.clone(), grad_output.clone()}, vec!{}, &[]);

        runner.append(sample);
    }
    let prep_duration = prep_init.elapsed();

    let first_runtime = Instant::now();
    runner.backward_grad().await;
    let first_duration = first_runtime.elapsed();

    let second_runtime = Instant::now();
    let _full_gpu_output = runner.backward_grad().await;
    let second_duration = second_runtime.elapsed();

    let start_duration = start_init.elapsed();

    println!("Cpu prep time: {:?}\nFirst operaiton runtime(init): {:?}\nSecond operation runtime(update): {:?}\nFull first gpu runtime: {:?}\nFull second gpu runtime: {:?}", prep_duration, first_duration, second_duration, start_duration - second_duration, start_duration - first_duration);
    println!("_____________________________________________________________");
}

async fn forward_propagation(iterations: u32, samples: u32, neurons: u32){
    println!("Forward propagation\nIterations: {}\nSamples: {}\nneurons: {}\n", iterations, samples, neurons);

    let start_init = Instant::now();

    let inputs: Tensor<f32> = Tensor::fill(0.69, &[neurons, samples]);

    let weights: Tensor<f32> = Tensor::fill(0.12412, &[neurons, neurons]);
    let biases: Tensor<f32> = Tensor::fill(-0.12631, &[neurons, 1]);


    let mut runner = GpuRunner::init(2, MemoryMetric::GB);

    let prep_init = Instant::now();

    for _i in 0..iterations{
        let sample = Sample::from_data(vec!{weights.clone(), inputs.clone(), biases.clone()}, vec!{}, &[]);

        runner.append(sample);
    }
    let prep_duration = prep_init.elapsed();

    let first_runtime = Instant::now();
    runner.forward_no_activ().await;
    let first_duration = first_runtime.elapsed();

    let second_runtime = Instant::now();
    let _full_gpu_output = runner.forward_no_activ().await;
    let second_duration = second_runtime.elapsed();

    let start_duration = start_init.elapsed();

    println!("Cpu prep time: {:?}\nFirst operaiton runtime(init): {:?}\nSecond operation runtime(update): {:?}\nFull first gpu runtime: {:?}\nFull second gpu runtime: {:?}", prep_duration, first_duration, second_duration, start_duration - second_duration, start_duration - first_duration);
   println!("_____________________________________________________________"); 
}