nncombinator 0.9.8

A library of neural networks that can be written type-safely
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125

use std::fmt::Debug;
use lazy_static::lazy_static;
use rand::Rng;
use nncombinator::arr::{Arr, Arr2, SerializedVec};
use nncombinator::device::input::DeviceInput;
use nncombinator::layer::{BackwardAll, BatchBackward, BatchDataType, BatchForward, BatchLoss, BatchPreTrain, ContinueForward, ForwardAll, Loss, OnStep, PartialForward, PersistProgress, PreTrain, Step, UpdateWeight};
use nncombinator::ope::UnitValue;
use nncombinator::cuda::allocator::{CudaAllocator, DeviceAlloc, MemoryPoolAllocator, MemoryPoolAllocatorInstantiation};
use nncombinator::cuda::{AsCudaMutPtr, AsCudaView, CudaMutPtr, CudaTensor1dPtr, CudaTensor2dPtr, CudaVec, WriteMemory};
use nncombinator::persistence::{Specialized, TextFilePersistence};

lazy_static! {
    pub static ref SHARED_MEMORY_POOL:MemoryPoolAllocator<DeviceAlloc> = MemoryPoolAllocator::with_size(8 * 1024 * 1024 * 1024,DeviceAlloc).unwrap();
}
pub fn assert_device<U,I,D: DeviceInput<U,I>>(_:&D)
    where U: UnitValue<U>,
          I: BatchDataType + Debug + 'static,
          <I as BatchDataType>::Type: Debug + 'static {
}
pub fn assert_forward_all<T: ForwardAll>(_:&T) {

}
pub fn assert_pre_train<U: UnitValue<U>,T: PreTrain<U>>(_:&T) {

}
pub fn assert_backward_all<U: UnitValue<U>,T: BackwardAll<U>>(_:&T) {

}
pub fn assert_loss<U: UnitValue<U>,T: Loss<U>>(_:&T) {

}
pub fn assert_update_weight<U: UnitValue<U>,T: UpdateWeight<U>>(_:&T) {

}
pub fn assert_partial_forward<T: PartialForward>(_:&T) {}
pub fn assert_continue_forward<T: ContinueForward>(_:&T) {}
pub fn assert_batch_forward<T: BatchForward>(_:&T) {

}
pub fn assert_batch_pre_train<U: UnitValue<U>,T: BatchPreTrain<U>>(_:&T) {

}
pub fn assert_batch_backward<U: UnitValue<U>,T: BatchBackward<U>>(_:&T) {

}
pub fn assert_batch_loss<U: UnitValue<U>,T: BatchLoss<U>>(_:&T) {

}

pub fn assert_step<T: Step>(_:&T) {

}

pub fn assert_on_step<T: OnStep>(_:&T) {

}

pub fn assert_text_persist_progress<T: PersistProgress<TextFilePersistence,Specialized>>(_:&T) {}
const NI: usize = 500;
const NO: usize = 600;
const BATCH: usize = 400;

pub fn gen_inputs() -> (Arr<f32,NO>,Arr2<f32,NI,NO>,SerializedVec<f32,Arr<f32,NI>>) {
    let mut rng = rand::thread_rng();

    let mut bias = Arr::<f32,NO>::new();
    for b in bias.iter_mut() { *b = rng.gen::<f32>() * 1e-3; }

    let mut units = Arr2::<f32,NI,NO>::new();
    for i in 0..NI {
        for j in 0..NO { units[(i,j)] = rng.gen::<f32>(); }
    }

    let mut inputs_host: Vec<Arr<f32,NI>> = Vec::with_capacity(BATCH);
    for _ in 0..BATCH {
        let mut v = Arr::<f32,NI>::new();
        for x in v.iter_mut() { *x = rng.gen::<f32>(); }
        inputs_host.push(v);
    }

    (bias,units,inputs_host.into())
}

pub fn approx_eq_slice(a: &[f32],b: &[f32],eps: f32) {
    assert_eq!(a.len(),b.len());
    for (i,(x,y)) in a.iter().zip(b.iter()).enumerate() {
        let d = (*x - *y).abs();
        assert!(d < eps,"diff[{}] = {} exceeds eps {} (x={},y={})",i,d,eps,x,y);
    }
}

pub fn upload_inputs_to_device<A: MemoryPoolAllocatorInstantiation<DeviceAlloc> + CudaAllocator>(
    alloc: &A,
    bias: &Arr<f32,NO>,units: &Arr2<f32,NI,NO>,
    batch_inputs: &SerializedVec<f32,Arr<f32,NI>>)
    -> (CudaTensor1dPtr<f32,A,NO>,CudaTensor2dPtr<f32,A,NI,NO>,CudaVec<f32,CudaTensor1dPtr<f32,A,NI>,A>)
    where CudaTensor1dPtr<f32,A,NO>: WriteMemory<f32>,
          CudaTensor2dPtr<f32,A,NI,NO>: WriteMemory<f32>,
          CudaVec<f32,CudaTensor1dPtr<f32,A,NI>,A>: AsCudaMutPtr<Pointee=f32,Allocator=A>,
          for<'a> CudaMutPtr<'a,f32,A>: WriteMemory<f32>,
          for<'a> &'a CudaVec<f32,CudaTensor1dPtr<f32,A,NI>,A>: AsCudaView<'a> {
    // Bias
    let mut d_bias = CudaTensor1dPtr::<f32,A,NO>::new(alloc).unwrap();
    d_bias.memcpy(bias.as_ptr(),NO).unwrap();

    // Units: flatten in (i,j) with leading dimension NO (calc_index(out=j,in=i,ld=NO) == i*NO+j)
    let mut flat_units: Vec<f32> = Vec::with_capacity(NI * NO);
    for i in 0..NI { for j in 0..NO { flat_units.push(units[(i,j)]); } }
    let mut d_units = CudaTensor2dPtr::<f32,A,NI,NO>::new(alloc).unwrap();
    d_units.memcpy(flat_units.as_ptr(),flat_units.len()).unwrap();

    // Inputs: layout is batch-major with leading dimension NI:
    // calc_index(x=i,y=batch_index,ld=NI) == batch_index*NI + i
    let mut flat_inputs: Vec<f32> = Vec::with_capacity(BATCH * NI);
    for b in batch_inputs.iter() {
        for &v in b.iter() { flat_inputs.push(v); }
    }
    let mut d_inputs = CudaVec::<f32,CudaTensor1dPtr<f32,A,NI>,A>::new(BATCH,alloc).unwrap();
    {
        let mut ptr = d_inputs.as_cuda_mut_ptr();
        ptr.memcpy(flat_inputs.as_ptr(),flat_inputs.len()).unwrap();
    }

    (d_bias,d_units,d_inputs)
}