boreholeio 0.1.0

A library for interacting with borehole.io, a subsurface data management, delivery and visualisation platform
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122

mod utils;

use boreholeio::strided_array_file::{ArrayProxy, StridedArrayFile};
use std::{path::PathBuf, str::FromStr};
use utils::{assert_eq_views, generate_array};

#[test]
fn self_generated_read_write() {
    let arr_i8 = generate_array(&vec![10]);
    let arr_u8 = generate_array(&vec![1, 1, 1]);
    let arr_u8_2 = generate_array(&vec![2, 3, 1, 4]).t().to_owned();
    let arr_i16 = generate_array(&vec![2, 1, 2]);
    let arr_u16 = generate_array(&vec![3, 3]);
    let arr_u16_2 = generate_array(&vec![2, 2, 2, 2, 2, 2]);
    let arr_i32 = generate_array(&vec![5]);
    let arr_u32 = generate_array(&vec![1, 1, 1, 1, 3, 4, 1, 2, 1, 1, 1]);
    let arr_i64 = generate_array(&vec![4, 4]);
    let arr_u64 = generate_array(&vec![3, 3]);
    let arr_f32 = generate_array(&vec![2, 4]);
    let arr_f64 = generate_array(&vec![2, 2]);
    let gt = vec![
        ArrayProxy::I8(arr_i8.view()),
        ArrayProxy::I8(arr_i8.view()), // Note duplicate
        ArrayProxy::U8(arr_u8.view()),
        ArrayProxy::I32(arr_i32.view()),   // Note some random order
        ArrayProxy::U8(arr_u8_2.view()),   // Note different (col-major order) strides
        ArrayProxy::U16(arr_u16_2.view()), // Note same value type but different shape
        ArrayProxy::U16(arr_u16.view()),
        ArrayProxy::F64(arr_f64.view()),
        ArrayProxy::U32(arr_u32.view()),
        ArrayProxy::I16(arr_i16.view()),
        ArrayProxy::I64(arr_i64.view()),
        ArrayProxy::U64(arr_u64.view()),
        ArrayProxy::F32(arr_f32.view()),
    ];
    let file_path = path("1.star");
    StridedArrayFile::write(&file_path, &gt).unwrap();

    let ours = StridedArrayFile::new(file_path);
    let mut i = 0;
    i += assert_eq_views::<i8>(&gt[i], &ours.get(i));
    i += assert_eq_views::<i8>(&gt[i], &ours.get(i));
    i += assert_eq_views::<u8>(&gt[i], &ours.get(i));
    i += assert_eq_views::<i32>(&gt[i], &ours.get(i));
    i += assert_eq_views::<u8>(&gt[i], &ours.get(i));
    i += assert_eq_views::<u16>(&gt[i], &ours.get(i));
    i += assert_eq_views::<u16>(&gt[i], &ours.get(i));
    i += assert_eq_views::<f64>(&gt[i], &ours.get(i));
    i += assert_eq_views::<u32>(&gt[i], &ours.get(i));
    i += assert_eq_views::<i16>(&gt[i], &ours.get(i));
    i += assert_eq_views::<i64>(&gt[i], &ours.get(i));
    i += assert_eq_views::<u64>(&gt[i], &ours.get(i));
    i += assert_eq_views::<f32>(&gt[i], &ours.get(i));
    assert_eq!(i, gt.len());
}

// import sys
// sys.path.append('./../src/')
// from borehole.io import StridedArrayFile
// import numpy as np
// if True:
//     arrs = [
//         np.random.rand(3, 4, 2),
//         np.identity(3, dtype=np.uint8),
//         np.random.randint(-100, 100, (1, 1, 1, 1, 2, 3, 1, 1, 4, 1, 1)),
//         np.zeros((10,), dtype=np.float32),
//     ]
//     f = StridedArrayFile('from-python.star')
//     f.write(arrs)
// else:
//     f = StridedArrayFile('1.star')
//     for i in range(len(f)):
//         print(f[i].dtype, f[i].shape, f[i].strides)
#[test]
fn python_generated_read() {
    let file_path = path("tests/strided_array_file/fixture/simple.star");
    let arrays = StridedArrayFile::new(file_path);
    assert_eq!(arrays.count().unwrap(), 4);
    assert_eq!(arrays.get(0).shape(), [3, 4, 2]);
    assert_eq!(arrays.get(0).element_type(), 9);
    let iden3f64 = ndarray::Array2::from_diag(&ndarray::arr1(&[1u8, 1, 1])).into_dyn();
    let view = ArrayProxy::U8(iden3f64.view());
    assert_eq_views::<u8>(&arrays.get(1), &view);
    assert_eq!(arrays.get(2).shape(), [1, 1, 1, 1, 2, 3, 1, 1, 4, 1, 1]);
    assert_eq!(arrays.get(2).element_type(), 4);
    let zeros10f32 = ndarray::Array1::from_elem([10], 0.0f32).into_dyn();
    let view = ArrayProxy::F32(zeros10f32.view());
    assert_eq_views::<f32>(&arrays.get(3), &view);
}

/// Strided array files contain strides in bytes, while `ndarray` uses strides
/// in array elements. This unit test verifies the correctness of the
/// conversion between these two formats.
#[test]
fn verify_strides() {
    // Generate random array and wrap it into a `proxy`.
    let data = generate_array(&vec![12]);
    let proxy = ArrayProxy::F64(data.view());
    // Create a `slice` that selects every 2nd element from the array.
    let step = 2;
    let slice_info = ndarray::SliceInfoElem::Slice {
        start: 0,
        end: None,
        step: step as isize,
    };
    let slice = proxy.slice(vec![slice_info]).unwrap();
    // Save this slice onto disk.
    let path = path("slice.star");
    StridedArrayFile::write(&path, &vec![slice.clone()]).unwrap();
    // Read it back into RAM.
    let file = StridedArrayFile::new(path);
    let view = file.get_as::<f64>(0);
    // Verify it's still the same data.
    for i in 0..view.len() {
        assert_eq!(view[i], data[i * step]);
    }
}

/// A shortcut for converting `str` to `PathBuf`
fn path(path: &str) -> PathBuf {
    PathBuf::from_str(path).unwrap()
}