bsread 0.1.1

Rust implementation of the BSREAD streaming protocol
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142

use crate::*;
use lz4::block::{decompress as lz4_decompress, CompressionMode};
use lz4::block::compress as lz4_compress;
use std::io::{Cursor};
use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt};
use std::cmp;


extern "C" {
    pub fn bshuf_decompress_lz4(
        input: *const u8,
        output: *mut u8,
        size: usize,
        elem_size: usize,
        block_size: usize,
    ) -> i64; // Corresponds to `int64_t` in C

    pub fn bshuf_compress_lz4(
        input: *const u8,
        output: *mut u8,
        size: usize,
        elem_size: usize,
        block_size: usize,
    ) -> i64;

    pub fn  bshuf_compress_lz4_bound(
        size: usize,
        elem_size: usize,
        block_size: usize) ->usize;
}

fn bshuf_untrans_bit_elem(input: &[u8],  elem_size: usize, ) -> Result<Vec<u8>, String> {
    let elem_size = elem_size;
    let mut c =  Cursor::new(input);
    let size_out =   c.read_u64::<BigEndian>().unwrap() as usize;
    let elements = size_out/elem_size;
    let block_size =c.read_u32::<BigEndian>().unwrap();
    let block_size = block_size / (elem_size as u32);
    let blob = &input[12..];
    let mut output = vec![0u8; size_out];

    let ret  = unsafe {
        bshuf_decompress_lz4(
            blob.as_ptr(),
            output.as_mut_ptr(),
            elements,
            elem_size,
            block_size as usize,
        )
    };

    // Check the return value for errors
    if ret < 0 {
        Err(format!("Decompression failed with error code {}", ret))
    } else {
        Ok(output)
    }
}


fn  bshuf_trans_bit_elem(input: &[u8],  elem_size: usize, ) -> Result<Vec<u8>, String> {
    let elem_size = elem_size;
    let blob_in = &input[0..];
    let target_block_size = 8192;
    let minimum_block_size = 128;
    let block_size_multiplier = 8;
    let block_size = target_block_size / elem_size;
    let block_size = (block_size / block_size_multiplier) * block_size_multiplier;
    let block_size = cmp::max(block_size, minimum_block_size);
    let elements= input.len() / elem_size;
    let output_bound = unsafe {
        bshuf_compress_lz4_bound(elements, elem_size, block_size)
    };
    let mut output = vec![0u8; output_bound+12];
    (&mut output[0..8]).write_u64::<BigEndian>((elements*elem_size)  as u64).unwrap();
    (&mut output[8..12]).write_u32::<BigEndian>((block_size*elem_size) as u32).unwrap();

    let blob_out = &mut output[12..];
    let ret  = unsafe {
        bshuf_compress_lz4(
            blob_in.as_ptr(),
            blob_out.as_mut_ptr(),
            elements ,
            elem_size,
            block_size,
        )
    };
    // Check the return value for errors
    if ret < 0 {
        Err(format!("Compression failed with error code {}", ret))
    } else {
        let size = ret as usize;
        output.truncate(size+12);
        Ok(output)
    }
}



pub fn decompress_bitshuffle_lz4(compressed_data: &[u8], element_size: usize) -> IOResult<Vec<u8>> {
    match bshuf_untrans_bit_elem(&compressed_data, element_size) {
        Ok(out) => {Ok(out)}
        Err(e) => {Err(new_error(ErrorKind::InvalidInput, e.as_str()))}
    }
}

pub fn decompress_lz4(compressed_data: &[u8], little_endian:bool) -> IOResult<Vec<u8>> {
    let size = if little_endian {
        (compressed_data[0] as i32) | (compressed_data[1] as i32) << 8 | (compressed_data[2] as i32) << 16 | (compressed_data[3] as i32) << 24
    } else {
        (compressed_data[3] as i32) | (compressed_data[2] as i32) << 8 | (compressed_data[1] as i32) << 16 | (compressed_data[0] as i32) << 24
    };
    let blob: &[u8] = &compressed_data[4..];
    let output = lz4_decompress(blob, Some(size))?;
    Ok(output)
}


pub fn compress_bitshuffle_lz4(data: &[u8], element_size: usize) -> IOResult<Vec<u8>> {
    match bshuf_trans_bit_elem(data, element_size){
        Ok(out) => {Ok(out)}
        Err(e) => {Err(new_error(ErrorKind::InvalidInput, e.as_str()))}
    }
}

pub fn compress_lz4(data: &[u8], little_endian:bool) -> IOResult<Vec<u8>> {
    let size = data.len();
    let header  = if little_endian {
        [size as u8,  (size >> 8) as u8, (size >> 16) as u8, (size >> 24) as u8]
    } else {
        [(size >> 24)  as u8,  (size >> 16) as u8, (size >> 8) as u8, size as u8]
    };

    let blob = lz4_compress(data, Some(CompressionMode::DEFAULT), false)?;

    let mut output = Vec::with_capacity(4 + blob.len());
    output.extend_from_slice(&header);
    output.extend_from_slice(&blob);
    Ok(output)
}