binary_vec_io/
lib.rs

1// Copyright (c) 2019 10X Genomics, Inc. All rights reserved.
2
3// Write and read functions to which one passes a File, a ref to a number type
4// defining the start of a 'vector' of entries, and the number of entries.
5//
6// See also crate memmap.
7
8use itertools::Itertools;
9use std::io::Write;
10use std::os::unix::fs::MetadataExt;
11
12pub trait BinaryInputOutputSafe {}
13impl BinaryInputOutputSafe for i8 {}
14impl BinaryInputOutputSafe for i16 {}
15impl BinaryInputOutputSafe for i32 {}
16impl BinaryInputOutputSafe for i64 {}
17impl BinaryInputOutputSafe for u8 {}
18impl BinaryInputOutputSafe for u16 {}
19impl BinaryInputOutputSafe for u32 {}
20impl BinaryInputOutputSafe for u64 {}
21impl BinaryInputOutputSafe for f32 {}
22impl BinaryInputOutputSafe for f64 {}
23// i128, u128?
24
25use std::io::Error;
26
27#[allow(dead_code)]
28pub fn binary_write_from_ref<T>(f: &mut std::fs::File, p: &T, n: usize) -> Result<(), Error> {
29    let raw = p as *const T as *const u8;
30    unsafe {
31        let sli: &[u8] = std::slice::from_raw_parts(raw, n * (std::mem::size_of::<T>()));
32        f.write_all(sli)?;
33        Ok(())
34    }
35}
36
37pub fn binary_read_to_ref<T>(f: &mut std::fs::File, p: &mut T, n: usize) -> Result<(), Error> {
38    let mut raw = p as *mut T as *mut u8;
39    unsafe {
40        use std::io::Read;
41        let bytes_to_read = n * std::mem::size_of::<T>();
42        let mut bytes_read = 0;
43        // Rarely, one must read twice (maybe, not necessarily proven).  Conceivably one needs
44        // to read more than twice on occasion.
45        const MAX_TRIES: usize = 10;
46        let mut reads = Vec::<usize>::new();
47        for _ in 0..MAX_TRIES {
48            if bytes_read == bytes_to_read {
49                break;
50            }
51            raw = raw.add(bytes_read);
52            let sli: &mut [u8] = std::slice::from_raw_parts_mut(raw, bytes_to_read - bytes_read);
53            let n = f.read(sli).unwrap();
54            reads.push(n);
55            bytes_read += n;
56        }
57        if bytes_read != bytes_to_read {
58            let metadata = f.metadata()?;
59            let msg = format!(
60                "Failure in binary_read_to_ref, bytes_read = {}, but \
61                bytes_to_read = {}.  Bytes read on successive\nattempts = {}.\n\
62                File has length {} and inode {}.",
63                bytes_read,
64                bytes_to_read,
65                reads.iter().format(","),
66                metadata.len(),
67                metadata.ino(),
68            );
69            panic!("{}", msg);
70        }
71    }
72    Ok(())
73}
74
75// The functions binary_write_vec and binary_read_vec append, either to a file,
76// in the first case, or to a vector, in the second case.
77
78#[allow(dead_code)]
79pub fn binary_write_vec<T>(f: &mut std::fs::File, x: &[T]) -> Result<(), Error>
80where
81    T: BinaryInputOutputSafe,
82{
83    let n = x.len();
84    binary_write_from_ref::<usize>(f, &n, 1)?;
85    binary_write_from_ref::<T>(f, &x[0], x.len())
86}
87
88#[allow(dead_code)]
89pub fn binary_read_vec<T>(f: &mut std::fs::File, x: &mut Vec<T>) -> Result<(), Error>
90where
91    T: BinaryInputOutputSafe,
92{
93    // Read the vector size.
94
95    let mut n: usize = 0;
96    binary_read_to_ref::<usize>(f, &mut n, 1)?;
97
98    // Resize the vector without setting any of its entries.
99    // (could use resize_without_setting)
100
101    let len = x.len();
102    if len + n > x.capacity() {
103        let extra: usize = len + n - x.capacity();
104        x.reserve(extra);
105    }
106    unsafe {
107        x.set_len(len + n);
108    }
109
110    // Read the vector entries.
111
112    binary_read_to_ref::<T>(f, &mut x[len], n)
113}
114
115pub fn binary_write_vec_vec<T>(f: &mut std::fs::File, x: &[Vec<T>]) -> Result<(), Error>
116where
117    T: BinaryInputOutputSafe,
118{
119    let n = x.len();
120    binary_write_from_ref::<usize>(f, &n, 1)?;
121    for i in 0..n {
122        binary_write_vec::<T>(f, &x[i])?;
123    }
124    Ok(())
125}
126
127pub fn binary_read_vec_vec<T>(f: &mut std::fs::File, x: &mut Vec<Vec<T>>) -> Result<(), Error>
128where
129    T: BinaryInputOutputSafe + Clone,
130{
131    let mut n: usize = 0;
132    binary_read_to_ref::<usize>(f, &mut n, 1)?;
133    let len = x.len();
134    if len + n > x.capacity() {
135        let extra: usize = len + n - x.capacity();
136        x.reserve(extra);
137    }
138    x.resize(len + n, Vec::<T>::new());
139    for i in 0..n {
140        binary_read_vec::<T>(f, &mut x[i])?;
141    }
142    Ok(())
143}
binary_vec_io/lib.rs

binary_vec_io/
lib.rs
