liboxen 0.48.0

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147

use crate::error::OxenError;

use rocksdb::{DBWithThreadMode, IteratorMode, MultiThreaded};
use std::collections::HashMap;
use std::mem;
use std::str;

pub fn insert_indices<S: AsRef<str>>(
    db: &DBWithThreadMode<MultiThreaded>,
    key: S,
    indices: Vec<u32>,
) -> Result<(), OxenError> {
    let key = key.as_ref();

    // Could not use the bytevec library here when inserting a larger set of indices
    let byte_indices = u32_to_u8(indices);

    db.put(key, byte_indices)?;
    Ok(())
}

pub fn get_indices<S: AsRef<str>>(
    db: &DBWithThreadMode<MultiThreaded>,
    key: S,
) -> Result<Option<Vec<u32>>, OxenError> {
    let key = key.as_ref();
    let bytes = key.as_bytes();
    match db.get(bytes) {
        Ok(Some(raw_indices)) => {
            // found it
            let decoded = u32_from_u8(raw_indices);

            Ok(Some(decoded))
        }
        Ok(None) => {
            // did not get val
            Ok(None)
        }
        Err(err) => {
            // error from the DB
            let err = format!("Err could not fetch value {key:?} from db: {err:?}",);
            Err(OxenError::basic_str(err))
        }
    }
}

pub fn list_indices(
    db: &DBWithThreadMode<MultiThreaded>,
) -> Result<Vec<(String, Vec<u32>)>, OxenError> {
    let iter = db.iterator(IteratorMode::Start);
    let mut results: Vec<(String, Vec<u32>)> = vec![];
    for item in iter {
        match item {
            Ok((key, value)) => match str::from_utf8(&key) {
                Ok(key) => {
                    let key = String::from(key);
                    let mut buffer: Vec<u8> = vec![];
                    for v in &*value {
                        buffer.push(*v);
                    }
                    let value = u32_from_u8(buffer);
                    results.push((key, value));
                }
                _ => {
                    return Err(OxenError::basic_str("Could not read utf8 val..."));
                }
            },
            _ => {
                return Err(OxenError::basic_str(
                    "Could not read iterate over db values",
                ));
            }
        }
    }
    Ok(results)
}

pub fn hash_map_indices(
    db: &DBWithThreadMode<MultiThreaded>,
) -> Result<HashMap<String, Vec<u32>>, OxenError> {
    let iter = db.iterator(IteratorMode::Start);
    let mut results: HashMap<String, Vec<u32>> = HashMap::new();
    for item in iter {
        match item {
            Ok((key, value)) => match str::from_utf8(&key) {
                Ok(key) => {
                    let key = String::from(key);
                    let mut buffer: Vec<u8> = vec![];
                    for v in &*value {
                        buffer.push(*v);
                    }
                    let value = u32_from_u8(buffer);
                    results.insert(key, value);
                }
                _ => {
                    log::error!("str_val_db::list() Could not decoded keys and values.")
                }
            },
            _ => {
                return Err(OxenError::basic_str(
                    "Could not read iterate over db values",
                ));
            }
        }
    }
    Ok(results)
}

pub fn u32_to_u8(mut indices: Vec<u32>) -> Vec<u8> {
    unsafe {
        let ratio = mem::size_of::<u32>() / mem::size_of::<u8>();

        let length = indices.len() * ratio;
        let capacity = indices.capacity() * ratio;
        let ptr = indices.as_mut_ptr() as *mut u8;

        // Don't run the destructor for vec32
        mem::forget(indices);

        // Construct new Vec
        Vec::from_raw_parts(ptr, length, capacity)
    }
}

pub fn u32_from_u8(v: Vec<u8>) -> Vec<u32> {
    let data = v.as_ptr();
    let len = v.len();
    let capacity = v.capacity();
    let element_size = mem::size_of::<u32>();

    // Make sure we have a proper amount of capacity (may be overkill)
    assert_eq!(capacity % element_size, 0);
    // Make sure we are going to read a full chunk of stuff
    assert_eq!(len % element_size, 0);

    unsafe {
        // Don't allow the current vector to be dropped
        // (which would invalidate the memory)
        mem::forget(v);

        Vec::from_raw_parts(
            data as *mut u32,
            len / element_size,
            capacity / element_size,
        )
    }
}