liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

use crate::error::OxenError;

use rocksdb::{DBWithThreadMode, IteratorMode, ThreadMode};
use std::str;

/// More efficient than get since it does not actual deserialize the value
pub fn has_key<T: ThreadMode, S: AsRef<str>>(db: &DBWithThreadMode<T>, key: S) -> bool {
    let key = key.as_ref();
    let bytes = key.as_bytes();
    match db.get_pinned(bytes) {
        Ok(Some(_value)) => true,
        Ok(None) => false,
        Err(err) => {
            log::error!("Error checking for entry: {err}");
            false
        }
    }
}

/// Count the number of entries in the db
pub fn count<T: ThreadMode>(db: &DBWithThreadMode<T>) -> Result<usize, OxenError> {
    let iter = db.iterator(IteratorMode::Start);
    Ok(iter.count())
}

/// Remove key from database
pub fn delete<T: ThreadMode, S: AsRef<str>>(
    db: &DBWithThreadMode<T>,
    key: S,
) -> Result<(), OxenError> {
    let key = key.as_ref();
    log::debug!("kv_db::delete {:?} from db: {:?}", key, db.path());

    db.delete(key)?;
    Ok(())
}

/// More efficient than `list` since it does not deserialize the values
pub fn list_keys<T: ThreadMode>(db: &DBWithThreadMode<T>) -> Result<Vec<String>, OxenError> {
    let iter = db.iterator(IteratorMode::Start);
    let mut keys: Vec<String> = vec![];
    for item in iter {
        match item {
            Ok((key, _value)) => {
                match str::from_utf8(&key) {
                    Ok(key) => {
                        // return full path
                        keys.push(String::from(key));
                    }
                    _ => {
                        log::error!("list_keys() Could not decode key {key:?}")
                    }
                }
            }
            _ => {
                return Err(OxenError::basic_str(
                    "Could not read iterate over db values",
                ));
            }
        }
    }
    Ok(keys)
}

/// Remove all values from the db
pub fn clear<T: ThreadMode>(db: &DBWithThreadMode<T>) -> Result<(), OxenError> {
    let iter = db.iterator(IteratorMode::Start);
    for item in iter {
        match item {
            Ok((key, _)) => {
                db.delete(key)?;
            }
            _ => {
                return Err(OxenError::basic_str(
                    "Could not read iterate over db values",
                ));
            }
        }
    }
    Ok(())
}