liboxen 0.46.7

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135

//! # oxen db list
//!
//! Print out values from an oxen database
//!

use crate::error::OxenError;
use crate::model::merkle_tree::node::StagedMerkleTreeNode;
use crate::util::progress_bar::spinner_with_msg;

use rocksdb::{DB, IteratorMode, LogLevel, Options};
use std::path::Path;
use std::str;

/// List the key -> value pairs in a database
pub fn list(path: impl AsRef<Path>, limit: Option<usize>) -> Result<(), OxenError> {
    let path = path.as_ref();
    let mut opts = Options::default();
    opts.set_log_level(LogLevel::Fatal);

    let db = DB::open_for_read_only(&opts, dunce::simplified(path), false)?;
    let iter = db.iterator(IteratorMode::Start);
    let mut count = 0;
    for item in iter {
        if let Some(limit) = limit
            && count >= limit
        {
            break;
        }

        match item {
            Ok((key, value)) => {
                let key = if let Ok(key) = str::from_utf8(&key) {
                    key.to_string()
                } else {
                    // deserialize as u128
                    if key.len() == 16 {
                        let key: [u8; 16] = (*key).try_into().map_err(|_| {
                            OxenError::basic_str("Could not convert key to [u8; 16]")
                        })?;
                        let key = u128::from_le_bytes(key);
                        format!("{key}")
                    } else {
                        return Err(OxenError::basic_str(
                            "Could not read iterate over db values",
                        ));
                    }
                };

                // try deserialize as StagedMerkleTreeNode
                let val: Result<StagedMerkleTreeNode, rmp_serde::decode::Error> =
                    rmp_serde::from_slice(&value);
                match val {
                    Ok(val) => {
                        println!("{key}\t{val}");
                    }
                    Err(_) => {
                        if let Ok(val) = str::from_utf8(&value) {
                            println!("{key}\t{val}");
                        } else {
                            println!("{key}\t<binary data>");
                        }
                    }
                }
            }
            _ => {
                return Err(OxenError::basic_str(
                    "Could not read iterate over db values",
                ));
            }
        }
        count += 1;
    }

    println!("{count} total entries");

    Ok(())
}

/// Count the values in a database
pub fn count(path: impl AsRef<Path>) -> Result<usize, OxenError> {
    let path = path.as_ref();
    let opts = Options::default();
    log::debug!("Opening db at {path:?}");
    let db = DB::open_for_read_only(&opts, dunce::simplified(path), false)?;
    log::debug!("Opened db at {path:?}");
    let iter = db.iterator(IteratorMode::Start);
    log::debug!("Iterating over db at {path:?}");
    let progress = spinner_with_msg(format!("Counting db at {path:?}"));
    let mut count = 0;
    for _ in iter {
        count += 1;
        progress.inc(1);
        progress.set_message(format!("{count} entries"));
    }
    progress.finish_and_clear();
    Ok(count)
}

/// Get a value from a database
pub fn get(
    path: impl AsRef<Path>,
    key: impl AsRef<str>,
    dtype: Option<&str>,
) -> Result<String, OxenError> {
    let path = path.as_ref();
    let str_key = key.as_ref();
    let mut opts = Options::default();
    opts.set_log_level(LogLevel::Fatal);

    let key = if let Some(dtype) = dtype {
        if dtype == "u128" {
            let key = str_key.parse::<u128>()?;
            key.to_le_bytes().to_vec()
        } else {
            str_key.as_bytes().to_vec()
        }
    } else {
        str_key.as_bytes().to_vec()
    };

    log::debug!("Opening db at {path:?}");
    let db = DB::open_for_read_only(&opts, dunce::simplified(path), false)?;
    log::debug!("Opened db at {path:?}");

    if let Some(value) = db.get(key)? {
        log::debug!("Got value from db at {path:?}");
        if let Ok(value) = str::from_utf8(&value) {
            Ok(value.to_string())
        } else {
            Ok(format!("<{} bytes>", value.len()))
        }
    } else {
        Err(OxenError::basic_str(format!("Key {str_key} not found")))
    }
}