liboxen 0.58.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116

use rayon::prelude::*;
use std::path::Path;

use crate::model::{LocalRepository, Namespace};
use crate::repositories;
use crate::repositories::size::{self, RepoSizeFile, SizeStatus};
use crate::sync_dir::{is_namespace, namespace_dirs};

pub fn list(path: &Path) -> Vec<String> {
    log::debug!("repositories::namespaces::list",);
    namespace_dirs(path)
        .unwrap_or_default()
        .iter()
        .filter_map(|dir| dir.file_name()?.to_str().map(str::to_string))
        .collect()
}

/// The named namespace, or `None` when it has no directory on disk. Starts a size recalculation for
/// every repository that has no figure to count, so the total is a lower bound that later reads
/// converge on, and a warning names how many repositories are counted at an unfinished figure.
pub fn get(data_dir: &Path, name: &str) -> Option<Namespace> {
    log::debug!("repositories::namespaces::get {name}");
    let namespace_path = data_dir.join(name);

    if !is_namespace(name) || !namespace_path.is_dir() {
        return None;
    }

    let repos: Vec<LocalRepository> =
        repositories::list_repos_in_namespace(&namespace_path).collect();
    // Get storage per repo in parallel and sum up
    let figures: Vec<RepoSizeFile> = repos.par_iter().map(size::get_size).collect();

    // A read reports a failed pass rather than starting another, so a repository left with no
    // figure would count as nothing on every later read. Start one here for those.
    for (repo, figure) in repos.iter().zip(&figures) {
        if matches!(figure.status, SizeStatus::Error)
            && figure.size == 0
            && let Err(cause) = size::update_size(repo)
        {
            tracing::warn!(
                repo = ?repo.path,
                ?cause,
                "Could not start a size recalculation for a repository counted as nothing"
            );
        }
    }

    let outstanding = figures
        .iter()
        .filter(|figure| !matches!(figure.status, SizeStatus::Done))
        .count();
    if outstanding > 0 {
        tracing::warn!(
            namespace = name,
            outstanding,
            repositories = figures.len(),
            "Reporting a storage total that counts some repositories at a figure no pass completed"
        );
    }

    Some(Namespace {
        name: name.to_string(),
        storage_usage_gb: figures.iter().map(|figure| figure.size).sum::<u64>() as f64
            / bytesize::GB as f64,
    })
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::error::OxenError;
    use crate::repositories::size::repo_size_path;
    use crate::test;
    use crate::util;
    use crate::util::fs::AtomicFile;

    /// Leave `record` as the size a repo at `path` has recorded, without going through a commit,
    /// since what is under test is the summation rather than how a figure gets computed.
    fn repo_recording(path: &Path, record: &str) -> Result<LocalRepository, OxenError> {
        let repo = repositories::init(path)?;
        AtomicFile::new(repo_size_path(&repo)).write(record.as_bytes())?;
        Ok(repo)
    }

    #[test]
    fn test_get_sums_the_recorded_size_of_every_repo() -> Result<(), OxenError> {
        test::run_empty_dir_test(|dir| {
            assert!(
                get(dir, "ox").is_none(),
                "a namespace with no directory on disk is reported as absent"
            );

            let namespace_path = dir.join("ox");
            repo_recording(&namespace_path.join("first"), "1500")?;
            let second = repo_recording(&namespace_path.join("second"), "2500")?;

            let namespace = get(dir, "ox").expect("namespace exists");
            assert_eq!(namespace.storage_usage_gb, 4000.0 / bytesize::GB as f64);

            util::fs::write_to_path(repo_size_path(&second), "not a size record")?;
            let namespace = get(dir, "ox").expect("namespace exists");
            assert_eq!(
                namespace.storage_usage_gb,
                1500.0 / bytesize::GB as f64,
                "a repository with no readable figure leaves the rest of the total intact"
            );
            assert_eq!(
                size::wait_for_recorded_size(&second)?,
                second.version_bytes()?,
                "reading a namespace starts a recalculation for a repository with no figure"
            );
            Ok(())
        })
    }
}