liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81

use rayon::prelude::*;
use std::path::Path;

use crate::error::OxenError;
use crate::model::{LocalRepository, Namespace};
use crate::repositories;
use crate::util;

pub fn list(path: &Path) -> Vec<String> {
    log::debug!("repositories::namespaces::list",);
    let mut results: Vec<String> = vec![];

    if let Ok(dir) = std::fs::read_dir(path) {
        for entry in dir.into_iter().filter_map(|e| e.ok()) {
            // if the directory has a .oxen dir, let's add it, otherwise ignore
            let path = entry.path();

            log::debug!("repositories::namespaces::list checking path {path:?}");

            if path.is_dir() && !util::fs::is_in_oxen_hidden_dir(&path) {
                results.push(path.file_name().unwrap().to_str().unwrap().to_string())
            }
        }
    }

    results
}

// return the named namespace object
pub fn get(data_dir: &Path, name: &str) -> Result<Option<Namespace>, OxenError> {
    log::debug!("repositories::namespaces::get {name}");
    let namespace_path = data_dir.join(name);

    if !namespace_path.is_dir() {
        return Ok(None);
    }

    let mut namespace = Namespace {
        name: name.to_string(),
        storage_usage_gb: 0.0,
    };

    let repos: Vec<LocalRepository> =
        repositories::list_repos_in_namespace(&namespace_path).collect();
    // Get storage per repo in parallel and sum up
    namespace.storage_usage_gb = repos
        .par_iter()
        .map(get_storage_for_repo)
        .sum::<Result<u64, OxenError>>()? as f64
        / bytesize::GB as f64;

    Ok(Some(namespace))
}

fn get_storage_for_repo(repo: &LocalRepository) -> Result<u64, OxenError> {
    log::debug!(
        "repositories::namespaces::get_storage_for_repo for repo {:?}",
        repo.path
    );

    match repositories::size::get_size(repo) {
        Ok(size_file) => match size_file.status {
            repositories::size::SizeStatus::Done => {
                log::debug!("Got repo size: {} bytes", size_file.size);
                Ok(size_file.size)
            }
            repositories::size::SizeStatus::Pending => {
                log::info!("Size calculation is still pending, returning previous size");
                Ok(size_file.size)
            }
            repositories::size::SizeStatus::Error => {
                tracing::error!(repo = ?repo.path, "Repo size calculation failed");
                Err(OxenError::basic_str("Size calculation failed"))
            }
        },
        Err(e) => {
            tracing::error!(repo = ?repo.path, cause = ?e, "Error getting repo size");
            Err(e)
        }
    }
}