liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117

use serde::{Deserialize, Serialize};
use std::fmt;

use crate::util::fs::AtomicFile;
use crate::{error::OxenError, model::LocalRepository, util};
use std::path::PathBuf;

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum SizeStatus {
    Pending,
    Done,
    Error,
}

impl fmt::Display for SizeStatus {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            SizeStatus::Pending => write!(f, "pending"),
            SizeStatus::Done => write!(f, "done"),
            SizeStatus::Error => write!(f, "error"),
        }
    }
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct RepoSizeFile {
    pub status: SizeStatus,
    pub size: u64,
}

impl fmt::Display for RepoSizeFile {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match serde_json::to_string(self) {
            Ok(s) => write!(f, "{s}"),
            Err(_) => write!(f, ""),
        }
    }
}

pub fn update_size(repo: &LocalRepository) -> Result<(), OxenError> {
    let path = repo_size_path(repo);
    let size = match util::fs::read_from_path(&path) {
        Ok(content) => match serde_json::from_str::<RepoSizeFile>(&content) {
            Ok(parsed) => RepoSizeFile {
                status: SizeStatus::Pending,
                size: parsed.size,
            },
            Err(e) => {
                return Err(OxenError::basic_str(format!(
                    "Failed to parse size file: {e}"
                )));
            }
        },
        Err(e) => {
            log::info!("Size file not found, creating it: {e}");

            RepoSizeFile {
                status: SizeStatus::Pending,
                size: 0,
            }
        }
    };

    AtomicFile::new(&path).write(size.to_string().as_bytes())?;

    let repo_path = repo.path.clone();
    let path_clone = path.clone();

    // Spawn background thread for size calculation
    std::thread::spawn(move || {
        let size_result = fs_extra::dir::get_size(&repo_path);
        match size_result {
            Ok(calculated_size) => {
                let size = RepoSizeFile {
                    status: SizeStatus::Done,
                    size: calculated_size,
                };
                if let Err(e) = AtomicFile::new(&path_clone).write(size.to_string().as_bytes()) {
                    log::error!("Failed to write size result: {e}");
                }
            }
            Err(e) => {
                log::error!("Failed to calculate repository size: {e}");
                let size = RepoSizeFile {
                    status: SizeStatus::Error,
                    size: 0,
                };
                if let Err(e) = AtomicFile::new(&path_clone).write(size.to_string().as_bytes()) {
                    log::error!("Failed to write size error status: {e}");
                }
            }
        }
    });

    Ok(())
}

pub fn get_size(repo: &LocalRepository) -> Result<RepoSizeFile, OxenError> {
    let path = repo_size_path(repo);
    let size = util::fs::read_from_path(&path);
    match size {
        Ok(size) => {
            let size: RepoSizeFile = serde_json::from_str(&size)?;
            Ok(size)
        }
        Err(e) => {
            log::info!("Size file not found, creating it: {e}");
            update_size(repo)?;
            get_size(repo)
        }
    }
}

pub fn repo_size_path(repo: &LocalRepository) -> PathBuf {
    util::fs::oxen_hidden_dir(&repo.path).join("repo_size.toml")
}