liboxen 0.46.12

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114

use serde::{Deserialize, Serialize};
use std::fmt;

use crate::{error::OxenError, model::LocalRepository, util};
use std::path::PathBuf;

#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum SizeStatus {
    Pending,
    Done,
    Error,
}

impl fmt::Display for SizeStatus {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match self {
            SizeStatus::Pending => write!(f, "pending"),
            SizeStatus::Done => write!(f, "done"),
            SizeStatus::Error => write!(f, "error"),
        }
    }
}

#[derive(Serialize, Deserialize, Debug, Clone)]
pub struct RepoSizeFile {
    pub status: SizeStatus,
    pub size: u64,
}

impl fmt::Display for RepoSizeFile {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        match serde_json::to_string(self) {
            Ok(s) => write!(f, "{s}"),
            Err(_) => write!(f, ""),
        }
    }
}

pub fn update_size(repo: &LocalRepository) -> Result<(), OxenError> {
    let path = repo_size_path(repo);
    let size = match util::fs::read_from_path(&path) {
        Ok(content) => match serde_json::from_str::<RepoSizeFile>(&content) {
            Ok(parsed) => RepoSizeFile {
                status: SizeStatus::Pending,
                size: parsed.size,
            },
            Err(e) => {
                return Err(OxenError::basic_str(format!(
                    "Failed to parse size file: {e}"
                )));
            }
        },
        Err(e) => {
            log::info!("Size file not found, creating it: {e}");

            RepoSizeFile {
                status: SizeStatus::Pending,
                size: 0,
            }
        }
    };

    util::fs::write_to_path(&path, size.to_string())?;

    let repo_path = repo.path.clone();
    let path_clone = path.clone();

    // Spawn background thread for size calculation
    std::thread::spawn(move || {
        let size_result = fs_extra::dir::get_size(&repo_path);
        match size_result {
            Ok(calculated_size) => {
                let size = RepoSizeFile {
                    status: SizeStatus::Done,
                    size: calculated_size,
                };
                if let Err(e) = util::fs::write_to_path(&path_clone, size.to_string()) {
                    log::error!("Failed to write size result: {e}");
                }
            }
            Err(e) => {
                log::error!("Failed to calculate repository size: {e}");
                let size = RepoSizeFile {
                    status: SizeStatus::Error,
                    size: 0,
                };
                let _ = util::fs::write_to_path(&path_clone, size.to_string());
            }
        }
    });

    Ok(())
}

pub fn get_size(repo: &LocalRepository) -> Result<RepoSizeFile, OxenError> {
    let path = repo_size_path(repo);
    let size = util::fs::read_from_path(&path);
    match size {
        Ok(size) => {
            let size: RepoSizeFile = serde_json::from_str(&size)?;
            Ok(size)
        }
        Err(e) => {
            log::info!("Size file not found, creating it: {e}");
            update_size(repo)?;
            get_size(repo)
        }
    }
}

pub fn repo_size_path(repo: &LocalRepository) -> PathBuf {
    util::fs::oxen_hidden_dir(&repo.path).join("repo_size.toml")
}