liboxen 0.9.4-beta3

Oxen is a fast, unstructured data version control, to help version datasets, written in Rust.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

use bytesize;
use fs_extra::dir::get_size;
use rayon::prelude::*;
use std::path::Path;

use crate::api;
use crate::core;
use crate::error::OxenError;
use crate::model::{LocalRepository, Namespace};
use crate::util;

pub fn list(path: &Path) -> Vec<String> {
    log::debug!("api::local::namespaces::list",);
    let mut results: Vec<String> = vec![];

    if let Ok(dir) = std::fs::read_dir(path) {
        for entry in dir.into_iter().filter_map(|e| e.ok()) {
            // if the directory has a .oxen dir, let's add it, otherwise ignore
            let path = entry.path();

            log::debug!("api::local::namespaces::list checking path {:?}", path);

            if path.is_dir() && !util::fs::is_in_oxen_hidden_dir(&path) {
                results.push(path.file_name().unwrap().to_str().unwrap().to_string())
            }
        }
    }

    results
}

// return the named namespace object
pub fn get(data_dir: &Path, name: &str) -> Result<Option<Namespace>, OxenError> {
    log::debug!("api::local::namespaces::get {}", name);
    let namespace_path = data_dir.join(name);

    if !namespace_path.is_dir() {
        return Ok(None);
    }

    let mut namespace = Namespace {
        name: name.to_string(),
        storage_usage_gb: 0.0,
    };

    let repos = api::local::repositories::list_repos_in_namespace(&namespace_path);
    // Get storage per repo in parallel and sum up
    namespace.storage_usage_gb = repos
        .par_iter()
        .map(get_storage_for_repo)
        .sum::<Result<u64, OxenError>>()? as f64
        / bytesize::GB as f64;

    Ok(Some(namespace))
}

fn get_storage_for_repo(repo: &LocalRepository) -> Result<u64, OxenError> {
    let latest_commit = match api::local::commits::latest_commit(repo) {
        Ok(commit) => commit,
        Err(e) => {
            log::warn!(
                "api::local::namespaces::get_storage_for_repo no latest commit for repo {:?}: {}",
                repo.path,
                e
            );
            return Ok(0);
        }
    };

    let cache_path = core::cache::cachers::repo_size::repo_size_path(repo, &latest_commit);
    log::debug!(
        "api::local::namespaces::get_storage_for_repo cache path {:?}",
        cache_path
    );
    match util::fs::read_from_path(&cache_path) {
        Ok(size) => {
            log::debug!(
                "api::local::namespaces::get_storage_for_repo got repo size {:?}",
                cache_path
            );
            match size.parse::<u64>() {
                Ok(size) => Ok(size),
                Err(e) => {
                    log::error!(
                        "api::local::namespaces::get_storage_for_repo error parsing size: {}",
                        e
                    );
                    Err(OxenError::basic_str(e.to_string()))
                }
            }
        }
        Err(_) => {
            log::warn!("api::local::namespaces::get_storage_for_repo cache file not found, calculating size");
            match get_size(&repo.path) {
                Ok(size) => Ok(size),
                Err(e) => {
                    log::error!(
                        "api::local::namespaces::get_storage_for_repo error getting size: {}",
                        e
                    );
                    Err(OxenError::basic_str(e.to_string()))
                }
            }
        }
    }
}