liboxen 0.48.3

Oxen is a fast, unstructured data version control, to help version large machine learning datasets written in Rust.
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121

use std::path::Path;

use super::Migrate;

use crate::core::workspaces::workspace_name_index;
use crate::error::OxenError;
use crate::model::LocalRepository;
use crate::model::Workspace;
use crate::repositories;
use crate::util;
use crate::util::progress_bar::{ProgressBarType, oxen_progress_bar};

pub struct AddWorkspaceNameIndexMigration;

impl Migrate for AddWorkspaceNameIndexMigration {
    fn name(&self) -> &'static str {
        "add_workspace_name_index"
    }

    fn description(&self) -> &'static str {
        "Creates a RocksDB index mapping workspace names to IDs for O(1) lookup"
    }

    fn up(&self, path: &Path, all: bool) -> Result<(), OxenError> {
        if all {
            run_on_all_repos(path)?;
        } else {
            let repo = LocalRepository::from_dir(path)?;
            run_on_one_repo(&repo)?;
        }
        Ok(())
    }

    fn down(&self, path: &Path, all: bool) -> Result<(), OxenError> {
        if all {
            revert_all_repos(path)?;
        } else {
            let repo = LocalRepository::from_dir(path)?;
            revert_one_repo(&repo)?;
        }
        Ok(())
    }

    fn is_needed(&self, repo: &LocalRepository) -> Result<bool, OxenError> {
        // Needed if workspace directory exists but index does not
        let workspaces_dir = Workspace::workspaces_dir(repo);
        Ok(workspaces_dir.exists() && !workspace_name_index::index_exists(repo))
    }
}

fn run_on_all_repos(path: &Path) -> Result<(), OxenError> {
    println!("🐂 Collecting namespaces to migrate...");
    let namespaces = repositories::list_namespaces(path)?;
    let bar = oxen_progress_bar(namespaces.len() as u64, ProgressBarType::Counter);
    println!("🐂 Migrating {} namespaces", namespaces.len());
    for namespace in namespaces {
        let namespace_path = path.join(namespace);
        let repos = repositories::list_repos_in_namespace(&namespace_path);
        for repo in repos {
            match run_on_one_repo(&repo) {
                Ok(_) => {}
                Err(err) => {
                    log::error!(
                        "Could not create workspace name index for repo {:?}\nErr: {}",
                        util::fs::canonicalize(repo.path),
                        err
                    );
                }
            }
        }
        bar.inc(1);
    }
    Ok(())
}

fn run_on_one_repo(repo: &LocalRepository) -> Result<(), OxenError> {
    let workspaces_dir = Workspace::workspaces_dir(repo);
    if !workspaces_dir.exists() {
        return Ok(());
    }

    log::info!("Creating workspace name index for repo: {:?}", repo.path);

    let idx = workspace_name_index::get_index(repo)?;
    idx.rebuild_from_disk(repo)?;
    Ok(())
}

fn revert_all_repos(path: &Path) -> Result<(), OxenError> {
    println!("🐂 Collecting namespaces to revert...");
    let namespaces = repositories::list_namespaces(path)?;
    let bar = oxen_progress_bar(namespaces.len() as u64, ProgressBarType::Counter);
    println!("🐂 Reverting {} namespaces", namespaces.len());
    for namespace in namespaces {
        let namespace_path = path.join(namespace);
        let repos = repositories::list_repos_in_namespace(&namespace_path);
        for repo in repos {
            match revert_one_repo(&repo) {
                Ok(_) => {}
                Err(err) => {
                    log::error!(
                        "Could not revert workspace name index for repo {:?}\nErr: {}",
                        util::fs::canonicalize(repo.path),
                        err
                    );
                }
            }
        }
        bar.inc(1);
    }
    Ok(())
}

fn revert_one_repo(repo: &LocalRepository) -> Result<(), OxenError> {
    let index_dir = workspace_name_index::index_dir(repo);
    workspace_name_index::remove_from_cache(repo);
    if index_dir.exists() {
        util::fs::remove_dir_all(index_dir)?;
    }
    Ok(())
}