liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68

use std::path::Path;

use crate::{
    constants::{LAST_MIGRATION_FILE, OXEN_HIDDEN_DIR},
    error::OxenError,
    namespaces, repositories,
    view::repository::RepositoryListView,
};

pub fn list_unmigrated(
    data_dir: &Path,
    migration_tstamp: String,
) -> Result<Vec<RepositoryListView>, OxenError> {
    let mut result: Vec<RepositoryListView> = vec![];

    let global_last_migration = data_dir.join(crate::constants::LAST_MIGRATION_FILE);

    if !global_last_migration.exists() {
        return Err(OxenError::basic_str(
            "No global migration file found on server.",
        ));
    }

    let global_last_migration = std::fs::read_to_string(&global_last_migration)?;

    if global_last_migration >= migration_tstamp {
        log::debug!(
            "Global last migration file indicates all files successfully migrated up to {migration_tstamp}"
        );
        return Ok(result);
    }

    let namespaces = namespaces::list(data_dir);
    for namespace in namespaces {
        let namespace_path = data_dir.join(&namespace);
        for repo in repositories::list_repos_in_namespace(&namespace_path) {
            let repo_name = repo.path.file_name().unwrap().to_str().unwrap();

            let repo_last_migration = repo.path.join(OXEN_HIDDEN_DIR).join(LAST_MIGRATION_FILE);

            // If no repo-level migration file, the repo is migrated up to the global migration.
            // If we're here, that's out of date.
            if !repo_last_migration.exists() {
                log::debug!(
                    "No repo migration file found for {} looking in {}",
                    repo_name,
                    repo_last_migration.to_string_lossy()
                );
                result.push(RepositoryListView {
                    namespace: namespace.clone(),
                    name: repo_name.to_string(),
                    min_version: None,
                });
            } else if let Ok(repo_last_migration) = std::fs::read_to_string(&repo_last_migration)
                && repo_last_migration <= migration_tstamp
            {
                log::debug!("Repo migration file found for {repo_name} is out of date");
                result.push(RepositoryListView {
                    namespace: namespace.clone(),
                    name: repo_name.to_string(),
                    min_version: None,
                });
            }
        }
    }

    Ok(result)
}