liboxen 0.61.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67

use std::path::Path;

use super::{Direction, Migrate};
use crate::error::OxenError;
use crate::model::LocalRepository;
use crate::repositories::placement::{ENV_CLOSE_WAIT, place_by_uuid};
use crate::sync_dir::NAME_TABLE_DIR;

pub struct PlaceRepositoryByUuidMigration;

impl Migrate for PlaceRepositoryByUuidMigration {
    fn name(&self) -> &'static str {
        "place_repository_by_uuid"
    }

    fn description(&self) -> &'static str {
        "Moves a server's repository to the directory its UUID places it in"
    }

    /// Moves the repository's directory, so `repo` names a directory that is gone once this
    /// returns. Run under the repository's exclusive lock.
    fn up(&self, repo: LocalRepository) -> Result<(), OxenError> {
        let repo_dir = repo.path.clone();
        // Dropped first, so its own LMDB env cannot hold off the move.
        drop(repo);
        let Some(sync_dir) = server_sync_dir(&repo_dir) else {
            return Err(OxenError::internal_error(format!(
                "{repo_dir:?} is not in a server's sync directory"
            )));
        };
        let placed_dir = place_by_uuid(sync_dir, &repo_dir, ENV_CLOSE_WAIT)?;
        log::info!("Moved repository {repo_dir:?} to {placed_dir:?}");
        Ok(())
    }

    fn down(&self, _repo: LocalRepository) -> Result<(), OxenError> {
        Err(OxenError::internal_error(
            "place_repository_by_uuid is up-only: the server creates every new repository where \
             its UUID places it, so there is nothing to move back to",
        ))
    }

    /// Always false: the server serves a repository from either layout, and a client-side
    /// repository is in neither.
    fn is_needed(&self, _repo: &LocalRepository) -> Result<bool, OxenError> {
        Ok(false)
    }

    /// Optional and up-only, for a repository at `{namespace}/{name}` in a server's sync directory.
    fn is_applicable(
        &self,
        direction: Direction,
        repo: &LocalRepository,
    ) -> Result<bool, OxenError> {
        match direction {
            Direction::Up => Ok(server_sync_dir(&repo.path).is_some()),
            Direction::Down => Ok(false),
        }
    }
}

/// The sync directory `repo_dir` sits at `{namespace}/{name}` in, recognized by the server's name
/// table beside the namespaces.
fn server_sync_dir(repo_dir: &Path) -> Option<&Path> {
    let sync_dir = repo_dir.parent()?.parent()?;
    sync_dir.join(NAME_TABLE_DIR).is_dir().then_some(sync_dir)
}