liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95

use crate::api;
use crate::error::OxenError;
use crate::model::LocalRepository;
use crate::repositories;

use crate::core::v_latest::fetch;
use crate::opts::fetch_opts::FetchOpts;

#[tracing::instrument(skip(repo), fields(repo_path = %repo.path.display()))]
pub async fn pull(repo: &LocalRepository) -> Result<(), OxenError> {
    let mut fetch_opts = FetchOpts::new();
    fetch_opts.depth = repo.depth();
    fetch_opts.subtree_paths = repo.subtree_paths();
    pull_remote_branch(repo, &fetch_opts).await
}

#[tracing::instrument(skip(repo), fields(repo_path = %repo.path.display()))]
pub async fn pull_all(repo: &LocalRepository) -> Result<(), OxenError> {
    let fetch_opts = FetchOpts {
        all: true,
        depth: repo.depth(),
        subtree_paths: repo.subtree_paths(),
        ..FetchOpts::new()
    };
    repositories::pull_remote_branch(repo, &fetch_opts).await
}

/// Pull a specific remote and branch
#[tracing::instrument(skip(repo, fetch_opts), fields(repo_path = %repo.path.display()))]
pub async fn pull_remote_branch(
    repo: &LocalRepository,
    fetch_opts: &FetchOpts,
) -> Result<(), OxenError> {
    let remote = &fetch_opts.remote;
    let branch = &fetch_opts.branch;
    let mut fetch_opts = fetch_opts.clone();
    println!("🐂 oxen pull {remote} {branch}");

    let remote = repo
        .get_remote(remote)
        .ok_or_else(|| OxenError::RemoteNotSet(remote.clone()))?;

    let remote_repo = api::client::repositories::get_by_remote(&remote).await?;

    api::client::repositories::pre_pull(&remote_repo).await?;

    let previous_head_commit = repositories::commits::head_commit_maybe(repo)?;

    // Fetch all the tree nodes and the entries
    fetch_opts.should_update_branch_head = false;
    let remote_branch = fetch::fetch_remote_branch(repo, &remote_repo, &fetch_opts).await?;

    let new_head_commit = repositories::revisions::get(repo, &remote_branch.commit_id)?
        .ok_or_else(|| OxenError::RevisionNotFound(remote_branch.commit_id.to_owned().into()))?;

    match previous_head_commit {
        Some(previous_head_commit) => {
            log::debug!(
                "checking if we need to merge previous {} new {}",
                previous_head_commit.id,
                new_head_commit.id
            );
            if previous_head_commit.id != new_head_commit.id {
                match repositories::merge::merge_commit_into_base(
                    repo,
                    &new_head_commit,
                    &previous_head_commit,
                )
                .await
                {
                    Ok(Some(commit)) => {
                        // Merge successful, update branch head
                        repositories::branches::update(repo, branch, commit.id)?;
                    }
                    Ok(None) => {
                        // Merge conflict, keep the previous commit
                        return Err(OxenError::UpstreamMergeConflict(
                            "There was a merge conflict, please resolve it before pulling.".into(),
                        ));
                    }
                    Err(e) => return Err(e),
                }
            }
        }

        None => {
            repositories::branches::update(repo, branch, new_head_commit.id)?;
            repositories::checkout::checkout(repo, branch).await?;
        }
    }

    api::client::repositories::post_pull(&remote_repo).await?;

    Ok(())
}