liboxen 0.53.0

Oxen is a fast data version control system, built with machine learning training data in mind. Designed to handle terabytes of data with ease, using a workflow similar to git. Version both structured and unstructured data of any modality: text, images, video, audio, CSV, Parquet, JSONL, model checkpoints, and more. liboxen is the embeddable core library behind the oxen CLI and server, which power fine tuning and inference pipelines for multimodal LLMs, image models, and video models on Oxen.ai.
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104

use std::path::{Path, PathBuf};

use crate::model::CommitEntry;
use serde::{Deserialize, Serialize};

use super::merkle_tree::node::FileNode;

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct EntryMergeConflict {
    pub lca_entry: CommitEntry,   // Least Common Ancestor Entry
    pub base_entry: CommitEntry,  // Entry that existed in the base commit
    pub merge_entry: CommitEntry, // Entry we are trying to merge in
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct NodeMergeConflict {
    pub lca_entry: (FileNode, PathBuf),  // Least Common Ancestor Entry
    pub base_entry: (FileNode, PathBuf), // Entry that existed in the base commit
    pub merge_entry: (FileNode, PathBuf), // Entry we are trying to merge in
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MergeConflict {
    pub lca_entry: MergeConflictEntry,
    pub base_entry: MergeConflictEntry,
    pub merge_entry: MergeConflictEntry,
}

impl MergeConflict {
    pub fn to_entry_merge_conflict(&self) -> EntryMergeConflict {
        EntryMergeConflict {
            lca_entry: self.lca_entry.to_commit_entry(),
            base_entry: self.base_entry.to_commit_entry(),
            merge_entry: self.merge_entry.to_commit_entry(),
        }
    }
}

#[derive(Deserialize, Serialize, Debug, Clone)]
pub struct MergeConflictEntry {
    pub path: PathBuf,
    pub filename: String,
    pub hash: String,
    pub commit_id: String,
}

impl MergeConflictEntry {
    pub fn to_commit_entry(&self) -> CommitEntry {
        CommitEntry {
            path: self.path.clone(),
            hash: self.hash.clone(),
            commit_id: self.commit_id.clone(),
            // TODO: We need to get rid of CommitEntry after the v19 migration
            num_bytes: 0,
            last_modified_seconds: 0,
            last_modified_nanoseconds: 0,
        }
    }
}

impl EntryMergeConflict {
    pub fn to_merge_conflict(&self) -> MergeConflict {
        MergeConflict {
            lca_entry: self.lca_entry.to_merge_conflict_entry(),
            base_entry: self.base_entry.to_merge_conflict_entry(),
            merge_entry: self.merge_entry.to_merge_conflict_entry(),
        }
    }
}

impl NodeMergeConflict {
    pub fn to_merge_conflict(&self) -> MergeConflict {
        MergeConflict {
            lca_entry: to_merge_conflict_entry(&self.lca_entry.0, &self.lca_entry.1),
            base_entry: to_merge_conflict_entry(&self.base_entry.0, &self.base_entry.1),
            merge_entry: to_merge_conflict_entry(&self.merge_entry.0, &self.merge_entry.1),
        }
    }
}

impl CommitEntry {
    fn to_merge_conflict_entry(&self) -> MergeConflictEntry {
        MergeConflictEntry {
            path: self.path.clone(),
            filename: self
                .path
                .file_name()
                .unwrap()
                .to_string_lossy()
                .into_owned(),
            hash: self.hash.clone(),
            commit_id: self.commit_id.clone(),
        }
    }
}

fn to_merge_conflict_entry(node: &FileNode, path: &Path) -> MergeConflictEntry {
    MergeConflictEntry {
        path: path.to_path_buf(),
        filename: path.file_name().unwrap().to_string_lossy().into_owned(),
        hash: node.hash().to_string(),
        commit_id: node.last_commit_id().to_string(),
    }
}